feat(cloud-workers): session placement, dispatch, and worker turn routing (#106332)

* feat(gateway-protocol): add session placement schema

Closed state discriminator for session execution placement (local/requested/provisioning/syncing/starting/active/draining/reconciling/reclaimed/failed), sessions.dispatch params, and worker-admission transcript/live cursor extensions. Swift protocol models mirror the schema.

* feat(state): add worker session placement table

worker_session_placements rows carry placement state, transition generation, worker ownership metadata, ACK cursors, and the turn claim columns used for atomic admission.

* feat(cloud-workers): add durable placement state machine store

SQLite-backed placement store split by concern: state table (placement-state), discriminated record types + shape invariants (placement-record), row codec + CAS transition values (placement-row-codec), atomic turn-claim admission/release/waiters (placement-turn-claims), and lifecycle CAS transitions (placement-store).

* feat(cloud-workers): sync workspaces and attach sessions to worker environments

Environment service session attachment + turn credentials, tunnel workspace commands over a dedicated SSH runner, and git/plain workspace sync into $HOME/.openclaw-worker/workspaces with an immutable manifest. Symlink escapes are rejected locally before transfer (macOS openrsync stat-fails them opaquely) and again by the remote manifest guard.

* feat(worker): run one-shot embedded turns from launch descriptors

Worker runtime executes a single embedded turn from a stdin launch descriptor and reports completed/failed/fenced on stdout for the gateway launcher. Terminal lifecycle live events are deferred past the final transcript flush; transcript projection helpers are shared via transcript-message instead of duplicated in the runtime.

* feat(cloud-workers): dispatch placements and route worker turns

Dispatch service drives local->requested->provisioning->syncing->starting->active with failure teardown (placement-dispatch-failure) and restart/runtime recovery incl. lost-worker reclaim (placement-dispatch-recovery). Worker turn launcher claims the placement turn atomically, builds a windowed launch descriptor (worker-turn-payload), runs the remote one-shot worker, and reconciles the committed transcript; agent runners route turns through the session placement admission provider.

* feat(gateway): expose session placement RPCs and startup reconciliation

sessions.dispatch RPC with lifecycle admission barriers, operator-facing placement projection on session listings, placement-aware session reset guard, and startup/interval reconciliation wiring for worker placements.
This commit is contained in:
Peter Steinberger
2026-07-13 08:03:53 -07:00
committed by GitHub
parent 06b27b9e1d
commit e98c7dfbcb
92 changed files with 13318 additions and 963 deletions

View File

@@ -4,126 +4,14 @@ import type { WorkerInferenceContext } from "../../packages/gateway-protocol/src
import { WORKER_INFERENCE_MAX_CONTEXT_MESSAGES } from "../../packages/gateway-protocol/src/schema/worker-inference.js";
import type { AgentMessage } from "../agents/runtime/index.js";
import type { AgentSessionWriteLockRunner } from "../agents/sessions/agent-session.js";
import type { AssistantMessage, Context, Message } from "../llm/types.js";
import { isWorkerTranscriptMessageFrameSafe } from "./transcript-message.js";
function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
return {
type: "text" as const,
text: part.text,
...(part.textSignature ? { textSignature: part.textSignature } : {}),
};
}
function cloneImageContent(part: { type: "image"; data: string; mimeType: string }) {
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
}
function cloneUsage(message: AssistantMessage): WorkerTranscriptMessage & { role: "assistant" } {
return {
role: "assistant",
content: message.content.map((part) => {
if (part.type === "text") {
return cloneTextContent(part);
}
if (part.type === "thinking") {
return {
type: "thinking" as const,
thinking: part.thinking,
...(part.thinkingSignature ? { thinkingSignature: part.thinkingSignature } : {}),
...(part.redacted === undefined ? {} : { redacted: part.redacted }),
};
}
return {
type: "toolCall" as const,
id: part.id,
name: part.name,
arguments: structuredClone(part.arguments),
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
...(part.executionMode ? { executionMode: part.executionMode } : {}),
};
}),
api: message.api,
provider: message.provider,
model: message.model,
...(message.responseModel ? { responseModel: message.responseModel } : {}),
...(message.responseId ? { responseId: message.responseId } : {}),
...(message.diagnostics
? {
diagnostics: message.diagnostics.map((diagnostic) => ({
type: diagnostic.type,
timestamp: diagnostic.timestamp,
...(diagnostic.error
? {
error: {
...(diagnostic.error.name ? { name: diagnostic.error.name } : {}),
message: diagnostic.error.message,
...(diagnostic.error.stack ? { stack: diagnostic.error.stack } : {}),
...(diagnostic.error.code === undefined ? {} : { code: diagnostic.error.code }),
},
}
: {}),
...(diagnostic.details ? { details: structuredClone(diagnostic.details) } : {}),
})),
}
: {}),
usage: {
input: message.usage.input,
output: message.usage.output,
cacheRead: message.usage.cacheRead,
cacheWrite: message.usage.cacheWrite,
...(message.usage.contextUsage
? { contextUsage: structuredClone(message.usage.contextUsage) }
: {}),
totalTokens: message.usage.totalTokens,
cost: {
input: message.usage.cost.input,
output: message.usage.cost.output,
cacheRead: message.usage.cost.cacheRead,
cacheWrite: message.usage.cost.cacheWrite,
total: message.usage.cost.total,
...(message.usage.cost.totalOrigin ? { totalOrigin: message.usage.cost.totalOrigin } : {}),
},
},
stopReason: message.stopReason,
...(message.errorMessage ? { errorMessage: message.errorMessage } : {}),
...(message.errorCode ? { errorCode: message.errorCode } : {}),
...(message.errorType ? { errorType: message.errorType } : {}),
...(message.errorBody ? { errorBody: message.errorBody } : {}),
timestamp: message.timestamp,
};
}
export function toWorkerTranscriptMessage(
message: AgentMessage,
): WorkerTranscriptMessage | undefined {
if (message.role === "user") {
const content =
typeof message.content === "string"
? [{ type: "text" as const, text: message.content }]
: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
);
return { role: "user", content, timestamp: message.timestamp };
}
if (message.role === "assistant") {
return cloneUsage(message);
}
if (message.role === "toolResult") {
return {
role: "toolResult",
toolCallId: message.toolCallId,
toolName: message.toolName,
content: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
),
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
isError: message.isError,
timestamp: message.timestamp,
};
}
return undefined;
}
import type { Context, Message } from "../llm/types.js";
import {
cloneImageContent,
cloneTextContent,
cloneUsage,
isWorkerTranscriptMessageFrameSafe,
toWorkerTranscriptMessage,
} from "./transcript-message.js";
export function toAgentMessage(message: WorkerTranscriptMessage): Message {
if (message.role === "user") {