mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-25 20:31:17 +00:00
feat(cloud-workers): session placement, dispatch, and worker turn routing (#106332)
* feat(gateway-protocol): add session placement schema Closed state discriminator for session execution placement (local/requested/provisioning/syncing/starting/active/draining/reconciling/reclaimed/failed), sessions.dispatch params, and worker-admission transcript/live cursor extensions. Swift protocol models mirror the schema. * feat(state): add worker session placement table worker_session_placements rows carry placement state, transition generation, worker ownership metadata, ACK cursors, and the turn claim columns used for atomic admission. * feat(cloud-workers): add durable placement state machine store SQLite-backed placement store split by concern: state table (placement-state), discriminated record types + shape invariants (placement-record), row codec + CAS transition values (placement-row-codec), atomic turn-claim admission/release/waiters (placement-turn-claims), and lifecycle CAS transitions (placement-store). * feat(cloud-workers): sync workspaces and attach sessions to worker environments Environment service session attachment + turn credentials, tunnel workspace commands over a dedicated SSH runner, and git/plain workspace sync into $HOME/.openclaw-worker/workspaces with an immutable manifest. Symlink escapes are rejected locally before transfer (macOS openrsync stat-fails them opaquely) and again by the remote manifest guard. * feat(worker): run one-shot embedded turns from launch descriptors Worker runtime executes a single embedded turn from a stdin launch descriptor and reports completed/failed/fenced on stdout for the gateway launcher. Terminal lifecycle live events are deferred past the final transcript flush; transcript projection helpers are shared via transcript-message instead of duplicated in the runtime. * feat(cloud-workers): dispatch placements and route worker turns Dispatch service drives local->requested->provisioning->syncing->starting->active with failure teardown (placement-dispatch-failure) and restart/runtime recovery incl. lost-worker reclaim (placement-dispatch-recovery). Worker turn launcher claims the placement turn atomically, builds a windowed launch descriptor (worker-turn-payload), runs the remote one-shot worker, and reconciles the committed transcript; agent runners route turns through the session placement admission provider. * feat(gateway): expose session placement RPCs and startup reconciliation sessions.dispatch RPC with lifecycle admission barriers, operator-facing placement projection on session listings, placement-aware session reset guard, and startup/interval reconciliation wiring for worker placements.
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
06b27b9e1d
commit
e98c7dfbcb
@@ -66,6 +66,7 @@ import type { AgentRunSessionTarget } from "../run-session-target.js";
|
||||
import { resolveAgentRunAbortLifecycleFields } from "../run-termination.js";
|
||||
import { buildAgentRuntimeAuthPlan } from "../runtime-plan/auth.js";
|
||||
import type { AgentMessage } from "../runtime/index.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../session-placement-admission.js";
|
||||
import { buildUsageWithNoCost } from "../stream-message-shared.js";
|
||||
import {
|
||||
buildClaudeCliFallbackContextPrelude,
|
||||
@@ -715,127 +716,139 @@ export function runAgentAttempt(params: {
|
||||
...mutableCliSessionStore,
|
||||
}
|
||||
: undefined;
|
||||
return runCliAgent({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionEntry: params.sessionEntry,
|
||||
agentId: params.sessionAgentId,
|
||||
trigger: "user",
|
||||
sessionFile: params.sessionFile,
|
||||
storePath: params.storePath,
|
||||
workspaceDir: params.workspaceDir,
|
||||
cwd: params.cwd,
|
||||
config: params.cfg,
|
||||
prompt: cliPrompt,
|
||||
transcriptPrompt: params.transcriptBody,
|
||||
modelProvider: params.providerOverride,
|
||||
provider: cliExecutionProvider,
|
||||
model: params.modelOverride,
|
||||
thinkLevel: params.resolvedThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runTimeoutOverrideMs: params.runTimeoutOverrideMs,
|
||||
runId: params.runId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
lane: params.opts.lane,
|
||||
extraSystemPrompt: params.opts.extraSystemPrompt,
|
||||
inputProvenance: params.opts.inputProvenance,
|
||||
sourceReplyDeliveryMode: params.opts.sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget:
|
||||
params.opts.requireExplicitMessageTarget ?? isSubagentSessionKey(params.sessionKey),
|
||||
cliSessionBindingFacts: params.opts.cliSessionBindingFacts,
|
||||
cliSessionId: nextCliSessionId,
|
||||
cliSessionBinding:
|
||||
nextCliSessionId === activeCliSessionBinding?.sessionId
|
||||
? activeCliSessionBinding
|
||||
: undefined,
|
||||
forkCliSessionOnResume,
|
||||
...(forkStoreParams
|
||||
? {
|
||||
claimCliSessionFork: async () => {
|
||||
const claimed = await consumeCliSessionForkInStore(forkStoreParams);
|
||||
if (claimed) {
|
||||
params.sessionEntry = claimed;
|
||||
}
|
||||
return Boolean(claimed);
|
||||
},
|
||||
restoreCliSessionFork: async () => {
|
||||
const restored = await restoreCliSessionForkInStore(forkStoreParams);
|
||||
if (restored) {
|
||||
params.sessionEntry = restored;
|
||||
}
|
||||
},
|
||||
persistCliSessionForkSuccessor: async (successorCliSessionId: string) => {
|
||||
const persisted = await persistCliSessionForkSuccessorInStore({
|
||||
...forkStoreParams,
|
||||
successorCliSessionId,
|
||||
});
|
||||
if (!persisted) {
|
||||
throw new Error("CLI session fork successor could not be persisted");
|
||||
}
|
||||
params.sessionEntry = persisted;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
authProfileId,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
// Image discovery must use the original turn, before retry/history decoration.
|
||||
imagePrompt: params.body,
|
||||
// Fallback prompts repeat the current task, so prompt-local images must
|
||||
// accompany every CLI process. Native dedupe requires a runtime receipt.
|
||||
images: params.opts.images,
|
||||
imageOrder: params.opts.imageOrder,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel: params.messageChannel,
|
||||
streamParams: params.opts.streamParams,
|
||||
messageProvider: params.opts.messageProvider ?? params.messageChannel,
|
||||
currentChannelId: params.runContext.currentChannelId,
|
||||
chatId: params.runContext.chatId,
|
||||
channelContext: params.runContext.channelContext,
|
||||
currentThreadTs: params.runContext.currentThreadTs,
|
||||
currentInboundAudio: params.runContext.currentInboundAudio,
|
||||
approvalReviewerDeviceId: params.opts.approvalReviewerDeviceId,
|
||||
agentAccountId: params.runContext.accountId,
|
||||
senderId: params.runContext.senderId,
|
||||
senderIsOwner: params.opts.senderIsOwner,
|
||||
bashElevated: params.opts.bashElevated,
|
||||
groupId: params.runContext.groupId,
|
||||
groupChannel: params.runContext.groupChannel,
|
||||
groupSpace: params.runContext.groupSpace,
|
||||
spawnedBy: params.spawnedBy,
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.opts.toolsAllow,
|
||||
params.opts.toolsAllowIsDefault,
|
||||
),
|
||||
cleanupBundleMcpOnRunEnd: params.opts.cleanupBundleMcpOnRunEnd,
|
||||
cleanupCliLiveSessionOnRunEnd: params.opts.cleanupCliLiveSessionOnRunEnd,
|
||||
oneShotCliRun: params.opts.oneShotCliRun,
|
||||
userTurnTranscriptRecorder: params.userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence: params.suppressPromptPersistenceOnRetry === true,
|
||||
...(mutableCliSessionStore && !forkCliSessionOnResume
|
||||
? {
|
||||
onBeforeFreshCliSessionRetry: async (retry) => {
|
||||
if (
|
||||
hasNewGeneratedMediaTaskForSessionKey(params.sessionKey, mediaTaskIdsBefore) ||
|
||||
retry.sessionId !== activeCliSessionBinding?.sessionId
|
||||
) {
|
||||
return false;
|
||||
return withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey ?? params.sessionId,
|
||||
agentId: params.sessionAgentId,
|
||||
runId: params.runId,
|
||||
},
|
||||
() =>
|
||||
runCliAgent({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionEntry: params.sessionEntry,
|
||||
agentId: params.sessionAgentId,
|
||||
trigger: "user",
|
||||
sessionFile: params.sessionFile,
|
||||
storePath: params.storePath,
|
||||
workspaceDir: params.workspaceDir,
|
||||
cwd: params.cwd,
|
||||
config: params.cfg,
|
||||
prompt: cliPrompt,
|
||||
transcriptPrompt: params.transcriptBody,
|
||||
modelProvider: params.providerOverride,
|
||||
provider: cliExecutionProvider,
|
||||
model: params.modelOverride,
|
||||
thinkLevel: params.resolvedThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runTimeoutOverrideMs: params.runTimeoutOverrideMs,
|
||||
runId: params.runId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
lane: params.opts.lane,
|
||||
extraSystemPrompt: params.opts.extraSystemPrompt,
|
||||
inputProvenance: params.opts.inputProvenance,
|
||||
sourceReplyDeliveryMode: params.opts.sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget:
|
||||
params.opts.requireExplicitMessageTarget ?? isSubagentSessionKey(params.sessionKey),
|
||||
cliSessionBindingFacts: params.opts.cliSessionBindingFacts,
|
||||
cliSessionId: nextCliSessionId,
|
||||
cliSessionBinding:
|
||||
nextCliSessionId === activeCliSessionBinding?.sessionId
|
||||
? activeCliSessionBinding
|
||||
: undefined,
|
||||
forkCliSessionOnResume,
|
||||
...(forkStoreParams
|
||||
? {
|
||||
claimCliSessionFork: async () => {
|
||||
const claimed = await consumeCliSessionForkInStore(forkStoreParams);
|
||||
if (claimed) {
|
||||
params.sessionEntry = claimed;
|
||||
}
|
||||
return Boolean(claimed);
|
||||
},
|
||||
restoreCliSessionFork: async () => {
|
||||
const restored = await restoreCliSessionForkInStore(forkStoreParams);
|
||||
if (restored) {
|
||||
params.sessionEntry = restored;
|
||||
}
|
||||
},
|
||||
persistCliSessionForkSuccessor: async (successorCliSessionId: string) => {
|
||||
const persisted = await persistCliSessionForkSuccessorInStore({
|
||||
...forkStoreParams,
|
||||
successorCliSessionId,
|
||||
});
|
||||
if (!persisted) {
|
||||
throw new Error("CLI session fork successor could not be persisted");
|
||||
}
|
||||
params.sessionEntry = persisted;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
authProfileId,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
// Image discovery must use the original turn, before retry/history decoration.
|
||||
imagePrompt: params.body,
|
||||
// Fallback prompts repeat the current task, so prompt-local images must
|
||||
// accompany every CLI process. Native dedupe requires a runtime receipt.
|
||||
images: params.opts.images,
|
||||
imageOrder: params.opts.imageOrder,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel: params.messageChannel,
|
||||
streamParams: params.opts.streamParams,
|
||||
messageProvider: params.opts.messageProvider ?? params.messageChannel,
|
||||
currentChannelId: params.runContext.currentChannelId,
|
||||
chatId: params.runContext.chatId,
|
||||
channelContext: params.runContext.channelContext,
|
||||
currentThreadTs: params.runContext.currentThreadTs,
|
||||
currentInboundAudio: params.runContext.currentInboundAudio,
|
||||
approvalReviewerDeviceId: params.opts.approvalReviewerDeviceId,
|
||||
agentAccountId: params.runContext.accountId,
|
||||
senderId: params.runContext.senderId,
|
||||
senderIsOwner: params.opts.senderIsOwner,
|
||||
bashElevated: params.opts.bashElevated,
|
||||
groupId: params.runContext.groupId,
|
||||
groupChannel: params.runContext.groupChannel,
|
||||
groupSpace: params.runContext.groupSpace,
|
||||
spawnedBy: params.spawnedBy,
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.opts.toolsAllow,
|
||||
params.opts.toolsAllowIsDefault,
|
||||
),
|
||||
cleanupBundleMcpOnRunEnd: params.opts.cleanupBundleMcpOnRunEnd,
|
||||
cleanupCliLiveSessionOnRunEnd: params.opts.cleanupCliLiveSessionOnRunEnd,
|
||||
oneShotCliRun: params.opts.oneShotCliRun,
|
||||
userTurnTranscriptRecorder: params.userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence: params.suppressPromptPersistenceOnRetry === true,
|
||||
...(mutableCliSessionStore && !forkCliSessionOnResume
|
||||
? {
|
||||
onBeforeFreshCliSessionRetry: async (retry) => {
|
||||
if (
|
||||
hasNewGeneratedMediaTaskForSessionKey(
|
||||
params.sessionKey,
|
||||
mediaTaskIdsBefore,
|
||||
) ||
|
||||
retry.sessionId !== activeCliSessionBinding?.sessionId
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`CLI session failed, clearing before fresh retry: provider=${sanitizeForLog(cliExecutionProvider)} sessionKey=${mutableCliSessionStore.sessionKey} reason=${sanitizeForLog(retry.reason)}`,
|
||||
);
|
||||
log.warn(
|
||||
`CLI session failed, clearing before fresh retry: provider=${sanitizeForLog(cliExecutionProvider)} sessionKey=${mutableCliSessionStore.sessionKey} reason=${sanitizeForLog(retry.reason)}`,
|
||||
);
|
||||
|
||||
params.sessionEntry =
|
||||
(await clearCliSessionInStore({
|
||||
provider: cliExecutionProvider,
|
||||
...mutableCliSessionStore,
|
||||
})) ?? params.sessionEntry;
|
||||
return true;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
params.sessionEntry =
|
||||
(await clearCliSessionInStore({
|
||||
provider: cliExecutionProvider,
|
||||
...mutableCliSessionStore,
|
||||
})) ?? params.sessionEntry;
|
||||
return true;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
};
|
||||
return resolveReusableCliSessionBinding().then(async (activeCliSessionBinding) => {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user