mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-19 18:51:34 +00:00
* feat(codex): scope app-server rate limits to the physical client Replace the process-global rate-limit cache with a WeakMap keyed by the physical app-server client, tracking per-limitId revisions. Rolling account/rateLimits/updated notifications merge sparsely per the protocol contract (credits/individualLimit/planType survive nulls), and usage-limit errors only trust a snapshot for auth-profile blocking when the same client observed a primary update during the failing turn's startup. Fixes cross-client rate-limit bleed in usage-limit error messages. A new client-runtime module installs the account/chatgptAuthTokens/refresh handler and the rate-limit observer once per physical client, replacing per-start inline handlers in shared-client, run-attempt, and side-question. * refactor(codex): split thread/resume subscription safety into thread-resume Move the thread/resume request out of thread-lifecycle into a dedicated thread-resume module that retires the exact physical client when resume acceptance is indeterminate: only a structured RPC rejection proves Codex holds no subscription, so any other failure abandons the client instead of returning it to the shared pool. Resume responses naming a different thread now fail closed (assertCodexThreadResumeSubscription), and the fresh-thread fallback requires a released subscription unless the resume was a proven RPC rejection. * refactor(codex): replace client-factory positional DI with shared-client factory Delete the lazy positional-argument CodexAppServerClientFactory and use the options-object factory type exported from shared-client. Callers in run-attempt, compact, bounded-turn, provider-capabilities, and the web-search provider now default to getLeasedSharedCodexAppServerClient directly; the lazy indirection was ineffective because those modules already import shared-client statically. * feat(codex): route app-server turn traffic through a keyed turn router Install one turn router per physical app-server client and replace the broad per-attempt notification/request fanout with explicit per-thread routes. Attempt startup reserves the thread route (before thread/resume on the resume path, so early notifications buffer instead of racing), run-attempt activates it with receive-time, queued, and request handlers, arms the route before turn/start, binds the accepted turn id to flush buffered traffic in wire order, and releases the route on cleanup. Requests for a pending turn wait for binding instead of being auto-declined, native turn completion waits use route state instead of scanning buffered notifications, and correlation readers now match the canonical v2 wire shapes only (top-level threadId, nested turn.id). The unscoped response-delta lease-count attribution and its client API are deleted along with the retired correlation predicates. * test(codex): reset the shared binding store between thread-lifecycle tests SQLite bindings are keyed by session identity rather than the per-test temp dir, so earlier tests leaked resumable threads into fresh-start expectations. The old silent resume-failure fallback masked the leak; subscription safety surfaces it. * test(codex): reset the binding store between delivery-hint iterations The loop reuses one session identity across iterations, so the previous iteration's thread would resume against a harness that cannot serve it.
298 lines
12 KiB
TypeScript
298 lines
12 KiB
TypeScript
/**
|
|
* State machine for Codex app-server turn notifications and idle-watch updates.
|
|
*/
|
|
import {
|
|
codexExecutionToolName,
|
|
describeNotificationActivity,
|
|
isAssistantCompletionReleaseNotification,
|
|
isCodexTurnAbortMarkerNotification,
|
|
isFileChangePatchUpdatedNotification,
|
|
isAssistantCommentaryCompletionNotification,
|
|
isNativeToolProgressNotification,
|
|
isPendingOpenClawDynamicToolCompletionNotification,
|
|
isRawAssistantProgressNotification,
|
|
isRawReasoningCompletionNotification,
|
|
isRawToolOutputCompletionNotification,
|
|
isReasoningProgressNotification,
|
|
isReasoningItemCompletionNotification,
|
|
isRetryableErrorNotification,
|
|
readCodexNotificationItem,
|
|
readNotificationItemId,
|
|
shouldDisarmAssistantCompletionIdleWatch,
|
|
updateActiveCompletionBlockerItemIds,
|
|
updateActiveTurnItemIds,
|
|
} from "./attempt-notifications.js";
|
|
import { CODEX_POST_REASONING_REPLY_IDLE_TIMEOUT_MS } from "./attempt-timeouts.js";
|
|
import type { CodexAttemptTurnWatchController } from "./attempt-turn-watches.js";
|
|
import { isCodexNotificationForTurn } from "./notification-correlation.js";
|
|
import type { CodexServerNotification } from "./protocol.js";
|
|
|
|
type CodexExecutionPhase =
|
|
| { phase: "turn_accepted" }
|
|
| { phase: "assistant_output_started" }
|
|
| { phase: "tool_execution_started"; itemId?: string; tool: string };
|
|
|
|
/** Emits coarse execution phases exactly once from app-server notifications. */
|
|
export function reportCodexExecutionNotification(params: {
|
|
notification: CodexServerNotification;
|
|
emitExecutionPhaseOnce: (key: string, info: CodexExecutionPhase) => void;
|
|
}): void {
|
|
const { notification } = params;
|
|
if (notification.method === "turn/started") {
|
|
params.emitExecutionPhaseOnce("turn_accepted", { phase: "turn_accepted" });
|
|
return;
|
|
}
|
|
if (notification.method === "item/agentMessage/delta") {
|
|
params.emitExecutionPhaseOnce("assistant_output_started", {
|
|
phase: "assistant_output_started",
|
|
});
|
|
return;
|
|
}
|
|
if (notification.method !== "item/started") {
|
|
return;
|
|
}
|
|
const item = readCodexNotificationItem(notification.params);
|
|
const tool = item ? codexExecutionToolName(item) : undefined;
|
|
if (!item || !tool) {
|
|
return;
|
|
}
|
|
params.emitExecutionPhaseOnce(`tool:${item.id}`, {
|
|
phase: "tool_execution_started",
|
|
tool,
|
|
itemId: item.id,
|
|
});
|
|
}
|
|
|
|
/** Returns true when a notification ends the current app-server turn. */
|
|
export function isTerminalCodexTurnNotificationForTurn(params: {
|
|
notification: CodexServerNotification;
|
|
threadId: string;
|
|
turnId: string;
|
|
currentPromptTexts: string[];
|
|
}): boolean {
|
|
if (!isCodexNotificationForTurn(params.notification.params, params.threadId, params.turnId)) {
|
|
return false;
|
|
}
|
|
return (
|
|
params.notification.method === "turn/completed" ||
|
|
isCodexTurnAbortMarkerNotification(params.notification, {
|
|
currentPromptTexts: params.currentPromptTexts,
|
|
})
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Applies one notification to active item tracking, idle watches, and terminal
|
|
* turn state.
|
|
*/
|
|
export function applyCodexTurnNotificationState(params: {
|
|
notification: CodexServerNotification;
|
|
threadId: string;
|
|
turnId: string;
|
|
currentPromptTexts: string[];
|
|
turnWatches: CodexAttemptTurnWatchController;
|
|
activeTurnItemIds: Set<string>;
|
|
activeCompletionBlockerItemIds: Set<string>;
|
|
activeAppServerTurnRequests: number;
|
|
pendingOpenClawDynamicToolCompletionIds: Set<string>;
|
|
turnCrossedToolHandoff: boolean;
|
|
postToolRawAssistantCompletionIdleTimeoutMs: number;
|
|
onScheduleTerminalDynamicToolReleaseCheck: () => void;
|
|
onReportExecutionNotification: (notification: CodexServerNotification) => void;
|
|
}): {
|
|
isCurrentTurnNotification: boolean;
|
|
isTurnAbortMarker: boolean;
|
|
isTurnTerminal: boolean;
|
|
turnCrossedToolHandoff: boolean;
|
|
} {
|
|
const { notification, turnWatches } = params;
|
|
const isCurrentTurnNotification = isCodexNotificationForTurn(
|
|
notification.params,
|
|
params.threadId,
|
|
params.turnId,
|
|
);
|
|
const isTurnCompletion = notification.method === "turn/completed" && isCurrentTurnNotification;
|
|
let turnCrossedToolHandoff = params.turnCrossedToolHandoff;
|
|
|
|
if (isCurrentTurnNotification) {
|
|
turnWatches.touchActivity(`notification:${notification.method}`, {
|
|
details: describeNotificationActivity(notification),
|
|
attemptProgress: true,
|
|
});
|
|
params.onReportExecutionNotification(notification);
|
|
updateActiveTurnItemIds(notification, params.activeTurnItemIds);
|
|
updateActiveCompletionBlockerItemIds(notification, params.activeCompletionBlockerItemIds);
|
|
if (notification.method === "item/completed" && params.activeTurnItemIds.size === 0) {
|
|
params.onScheduleTerminalDynamicToolReleaseCheck();
|
|
}
|
|
}
|
|
|
|
const unblockedAssistantCompletionRelease =
|
|
isCurrentTurnNotification &&
|
|
turnWatches.isAssistantCompletionIdleWatchArmed() &&
|
|
notification.method === "item/completed" &&
|
|
params.activeTurnItemIds.size === 0;
|
|
const trackedDynamicToolCompletion = isPendingOpenClawDynamicToolCompletionNotification(
|
|
notification,
|
|
params.pendingOpenClawDynamicToolCompletionIds,
|
|
);
|
|
const rawToolOutputCompletion = isRawToolOutputCompletionNotification(notification);
|
|
if (
|
|
isCurrentTurnNotification &&
|
|
(rawToolOutputCompletion || isNativeToolProgressNotification(notification))
|
|
) {
|
|
turnCrossedToolHandoff = true;
|
|
}
|
|
const assistantCompletionCanRelease = isAssistantCompletionReleaseNotification(
|
|
notification,
|
|
turnCrossedToolHandoff,
|
|
);
|
|
const postToolProgressNeedsTerminalGuard =
|
|
isCurrentTurnNotification &&
|
|
turnCrossedToolHandoff &&
|
|
(((isRawAssistantProgressNotification(notification) ||
|
|
isRawReasoningCompletionNotification(notification)) &&
|
|
params.activeTurnItemIds.size === 0) ||
|
|
isReasoningProgressNotification(notification));
|
|
const postToolPatchUpdateNeedsTerminalGuard =
|
|
isCurrentTurnNotification &&
|
|
turnCrossedToolHandoff &&
|
|
isFileChangePatchUpdatedNotification(notification);
|
|
const rawResponseItemCompletedWithNoActiveItems =
|
|
isCurrentTurnNotification &&
|
|
notification.method === "rawResponseItem/completed" &&
|
|
params.activeTurnItemIds.size === 0 &&
|
|
params.activeAppServerTurnRequests === 0 &&
|
|
!assistantCompletionCanRelease &&
|
|
!postToolProgressNeedsTerminalGuard &&
|
|
!rawToolOutputCompletion;
|
|
const shouldArmNoToolPostProgressReplyWatch =
|
|
isCurrentTurnNotification &&
|
|
!turnCrossedToolHandoff &&
|
|
params.activeTurnItemIds.size === 0 &&
|
|
(isReasoningItemCompletionNotification(notification) ||
|
|
isAssistantCommentaryCompletionNotification(notification));
|
|
const shouldArmNoToolPostRawProgressReplyWatch =
|
|
!turnCrossedToolHandoff &&
|
|
rawResponseItemCompletedWithNoActiveItems &&
|
|
(isRawReasoningCompletionNotification(notification) ||
|
|
isRawAssistantProgressNotification(notification));
|
|
const shouldRearmCompletionIdleWatchAfterLastCurrentTurnItem =
|
|
isCurrentTurnNotification &&
|
|
notification.method === "item/completed" &&
|
|
params.activeTurnItemIds.size === 0 &&
|
|
!trackedDynamicToolCompletion &&
|
|
!assistantCompletionCanRelease &&
|
|
!shouldArmNoToolPostProgressReplyWatch;
|
|
const shouldUsePostToolContinuationWatch =
|
|
turnCrossedToolHandoff &&
|
|
(postToolProgressNeedsTerminalGuard ||
|
|
postToolPatchUpdateNeedsTerminalGuard ||
|
|
rawToolOutputCompletion ||
|
|
trackedDynamicToolCompletion ||
|
|
shouldRearmCompletionIdleWatchAfterLastCurrentTurnItem);
|
|
const armPostToolContinuationWatch = () => {
|
|
turnWatches.armCompletionIdleWatch({
|
|
timeoutMs: params.postToolRawAssistantCompletionIdleTimeoutMs,
|
|
});
|
|
turnWatches.extendAttemptIdleWatch(params.postToolRawAssistantCompletionIdleTimeoutMs);
|
|
};
|
|
const armPostProgressReplyWatch = () => {
|
|
turnWatches.armCompletionIdleWatch({
|
|
timeoutMs: CODEX_POST_REASONING_REPLY_IDLE_TIMEOUT_MS,
|
|
});
|
|
turnWatches.extendAttemptIdleWatch(CODEX_POST_REASONING_REPLY_IDLE_TIMEOUT_MS);
|
|
};
|
|
|
|
if (isCurrentTurnNotification && notification.method === "error") {
|
|
if (isRetryableErrorNotification(notification.params)) {
|
|
turnWatches.disarmCompletionIdleWatch();
|
|
} else {
|
|
turnWatches.armCompletionIdleWatch({ pinnedByTerminalError: true });
|
|
}
|
|
turnWatches.disarmAssistantCompletionIdleWatch();
|
|
} else if (isTurnCompletion) {
|
|
turnWatches.disarmAssistantCompletionIdleWatch();
|
|
} else if (isCurrentTurnNotification && assistantCompletionCanRelease) {
|
|
turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification));
|
|
} else if (postToolProgressNeedsTerminalGuard || postToolPatchUpdateNeedsTerminalGuard) {
|
|
// Post-tool assistant/reasoning status and patch snapshots can be followed
|
|
// by more native edit streaming. Keep the short guard alive until Codex
|
|
// reports a terminal turn state instead of falling back to the long
|
|
// terminal watch.
|
|
armPostToolContinuationWatch();
|
|
} else if (shouldArmNoToolPostProgressReplyWatch || shouldArmNoToolPostRawProgressReplyWatch) {
|
|
armPostProgressReplyWatch();
|
|
} else if (trackedDynamicToolCompletion) {
|
|
armPostToolContinuationWatch();
|
|
} else if (unblockedAssistantCompletionRelease) {
|
|
turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification));
|
|
} else if (shouldRearmCompletionIdleWatchAfterLastCurrentTurnItem) {
|
|
// If a non-assistant current-turn item is the last active item and the
|
|
// bridge then goes quiet, reset the short completion-idle guard from that
|
|
// final completion so the remaining silent-turn gap fails fast.
|
|
if (shouldUsePostToolContinuationWatch) {
|
|
armPostToolContinuationWatch();
|
|
} else {
|
|
turnWatches.armCompletionIdleWatch();
|
|
}
|
|
} else if (rawResponseItemCompletedWithNoActiveItems) {
|
|
turnWatches.armCompletionIdleWatch();
|
|
} else if (isCurrentTurnNotification && rawToolOutputCompletion) {
|
|
// Raw OpenAI response streams can report the tool-output handoff without
|
|
// a matching app-server `item/completed`; keep the post-tool guard alive.
|
|
armPostToolContinuationWatch();
|
|
} else if (isCurrentTurnNotification && shouldDisarmAssistantCompletionIdleWatch(notification)) {
|
|
turnWatches.disarmAssistantCompletionIdleWatch();
|
|
}
|
|
|
|
if (
|
|
turnWatches.isCompletionIdleWatchArmed() &&
|
|
!turnWatches.isCompletionIdleWatchPinnedByTerminalError() &&
|
|
notification.method !== "turn/completed" &&
|
|
isCurrentTurnNotification &&
|
|
!trackedDynamicToolCompletion &&
|
|
!rawToolOutputCompletion &&
|
|
!postToolProgressNeedsTerminalGuard &&
|
|
!postToolPatchUpdateNeedsTerminalGuard &&
|
|
!rawResponseItemCompletedWithNoActiveItems &&
|
|
!shouldArmNoToolPostProgressReplyWatch &&
|
|
!shouldArmNoToolPostRawProgressReplyWatch &&
|
|
!shouldRearmCompletionIdleWatchAfterLastCurrentTurnItem
|
|
) {
|
|
// The short completion-idle watchdog guards blind gaps after Codex
|
|
// accepts a turn or after OpenClaw hands a turn-scoped request result
|
|
// back to Codex. Bookkeeping that closes the just-served OpenClaw
|
|
// dynamic tool item is still part of that handoff, so keep the short
|
|
// watchdog armed for that notification.
|
|
turnWatches.disarmCompletionIdleWatch();
|
|
}
|
|
|
|
if (trackedDynamicToolCompletion) {
|
|
const itemId = readNotificationItemId(notification);
|
|
if (itemId) {
|
|
params.pendingOpenClawDynamicToolCompletionIds.delete(itemId);
|
|
params.onScheduleTerminalDynamicToolReleaseCheck();
|
|
}
|
|
}
|
|
|
|
const isTurnAbortMarker =
|
|
isCurrentTurnNotification &&
|
|
isCodexTurnAbortMarkerNotification(notification, {
|
|
currentPromptTexts: params.currentPromptTexts,
|
|
});
|
|
const isTurnTerminal = isTerminalCodexTurnNotificationForTurn({
|
|
notification,
|
|
threadId: params.threadId,
|
|
turnId: params.turnId,
|
|
currentPromptTexts: params.currentPromptTexts,
|
|
});
|
|
|
|
return {
|
|
isCurrentTurnNotification,
|
|
isTurnAbortMarker,
|
|
isTurnTerminal,
|
|
turnCrossedToolHandoff,
|
|
};
|
|
}
|