diff --git a/.agents/skills/autoreview/scripts/autoreview b/.agents/skills/autoreview/scripts/autoreview index 79169878bc00..ca3b8775c29e 100755 --- a/.agents/skills/autoreview/scripts/autoreview +++ b/.agents/skills/autoreview/scripts/autoreview @@ -201,7 +201,8 @@ SECRET_ASSIGNMENT_PATTERN = re.compile( r"(?:(?:\?\.|\.)[A-Za-z_$][A-Za-z0-9_$]*)*)(?=[ \t]*\()|" r"(?P[A-Za-z_$][A-Za-z0-9_$]*" r"(?:(?:\?\.|\.)[A-Za-z_$][A-Za-z0-9_$]*" - r"|\[(?:[\"'][A-Za-z_$][A-Za-z0-9_$]*[\"']|[0-9]+)\])+)" + r"|(?:\?\.)?\[(?:[A-Za-z_$][A-Za-z0-9_$]*" + r"|[\"'][A-Za-z_$][A-Za-z0-9_$]*[\"']|[0-9]+)\])+)" r"(?![A-Za-z0-9_./+=:@#$%&*!?-])|" r"(?P[A-Za-z0-9_./+=:@#$%&*!?-]{8,}))" ) @@ -411,6 +412,20 @@ CSHARP_METHOD_PREFIX_PATTERN = ( r"(?:where\s+[^{;]+)?\{" ) CSHARP_EVIDENCE_WINDOW = 8192 +SOURCE_CODE_REFERENCE_ROOT_VALUES = { + "attemptAuthProfileStore", +} +SOURCE_CODE_REFERENCE_ROOT_PATTERN = re.compile( + r"(? bool: ) -def secret_literal_risk(expression: str, minimum_length: int = 12) -> bool: +def secret_literal_risk( + expression: str, + minimum_length: int = 12, + *, + javascript_dialect: str | None = None, +) -> bool: if credentialed_uri_risk(expression) or basic_authorization_risk(expression) or any( pattern.search(expression) for pattern in SECRET_VALUE_PATTERNS ): @@ -3794,6 +3823,11 @@ def secret_literal_risk(expression: str, minimum_length: int = 12) -> bool: if match.group("bare") is not None: if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): continue + if ( + javascript_dialect is not None + and SOURCE_CODE_REFERENCE_ROOT_PATTERN.fullmatch(value) + ): + continue suffix = expression[match.end() :].lstrip() if suffix.startswith("("): continue @@ -3805,15 +3839,24 @@ def fallback_secret_risk( text: str, minimum_length: int = 8, *, - typescript: bool = False, + javascript_dialect: str | None = None, ) -> bool: return secret_literal_risk( - fallback_expression(text, typescript=typescript), + fallback_expression( + text, + typescript=javascript_dialect == "typescript", + ), minimum_length=minimum_length, + javascript_dialect=javascript_dialect, ) -def safe_secret_assignment_suffix(text: str, end: int) -> bool: +def safe_secret_assignment_suffix( + text: str, + end: int, + *, + javascript_dialect: str | None = None, +) -> bool: cursor = end raw_diff = text.startswith("diff --git ") while cursor < len(text): @@ -3847,7 +3890,10 @@ def safe_secret_assignment_suffix(text: str, end: int) -> bool: or (suffix.startswith("?") and not suffix.startswith("?.")) or re.match(r"(?:or|and)\b", suffix) is not None ): - return not fallback_secret_risk(suffix) + return not fallback_secret_risk( + suffix, + javascript_dialect=javascript_dialect, + ) return True if text.startswith("//", cursor) or text.startswith("#", cursor): cursor = text.find("\n", cursor) @@ -3866,7 +3912,10 @@ def safe_secret_assignment_suffix(text: str, end: int) -> bool: or (suffix.startswith("?") and not suffix.startswith("?.")) or re.match(r"(?:or|and|if|unless)\b", suffix) is not None ): - return not fallback_secret_risk(suffix) + return not fallback_secret_risk( + suffix, + javascript_dialect=javascript_dialect, + ) if text[cursor] in ",;)]}": return True if text[cursor] in {'"', "'", "`"}: @@ -4561,7 +4610,12 @@ def public_call_argument_risk( return False if generic_credential_prompt_is_safe(value) else None -def call_arguments_risk(arguments: str, call_target: str) -> bool: +def call_arguments_risk( + arguments: str, + call_target: str, + *, + javascript_dialect: str | None = None, +) -> bool: for index, argument in enumerate(split_top_level_call_arguments(arguments)): public_risk = public_call_argument_risk(call_target, argument, index) if public_risk is not None: @@ -4570,12 +4624,22 @@ def call_arguments_risk(arguments: str, call_target: str) -> bool: continue if not safe_credential_lookup_argument( call_target, argument, index - ) and fallback_secret_risk(argument, minimum_length=12): + ) and fallback_secret_risk( + argument, + minimum_length=12, + javascript_dialect=javascript_dialect, + ): return True return False -def safe_secret_call_suffix(text: str, end: int, call_target: str) -> bool: +def safe_secret_call_suffix( + text: str, + end: int, + call_target: str, + *, + javascript_dialect: str | None = None, +) -> bool: if end >= len(text) or text[end] != "(": return False @@ -4647,7 +4711,15 @@ def safe_secret_call_suffix(text: str, end: int, call_target: str) -> bool: if cursor is None: return None arguments = text[start + 1 : cursor - 1] - return None if call_arguments_risk(arguments, target) else cursor + return ( + None + if call_arguments_risk( + arguments, + target, + javascript_dialect=javascript_dialect, + ) + else cursor + ) cursor = safe_call_end(end, call_target) if cursor is None: @@ -4867,7 +4939,12 @@ def mask_reference_declaration_evidence(text: str) -> str: def javascript_reference_spans(text: str) -> frozenset[tuple[int, int]]: return frozenset( (match.start(), match.end()) - for match in SOURCE_CODE_REFERENCE_PATTERN.finditer(text) + for pattern in ( + SOURCE_CODE_REFERENCE_PATTERN, + SOURCE_CODE_REFERENCE_ROOT_PATTERN, + SOURCE_CODE_LIFECYCLE_REFERENCE_PATTERN, + ) + for match in pattern.finditer(text) ) @@ -4934,7 +5011,7 @@ def safe_javascript_reference_suffix( if continuation: return not fallback_secret_risk( text[cursor:], - typescript=typescript, + javascript_dialect="typescript" if typescript else "javascript", ) return saw_newline @@ -5061,7 +5138,10 @@ def secret_text_risk( and prefix.start() in chained_assignment_positions ), ) - if fallback is not None and fallback_secret_risk(fallback): + if fallback is not None and fallback_secret_risk( + fallback, + javascript_dialect=javascript_dialect, + ): return True for match in secret_assignment_matches( SECRET_ASSIGNMENT_PATTERN, @@ -5090,7 +5170,11 @@ def secret_text_risk( if ( key.strip("\"'").lower() == "credentials" and value.lower() in FETCH_CREDENTIAL_MODE_VALUES - and safe_secret_assignment_suffix(text, match.end()) + and safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ) ): continue if ( @@ -5102,11 +5186,19 @@ def secret_text_risk( ) or safe_backtick_secret_template(value) ) - and safe_secret_assignment_suffix(text, match.end()) + and safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ) ): continue if value.lower() in SECRET_PLACEHOLDER_VALUES: - if safe_secret_assignment_suffix(text, match.end()): + if safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ): continue return True if ( @@ -5114,7 +5206,11 @@ def secret_text_risk( and len(value) < 12 and re.fullmatch(r"[A-Za-z_$][A-Za-z0-9_$]*", value) ): - if safe_secret_assignment_suffix(text, match.end()): + if safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ): continue return True if ( @@ -5126,14 +5222,24 @@ def secret_text_risk( separator, value, ) - and safe_secret_assignment_suffix(text, match.end()) + and safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ) ): continue if not quoted: source_start = match.end() - len(value) source_end = match.end() - (1 if value.endswith("!") else 0) if ( - (source_start, source_end) in source_reference_spans + ( + (source_start, source_end) in source_reference_spans + or ( + javascript_dialect is not None + and SOURCE_CODE_REFERENCE_ROOT_PATTERN.fullmatch(value) + ) + ) and safe_javascript_reference_suffix( text, match.end(), @@ -5167,6 +5273,7 @@ def secret_text_risk( text, call_start, call_target.group(0), + javascript_dialect=javascript_dialect, ) ): continue @@ -5176,11 +5283,20 @@ def secret_text_risk( and call_target and text[call_start : call_start + 1] == "(" ): - if safe_secret_call_suffix(text, call_start, value): + if safe_secret_call_suffix( + text, + call_start, + value, + javascript_dialect=javascript_dialect, + ): continue return True if any(pattern.fullmatch(value) for pattern in reference_patterns): - if safe_secret_assignment_suffix(text, match.end()): + if safe_secret_assignment_suffix( + text, + match.end(), + javascript_dialect=javascript_dialect, + ): continue return True return True diff --git a/.agents/skills/autoreview/tests/test_autoreview_hardening.py b/.agents/skills/autoreview/tests/test_autoreview_hardening.py index 8973584b35cf..10688ddb2530 100644 --- a/.agents/skills/autoreview/tests/test_autoreview_hardening.py +++ b/.agents/skills/autoreview/tests/test_autoreview_hardening.py @@ -2210,6 +2210,115 @@ class AutoreviewHardeningTests(unittest.TestCase): ) ) + def test_secret_detector_allows_lifecycle_named_typescript_references(self) -> None: + key_term = "Api" + "Key" + key_field = key_term[0].lower() + key_term[1:] + credential_term = "Cred" + "ential" + source = ( + f"const resolvedStream{key_term} = resolveAttemptDispatch{key_term}({{\n" + f" {key_field}Info,\n" + " runtimeAuthState,\n" + "});\n" + f"const successful{credential_term} = successfulProfileId\n" + " ? attemptAuthProfileStore.profiles[successfulProfileId]\n" + " : undefined;\n" + f"const successful{key_term}Info = get{key_term}Info();\n" + f"const {key_field} = successful{key_term}Info?.{key_field};\n" + f"const resolved{key_term} = resolveSecretSentinel({key_field});\n" + "return {\n" + f" resolved{key_term}: resolvedStream{key_term},\n" + f" {credential_term.lower()}: successful{credential_term},\n" + f" {key_field}: resolved{key_term},\n" + "};\n" + ) + literal_value = "actual-production-" + "secret" + unsafe_sources = ( + f'const resolved{key_term} = "' + literal_value + '";', + "const config = { pass" + + "word: resolved" + + key_term + + ' + "' + + literal_value + + "\" };", + "const config = { pass" + + "word: Abcdefghijklmnop.Qrstuvwxyzabcdef };", + ) + + self.assertFalse( + self.helper["secret_text_risk"]( + source, + javascript_dialect="typescript", + ) + ) + for unsafe_source in unsafe_sources: + with self.subTest(unsafe_source=unsafe_source): + self.assertTrue( + self.helper["secret_text_risk"]( + unsafe_source, + javascript_dialect="typescript", + ) + ) + + store_reference = ( + "const cred" + + "ential = attemptAuthProfileStore.profiles[successfulProfileId];" + ) + optional_store_reference = ( + "const cred" + + "ential = attemptAuthProfileStore?.[successfulProfileId];" + ) + quoted_store_reference = ( + "const cred" + + 'ential = attemptAuthProfileStore["profiles"][successfulProfileId];' + ) + yaml_store_literal = ( + "pass" + + 'word: attemptAuthProfileStore["' + + literal_value + + '"]' + ) + quoted_secret_key = "N7xQ2mP9vK4r" + "T8wZ" + typescript_store_literal = ( + "const pass" + + 'word = attemptAuthProfileStore["' + + quoted_secret_key + + '"];' + ) + self.assertFalse( + self.helper["secret_text_risk"]( + store_reference, + javascript_dialect="typescript", + ) + ) + self.assertFalse( + self.helper["secret_text_risk"]( + optional_store_reference, + javascript_dialect="typescript", + ) + ) + self.assertFalse( + self.helper["secret_text_risk"]( + quoted_store_reference, + javascript_dialect="typescript", + ) + ) + self.assertTrue(self.helper["secret_text_risk"](yaml_store_literal)) + self.assertTrue( + self.helper["secret_text_risk"]( + typescript_store_literal, + javascript_dialect="typescript", + ) + ) + + def test_lifecycle_reference_scan_is_bounded_for_non_matching_identifier(self) -> None: + source = "const value = resolved" + "A" * 100_000 + "X;" + + started = time.monotonic() + spans = self.helper["javascript_reference_spans"](source) + + self.assertEqual(spans, frozenset()) + self.assertLess(time.monotonic() - started, 5.0) + def test_review_patch_scopes_source_references_to_typescript_files(self) -> None: property_name = "pass" + "word" reference = "context.driverPass" + "word" diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index fdab83c68fc8..8fc73a371a93 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -477,7 +477,6 @@ src/agents/embedded-agent-runner/run.incomplete-turn.test.ts src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts src/agents/embedded-agent-runner/run.overflow-compaction.test.ts -src/agents/embedded-agent-runner/run.ts src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts src/agents/embedded-agent-runner/run/attempt.session-lock.test.ts diff --git a/src/agents/embedded-agent-runner/run-execution.ts b/src/agents/embedded-agent-runner/run-execution.ts new file mode 100644 index 000000000000..822b8cd53c4b --- /dev/null +++ b/src/agents/embedded-agent-runner/run-execution.ts @@ -0,0 +1,10 @@ +import { runPreparedEmbeddedLoop } from "./run-loop.js"; +import type { PreparedEmbeddedRunInput } from "./run/execution-context.js"; +import type { EmbeddedAgentRunResult } from "./types.js"; + +/** Runs one fully prepared embedded-agent request. */ +export function executePreparedEmbeddedRun( + input: PreparedEmbeddedRunInput, +): Promise { + return runPreparedEmbeddedLoop(input); +} diff --git a/src/agents/embedded-agent-runner/run-loop.ts b/src/agents/embedded-agent-runner/run-loop.ts new file mode 100644 index 000000000000..2174cf8cf0ff --- /dev/null +++ b/src/agents/embedded-agent-runner/run-loop.ts @@ -0,0 +1,669 @@ +/** Prepared embedded-agent loop and cleanup. */ +import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js"; +import { ensureContextEnginesInitialized } from "../../context-engine/init.js"; +import { + resolveContextEngine, + resolveContextEngineOwnerPluginId, +} from "../../context-engine/registry.js"; +import { buildContextEngineRuntimeSettings } from "../../context-engine/runtime-settings.js"; +import { formatErrorMessage } from "../../infra/errors.js"; +import { + retireSessionMcpRuntime, + retireSessionMcpRuntimeForSessionKey, +} from "../agent-bundle-mcp-tools.js"; +import { resolveSessionAgentIds } from "../agent-scope.js"; +import type { ToolOutcomeObservation } from "../agent-tools.before-tool-call.js"; +import type { FailoverReason } from "../embedded-agent-helpers.js"; +import { isStrictAgenticExecutionContractActive } from "../execution-contract.js"; +import { runAgentCleanupStep } from "../run-cleanup-timeout.js"; +import { resolveToolLoopDetectionConfig } from "../tool-loop-detection-config.js"; +import { normalizeUsage } from "../usage.js"; +import { log } from "./logger.js"; +import { + createPostCompactionLoopGuard, + PostCompactionLoopPersistedError, +} from "./post-compaction-loop-guard.js"; +import { createEmbeddedRunReplayState } from "./replay-state.js"; +import { handleEmbeddedAssistantFailure } from "./run/assistant-failure.js"; +import { prepareAndDispatchEmbeddedRunAttempt } from "./run/attempt-dispatch-preparation.js"; +import { normalizeEmbeddedRunAttempt } from "./run/attempt-normalization.js"; +import { recoverEmbeddedRunAttempt } from "./run/attempt-recovery.js"; +import { forgetPromptBuildDrainCacheForRun } from "./run/attempt.prompt-helpers.js"; +import { hasCodexAppServerRecoveryRetryBudget } from "./run/codex-app-server-recovery.js"; +import { createEmbeddedRunCompactionRuntime } from "./run/compaction-runtime.js"; +import { createEmbeddedRunContextRecoveryState } from "./run/context-recovery-state.js"; +import type { PreparedEmbeddedRunInput } from "./run/execution-context.js"; +import { resolveRunFailoverDecision } from "./run/failover-policy.js"; +import { createEmbeddedRunFailoverRetryController } from "./run/failover-retry-controller.js"; +import { buildErrorAgentMeta, resolveMaxRunRetryIterations } from "./run/helpers.js"; +import { createIdleTimeoutBreakerState } from "./run/idle-timeout-breaker.js"; +import { + DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT, + DEFAULT_REASONING_ONLY_RETRY_LIMIT, +} from "./run/incomplete-turn.js"; +import { handleRetryLimitExhaustion } from "./run/retry-limit.js"; +import { prepareEmbeddedRunRuntime } from "./run/runtime-preparation.js"; +import { createEmbeddedRunSessionPromptState } from "./run/session-prompt-state.js"; +import { prepareEmbeddedRunTerminal } from "./run/terminal-preparation.js"; +import { resolveEmbeddedRunTerminal } from "./run/terminal-resolution.js"; +import { createEmbeddedRunTerminalRetryState } from "./run/terminal-retry-state.js"; +import { resolveEmbeddedRunTerminalTimeout } from "./run/terminal-timeout.js"; +import type { EmbeddedAgentRunResult, TraceAttempt } from "./types.js"; +import { createUsageAccumulator } from "./usage-accumulator.js"; + +export async function runPreparedEmbeddedLoop( + input: PreparedEmbeddedRunInput, +): Promise { + const params = input.runParams; + let { provider, modelId } = input; + const { + agentDir, + workspaceDir: resolvedWorkspace, + globalLane, + hookRunner, + hookContext: hookCtx, + fallbackConfigured, + isProbeSession, + resolvedSessionKey, + resolvedToolResultFormat, + startedAtMs: started, + startupStages, + lifecycleGeneration, + suspendForFailure, + } = input; + const { maybeEmitFastModeAutoResetBestEffort, notifyExecutionPhase } = input.progressController; + const { laneTaskAbortController } = input.laneController; + let startupStagesEmitted = false; + const preparedRuntime = await prepareEmbeddedRunRuntime({ + runParams: params, + provider, + modelId, + agentDir, + workspaceDir: resolvedWorkspace, + globalLane, + hookRunner, + hookContext: hookCtx, + markStartupStage: (stage) => startupStages.mark(stage), + notifyExecutionPhase, + fallbackConfigured, + }); + provider = preparedRuntime.provider; + modelId = preparedRuntime.modelId; + const { + requestedModelId, + model, + attemptAuthProfileStore, + profileCandidates, + profileFailureStore, + pluginHarnessOwnsAuthBootstrap, + attemptedThinking, + advanceAttemptAuthProfile, + maybeRefreshRuntimeAuthForAuthError, + stopRuntimeAuthRefreshTimer, + getApiKeyInfo, + } = preparedRuntime; + let { + agentHarness, + pluginHarnessOwnsTransport, + effectiveModel, + outerContextTokenMeta, + thinkLevel, + lastProfileId, + } = preparedRuntime.snapshot(); + const refreshPreparedRuntimeSnapshot = () => { + ({ + agentHarness, + pluginHarnessOwnsTransport, + effectiveModel, + outerContextTokenMeta, + thinkLevel, + lastProfileId, + } = preparedRuntime.snapshot()); + }; + const traceAttempts: TraceAttempt[] = []; + const traceAttemptUsesFallback = (attempt: TraceAttempt): boolean => + attempt.result === "rotate_profile" || attempt.result === "fallback_model"; + const resolveRuntimeFallbackReason = (): string | null => { + const fallbackAttempt = traceAttempts.findLast( + (attempt) => attempt.result === "fallback_model" && typeof attempt.reason === "string", + ); + return fallbackAttempt?.reason ?? lastRetryFailoverReason ?? null; + }; + const buildEmbeddedContextEngineRuntimeSettings = (settingsParams: { + tokenBudget?: number | null; + maxOutputTokens?: number | null; + degradedReason?: string | null; + }) => { + const fallbackReason = resolveRuntimeFallbackReason(); + return buildContextEngineRuntimeSettings({ + contextEngineHost: OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST, + provider, + requestedModel: requestedModelId, + resolvedModel: modelId, + selectedContextEngineId: contextEngine.info.id, + contextEngineSelectionSource: contextEngine.info.id === "legacy" ? "default" : "configured", + promptTokenBudget: settingsParams.tokenBudget, + maxOutputTokens: settingsParams.maxOutputTokens, + fallbackReason, + degradedReason: settingsParams.degradedReason, + }); + }; + const { sessionAgentId } = resolveSessionAgentIds({ + sessionKey: params.sessionKey, + config: params.config, + agentId: params.agentId, + }); + const strictAgenticActive = isStrictAgenticExecutionContractActive({ + config: params.config, + sessionKey: params.sessionKey, + agentId: params.agentId, + provider, + modelId, + }); + const executionContract = strictAgenticActive ? "strict-agentic" : "default"; + const maxReasoningOnlyRetryAttempts = DEFAULT_REASONING_ONLY_RETRY_LIMIT; + const maxEmptyResponseRetryAttempts = DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT; + + const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations( + profileCandidates.length, + params.config, + sessionAgentId, + ); + const contextRecoveryState = createEmbeddedRunContextRecoveryState(); + let bootstrapPromptWarningSignaturesSeen = + params.bootstrapPromptWarningSignaturesSeen ?? + (params.bootstrapPromptWarningSignature ? [params.bootstrapPromptWarningSignature] : []); + const usageAccumulator = createUsageAccumulator(); + let lastRunPromptUsage: ReturnType | undefined; + let runLoopIterations = 0; + let overloadProfileRotations = 0; + const terminalRetryState = createEmbeddedRunTerminalRetryState(); + let sameModelIdleTimeoutRetries = 0; + // Cost-runaway breaker for #76293. State lives at the run-loop level + // on purpose so it survives across attempt boundaries and across + // profile/auth retries within this embedded run (a wrapper-local + // counter would reset on every iteration). The helper is pure and + // unit-tested in run/idle-timeout-breaker.test.ts; the run loop just + // feeds it the outcome of each attempt. + const idleTimeoutBreakerState = createIdleTimeoutBreakerState(); + // Post-compaction loop guard for #77474. Armed at each compaction-success + // site below; observed from the live tool-outcome path so it can abort + // while the post-compaction prompt is still running. + const resolvedLoopDetectionConfig = resolveToolLoopDetectionConfig({ + cfg: params.config, + agentId: sessionAgentId, + }); + const postCompactionGuard = createPostCompactionLoopGuard( + resolvedLoopDetectionConfig?.postCompactionGuard, + { enabled: resolvedLoopDetectionConfig?.enabled !== false }, + ); + let postCompactionAbortController: AbortController | undefined; + let postCompactionAbortError: PostCompactionLoopPersistedError | undefined; + const attemptTerminalToolPresentation = { + ordinal: -1, + value: undefined as string | undefined, + }; + let nextToolOutcomeOrdinal = 0; + const allocateToolOutcomeOrdinal = (): number => nextToolOutcomeOrdinal++; + const readAttemptTerminalToolPresentation = (): string | undefined => + attemptTerminalToolPresentation.value; + const observeToolOutcome = (observation: ToolOutcomeObservation): void => { + const observationOrdinal = + observation.toolCallOrdinal ?? attemptTerminalToolPresentation.ordinal + 1; + if (observationOrdinal >= attemptTerminalToolPresentation.ordinal) { + attemptTerminalToolPresentation.ordinal = observationOrdinal; + attemptTerminalToolPresentation.value = observation.terminalPresentation; + } + if (observation.presentationOnly) { + return; + } + const verdict = postCompactionGuard.observe(observation); + if (verdict.shouldAbort) { + postCompactionAbortError ??= PostCompactionLoopPersistedError.fromVerdict(verdict); + laneTaskAbortController.abort(postCompactionAbortError); + postCompactionAbortController?.abort(postCompactionAbortError); + } + }; + let lastRetryFailoverReason: FailoverReason | null = null; + let codexAppServerRecoveryRetries = 0; + // Silent-error retry: non-strict-agentic models (e.g. ollama/glm-5.1) can + // end a turn with stopReason="error" + zero output tokens, producing no + // user-visible text. This is an orthogonal, model-agnostic resubmission + // for errored turns; stopReason="stop" empty zero-token turns use the + // visible-answer retry instruction instead. + let emptyErrorRetries = 0; + const sessionPromptState = createEmbeddedRunSessionPromptState({ + runParams: params, + sessionAgentId, + resolvedSessionKey, + lifecycleGeneration, + }); + const failoverRetryController = createEmbeddedRunFailoverRetryController({ + runParams: params, + provider, + modelId, + globalLane, + agentDir, + fallbackConfigured, + profileFailureStore, + getLastProfileId: () => preparedRuntime.snapshot().lastProfileId, + getSessionId: () => sessionPromptState.sessionId, + harnessOwnsTransport: () => preparedRuntime.snapshot().pluginHarnessOwnsTransport, + }); + // Resolve the context engine once and reuse across retries to avoid + // repeated initialization/connection overhead per attempt. + ensureContextEnginesInitialized(); + const contextEngine = await resolveContextEngine(params.config, { + agentDir, + workspaceDir: resolvedWorkspace, + }); + const resolveContextEnginePluginId = () => resolveContextEngineOwnerPluginId(contextEngine); + startupStages.mark("context-engine"); + notifyExecutionPhase("context_engine", { provider, model: modelId }); + try { + const compactionRuntime = createEmbeddedRunCompactionRuntime({ + runParams: params, + contextEngine, + hookRunner, + hookContext: hookCtx, + sessionPromptState, + }); + let authRetryPending = false; + let accumulatedReplayState = createEmbeddedRunReplayState(); + // Hoisted so the retry-limit error path can use the most recent API total. + let lastTurnTotal: number | undefined; + while (true) { + refreshPreparedRuntimeSnapshot(); + if (runLoopIterations >= MAX_RUN_LOOP_ITERATIONS) { + const message = + `Exceeded retry limit after ${runLoopIterations} attempts ` + + `(max=${MAX_RUN_LOOP_ITERATIONS}).`; + log.error( + `[run-retry-limit] sessionKey=${params.sessionKey ?? params.sessionId} ` + + `provider=${provider}/${modelId} attempts=${runLoopIterations} ` + + `maxAttempts=${MAX_RUN_LOOP_ITERATIONS}`, + ); + const retryLimitDecision = resolveRunFailoverDecision({ + stage: "retry_limit", + fallbackConfigured, + failoverReason: lastRetryFailoverReason, + }); + return handleRetryLimitExhaustion({ + message, + decision: retryLimitDecision, + provider, + model: modelId, + profileId: lastProfileId, + durationMs: Date.now() - started, + agentMeta: buildErrorAgentMeta({ + sessionId: sessionPromptState.sessionId, + sessionFile: sessionPromptState.sessionFile, + provider, + model: model.id, + ...outerContextTokenMeta, + usageAccumulator, + lastRunPromptUsage, + lastTurnTotal, + }), + replayInvalid: accumulatedReplayState.replayInvalid ? true : undefined, + livenessState: "blocked", + }); + } + runLoopIterations += 1; + const runtimeAuthRetry: boolean = authRetryPending; + authRetryPending = false; + attemptedThinking.add(thinkLevel); + const codexAppServerRecoveryRetryAvailable = hasCodexAppServerRecoveryRetryBudget({ + alreadyRetried: codexAppServerRecoveryRetries > 0, + runLoopIterations, + maxRunLoopIterations: MAX_RUN_LOOP_ITERATIONS, + }); + const dispatch = await prepareAndDispatchEmbeddedRunAttempt({ + runInput: input, + preparedRuntime, + contextEngine, + sessionPromptState, + terminalRetryState, + replayState: accumulatedReplayState, + provider, + modelId, + startupStagesEmitted, + bootstrapPromptWarningSignaturesSeen, + resolveRuntimeFallbackReason, + observeToolOutcome, + allocateToolOutcomeOrdinal, + getPostCompactionAbortError: () => postCompactionAbortError, + setPostCompactionAbortController: (controller) => { + postCompactionAbortController = controller; + }, + clearPostCompactionAbortController: (controller) => { + if (postCompactionAbortController === controller) { + postCompactionAbortController = undefined; + } + }, + }); + startupStagesEmitted = dispatch.startupStagesEmitted; + const { dispatchedAttempt, runtimePlan } = dispatch; + const normalizedAttempt = await normalizeEmbeddedRunAttempt({ + runInput: input, + preparedRuntime, + dispatchedAttempt, + sessionPromptState, + provider, + modelId, + bootstrapPromptWarningSignaturesSeen, + usageAccumulator, + lastRunPromptUsage, + lastTurnTotal, + idleTimeoutBreakerState, + contextRecoveryState, + replayState: accumulatedReplayState, + lastRetryFailoverReason, + }); + if (normalizedAttempt.action === "complete") { + return normalizedAttempt.result; + } + if (normalizedAttempt.action === "retry") { + bootstrapPromptWarningSignaturesSeen = + normalizedAttempt.bootstrapPromptWarningSignaturesSeen; + lastRunPromptUsage = normalizedAttempt.lastRunPromptUsage; + lastTurnTotal = normalizedAttempt.lastTurnTotal; + accumulatedReplayState = normalizedAttempt.replayState; + continue; + } + bootstrapPromptWarningSignaturesSeen = normalizedAttempt.bootstrapPromptWarningSignaturesSeen; + lastRunPromptUsage = normalizedAttempt.lastRunPromptUsage; + lastTurnTotal = normalizedAttempt.lastTurnTotal; + accumulatedReplayState = normalizedAttempt.replayState; + const { + attempt, + aborted, + externalAbort, + promptError, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + sessionIdUsed, + sessionFileUsed, + currentAttemptAssistant, + attemptAssistant, + terminalOutcome, + terminalAborted, + terminalTimedOut, + terminalInterrupted, + signalOwnedInterruption, + setTerminalLifecycleMeta, + attemptCompactionCount, + activeErrorContext, + resolveReplayInvalidForAttempt, + canRestartForLiveSwitch, + } = normalizedAttempt; + const recovery = await recoverEmbeddedRunAttempt({ + runInput: input, + preparedRuntime, + normalizedAttempt, + runtimePlan, + sessionPromptState, + failoverRetryController, + compactionRuntime, + contextEngine, + contextRecoveryState, + resolveContextEnginePluginId, + buildRuntimeSettings: buildEmbeddedContextEngineRuntimeSettings, + armPostCompactionGuard: () => postCompactionGuard.armPostCompaction(), + usageAccumulator, + lastRunPromptUsage, + lastTurnTotal, + runtimeAuthRetry, + codexAppServerRecoveryRetryAvailable, + codexAppServerRecoveryRetries, + lastRetryFailoverReason, + traceAttempts, + sessionAgentId, + }); + if (recovery.action === "complete") { + return recovery.result; + } + if (recovery.action === "retry") { + thinkLevel = recovery.thinkLevel; + authRetryPending = recovery.authRetryPending; + codexAppServerRecoveryRetries = recovery.codexAppServerRecoveryRetries; + lastRetryFailoverReason = recovery.lastRetryFailoverReason; + continue; + } + const { shouldSurfaceCodexCompletionTimeout } = recovery; + + const assistantFailureOutcome = await handleEmbeddedAssistantFailure({ + runParams: params, + attempt, + attemptAssistant, + currentAttemptAssistant, + terminalProviderStarted: terminalOutcome.providerStarted === true, + terminalInterrupted, + promptError, + activeErrorContext, + provider, + modelId, + model: model.id, + thinkLevel, + getThinkLevel: () => preparedRuntime.snapshot().thinkLevel, + attemptedThinking, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + signalOwnedInterruption, + externalAbort, + aborted, + fallbackConfigured, + pluginHarnessOwnsTransport, + canRestartForLiveSwitch, + authProfileId: lastProfileId, + authProfileStore: attemptAuthProfileStore, + runtimeAuthRetry, + maybeRefreshRuntimeAuthForAuthError, + resolveAuthProfileFailureReason: failoverRetryController.resolveAuthProfileFailureReason, + emptyErrorRetries, + overloadProfileRotations, + overloadProfileRotationLimit: failoverRetryController.overloadProfileRotationLimit, + rateLimitProfileRotations: failoverRetryController.rateLimitProfileRotations, + rateLimitProfileRotationLimit: failoverRetryController.rateLimitProfileRotationLimit, + sameModelIdleTimeoutRetries, + previousRetryFailoverReason: lastRetryFailoverReason, + maybeMarkAuthProfileFailure: failoverRetryController.maybeMarkAuthProfileFailure, + maybeEscalateRateLimitProfileFallback: + failoverRetryController.maybeEscalateRateLimitProfileFallback, + maybeRetrySameModelRateLimit: failoverRetryController.maybeRetrySameModelRateLimit, + maybeBackoffBeforeOverloadFailover: + failoverRetryController.maybeBackoffBeforeOverloadFailover, + advanceAttemptAuthProfile, + traceAttempts, + suspendForFailure, + suspensionSessionId: sessionPromptState.sessionId ?? params.sessionId, + agentDir, + isProbeSession, + }); + thinkLevel = assistantFailureOutcome.thinkLevel; + preparedRuntime.setThinkLevel(thinkLevel); + authRetryPending = assistantFailureOutcome.authRetryPending; + emptyErrorRetries = assistantFailureOutcome.emptyErrorRetries; + overloadProfileRotations = assistantFailureOutcome.overloadProfileRotations; + sameModelIdleTimeoutRetries = assistantFailureOutcome.sameModelIdleTimeoutRetries; + lastRetryFailoverReason = assistantFailureOutcome.lastRetryFailoverReason; + if (!assistantFailureOutcome.preserveSameModelRateLimitRetryCount) { + failoverRetryController.resetSameModelRateLimitRetries(); + } + if (assistantFailureOutcome.action === "retry") { + continue; + } + const assistantProfileFailureReason = assistantFailureOutcome.assistantProfileFailureReason; + const { + agentMeta, + reportedModelRef, + finalAssistantVisibleText, + finalAssistantRawText, + payloads, + payloadsWithToolMedia, + timedOutDuringPrompt, + recoveredFinalAssistantPayloadsAfterPromptTimeout, + hasSuccessfulFinalAssistantAfterPromptTimeout, + hasPartialAssistantTextAfterPromptTimeout, + attemptToolSummary, + failureSignal, + } = prepareEmbeddedRunTerminal({ + runParams: params, + attempt, + attemptAssistant, + currentAttemptAssistant, + provider, + model: model.id, + activeErrorContext, + authProfileStore: attemptAuthProfileStore, + authProfileId: lastProfileId, + sessionIdUsed, + sessionFileUsed, + outerContextTokenMeta, + usageAccumulator, + lastRunPromptUsage, + lastTurnTotal, + contextRecoveryState, + resolvedToolResultFormat, + terminalInterrupted, + terminalTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + }); + + const terminalTimeoutResult = resolveEmbeddedRunTerminalTimeout({ + timedOutDuringPrompt, + hasSuccessfulFinalAssistantAfterPromptTimeout, + shouldSurfaceCodexCompletionTimeout, + idleTimedOut, + attempt, + hasPartialAssistantTextAfterPromptTimeout, + payloads, + payloadsWithToolMedia, + terminalAborted, + terminalTimedOut, + terminalOutcome, + resolveReplayInvalid: resolveReplayInvalidForAttempt, + setTerminalLifecycleMeta, + startedAtMs: started, + agentMeta, + finalAssistantVisibleText, + finalAssistantRawText, + attemptToolSummary, + failureSignal, + }); + if (terminalTimeoutResult) { + return terminalTimeoutResult; + } + + const terminalResolution = await resolveEmbeddedRunTerminal({ + runParams: params, + retryState: terminalRetryState, + attempt, + attemptAssistant, + activeErrorContext, + modelApi: effectiveModel.api, + executionContract, + terminalAborted, + terminalTimedOut, + terminalInterrupted, + externalAbort, + signalOwnedInterruption, + promptError, + payloadsWithToolMedia, + recoveredFinalAssistantPayloadsAfterPromptTimeout, + finalAssistantVisibleText, + finalAssistantRawText, + agentMeta, + attemptToolSummary, + failureSignal, + maxReasoningOnlyRetryAttempts, + maxEmptyResponseRetryAttempts, + attemptCompactionCount, + replayState: accumulatedReplayState, + activePromptPersisted: sessionPromptState.activePrompt.persisted, + activateInternalPrompt: sessionPromptState.activateInternalPrompt, + setSuppressNextUserMessagePersistence: (value) => { + sessionPromptState.suppressNextUserMessagePersistence = value; + }, + armPostCompactionGuard: () => postCompactionGuard.armPostCompaction(), + readTerminalToolPresentation: readAttemptTerminalToolPresentation, + resolveReplayInvalid: resolveReplayInvalidForAttempt, + setTerminalLifecycleMeta, + maybeMarkAuthProfileFailure: failoverRetryController.maybeMarkAuthProfileFailure, + assistantProfileFailureReason, + startedAtMs: started, + provider, + modelId, + authProfileId: lastProfileId, + profileFailureStore, + attemptAuthProfileStore, + apiKeyInfo: getApiKeyInfo(), + agentHarnessId: agentHarness.id, + pluginHarnessOwnsTransport, + pluginHarnessOwnsAuthBootstrap, + reportedModelRef, + traceAttempts, + traceAttemptUsesFallback, + thinkLevel, + contextRecoveryState, + }); + if (terminalResolution.action === "retry") { + continue; + } + return terminalResolution.result; + } + } finally { + if (params.isFinalFallbackAttempt !== false) { + await maybeEmitFastModeAutoResetBestEffort(); + } + forgetPromptBuildDrainCacheForRun(params.runId); + stopRuntimeAuthRefreshTimer(); + await runAgentCleanupStep({ + runId: params.runId, + sessionId: params.sessionId, + step: "context-engine-dispose", + log, + cleanup: async () => { + await contextEngine.dispose?.(); + }, + }); + if (params.cleanupBundleMcpOnRunEnd === true) { + await runAgentCleanupStep({ + runId: params.runId, + sessionId: params.sessionId, + step: "bundle-mcp-retire", + log, + cleanup: async () => { + const onError = (errorLocal: unknown, sessionId: string) => { + log.warn( + `bundle-mcp cleanup failed after run for ${sessionId}: ${formatErrorMessage(errorLocal)}`, + ); + }; + const retiredBySessionKey = await retireSessionMcpRuntimeForSessionKey({ + sessionKey: params.sessionKey, + reason: "embedded-run-end", + // MCP App views hold bounded leases so their bridge can remain + // usable after a one-shot gateway run returns. + preserveActiveLeases: true, + onError, + }); + if (!retiredBySessionKey) { + await retireSessionMcpRuntime({ + sessionId: params.sessionId, + reason: "embedded-run-end", + preserveActiveLeases: true, + onError, + }); + } + }, + }); + } + } +} diff --git a/src/agents/embedded-agent-runner/run-orchestrator.ts b/src/agents/embedded-agent-runner/run-orchestrator.ts new file mode 100644 index 000000000000..893230b84e96 --- /dev/null +++ b/src/agents/embedded-agent-runner/run-orchestrator.ts @@ -0,0 +1,299 @@ +/** + * Embedded-agent run orchestration implementation. + */ +import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { SILENT_REPLY_TOKEN } from "../../auto-reply/tokens.js"; +import { getRuntimeConfigSnapshot } from "../../config/config.js"; +import { + captureAgentRunLifecycleGeneration, + getAgentEventLifecycleGeneration, + withAgentRunLifecycleGeneration, +} from "../../infra/agent-events.js"; +import { buildAgentHookContextChannelFields } from "../../plugins/hook-agent-context.js"; +import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; +import { resolveUserPath } from "../../utils.js"; +import { isMarkdownCapableMessageChannel } from "../../utils/message-channel.js"; +import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agent-scope.js"; +import { + applyAgentRunSessionTargetIdentity, + resolveAgentRunSessionTarget, +} from "../run-session-target.js"; +import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js"; +import { + resolveSessionSuspensionTarget, + suspendSession, + type SessionSuspensionParams, +} from "../session-suspension.js"; +import { redactRunIdentifier, resolveRunWorkspaceDir } from "../workspace-run.js"; +import { waitForDeferredTurnMaintenanceForSession } from "./context-engine-maintenance.js"; +import { resolveGlobalLane, resolveSessionLane } from "./lanes.js"; +import { log } from "./logger.js"; +import { executePreparedEmbeddedRun } from "./run-execution.js"; +import { + createEmbeddedRunStageTracker, + formatEmbeddedRunStageSummary, + shouldWarnEmbeddedRunStageSummary, +} from "./run/attempt-stage-timing.js"; +import { hasEmbeddedRunConfiguredModelFallbacks } from "./run/fallbacks.js"; +import { buildHandledReplyPayloads } from "./run/handled-reply.js"; +import type { + RunEmbeddedAgentInternalParams, + RunEmbeddedAgentParamsWithSessionFile, +} from "./run/internal-params.js"; +import { createEmbeddedRunLaneController } from "./run/lane-controller.js"; +import { + EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, + resolveEmbeddedRunLaneTimeoutMs, +} from "./run/lane-runtime.js"; +import type { RunEmbeddedAgentParams } from "./run/params.js"; +import { createEmbeddedRunProgressController } from "./run/progress-controller.js"; +import { resolveInitialEmbeddedRunModel } from "./run/runtime-resolution.js"; +import { assertAgentHarnessRunAdmission, backfillSessionKey } from "./run/session-bootstrap.js"; +import type { EmbeddedAgentRunResult } from "./types.js"; + +export function runEmbeddedAgent( + paramsInput: RunEmbeddedAgentParams, +): Promise { + const internalParamsInput = paramsInput as RunEmbeddedAgentInternalParams; + const requestedProvider = normalizeOptionalString(internalParamsInput.provider); + const requestedModel = normalizeOptionalString(internalParamsInput.model); + const needsConfiguredDefault = + !internalParamsInput.config && !requestedProvider && !requestedModel; + const config = + internalParamsInput.config ?? + (needsConfiguredDefault ? (getRuntimeConfigSnapshot() ?? undefined) : undefined); + const lifecycleGeneration = + internalParamsInput.lifecycleGeneration ?? + captureAgentRunLifecycleGeneration(internalParamsInput.runId); + return withAgentRunLifecycleGeneration(lifecycleGeneration, () => + runEmbeddedAgentInternal({ + ...internalParamsInput, + config, + lifecycleGeneration, + }), + ); +} + +async function runEmbeddedAgentInternal( + paramsInput: RunEmbeddedAgentInternalParams, +): Promise { + const paramsBase = applyAgentRunSessionTargetIdentity(paramsInput); + const skillWorkshopProposalMutationBudget = paramsBase.skillWorkshopProposalOnly + ? (paramsBase.skillWorkshopProposalMutationBudget ?? { remaining: 1 }) + : undefined; + let lifecycleGeneration = paramsBase.lifecycleGeneration!; + const queuedLifecycleGeneration = getAgentEventLifecycleGeneration(); + // Resolve sessionKey early so all downstream consumers (hooks, LCM, compaction) + // receive a non-null key even when callers omit it. See #60552. + const effectiveSessionKey = backfillSessionKey({ + config: paramsBase.config, + sessionId: paramsBase.sessionId, + sessionKey: paramsBase.sessionKey, + agentId: paramsBase.agentId, + }); + assertAgentHarnessRunAdmission({ ...paramsBase, sessionKey: effectiveSessionKey }); + const runSessionTarget = await resolveAgentRunSessionTarget({ + ...paramsBase, + sessionKey: effectiveSessionKey, + }); + let params: RunEmbeddedAgentParamsWithSessionFile = { + ...paramsBase, + agentId: paramsBase.agentId ?? runSessionTarget.agentId, + sessionId: runSessionTarget.sessionId, + sessionKey: normalizeOptionalString(effectiveSessionKey ?? runSessionTarget.sessionKey), + sessionFile: runSessionTarget.sessionFile, + skillWorkshopProposalMutationBudget, + }; + const sessionLane = resolveSessionLane(params.sessionKey?.trim() || params.sessionId); + const globalLane = resolveGlobalLane(params.lane); + // Outer fallback attempts defer session suspension only while another + // candidate remains. Direct and final-candidate runs suspend normally. + const failureSuspension = resolveSessionSuspensionTarget(); + const suspendForFailure = (suspensionParams: Omit) => { + const suspension = { ...suspensionParams, laneId: globalLane }; + if (failureSuspension.mode === "defer") { + failureSuspension.defer(suspension); + return; + } + void suspendSession(suspension); + }; + const laneController = createEmbeddedRunLaneController({ + getLifecycleGeneration: () => lifecycleGeneration, + getParams: () => params, + globalLane, + initialQueuedLifecycleGeneration: queuedLifecycleGeneration, + sessionLane, + setLifecycleGeneration: (generation) => { + lifecycleGeneration = generation; + }, + setParams: (nextParams) => { + params = nextParams; + }, + }); + const { enqueueGlobal, enqueueSession, noteLaneTaskProgress, throwIfAborted } = laneController; + const channelHint = params.messageChannel ?? params.messageProvider; + const resolvedToolResultFormat = + params.toolResultFormat ?? + (channelHint + ? isMarkdownCapableMessageChannel(channelHint) + ? "markdown" + : "plain" + : "markdown"); + const isProbeSession = params.sessionId?.startsWith("probe-") ?? false; + + throwIfAborted(); + + return enqueueSession(async () => { + throwIfAborted(); + // Same-session reads below must see any prior deferred transcript rewrite. + // Checkpoint before the global lane so unrelated sessions can still start + // while this session waits on its own maintenance lane. + params.replyOperation?.markWaitingForDeferredMaintenance(); + try { + await waitForDeferredTurnMaintenanceForSession(params.sessionKey); + } finally { + params.replyOperation?.markDeferredMaintenanceWaitEnded(); + } + throwIfAborted(); + return enqueueGlobal(async () => { + throwIfAborted(); + const started = Date.now(); + const startupStages = createEmbeddedRunStageTracker(); + const progressController = createEmbeddedRunProgressController({ + attempt: params, + noteLaneTaskProgress, + startedAtMs: started, + }); + const { notifyExecutionPhase } = progressController; + const emitStartupStageSummary = (phase: string) => { + const summary = startupStages.snapshot(); + const shouldWarn = shouldWarnEmbeddedRunStageSummary(summary); + if (!shouldWarn && !log.isEnabled("trace")) { + return; + } + const message = formatEmbeddedRunStageSummary( + `[trace:embedded-run] startup stages: runId=${params.runId} sessionId=${params.sessionId} phase=${phase}`, + summary, + ); + if (shouldWarn) { + log.warn(message); + } else { + log.trace(message); + } + }; + params.onExecutionStarted?.({ lifecycleGeneration }); + notifyExecutionPhase("runner_entered"); + const workspaceResolution = resolveRunWorkspaceDir({ + workspaceDir: params.workspaceDir, + sessionKey: params.sessionKey, + agentId: params.agentId, + config: params.config, + }); + const resolvedWorkspace = workspaceResolution.workspaceDir; + const canonicalWorkspace = resolveUserPath( + resolveAgentWorkspaceDir(params.config ?? {}, workspaceResolution.agentId), + ); + const isCanonicalWorkspace = canonicalWorkspace === resolvedWorkspace; + const redactedSessionId = redactRunIdentifier(params.sessionId); + const redactedSessionKey = redactRunIdentifier(params.sessionKey); + const redactedWorkspace = redactRunIdentifier(resolvedWorkspace); + if (workspaceResolution.usedFallback) { + log.warn( + `[workspace-fallback] caller=runEmbeddedAgent reason=${workspaceResolution.fallbackReason} run=${params.runId} session=${redactedSessionId} sessionKey=${redactedSessionKey} agent=${workspaceResolution.agentId} workspace=${redactedWorkspace}`, + ); + } + startupStages.mark("workspace"); + notifyExecutionPhase("workspace"); + ensureRuntimePluginsLoaded({ + config: params.config, + workspaceDir: resolvedWorkspace, + allowGatewaySubagentBinding: params.allowGatewaySubagentBinding, + }); + startupStages.mark("runtime-plugins"); + notifyExecutionPhase("runtime_plugins"); + + let { provider, modelId } = resolveInitialEmbeddedRunModel({ + config: params.config, + agentId: workspaceResolution.agentId, + provider: params.provider, + model: params.model, + }); + const agentDir = + params.agentDir ?? resolveAgentDir(params.config ?? {}, workspaceResolution.agentId); + const normalizedSessionKey = params.sessionKey?.trim(); + const fallbackConfigured = hasEmbeddedRunConfiguredModelFallbacks({ + cfg: params.config, + agentId: params.agentId, + sessionKey: normalizedSessionKey, + modelFallbacksOverride: params.modelFallbacksOverride, + }); + const resolvedSessionKey = + normalizedSessionKey ?? params.sessionTarget?.sessionKey ?? params.sessionId; + const hookRunner = getGlobalHookRunner(); + const hookCtx = { + runId: params.runId, + jobId: params.jobId, + agentId: workspaceResolution.agentId, + sessionKey: resolvedSessionKey, + sessionId: params.sessionId, + workspaceDir: resolvedWorkspace, + modelProviderId: provider, + modelId, + trigger: params.trigger, + ...buildAgentHookContextChannelFields(params), + }; + if (params.trigger === "cron" && hookRunner?.hasHooks("before_agent_reply")) { + notifyExecutionPhase("before_agent_reply", { provider, model: modelId }); + const hookResult = await hookRunner.runBeforeAgentReply( + { cleanedBody: params.prompt }, + hookCtx, + ); + if (hookResult?.handled) { + return { + payloads: buildHandledReplyPayloads(hookResult.reply), + meta: { + durationMs: Date.now() - started, + agentMeta: { + sessionId: params.sessionId, + provider, + model: modelId, + }, + finalAssistantVisibleText: hookResult.reply?.text ?? SILENT_REPLY_TOKEN, + finalAssistantRawText: hookResult.reply?.text ?? SILENT_REPLY_TOKEN, + }, + }; + } + notifyExecutionPhase("runtime_plugins", { provider, model: modelId }); + } + + return executePreparedEmbeddedRun({ + runParams: params, + provider, + modelId, + agentDir, + workspaceResolution, + workspaceDir: resolvedWorkspace, + isCanonicalWorkspace, + globalLane, + hookRunner, + hookContext: hookCtx, + fallbackConfigured, + isProbeSession, + resolvedSessionKey, + resolvedToolResultFormat, + startedAtMs: started, + startupStages, + emitStartupStageSummary, + progressController, + laneController, + lifecycleGeneration, + suspendForFailure, + }); + }); + }); +} + +export const testing = { + EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, + resolveEmbeddedRunLaneTimeoutMs, +}; diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts index 48fc59e57390..1b1f9e947829 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts @@ -755,6 +755,140 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { }); }); + it("does not recover a stale prior assistant after the current prompt times out", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const staleAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [{ type: "text", text: "Stale answer from the prior attempt." }], + } as unknown as NonNullable; + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + timedOut: true, + lastAssistant: staleAssistant, + currentAttemptAssistant: undefined, + }), + ); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId: "run-prompt-timeout-stale-assistant", + }); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.some((payload) => payload.text?.includes("timed out"))).toBe(true); + expect(result.payloads?.some((payload) => payload.text?.includes("Stale answer"))).toBe(false); + expect(result.meta.finalAssistantVisibleText).toBeUndefined(); + }); + + it("recovers a completed prompt-timeout assistant without collected assistant text", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const finalText = "Completed answer after the timeout race."; + const finalAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [{ type: "text", text: finalText }], + } as unknown as NonNullable; + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: undefined as unknown as string[], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId: "run-prompt-timeout-no-assistant-texts", + }); + + expect(result.payloads).toEqual([{ text: finalText }]); + }); + + it("preserves tool media when prompt-timeout recovery replaces partial assistant text", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const partialText = "Partial answer before the timeout race."; + const finalText = "Complete answer after the timeout race."; + const finalAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [{ type: "text", text: finalText }], + } as unknown as NonNullable; + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [partialText], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + toolMediaUrls: ["https://example.test/recovered-output.png"], + }), + ); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId: "run-prompt-timeout-final-assistant-media", + }); + + expect(result.payloads).toEqual([ + { + mediaUrl: "https://example.test/recovered-output.png", + mediaUrls: ["https://example.test/recovered-output.png"], + audioAsVoice: undefined, + trustedLocalMedia: undefined, + }, + { text: finalText }, + ]); + }); + + it("replaces the latest partial assistant payload after prompt-timeout recovery", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const completedText = "Completed answer block before the final response."; + const partialText = "Partial final response before the timeout race."; + const finalText = "Complete final response after the timeout race."; + mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([ + { text: completedText }, + { text: partialText }, + ]); + const finalAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [{ type: "text", text: finalText }], + } as unknown as NonNullable; + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [completedText, partialText], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId: "run-prompt-timeout-latest-partial", + }); + + expect(result.payloads).toEqual([{ text: completedText }, { text: finalText }]); + }); + it("records same-model rate-limit retries without a profile-rotation trace", async () => { const rateLimitMessage = "429 rate_limit_exceeded: requests per minute exceeded; Retry-After: 30"; diff --git a/src/agents/embedded-agent-runner/run.ts b/src/agents/embedded-agent-runner/run.ts index c8da6ca749b5..e9d58d5059b5 100644 --- a/src/agents/embedded-agent-runner/run.ts +++ b/src/agents/embedded-agent-runner/run.ts @@ -1,4082 +1,4 @@ /** - * Top-level embedded-agent run orchestration entrypoint. + * Public embedded-agent run entrypoint. */ -import { randomBytes } from "node:crypto"; -import fs from "node:fs/promises"; -import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; -import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js"; -import type { ThinkLevel } from "../../auto-reply/thinking.js"; -import { SILENT_REPLY_TOKEN } from "../../auto-reply/tokens.js"; -import { getRuntimeConfigSnapshot } from "../../config/config.js"; -import { resolveStorePath } from "../../config/sessions.js"; -import { resolveSessionTranscriptRuntimeReadTarget } from "../../config/sessions/session-accessor.js"; -import { parseSqliteSessionFileMarker } from "../../config/sessions/sqlite-marker.js"; -import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../context-engine/host-compat.js"; -import { ensureContextEnginesInitialized } from "../../context-engine/init.js"; -import { - resolveContextEngine, - resolveContextEngineOwnerPluginId, -} from "../../context-engine/registry.js"; -import { buildContextEngineRuntimeSettings } from "../../context-engine/runtime-settings.js"; -import { - type ContextEngineSessionTarget, - resolveCompactionSuccessorTranscript, -} from "../../context-engine/types.js"; -import { - captureAgentRunLifecycleGeneration, - getAgentEventLifecycleGeneration, - registerAgentRunContext, - withAgentRunLifecycleGeneration, -} from "../../infra/agent-events.js"; -import { sleepWithAbort } from "../../infra/backoff.js"; -import { freezeDiagnosticTraceContext } from "../../infra/diagnostic-trace-context.js"; -import { formatErrorMessage, toErrorObject } from "../../infra/errors.js"; -import { redactIdentifier } from "../../logging/redact-identifier.js"; -import { buildAgentHookContextChannelFields } from "../../plugins/hook-agent-context.js"; -import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; -import { looksLikeSecretSentinel, resolveSecretSentinel } from "../../secrets/sentinel.js"; -import { createAgentHarnessTaskRuntimeScope } from "../../tasks/agent-harness-task-runtime-scope.js"; -import { createTrajectoryRuntimeRecorder } from "../../trajectory/runtime.js"; -import { resolveUserPath } from "../../utils.js"; -import { isMarkdownCapableMessageChannel } from "../../utils/message-channel.js"; -import { - retireSessionMcpRuntime, - retireSessionMcpRuntimeForSessionKey, -} from "../agent-bundle-mcp-tools.js"; -import { - resolveAgentDir, - resolveSessionAgentIds, - resolveAgentWorkspaceDir, -} from "../agent-scope.js"; -import type { ToolOutcomeObservation } from "../agent-tools.before-tool-call.js"; -import { resolveProcessToolScopeKey } from "../agent-tools.js"; -import { - type AuthProfileFailureReason, - type AuthProfileStore, - isProfileInCooldown, - markAuthProfileFailure, - markAuthProfileSuccess, -} from "../auth-profiles.js"; -import { listActiveProcessSessionReferences } from "../bash-process-references.js"; -import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js"; -import { - classifyAssistantFailoverReason, - classifyFailoverReason, - extractObservedOverflowTokenCount, - type FailoverReason, - formatAssistantErrorText, - isAuthAssistantError, - isBillingAssistantError, - isCompactionFailureError, - isFailoverAssistantError, - isFailoverErrorMessage, - isGenericUnknownStreamErrorMessage, - isLikelyContextOverflowError, - isRateLimitAssistantError, - parseImageDimensionError, - parseImageSizeError, - pickFallbackThinkingLevel, -} from "../embedded-agent-helpers.js"; -import { - fingerprintAuthProfileOwnerShape, - fingerprintAwsSdkRuntimeOwner, - fingerprintOpaqueRuntimeOwner, - fingerprintResolvedAuthProfileCredential, - fingerprintResolvedProviderAuth, -} from "../execution-auth-binding.js"; -import { isStrictAgenticExecutionContractActive } from "../execution-contract.js"; -import { - coerceToFailoverError, - describeFailoverError, - FailoverError, - resolveFailoverStatus, -} from "../failover-error.js"; -import { agentHarnessBuildsOpenClawTools } from "../harness/selection.js"; -import { LiveSessionModelSwitchError } from "../live-model-switch-error.js"; -import { shouldSwitchToLiveModel, clearLiveModelSwitchPending } from "../live-model-switch.js"; -import { - applyAuthHeaderOverride, - applyLocalNoAuthHeaderOverride, - type ResolvedProviderAuth, -} from "../model-auth.js"; -import { hasOnlyAssistantReasoningContent } from "../replay-turn-classification.js"; -import { runAgentCleanupStep } from "../run-cleanup-timeout.js"; -import { - applyAgentRunSessionTargetIdentity, - resolveAgentRunSessionTarget, -} from "../run-session-target.js"; -import { createAgentRunDirectAbortError } from "../run-termination.js"; -import { buildAgentRuntimePlan } from "../runtime-plan/build.js"; -import { - hasPreparedAuthAttemptModelMetadata, - resolveCredentialScopedAuthAttemptModelDecision, -} from "../runtime-plan/credential-scoped-model.js"; -import { - canRunPreparedAgentRuntimeAuthAttempt, - type PreparedAgentRuntimeAuthAttempt, -} from "../runtime-plan/prepare-auth.js"; -import type { AgentRuntimeAuthPlan } from "../runtime-plan/types.js"; -import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js"; -import { - resolveSessionSuspensionReason, - resolveSessionSuspensionTarget, - suspendSession, - type SessionSuspensionParams, -} from "../session-suspension.js"; -import { resolveCandidateThinkingLevel } from "../thinking-runtime.js"; -import { resolveToolLoopDetectionConfig } from "../tool-loop-detection-config.js"; -import { deriveContextPromptTokens, normalizeUsage, type UsageLike } from "../usage.js"; -import { redactRunIdentifier, resolveRunWorkspaceDir } from "../workspace-run.js"; -import { runPostCompactionSideEffects } from "./compaction-hooks.js"; -import { buildEmbeddedCompactionRuntimeContext } from "./compaction-runtime-context.js"; -import { - compactContextEngineWithSafetyTimeout, - resolveCompactionTimeoutMs, -} from "./compaction-safety-timeout.js"; -import { resolveContextEngineCapabilities } from "./context-engine-capabilities.js"; -import { - runContextEngineMaintenance, - waitForDeferredTurnMaintenanceForSession, -} from "./context-engine-maintenance.js"; -import { - hasMessagingToolDeliveryEvidence, - hasOutboundDeliveryEvidence, -} from "./delivery-evidence.js"; -import { resolveEmbeddedRunFailureSignal } from "./failure-signal.js"; -import { resolveGlobalLane, resolveSessionLane } from "./lanes.js"; -import { log } from "./logger.js"; -import { - createPostCompactionLoopGuard, - PostCompactionLoopPersistedError, -} from "./post-compaction-loop-guard.js"; -import { createEmbeddedRunReplayState, observeReplayMetadata } from "./replay-state.js"; -import { - handleAssistantFailover, - isShortWindowRateLimitMessage, -} from "./run/assistant-failover.js"; -import { - createEmbeddedRunStageTracker, - EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE, - formatEmbeddedRunStageSummary, - shouldWarnEmbeddedRunStageSummary, -} from "./run/attempt-stage-timing.js"; -import { forgetPromptBuildDrainCacheForRun } from "./run/attempt.prompt-helpers.js"; -import { - createEmbeddedRunAuthController, - resolveEmbeddedAuthCooldownProbePolicy, -} from "./run/auth-controller.js"; -import { prepareEmbeddedRunAuthPlan } from "./run/auth-plan.js"; -import { resolveAuthProfileFailureReason } from "./run/auth-profile-failure-policy.js"; -import { createScopedAuthProfileStore, resolveAttemptDispatchApiKey } from "./run/auth-store.js"; -import { runEmbeddedAttemptWithBackend } from "./run/backend.js"; -import { - hasCodexAppServerRecoveryRetryBudget, - resolveCodexAppServerRecoveryRetry, -} from "./run/codex-app-server-recovery.js"; -import { createFailoverDecisionLogger } from "./run/failover-observation.js"; -import { mergeRetryFailoverReason, resolveRunFailoverDecision } from "./run/failover-policy.js"; -import { hasEmbeddedRunConfiguredModelFallbacks } from "./run/fallbacks.js"; -import { buildHandledReplyPayloads } from "./run/handled-reply.js"; -import { - buildErrorAgentMeta, - buildUsageAgentMetaFields, - createCompactionDiagId, - isAssistantForModelRef, - resolveActiveErrorContext, - resolveFinalAssistantRawText, - resolveFinalAssistantVisibleText, - resolveLatestCallUsage, - resolveMaxRunRetryIterations, - resolveReportedModelRef, - MAX_SAME_MODEL_RATE_LIMIT_RETRIES, - resolveOverloadFailoverBackoffMs, - resolveOverloadProfileRotationLimit, - resolveRateLimitProfileRotationLimit, - resolveEmbeddedAttemptBasePrompt, - resolveNextSameModelRateLimitRetryCount, - resolveSameModelRateLimitRetryDelayMs, - type RuntimeAuthState, -} from "./run/helpers.js"; -import { - MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT, - createIdleTimeoutBreakerState, - stepIdleTimeoutBreaker, -} from "./run/idle-timeout-breaker.js"; -import { - DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT, - DEFAULT_REASONING_ONLY_RETRY_LIMIT, - hasAttemptTerminalState, - resolveAttemptReplayMetadata, - resolveEmptyResponseRetryInstruction, - resolveIncompleteTurnPayloadText, - resolveReasoningOnlyRetryInstruction, - resolveSilentToolResultReplyPayload, - resolveReplayInvalidFlag, - resolveRunLivenessState, - shouldRetryMissingAssistantTurn, - shouldRetrySilentErrorAssistantTurn, - shouldTreatEmptyAssistantReplyAsSilent, -} from "./run/incomplete-turn.js"; -import { createEmbeddedRunLaneController } from "./run/lane-controller.js"; -import { - EMBEDDED_RUN_LANE_HEARTBEAT_MS, - EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, - resolveEmbeddedRunLaneTimeoutMs, -} from "./run/lane-runtime.js"; -import { - resolveEmbeddedRunEffectiveModel, - selectEmbeddedRunHarness, - selectEmbeddedRunHarnessForPreparedAttempts, -} from "./run/model-harness.js"; -import { resolveEmbeddedRunModelSetup } from "./run/model-setup.js"; -import type { RunEmbeddedAgentParams } from "./run/params.js"; -import { buildEmbeddedRunPayloads } from "./run/payloads.js"; -import { createEmbeddedRunProgressController } from "./run/progress-controller.js"; -import { handleRetryLimitExhaustion } from "./run/retry-limit.js"; -import { - buildTraceToolSummary, - hasCompletedModelProgressForIdleBreaker, - normalizeEmbeddedRunAttemptResult, -} from "./run/run-attempt-result.js"; -import { - CODEX_HARNESS_ID, - resolveAttemptTrajectoryAttribution, - resolveInitialEmbeddedRunModel, - resolveInitialThinkLevel, -} from "./run/runtime-resolution.js"; -import { - assertAgentHarnessRunAdmission, - backfillSessionKey, - buildContextEngineCompactionSessionTarget, - isNoRealConversationCompactionNoop, - resetNoRealConversationTokenSnapshot, -} from "./run/session-bootstrap.js"; -import { resolveSkillWorkshopAttemptParams } from "./run/skill-workshop-attempt-params.js"; -import { - isEmbeddedRunTerminalAbort, - isEmbeddedRunTerminalInterrupted, - isEmbeddedRunTerminalTimeout, - resolveEmbeddedRunAttemptTerminalOutcome, -} from "./run/terminal-outcome.js"; -import { mergeAttemptToolMediaPayloads } from "./run/tool-media-payloads.js"; -import { - resolveLiveToolResultMaxChars, - sessionLikelyHasOversizedToolResults, - truncateOversizedToolResultsInActiveTarget, -} from "./tool-result-truncation.js"; -import type { EmbeddedAgentMeta, EmbeddedAgentRunResult, TraceAttempt } from "./types.js"; -import { createUsageAccumulator, mergeUsageIntoAccumulator } from "./usage-accumulator.js"; -import { mapThinkingLevelForProvider } from "./utils.js"; - -type ApiKeyInfo = ResolvedProviderAuth; - -const MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES = 1; -const MID_TURN_PRECHECK_CONTINUATION_PROMPT = - "Continue from the current transcript after the latest tool result. Do not repeat the original user request, and do not rerun completed tools unless the transcript shows they are still needed."; -const COMPACTION_CONTINUATION_RETRY_INSTRUCTION = - "The previous attempt compacted the conversation context before producing a final user-visible answer. Continue from the compacted transcript and produce the final answer now. Do not restart from scratch, do not repeat completed work, and do not rerun tools unless the transcript clearly lacks required evidence."; -const BEFORE_AGENT_FINALIZE_RETRY_PROMPT_PREFIX = - "Before accepting the previous final answer, apply this revision request and produce the revised final answer. Do not repeat completed work or rerun tools unless the request explicitly requires it."; -const MAX_BEFORE_AGENT_FINALIZE_REVISIONS = 3; -type RunEmbeddedAgentInternalParams = RunEmbeddedAgentParams & { - onSuccessfulAuthBinding?: ( - binding: import("../execution-auth-binding.js").AgentExecutionAuthBinding, - ) => void; - authProfileStateMode?: "read-write" | "read-only"; - /** Ring-zero tool override, supplied only by the OpenClaw orchestrator. */ - systemAgentTool?: import("../tools/system-agent-tool.js").SystemAgentToolOptions; -}; -type RunEmbeddedAgentParamsWithSessionFile = RunEmbeddedAgentInternalParams & { - sessionFile: string; -}; - -function buildBeforeAgentFinalizeRetryPrompt(reason: string): string { - return `${BEFORE_AGENT_FINALIZE_RETRY_PROMPT_PREFIX}\n\n${reason}`; -} - -const POST_RUN_AUTH_PROFILE_SUCCESS_SLOW_MS = 1_000; - -export function runEmbeddedAgent( - paramsInput: RunEmbeddedAgentParams, -): Promise { - const internalParamsInput = paramsInput as RunEmbeddedAgentInternalParams; - const requestedProvider = normalizeOptionalString(internalParamsInput.provider); - const requestedModel = normalizeOptionalString(internalParamsInput.model); - const needsConfiguredDefault = - !internalParamsInput.config && !requestedProvider && !requestedModel; - const config = - internalParamsInput.config ?? - (needsConfiguredDefault ? (getRuntimeConfigSnapshot() ?? undefined) : undefined); - const lifecycleGeneration = - internalParamsInput.lifecycleGeneration ?? - captureAgentRunLifecycleGeneration(internalParamsInput.runId); - return withAgentRunLifecycleGeneration(lifecycleGeneration, () => - runEmbeddedAgentInternal({ - ...internalParamsInput, - config, - lifecycleGeneration, - }), - ); -} - -async function runEmbeddedAgentInternal( - paramsInput: RunEmbeddedAgentInternalParams, -): Promise { - const paramsBase = applyAgentRunSessionTargetIdentity(paramsInput); - const skillWorkshopProposalMutationBudget = paramsBase.skillWorkshopProposalOnly - ? (paramsBase.skillWorkshopProposalMutationBudget ?? { remaining: 1 }) - : undefined; - let lifecycleGeneration = paramsBase.lifecycleGeneration!; - const queuedLifecycleGeneration = getAgentEventLifecycleGeneration(); - // Resolve sessionKey early so all downstream consumers (hooks, LCM, compaction) - // receive a non-null key even when callers omit it. See #60552. - const effectiveSessionKey = backfillSessionKey({ - config: paramsBase.config, - sessionId: paramsBase.sessionId, - sessionKey: paramsBase.sessionKey, - agentId: paramsBase.agentId, - }); - assertAgentHarnessRunAdmission({ ...paramsBase, sessionKey: effectiveSessionKey }); - const runSessionTarget = await resolveAgentRunSessionTarget({ - ...paramsBase, - sessionKey: effectiveSessionKey, - }); - let params: RunEmbeddedAgentParamsWithSessionFile = { - ...paramsBase, - agentId: paramsBase.agentId ?? runSessionTarget.agentId, - sessionId: runSessionTarget.sessionId, - sessionKey: normalizeOptionalString(effectiveSessionKey ?? runSessionTarget.sessionKey), - sessionFile: runSessionTarget.sessionFile, - skillWorkshopProposalMutationBudget, - }; - const sessionLane = resolveSessionLane(params.sessionKey?.trim() || params.sessionId); - const globalLane = resolveGlobalLane(params.lane); - // Outer fallback attempts defer session suspension only while another - // candidate remains. Direct and final-candidate runs suspend normally. - const failureSuspension = resolveSessionSuspensionTarget(); - const suspendForFailure = (suspensionParams: Omit) => { - const suspension = { ...suspensionParams, laneId: globalLane }; - if (failureSuspension.mode === "defer") { - failureSuspension.defer(suspension); - return; - } - void suspendSession(suspension); - }; - const { - enqueueGlobal, - enqueueSession, - laneTaskAbortController, - laneTaskReleaseController, - noteLaneTaskProgress, - throwIfAborted, - } = createEmbeddedRunLaneController({ - getLifecycleGeneration: () => lifecycleGeneration, - getParams: () => params, - globalLane, - initialQueuedLifecycleGeneration: queuedLifecycleGeneration, - sessionLane, - setLifecycleGeneration: (generation) => { - lifecycleGeneration = generation; - }, - setParams: (nextParams) => { - params = nextParams; - }, - }); - const channelHint = params.messageChannel ?? params.messageProvider; - const resolvedToolResultFormat = - params.toolResultFormat ?? - (channelHint - ? isMarkdownCapableMessageChannel(channelHint) - ? "markdown" - : "plain" - : "markdown"); - const isProbeSession = params.sessionId?.startsWith("probe-") ?? false; - - throwIfAborted(); - - return enqueueSession(async () => { - throwIfAborted(); - // Same-session reads below must see any prior deferred transcript rewrite. - // Checkpoint before the global lane so unrelated sessions can still start - // while this session waits on its own maintenance lane. - params.replyOperation?.markWaitingForDeferredMaintenance(); - try { - await waitForDeferredTurnMaintenanceForSession(params.sessionKey); - } finally { - params.replyOperation?.markDeferredMaintenanceWaitEnded(); - } - throwIfAborted(); - return enqueueGlobal(async () => { - throwIfAborted(); - const started = Date.now(); - const startupStages = createEmbeddedRunStageTracker(); - let startupStagesEmitted = false; - const { - fastModeAutoOnSeconds, - fastModeAutoProgressState, - fastModeStartedAtMs: fastModeStarted, - maybeAnnounceFastModeAutoOff, - maybeEmitFastModeAutoResetBestEffort, - notifyAgentEvent, - notifyExecutionPhase, - notifyRunProgress, - notifyToolResult, - resolveAttemptFastModeParam, - } = createEmbeddedRunProgressController({ - attempt: params, - noteLaneTaskProgress, - startedAtMs: started, - }); - const emitStartupStageSummary = (phase: string) => { - const summary = startupStages.snapshot(); - const shouldWarn = shouldWarnEmbeddedRunStageSummary(summary); - if (!shouldWarn && !log.isEnabled("trace")) { - return; - } - const message = formatEmbeddedRunStageSummary( - `[trace:embedded-run] startup stages: runId=${params.runId} sessionId=${params.sessionId} phase=${phase}`, - summary, - ); - if (shouldWarn) { - log.warn(message); - } else { - log.trace(message); - } - }; - params.onExecutionStarted?.({ lifecycleGeneration }); - notifyExecutionPhase("runner_entered"); - const workspaceResolution = resolveRunWorkspaceDir({ - workspaceDir: params.workspaceDir, - sessionKey: params.sessionKey, - agentId: params.agentId, - config: params.config, - }); - const resolvedWorkspace = workspaceResolution.workspaceDir; - const canonicalWorkspace = resolveUserPath( - resolveAgentWorkspaceDir(params.config ?? {}, workspaceResolution.agentId), - ); - const isCanonicalWorkspace = canonicalWorkspace === resolvedWorkspace; - const redactedSessionId = redactRunIdentifier(params.sessionId); - const redactedSessionKey = redactRunIdentifier(params.sessionKey); - const redactedWorkspace = redactRunIdentifier(resolvedWorkspace); - if (workspaceResolution.usedFallback) { - log.warn( - `[workspace-fallback] caller=runEmbeddedAgent reason=${workspaceResolution.fallbackReason} run=${params.runId} session=${redactedSessionId} sessionKey=${redactedSessionKey} agent=${workspaceResolution.agentId} workspace=${redactedWorkspace}`, - ); - } - startupStages.mark("workspace"); - notifyExecutionPhase("workspace"); - ensureRuntimePluginsLoaded({ - config: params.config, - workspaceDir: resolvedWorkspace, - allowGatewaySubagentBinding: params.allowGatewaySubagentBinding, - }); - startupStages.mark("runtime-plugins"); - notifyExecutionPhase("runtime_plugins"); - - let { provider, modelId } = resolveInitialEmbeddedRunModel({ - config: params.config, - agentId: workspaceResolution.agentId, - provider: params.provider, - model: params.model, - }); - const agentDir = - params.agentDir ?? resolveAgentDir(params.config ?? {}, workspaceResolution.agentId); - const normalizedSessionKey = params.sessionKey?.trim(); - const fallbackConfigured = hasEmbeddedRunConfiguredModelFallbacks({ - cfg: params.config, - agentId: params.agentId, - sessionKey: normalizedSessionKey, - modelFallbacksOverride: params.modelFallbacksOverride, - }); - const resolvedSessionKey = - normalizedSessionKey ?? params.sessionTarget?.sessionKey ?? params.sessionId; - const hookRunner = getGlobalHookRunner(); - const hookCtx = { - runId: params.runId, - jobId: params.jobId, - agentId: workspaceResolution.agentId, - sessionKey: resolvedSessionKey, - sessionId: params.sessionId, - workspaceDir: resolvedWorkspace, - modelProviderId: provider, - modelId, - trigger: params.trigger, - ...buildAgentHookContextChannelFields(params), - }; - if (params.trigger === "cron" && hookRunner?.hasHooks("before_agent_reply")) { - notifyExecutionPhase("before_agent_reply", { provider, model: modelId }); - const hookResult = await hookRunner.runBeforeAgentReply( - { cleanedBody: params.prompt }, - hookCtx, - ); - if (hookResult?.handled) { - return { - payloads: buildHandledReplyPayloads(hookResult.reply), - meta: { - durationMs: Date.now() - started, - agentMeta: { - sessionId: params.sessionId, - provider, - model: modelId, - }, - finalAssistantVisibleText: hookResult.reply?.text ?? SILENT_REPLY_TOKEN, - finalAssistantRawText: hookResult.reply?.text ?? SILENT_REPLY_TOKEN, - }, - }; - } - notifyExecutionPhase("runtime_plugins", { provider, model: modelId }); - } - - const modelSetup = await resolveEmbeddedRunModelSetup({ - runParams: params, - provider, - modelId, - agentDir, - workspaceDir: resolvedWorkspace, - globalLane, - hookRunner, - hookContext: hookCtx, - onHooksResolved: () => startupStages.mark("hooks"), - }); - provider = modelSetup.provider; - modelId = modelSetup.modelId; - const { - requestedModelId, - modelSelectionChangedByHook, - beforeAgentStartResult, - requestStreamTransportOverrides, - expectedHarnessArtifact, - nativeModelOwnedHarnessId, - nativeModelOwned, - modelConfigProvider, - model, - authStorage, - modelRegistry, - } = modelSetup; - let agentHarness = modelSetup.agentHarness; - let pluginHarnessOwnsTransport = modelSetup.pluginHarnessOwnsTransport; - let runtimeModel = model; - const resolveEffectiveModel = (candidate: typeof runtimeModel) => - resolveEmbeddedRunEffectiveModel({ - runParams: params, - provider, - modelConfigProvider, - modelId, - agentHarnessId: agentHarness.id, - runtimeModel: candidate, - nativeModelOwned, - requestStreamTransportOverrides, - nativeModelOwnedHarnessId, - }); - const initialResolvedRuntimeModel = resolveEffectiveModel(runtimeModel); - let contextTokenBudget = initialResolvedRuntimeModel.contextTokenBudget; - let contextWindowInfo = initialResolvedRuntimeModel.contextWindowInfo; - let outerContextTokenMeta: { contextTokens?: number } = - contextTokenBudget === undefined ? {} : { contextTokens: contextTokenBudget }; - let effectiveModel = initialResolvedRuntimeModel.effectiveModel; - const applyResolvedRuntimeModel = ( - candidate: typeof runtimeModel, - resolved = resolveEffectiveModel(candidate), - ) => { - runtimeModel = candidate; - effectiveModel = resolved.effectiveModel; - contextTokenBudget = resolved.contextTokenBudget; - contextWindowInfo = resolved.contextWindowInfo; - outerContextTokenMeta = - contextTokenBudget === undefined ? {} : { contextTokens: contextTokenBudget }; - }; - const selectHarnessForModel = ( - candidate: typeof effectiveModel, - plan?: AgentRuntimeAuthPlan, - preparedAuthAttempt?: PreparedAgentRuntimeAuthAttempt, - ) => - selectEmbeddedRunHarness({ - runParams: params, - provider, - modelId, - model: candidate, - plan, - preparedAuthAttempt, - requestStreamTransportOverrides, - nativeModelOwnedHarnessId, - }); - const selectHarnessForPreparedAttempts = ( - candidate: typeof effectiveModel, - attempts: readonly PreparedAgentRuntimeAuthAttempt[], - ) => - selectEmbeddedRunHarnessForPreparedAttempts({ - runParams: params, - provider, - modelId, - model: candidate, - attempts, - requestStreamTransportOverrides, - nativeModelOwnedHarnessId, - }); - startupStages.mark("model-resolution"); - notifyExecutionPhase("model_resolution", { provider, model: modelId }); - - // Route-aware support settles before the canonical auth decision. The - // materialized route below may confirm this choice, but cannot create a - // second profile/endpoint planner in the primary runner. - agentHarness = selectHarnessForModel(effectiveModel); - pluginHarnessOwnsTransport = agentHarness.id !== "openclaw"; - - const authStages = log.isEnabled("trace") ? createEmbeddedRunStageTracker() : undefined; - const preparedAuthPlan = await prepareEmbeddedRunAuthPlan({ - runParams: params, - provider, - modelId, - model, - agentDir, - workspaceDir: resolvedWorkspace, - requestStreamTransportOverrides, - nativeModelOwned, - authStorage, - modelRegistry, - getAgentHarness: () => agentHarness, - setAgentHarness: (nextHarness) => { - agentHarness = nextHarness; - pluginHarnessOwnsTransport = agentHarness.id !== "openclaw"; - }, - getRuntimeModel: () => runtimeModel, - getEffectiveModel: () => effectiveModel, - applyResolvedRuntimeModel, - selectHarnessForPreparedAttempts, - markStage: (stage) => authStages?.mark(stage), - }); - const { - usesOpenAIAuthRouting, - attemptAuthProfileStore, - lockedProfileId, - preferredProfileId, - providerUsesProfileScopedModelMetadata, - materializeAuthPlan, - materializeAuthPlanUncached, - preparedAuthAttempts, - } = preparedAuthPlan; - let { activePreparedAuthPlan } = preparedAuthPlan; - // A selected plugin harness owns context pressure with its native transcript, - // even if it cannot expose manual compaction. Generic recovery is OpenClaw-only. - const genericCompactionRecoveryAllowed = !pluginHarnessOwnsTransport; - const profileCandidates = preparedAuthAttempts.map((attempt) => attempt.profileId); - const forwardedPluginHarnessProfileId = pluginHarnessOwnsTransport - ? activePreparedAuthPlan.forwardedAuthProfileId - : undefined; - const profileFailureStore = attemptAuthProfileStore; - let profileIndex = 0; - const traceAttempts: TraceAttempt[] = []; - const traceAttemptUsesFallback = (attempt: TraceAttempt): boolean => - attempt.result === "rotate_profile" || attempt.result === "fallback_model"; - const resolveRuntimeFallbackReason = (): string | null => { - const fallbackAttempt = traceAttempts.findLast( - (attempt) => attempt.result === "fallback_model" && typeof attempt.reason === "string", - ); - return fallbackAttempt?.reason ?? lastRetryFailoverReason ?? null; - }; - const buildEmbeddedContextEngineRuntimeSettings = (settingsParams: { - tokenBudget?: number | null; - maxOutputTokens?: number | null; - degradedReason?: string | null; - }) => { - const fallbackReason = resolveRuntimeFallbackReason(); - return buildContextEngineRuntimeSettings({ - contextEngineHost: OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST, - provider, - requestedModel: requestedModelId, - resolvedModel: modelId, - selectedContextEngineId: contextEngine.info.id, - contextEngineSelectionSource: - contextEngine.info.id === "legacy" ? "default" : "configured", - promptTokenBudget: settingsParams.tokenBudget, - maxOutputTokens: settingsParams.maxOutputTokens, - fallbackReason, - degradedReason: settingsParams.degradedReason, - }); - }; - - const requestedThinkLevel = resolveInitialThinkLevel({ - requested: params.thinkLevel, - config: params.config, - provider, - modelId, - model: effectiveModel, - }); - // Hooks can replace the model after outer selection. Revalidate here so the - // final model/runtime never receives an unsupported thinking level. - const initialThinkLevel = modelSelectionChangedByHook - ? (resolveCandidateThinkingLevel({ - cfg: params.config, - provider, - modelId, - level: requestedThinkLevel, - catalog: [ - { - provider, - id: modelId, - api: effectiveModel.api, - reasoning: effectiveModel.reasoning, - params: effectiveModel.params, - compat: effectiveModel.compat, - }, - ], - agentId: params.agentId, - sessionKey: params.sessionKey, - agentRuntime: agentHarness.id, - }) ?? requestedThinkLevel) - : requestedThinkLevel; - let thinkLevel = initialThinkLevel; - const attemptedThinking = new Set(); - let apiKeyInfo: ApiKeyInfo | null = null; - const getApiKeyInfo = (): ApiKeyInfo | null => apiKeyInfo; - let lastProfileId: string | undefined; - let runtimeAuthState: RuntimeAuthState | null = null; - let runtimeAuthRefreshCancelled = false; - const pluginHarnessOwnsAuthBootstrap = - pluginHarnessOwnsTransport && agentHarness.authBootstrap === "harness"; - const preparedApiKeyRoute = activePreparedAuthPlan.modelRoute?.authRequirement === "api-key"; - const pluginHarnessHasPreparedApiKeyAttempt = preparedAuthAttempts.some( - (attempt) => attempt.plan.modelRoute?.authRequirement === "api-key", - ); - const pluginHarnessNeedsOpenClawAuthBootstrap = - pluginHarnessOwnsTransport && - usesOpenAIAuthRouting && - (preparedApiKeyRoute || - (!pluginHarnessOwnsAuthBootstrap && - profileCandidates.some((profileId) => Boolean(profileId)))); - const findPreparedAuthAttempt = (profileId: string | undefined, attemptIndex?: number) => { - const attempt = - attemptIndex === undefined - ? preparedAuthAttempts.find((candidate) => candidate.profileId === profileId) - : preparedAuthAttempts[attemptIndex]; - return attempt?.profileId === profileId ? attempt : undefined; - }; - let preparedProfileAttempted = false; - const prepareAuthAttempt = async (attempt: (typeof preparedAuthAttempts)[number]) => { - if ( - !canRunPreparedAgentRuntimeAuthAttempt({ - attempt, - priorProfileAttempted: preparedProfileAttempted, - }) - ) { - throw new Error( - `Prepared direct auth fallback cannot bypass unavailable profiles for ${provider}/${modelId}.`, - ); - } - const modelDecision = resolveCredentialScopedAuthAttemptModelDecision({ - attempt, - priorProfileAttempted: preparedProfileAttempted, - requestedProfileId: params.authProfileId, - providerUsesProfileScopedModelMetadata, - }); - const nextRuntimeModel = modelDecision.shouldMaterialize - ? modelDecision.forceResolve - ? await materializeAuthPlanUncached(attempt.plan, true) - : await materializeAuthPlan(attempt.plan) - : runtimeModel; - const nextResolvedModel = resolveEffectiveModel(nextRuntimeModel); - const nextHarness = selectHarnessForPreparedAttempts( - nextResolvedModel.effectiveModel, - preparedAuthAttempts, - ); - if (nextHarness.id !== agentHarness.id) { - throw new Error( - `Prepared auth retry changed the selected agent harness for ${provider}/${modelId}.`, - ); - } - preparedProfileAttempted ||= attempt.kind === "profile"; - return { - runtimeModel: nextRuntimeModel, - authRequirement: modelDecision.authRequirement, - allowAuthProfileFallback: attempt.allowAuthProfileFallback, - commit() { - // Model metadata and its prepared route/profile become active in - // the same auth-controller transition before dispatch. - applyResolvedRuntimeModel(nextRuntimeModel, nextResolvedModel); - activePreparedAuthPlan = attempt.plan; - }, - }; - }; - const hasPreparedAuthAttemptMetadata = hasPreparedAuthAttemptModelMetadata({ - attempts: preparedAuthAttempts, - providerUsesProfileScopedModelMetadata, - }); - const prepareModelForAuthProfile = - hasPreparedAuthAttemptMetadata && - (!pluginHarnessOwnsAuthBootstrap || pluginHarnessHasPreparedApiKeyAttempt) - ? async (profileId: string | undefined, attemptIndex?: number) => { - const attempt = findPreparedAuthAttempt(profileId, attemptIndex); - if (!attempt) { - throw new Error( - `Auth profile "${profileId ?? "(none)"}" is outside the prepared attempts for ${provider}/${modelId}.`, - ); - } - const prepared = await prepareAuthAttempt(attempt); - if (attempt.plan.modelRoute && !prepared.authRequirement) { - throw new Error(`Prepared route metadata is missing for ${provider}/${modelId}.`); - } - return { - runtimeModel: prepared.runtimeModel, - authRequirement: prepared.authRequirement, - allowAuthProfileFallback: prepared.allowAuthProfileFallback, - commit: () => prepared.commit(), - }; - } - : undefined; - const { - applyAuthProfileCandidate, - advanceAuthProfile, - initializeAuthProfile, - maybeRefreshRuntimeAuthForAuthError, - stopRuntimeAuthRefreshTimer, - } = createEmbeddedRunAuthController({ - config: params.config, - agentDir, - workspaceDir: resolvedWorkspace, - authStore: attemptAuthProfileStore, - authStorage, - profileCandidates, - lockedProfileId, - initialThinkLevel, - attemptedThinking, - fallbackConfigured, - allowTransientCooldownProbe: params.allowTransientCooldownProbe === true, - getProvider: () => provider, - getModelId: () => modelId, - getRuntimeModel: () => runtimeModel, - setRuntimeModel: (next) => { - runtimeModel = next; - }, - getEffectiveModel: () => effectiveModel, - setEffectiveModel: (next) => { - effectiveModel = next; - }, - getApiKeyInfo, - setApiKeyInfo: (next) => { - apiKeyInfo = next; - }, - getLastProfileId: () => lastProfileId, - setLastProfileId: (next) => { - lastProfileId = next; - }, - getRuntimeAuthState: () => runtimeAuthState, - setRuntimeAuthState: (next) => { - runtimeAuthState = next; - }, - getRuntimeAuthRefreshCancelled: () => runtimeAuthRefreshCancelled, - setRuntimeAuthRefreshCancelled: (next) => { - runtimeAuthRefreshCancelled = next; - }, - getProfileIndex: () => profileIndex, - setProfileIndex: (next) => { - profileIndex = next; - }, - ...(prepareModelForAuthProfile ? { prepareModelForAuthProfile } : {}), - setThinkLevel: (next) => { - thinkLevel = next; - }, - log, - }); - authStages?.mark("controller"); - const advancePluginHarnessAuthAttempt = async (): Promise => { - if (!pluginHarnessOwnsTransport || lockedProfileId) { - return false; - } - let nextIndex = profileIndex + 1; - while (nextIndex < preparedAuthAttempts.length) { - const candidateAttempt = preparedAuthAttempts[nextIndex]; - if (!candidateAttempt) { - nextIndex += 1; - continue; - } - const candidate = candidateAttempt.profileId; - if ( - candidate && - isProfileInCooldown(attemptAuthProfileStore, candidate, undefined, modelId) - ) { - nextIndex += 1; - continue; - } - if ( - !canRunPreparedAgentRuntimeAuthAttempt({ - attempt: candidateAttempt, - priorProfileAttempted: preparedProfileAttempted, - }) - ) { - return false; - } - if (candidateAttempt.plan.modelRoute?.authRequirement === "api-key") { - try { - await applyAuthProfileCandidate(candidate, nextIndex); - profileIndex = nextIndex; - thinkLevel = initialThinkLevel; - attemptedThinking.clear(); - return true; - } catch { - nextIndex += 1; - continue; - } - } - if (!candidate || candidateAttempt.plan.forwardedAuthProfileId !== candidate) { - nextIndex += 1; - continue; - } - const prepared = await prepareAuthAttempt(candidateAttempt); - stopRuntimeAuthRefreshTimer(); - apiKeyInfo = null; - runtimeAuthState = null; - prepared.commit(); - profileIndex = nextIndex; - lastProfileId = candidate; - thinkLevel = initialThinkLevel; - attemptedThinking.clear(); - return true; - } - return false; - }; - const advanceAttemptAuthProfile = pluginHarnessOwnsAuthBootstrap - ? advancePluginHarnessAuthAttempt - : advanceAuthProfile; - - // Plugin harnesses own their model transport/auth. Running OpenClaw's generic - // auth bootstrap here can turn synthetic provider markers into real - // vendor-token refresh attempts before the plugin gets control. - if (!pluginHarnessOwnsTransport || pluginHarnessNeedsOpenClawAuthBootstrap) { - await initializeAuthProfile(); - } else if (lockedProfileId) { - lastProfileId = lockedProfileId; - } else if (forwardedPluginHarnessProfileId) { - const initialAttempt = preparedAuthAttempts[profileIndex]; - const initialProfileInCooldown = - initialAttempt?.kind === "profile" && - isProfileInCooldown( - attemptAuthProfileStore, - initialAttempt.profileId, - undefined, - modelId, - ); - const cooldownProbePolicy = resolveEmbeddedAuthCooldownProbePolicy({ - authStore: attemptAuthProfileStore, - profileCandidates, - lockedProfileId, - modelId, - allowTransientCooldownProbe: params.allowTransientCooldownProbe === true, - }); - if (initialProfileInCooldown && !cooldownProbePolicy.allowProbe) { - if (!(await advancePluginHarnessAuthAttempt())) { - throw new Error( - `Prepared auth profiles are temporarily unavailable for ${provider}/${modelId}.`, - ); - } - } else { - if (initialProfileInCooldown) { - log.warn( - `probing cooldowned auth profile for ${provider}/${modelId} due to ${cooldownProbePolicy.unavailableReason ?? "transient"} unavailability`, - ); - } - preparedProfileAttempted = initialAttempt?.kind === "profile"; - lastProfileId = forwardedPluginHarnessProfileId; - } - } - authStages?.mark("initialize"); - if (authStages) { - log.trace( - formatEmbeddedRunStageSummary( - `[trace:embedded-run] auth stages: runId=${params.runId} sessionId=${params.sessionId} phase=auth`, - authStages.snapshot(), - ), - ); - } - startupStages.mark("auth"); - notifyExecutionPhase("auth", { provider, model: modelId }); - const resolveRunAttemptAuthProfileStore = (): AuthProfileStore => { - if (!pluginHarnessOwnsTransport) { - return attemptAuthProfileStore; - } - const activePlan = activePreparedAuthPlan; - const activeProfileIds = activePlan.modelRoute - ? [ - activePlan.forwardedAuthProfileId, - ...(activePlan.forwardedAuthProfileCandidateIds ?? []), - ] - : [lastProfileId]; - return createScopedAuthProfileStore( - attemptAuthProfileStore, - activeProfileIds.filter((profileId): profileId is string => Boolean(profileId)), - ); - }; - const harnessBuildsOpenClawTools = agentHarnessBuildsOpenClawTools(agentHarness.id); - const { sessionAgentId } = resolveSessionAgentIds({ - sessionKey: params.sessionKey, - config: params.config, - agentId: params.agentId, - }); - const strictAgenticActive = isStrictAgenticExecutionContractActive({ - config: params.config, - sessionKey: params.sessionKey, - agentId: params.agentId, - provider, - modelId, - }); - const executionContract = strictAgenticActive ? "strict-agentic" : "default"; - const maxReasoningOnlyRetryAttempts = DEFAULT_REASONING_ONLY_RETRY_LIMIT; - const maxEmptyResponseRetryAttempts = DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT; - - const MAX_TIMEOUT_COMPACTION_ATTEMPTS = 2; - const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3; - const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations( - profileCandidates.length, - params.config, - sessionAgentId, - ); - let overflowCompactionAttempts = 0; - let toolResultTruncationAttempted = false; - let bootstrapPromptWarningSignaturesSeen = - params.bootstrapPromptWarningSignaturesSeen ?? - (params.bootstrapPromptWarningSignature ? [params.bootstrapPromptWarningSignature] : []); - const usageAccumulator = createUsageAccumulator(); - let lastRunPromptUsage: ReturnType | undefined; - let autoCompactionCount = 0; - let lastCompactionTokensAfter: number | undefined; - let lastContextBudgetStatus: EmbeddedAgentMeta["contextBudgetStatus"]; - let runLoopIterations = 0; - let overloadProfileRotations = 0; - let consecutiveSameModelRateLimitRetries = 0; - let reasoningOnlyRetryAttempts = 0; - let emptyResponseRetryAttempts = 0; - let compactionContinuationRetryAttempts = 0; - let beforeAgentFinalizeRevisionAttempts = 0; - let sameModelIdleTimeoutRetries = 0; - // Cost-runaway breaker for #76293. State lives at the run-loop level - // on purpose so it survives across attempt boundaries and across - // profile/auth retries within this embedded run (a wrapper-local - // counter would reset on every iteration). The helper is pure and - // unit-tested in run/idle-timeout-breaker.test.ts; the run loop just - // feeds it the outcome of each attempt. - const idleTimeoutBreakerState = createIdleTimeoutBreakerState(); - // Post-compaction loop guard for #77474. Armed at each compaction-success - // site below; observed from the live tool-outcome path so it can abort - // while the post-compaction prompt is still running. - const resolvedLoopDetectionConfig = resolveToolLoopDetectionConfig({ - cfg: params.config, - agentId: sessionAgentId, - }); - const postCompactionGuard = createPostCompactionLoopGuard( - resolvedLoopDetectionConfig?.postCompactionGuard, - { enabled: resolvedLoopDetectionConfig?.enabled !== false }, - ); - let postCompactionAbortController: AbortController | undefined; - let postCompactionAbortError: PostCompactionLoopPersistedError | undefined; - const attemptTerminalToolPresentation = { - ordinal: -1, - value: undefined as string | undefined, - }; - let nextToolOutcomeOrdinal = 0; - const allocateToolOutcomeOrdinal = (): number => nextToolOutcomeOrdinal++; - const readAttemptTerminalToolPresentation = (): string | undefined => - attemptTerminalToolPresentation.value; - const observeToolOutcome = (observation: ToolOutcomeObservation): void => { - const observationOrdinal = - observation.toolCallOrdinal ?? attemptTerminalToolPresentation.ordinal + 1; - if (observationOrdinal >= attemptTerminalToolPresentation.ordinal) { - attemptTerminalToolPresentation.ordinal = observationOrdinal; - attemptTerminalToolPresentation.value = observation.terminalPresentation; - } - if (observation.presentationOnly) { - return; - } - const verdict = postCompactionGuard.observe(observation); - if (verdict.shouldAbort) { - postCompactionAbortError ??= PostCompactionLoopPersistedError.fromVerdict(verdict); - laneTaskAbortController.abort(postCompactionAbortError); - postCompactionAbortController?.abort(postCompactionAbortError); - } - }; - let lastRetryFailoverReason: FailoverReason | null = null; - let compactionContinuationRetryInstruction: string | null = null; - let rateLimitProfileRotations = 0; - let timeoutCompactionAttempts = 0; - let codexAppServerRecoveryRetries = 0; - // Silent-error retry: non-strict-agentic models (e.g. ollama/glm-5.1) can - // end a turn with stopReason="error" + zero output tokens, producing no - // user-visible text. This is an orthogonal, model-agnostic resubmission - // for errored turns; stopReason="stop" empty zero-token turns use the - // visible-answer retry instruction instead. - const MAX_EMPTY_ERROR_RETRIES = 3; - let emptyErrorRetries = 0; - const MAX_MISSING_ASSISTANT_RETRIES = 1; - let missingAssistantRetryAttempts = 0; - const overloadFailoverBackoffMs = resolveOverloadFailoverBackoffMs(params.config); - const overloadProfileRotationLimit = resolveOverloadProfileRotationLimit(params.config); - const rateLimitProfileRotationLimit = resolveRateLimitProfileRotationLimit(params.config); - let activeSessionId = params.sessionId; - let activeSessionFile = params.sessionFile; - let activeSessionTarget: ContextEngineSessionTarget | undefined = - buildContextEngineCompactionSessionTarget({ - agentId: params.agentId ?? sessionAgentId, - config: params.config, - sessionFile: activeSessionFile, - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - sessionTarget: params.sessionTarget, - }); - const adoptActiveSessionId = (nextSessionId: string | undefined) => { - if (!nextSessionId || nextSessionId === activeSessionId) { - return; - } - activeSessionId = nextSessionId; - // Keep every active-run owner on the rotated identity. Restart recovery - // uses the reply registry while lifecycle persistence uses run context. - params.replyOperation?.updateSessionId(activeSessionId); - params.onSessionIdChanged?.(activeSessionId); - registerAgentRunContext(params.runId, { - sessionId: activeSessionId, - lifecycleGeneration, - }); - }; - const adoptActiveSessionTarget = async ( - nextSessionTarget: ContextEngineSessionTarget | undefined, - ) => { - if (!nextSessionTarget) { - return; - } - const resolvedTarget = await resolveAgentRunSessionTarget({ - agentId: nextSessionTarget.agentId ?? sessionAgentId, - config: params.config, - sessionId: nextSessionTarget.sessionId ?? activeSessionId, - sessionKey: nextSessionTarget.sessionKey ?? resolvedSessionKey, - sessionTarget: nextSessionTarget, - }); - activeSessionTarget = nextSessionTarget; - activeSessionFile = resolvedTarget.sessionFile; - adoptActiveSessionId(resolvedTarget.sessionId); - }; - let suppressNextUserMessagePersistence = params.suppressNextUserMessagePersistence ?? false; - let activePrompt: { - override?: string; - persisted: boolean; - internal: boolean; - } = { - persisted: suppressNextUserMessagePersistence, - internal: false, - }; - const activateInternalPrompt = (prompt: string, persisted: boolean) => { - activePrompt = { override: prompt, persisted, internal: true }; - suppressNextUserMessagePersistence = persisted; - }; - const onUserMessagePersisted: RunEmbeddedAgentParams["onUserMessagePersisted"] = ( - message, - ) => { - const messageMetadata = message as { - __openclaw?: { beforeAgentRunBlocked?: unknown }; - }; - const blockedBeforeAgentRun = messageMetadata["__openclaw"]?.beforeAgentRunBlocked; - const markCurrentUserMessagePersisted = () => { - activePrompt.persisted = true; - params.onUserMessagePersisted?.(message); - }; - const recorder = params.userTurnTranscriptRecorder; - if (!recorder) { - markCurrentUserMessagePersisted(); - return; - } - const markWhenPersisted = (persisted: { message?: unknown } | undefined) => { - if (persisted?.message || recorder.hasPersisted()) { - markCurrentUserMessagePersisted(); - } - }; - if (blockedBeforeAgentRun !== undefined) { - const canonicalPersistence = recorder - .persistBlocked(message) - .then(markWhenPersisted) - .catch((persistError: unknown) => { - log.warn( - `failed to persist canonical blocked embedded user turn transcript: ${formatErrorMessage( - persistError, - )}`, - ); - }); - recorder.markRuntimePersistencePending(canonicalPersistence); - return; - } - const canonicalPersistence = recorder - .persistApproved() - .then(markWhenPersisted) - .catch((persistError: unknown) => { - log.warn( - `failed to persist canonical embedded user turn transcript: ${formatErrorMessage( - persistError, - )}`, - ); - }); - recorder.markRuntimePersistencePending(canonicalPersistence); - }; - const continueFromCurrentTranscript = () => { - activateInternalPrompt(MID_TURN_PRECHECK_CONTINUATION_PROMPT, true); - }; - const waitForCurrentUserMessagePersistence = async () => { - if (params.userTurnTranscriptRecorder?.hasRuntimePersistencePending() === true) { - await params.userTurnTranscriptRecorder.waitForRuntimePersistence(); - } - }; - const maybeEscalateRateLimitProfileFallback = (paramsLocal: { - failoverProvider: string; - failoverModel: string; - logFallbackDecision: (decision: "fallback_model", extra?: { status?: number }) => void; - }) => { - rateLimitProfileRotations += 1; - if (rateLimitProfileRotations <= rateLimitProfileRotationLimit || !fallbackConfigured) { - return; - } - const status = resolveFailoverStatus("rate_limit"); - log.warn( - `rate-limit profile rotation cap reached for ${sanitizeForLog(provider)}/${sanitizeForLog(modelId)} after ${rateLimitProfileRotations} rotations; escalating to model fallback`, - ); - paramsLocal.logFallbackDecision("fallback_model", { status }); - throw new FailoverError( - "The AI service is temporarily rate-limited. Please try again in a moment.", - { - reason: "rate_limit", - provider: paramsLocal.failoverProvider, - model: paramsLocal.failoverModel, - profileId: lastProfileId, - sessionId: activeSessionId, - lane: globalLane, - status, - }, - ); - }; - const maybeMarkAuthProfileFailure = async (failure: { - profileId?: string; - reason?: AuthProfileFailureReason | null; - config?: RunEmbeddedAgentParams["config"]; - agentDir?: RunEmbeddedAgentParams["agentDir"]; - modelId?: string; - }) => { - if (params.authProfileStateMode === "read-only") { - return; - } - const { profileId, reason } = failure; - if (!profileId || !reason) { - return; - } - if (pluginHarnessOwnsTransport && reason === "timeout") { - // Harness-owned transport timeouts are lifecycle failures, not - // credential evidence. Do not poison OpenClaw auth cooldowns. - return; - } - await markAuthProfileFailure({ - store: profileFailureStore, - profileId, - reason, - cfg: params.config, - agentDir, - runId: params.runId, - modelId: failure.modelId, - }); - }; - const markAuthProfileSuccessAfterRun = () => { - if (params.authProfileStateMode === "read-only" || !lastProfileId) { - return; - } - const successProfileId = lastProfileId; - const safeSuccessProfileId = redactIdentifier(successProfileId, { len: 12 }); - const successProvider = resolveAuthProfileStateProvider( - profileFailureStore, - successProfileId, - provider, - ); - const successStarted = Date.now(); - void markAuthProfileSuccess({ - store: profileFailureStore, - provider: successProvider, - profileId: successProfileId, - agentDir: params.agentDir, - }) - .then(() => { - const durationMs = Date.now() - successStarted; - if (durationMs >= POST_RUN_AUTH_PROFILE_SUCCESS_SLOW_MS) { - log.warn( - `post-run auth-profile success bookkeeping completed after ${durationMs}ms: ` + - `runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${sanitizeForLog(successProvider)} profileId=${safeSuccessProfileId}`, - ); - } else if (log.isEnabled("trace")) { - log.trace( - `post-run auth-profile success bookkeeping completed: ` + - `runId=${params.runId} sessionId=${params.sessionId} durationMs=${durationMs}`, - ); - } - }) - .catch((err: unknown) => { - log.warn( - `post-run auth-profile success bookkeeping failed: ` + - `runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${sanitizeForLog(successProvider)} profileId=${safeSuccessProfileId} ` + - `error=${formatErrorMessage(err)}`, - ); - }); - }; - const resolveRunAuthProfileFailureReason = ( - failoverReason: FailoverReason | null, - opts?: { providerStarted?: boolean; transientRateLimit?: boolean }, - ) => - resolveAuthProfileFailureReason({ - failoverReason, - providerStarted: opts?.providerStarted, - transientRateLimit: opts?.transientRateLimit, - policy: params.authProfileFailurePolicy, - }); - const maybeBackoffBeforeOverloadFailover = async (reason: FailoverReason | null) => { - if (reason !== "overloaded" || overloadFailoverBackoffMs <= 0) { - return; - } - log.warn( - `overload backoff before failover for ${provider}/${modelId}: delayMs=${overloadFailoverBackoffMs}`, - ); - try { - await sleepWithAbort(overloadFailoverBackoffMs, params.abortSignal); - } catch (err) { - if (params.abortSignal?.aborted) { - const abortErr = new Error("Operation aborted", { cause: err }); - abortErr.name = "AbortError"; - throw abortErr; - } - throw err; - } - }; - const maybeRetrySameModelRateLimit = async (retry?: { - retryAfterSeconds?: number; - }): Promise => { - if (consecutiveSameModelRateLimitRetries >= MAX_SAME_MODEL_RATE_LIMIT_RETRIES) { - return false; - } - const delayMs = resolveSameModelRateLimitRetryDelayMs({ - retriesSoFar: consecutiveSameModelRateLimitRetries, - retryAfterSeconds: retry?.retryAfterSeconds, - }); - log.warn( - `rate-limit same-model retry ${consecutiveSameModelRateLimitRetries + 1}/${MAX_SAME_MODEL_RATE_LIMIT_RETRIES} for ${sanitizeForLog(provider)}/${sanitizeForLog(modelId)}: delayMs=${delayMs}`, - ); - try { - await sleepWithAbort(delayMs, params.abortSignal); - } catch (err) { - if (params.abortSignal?.aborted) { - const abortErr = new Error("Operation aborted", { cause: err }); - abortErr.name = "AbortError"; - throw abortErr; - } - throw err; - } - consecutiveSameModelRateLimitRetries = resolveNextSameModelRateLimitRetryCount({ - retriesSoFar: consecutiveSameModelRateLimitRetries, - retriedSameModelRateLimit: true, - }); - return true; - }; - // Resolve the context engine once and reuse across retries to avoid - // repeated initialization/connection overhead per attempt. - ensureContextEnginesInitialized(); - const contextEngine = await resolveContextEngine(params.config, { - agentDir, - workspaceDir: resolvedWorkspace, - }); - const resolveContextEnginePluginId = () => resolveContextEngineOwnerPluginId(contextEngine); - startupStages.mark("context-engine"); - notifyExecutionPhase("context_engine", { provider, model: modelId }); - try { - const resolveActiveHookContext = () => ({ - ...hookCtx, - sessionId: activeSessionId, - }); - const adoptCompactionTranscript = async ( - compactResult: Awaited>, - ): Promise => { - const previousSessionId = activeSessionId; - const nextSessionTarget = compactResult.result?.sessionTarget; - const successor = resolveCompactionSuccessorTranscript(compactResult); - await adoptActiveSessionTarget( - nextSessionTarget && successor.sessionId - ? { - ...nextSessionTarget, - sessionId: nextSessionTarget.sessionId ?? successor.sessionId, - } - : nextSessionTarget, - ); - if ( - !nextSessionTarget && - successor.sessionFile && - successor.sessionFile !== activeSessionFile - ) { - activeSessionFile = successor.sessionFile; - } - adoptActiveSessionId(successor.sessionId); - return successor.sessionId && successor.sessionId !== previousSessionId - ? previousSessionId - : undefined; - }; - const onCompactionHookMessages = async (payload: { - phase: "before" | "after"; - messages: string[]; - }) => { - const messages = payload.messages.filter((message) => message.trim().length > 0); - if (messages.length === 0) { - return; - } - await params.onAgentEvent?.({ - stream: "compaction", - data: { - phase: payload.phase === "before" ? "start" : "end", - ...(payload.phase === "after" ? { completed: true } : {}), - messages, - }, - ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), - }); - }; - // When the engine owns compaction, compactEmbeddedAgentSessionDirect is - // bypassed. Fire lifecycle hooks here so recovery paths still notify - // subscribers like memory extensions and usage trackers. - const runOwnsCompactionBeforeHook = async (reason: string) => { - if ( - contextEngine.info.ownsCompaction !== true || - !hookRunner?.hasHooks("before_compaction") - ) { - return; - } - try { - await hookRunner.runBeforeCompaction( - { messageCount: -1, sessionFile: activeSessionFile }, - resolveActiveHookContext(), - ); - } catch (hookErr) { - log.warn(`before_compaction hook failed during ${reason}: ${String(hookErr)}`); - } - }; - const runOwnsCompactionAfterHook = async ( - reason: string, - compactResult: Awaited>, - previousSessionId?: string, - ) => { - if ( - contextEngine.info.ownsCompaction !== true || - !compactResult.ok || - !compactResult.compacted || - !hookRunner?.hasHooks("after_compaction") - ) { - return; - } - try { - await hookRunner.runAfterCompaction( - { - messageCount: -1, - compactedCount: -1, - tokenCount: compactResult.result?.tokensAfter, - sessionFile: - resolveCompactionSuccessorTranscript(compactResult).sessionFile ?? - activeSessionFile, - ...(previousSessionId ? { previousSessionId } : {}), - }, - resolveActiveHookContext(), - ); - } catch (hookErr) { - log.warn(`after_compaction hook failed during ${reason}: ${String(hookErr)}`); - } - }; - let authRetryPending = false; - let accumulatedReplayState = createEmbeddedRunReplayState(); - // Hoisted so the retry-limit error path can use the most recent API total. - let lastTurnTotal: number | undefined; - while (true) { - if (runLoopIterations >= MAX_RUN_LOOP_ITERATIONS) { - const message = - `Exceeded retry limit after ${runLoopIterations} attempts ` + - `(max=${MAX_RUN_LOOP_ITERATIONS}).`; - log.error( - `[run-retry-limit] sessionKey=${params.sessionKey ?? params.sessionId} ` + - `provider=${provider}/${modelId} attempts=${runLoopIterations} ` + - `maxAttempts=${MAX_RUN_LOOP_ITERATIONS}`, - ); - const retryLimitDecision = resolveRunFailoverDecision({ - stage: "retry_limit", - fallbackConfigured, - failoverReason: lastRetryFailoverReason, - }); - return handleRetryLimitExhaustion({ - message, - decision: retryLimitDecision, - provider, - model: modelId, - profileId: lastProfileId, - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: activeSessionId, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastTurnTotal, - }), - replayInvalid: accumulatedReplayState.replayInvalid ? true : undefined, - livenessState: "blocked", - }); - } - runLoopIterations += 1; - const runtimeAuthRetry = authRetryPending; - authRetryPending = false; - attemptedThinking.add(thinkLevel); - await fs.mkdir(resolvedWorkspace, { recursive: true }); - if (!startupStagesEmitted) { - startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.workspace); - } - - const basePrompt = - activePrompt.override ?? - resolveEmbeddedAttemptBasePrompt({ - nativeModelOwned, - provider, - prompt: params.prompt, - }); - const prompt = compactionContinuationRetryInstruction - ? `${basePrompt}\n\n${compactionContinuationRetryInstruction}` - : basePrompt; - const resolvedStreamApiKey = resolveAttemptDispatchApiKey({ - apiKeyInfo, - runtimeAuthState, - }); - const attemptFastMode = resolveAttemptFastModeParam(); - const trajectorySessionFile = resolvedSessionKey - ? ( - await resolveSessionTranscriptRuntimeReadTarget({ - agentId: workspaceResolution.agentId, - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - storePath: resolveStorePath(params.config?.session?.store, { - agentId: workspaceResolution.agentId, - }), - }) - ).sessionFile - : activeSessionFile; - if (!startupStagesEmitted) { - startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.prompt); - } - const runtimePlan = buildAgentRuntimePlan({ - provider, - modelId, - model: effectiveModel, - modelApi: effectiveModel.api, - harnessId: agentHarness.id, - harnessRuntime: agentHarness.id, - preparedAuthPlan: activePreparedAuthPlan, - config: params.config, - workspaceDir: resolvedWorkspace, - agentDir, - agentId: workspaceResolution.agentId, - thinkingLevel: mapThinkingLevelForProvider(thinkLevel), - extraParamsOverride: { - ...params.streamParams, - fastMode: attemptFastMode, - }, - }); - const trajectoryAttribution = resolveAttemptTrajectoryAttribution({ - model: effectiveModel, - modelId, - provider, - runtimePlan, - }); - const hostTrajectoryRecorder = - agentHarness.id === CODEX_HARNESS_ID && !params.disableTrajectory - ? createTrajectoryRuntimeRecorder({ - cfg: params.config, - env: process.env, - runId: params.runId, - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - sessionFile: trajectorySessionFile, - provider: trajectoryAttribution.provider, - modelId: trajectoryAttribution.modelId, - modelApi: trajectoryAttribution.modelApi, - workspaceDir: resolvedWorkspace, - }) - : undefined; - const runAttemptAuthProfileStore = resolveRunAttemptAuthProfileStore(); - if (!startupStagesEmitted) { - startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.runtimePlan); - startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.dispatch); - notifyExecutionPhase("attempt_dispatch", { provider, model: modelId }); - emitStartupStageSummary(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.dispatch); - startupStagesEmitted = true; - } - - const attemptAbortController = new AbortController(); - postCompactionAbortController = attemptAbortController; - const parentAbortSignal = params.abortSignal; - const relayParentAbort = (): void => { - laneTaskAbortController.abort(parentAbortSignal?.reason); - attemptAbortController.abort(parentAbortSignal?.reason); - }; - if (parentAbortSignal?.aborted) { - relayParentAbort(); - } else { - parentAbortSignal?.addEventListener("abort", relayParentAbort, { once: true }); - } - // Native attempts start the heartbeat only after their own timeout - // watchdog is armed, keeping preflight inside the requested deadline. - let progressInterval: ReturnType | undefined; - const stopLaneProgressHeartbeat = () => { - if (progressInterval) { - clearInterval(progressInterval); - progressInterval = undefined; - } - attemptAbortController.signal.removeEventListener("abort", stopLaneProgressHeartbeat); - }; - const startLaneProgressHeartbeat = () => { - if (progressInterval || attemptAbortController.signal.aborted) { - return; - } - progressInterval = setInterval( - () => noteLaneTaskProgress(), - EMBEDDED_RUN_LANE_HEARTBEAT_MS, - ); - progressInterval.unref?.(); - attemptAbortController.signal.addEventListener("abort", stopLaneProgressHeartbeat, { - once: true, - }); - }; - // Timeout recovery can continue after an attempt returns, but a native - // transport that ignores its timeout releases the lane after one grace. - let timeoutReleaseTimer: ReturnType | undefined; - const clearAttemptTimeoutRelease = () => { - if (timeoutReleaseTimer) { - clearTimeout(timeoutReleaseTimer); - timeoutReleaseTimer = undefined; - } - }; - const armAttemptTimeoutRelease = (reason: Error) => { - if (timeoutReleaseTimer) { - return; - } - timeoutReleaseTimer = setTimeout( - () => laneTaskReleaseController.abort(reason), - EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, - ); - timeoutReleaseTimer.unref?.(); - }; - let attemptCancellationRequested = false; - const codexAppServerRecoveryRetryAvailable = hasCodexAppServerRecoveryRetryBudget({ - alreadyRetried: codexAppServerRecoveryRetries > 0, - runLoopIterations, - maxRunLoopIterations: MAX_RUN_LOOP_ITERATIONS, - }); - const rawAttempt = await runEmbeddedAttemptWithBackend({ - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - promptCacheKey: params.promptCacheKey, - sandboxSessionKey: params.sandboxSessionKey, - trigger: params.trigger, - memoryFlushWritePath: params.memoryFlushWritePath, - messageChannel: params.messageChannel, - messageProvider: params.messageProvider, - clientCaps: params.clientCaps, - chatType: params.chatType, - agentAccountId: params.agentAccountId, - messageTo: params.messageTo, - messageThreadId: params.messageThreadId, - messageActionTurnCapability: params.messageActionTurnCapability, - groupId: params.groupId, - groupChannel: params.groupChannel, - groupSpace: params.groupSpace, - memberRoleIds: params.memberRoleIds, - spawnedBy: params.spawnedBy, - isCanonicalWorkspace, - senderId: params.senderId, - senderName: params.senderName, - senderUsername: params.senderUsername, - senderE164: params.senderE164, - senderIsOwner: params.senderIsOwner, - approvalReviewerDeviceId: params.approvalReviewerDeviceId, - currentChannelId: params.currentChannelId, - chatId: params.chatId, - channelContext: params.channelContext, - currentMessagingTarget: params.currentMessagingTarget, - currentThreadTs: params.currentThreadTs, - currentMessageId: params.currentMessageId, - currentInboundAudio: params.currentInboundAudio, - replyToMode: params.replyToMode, - hasRepliedRef: params.hasRepliedRef, - sessionFile: activeSessionFile, - sessionTarget: activeSessionTarget, - trajectorySessionFile, - trajectoryRecorder: hostTrajectoryRecorder, - workspaceDir: resolvedWorkspace, - cwd: params.cwd, - agentDir, - config: params.config, - allowGatewaySubagentBinding: params.allowGatewaySubagentBinding, - ...(nativeModelOwned - ? {} - : { - contextEngine, - contextTokenBudget, - contextWindowInfo, - }), - skillsSnapshot: params.skillsSnapshot, - prompt, - transcriptPrompt: params.transcriptPrompt, - userTurnTranscriptRecorder: params.userTurnTranscriptRecorder, - skipPreparedUserTurnMessage: activePrompt.internal, - currentInboundEventKind: params.currentInboundEventKind, - currentInboundContext: params.currentInboundContext, - images: params.images, - imageOrder: params.imageOrder, - clientTools: params.clientTools, - disableTools: params.disableTools, - provider, - modelId, - requestedModelId, - fallbackActive: modelId !== requestedModelId || Boolean(resolveRuntimeFallbackReason()), - fallbackReason: resolveRuntimeFallbackReason(), - isFinalFallbackAttempt: params.isFinalFallbackAttempt, - // Use the harness selected before model/auth setup for the actual - // attempt too. Otherwise plugin-owned transports can skip OpenClaw auth - // bootstrap but drift back to OpenClaw when the attempt is created. - agentHarnessId: agentHarness.id, - agentHarnessRuntimeOverride: agentHarness.id, - modelSelectionLocked: params.modelSelectionLocked, - ...(params.onSuccessfulAuthBinding || expectedHarnessArtifact - ? { captureRuntimeArtifact: true } - : {}), - ...(expectedHarnessArtifact - ? { expectedRuntimeArtifact: expectedHarnessArtifact.artifact } - : {}), - ...(params.sessionKey - ? { - agentHarnessTaskRuntimeScope: createAgentHarnessTaskRuntimeScope({ - requesterSessionKey: params.sessionKey, - }), - } - : {}), - runtimePlan, - model: applyAuthHeaderOverride( - applyLocalNoAuthHeaderOverride(effectiveModel, apiKeyInfo), - // When runtime auth exchange produced a different credential - // (runtimeAuthState is set), the exchanged token lives in - // authStorage and the SDK will pick it up automatically. - // Skip header injection to avoid leaking the pre-exchange key. - runtimeAuthState ? null : apiKeyInfo, - params.config, - ), - resolvedApiKey: resolvedStreamApiKey, - authProfileId: lastProfileId, - authProfileIdSource: lockedProfileId ? "user" : "auto", - initialReplayState: accumulatedReplayState, - authStorage, - authProfileStore: runAttemptAuthProfileStore, - // These harnesses build OpenClaw tools internally. Keep transport auth - // scoped while letting tool construction see plugin/provider creds. - toolAuthProfileStore: harnessBuildsOpenClawTools ? attemptAuthProfileStore : undefined, - modelRegistry, - agentId: workspaceResolution.agentId, - beforeAgentStartResult, - thinkLevel, - onToolOutcome: observeToolOutcome, - allocateToolOutcomeOrdinal, - onToolStreamBoundary: maybeAnnounceFastModeAutoOff, - onRunProgress: notifyRunProgress, - fastMode: attemptFastMode, - fastModeAuto: params.fastMode === "auto", - ...(params.fastMode === "auto" - ? { - fastModeStartedAtMs: fastModeStarted, - fastModeAutoOnSeconds, - fastModeAutoProgressState, - } - : {}), - verboseLevel: params.verboseLevel, - reasoningLevel: params.reasoningLevel, - toolResultFormat: resolvedToolResultFormat, - toolProgressDetail: params.toolProgressDetail, - execOverrides: params.execOverrides, - bashElevated: params.bashElevated, - timeoutMs: params.timeoutMs, - runTimeoutOverrideMs: params.runTimeoutOverrideMs, - runId: params.runId, - lifecycleGeneration, - abortSignal: attemptAbortController.signal, - onAttemptTimeoutArmed: pluginHarnessOwnsTransport - ? undefined - : startLaneProgressHeartbeat, - onAttemptTimeout: pluginHarnessOwnsTransport ? undefined : armAttemptTimeoutRelease, - onAttemptAbort: () => { - attemptCancellationRequested = true; - if (!params.abortSignal?.aborted) { - params.replyOperation?.abortByUser(); - } - if (!pluginHarnessOwnsTransport) { - stopLaneProgressHeartbeat(); - laneTaskAbortController.abort(); - } - }, - replyOperation: params.replyOperation, - shouldEmitToolResult: params.shouldEmitToolResult, - shouldEmitToolOutput: params.shouldEmitToolOutput, - onPartialReply: params.onPartialReply, - onAssistantMessageStart: params.onAssistantMessageStart, - onBlockReply: params.onBlockReply, - onBlockReplyFlush: params.onBlockReplyFlush, - blockReplyBreak: params.blockReplyBreak, - blockReplyChunking: params.blockReplyChunking, - onReasoningStream: params.onReasoningStream, - streamReasoningInNonStreamModes: params.streamReasoningInNonStreamModes, - onReasoningEnd: params.onReasoningEnd, - onToolResult: notifyToolResult, - onAgentToolResult: params.onAgentToolResult, - onAgentEvent: notifyAgentEvent, - deferTerminalLifecycle: - params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd, - deferTerminalLifecycleEnd: - params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd, - onExecutionPhase: params.onExecutionPhase, - extraSystemPrompt: params.extraSystemPrompt, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - taskSuggestionDeliveryMode: params.taskSuggestionDeliveryMode, - inputProvenance: params.inputProvenance, - streamParams: params.streamParams, - modelRun: params.modelRun, - disableTrajectory: params.disableTrajectory, - ...resolveSkillWorkshopAttemptParams(params), - promptMode: params.promptMode, - ownerNumbers: params.ownerNumbers, - enforceFinalTag: params.enforceFinalTag, - silentExpected: params.silentExpected, - suppressLiveStreamOutput: params.suppressLiveStreamOutput, - bootstrapContextMode: params.bootstrapContextMode, - bootstrapContextRunKind: params.bootstrapContextRunKind, - jobId: params.jobId, - toolsAllow: params.toolsAllow, - ...(params.systemAgentTool ? { systemAgentTool: params.systemAgentTool } : {}), - cleanupBundleMcpOnRunEnd: params.cleanupBundleMcpOnRunEnd, - disableMessageTool: params.disableMessageTool, - forceRestartSafeTools: params.forceRestartSafeTools, - forceMessageTool: params.forceMessageTool, - enableHeartbeatTool: params.enableHeartbeatTool, - forceHeartbeatTool: params.forceHeartbeatTool, - requireExplicitMessageTarget: params.requireExplicitMessageTarget, - internalEvents: params.internalEvents, - bootstrapPromptWarningSignaturesSeen, - bootstrapPromptWarningSignature: - bootstrapPromptWarningSignaturesSeen[bootstrapPromptWarningSignaturesSeen.length - 1], - suppressNextUserMessagePersistence, - beforeAgentFinalizeRevisionAttempts, - maxBeforeAgentFinalizeRevisions: MAX_BEFORE_AGENT_FINALIZE_REVISIONS, - suppressTranscriptOnlyAssistantPersistence: - params.suppressTranscriptOnlyAssistantPersistence, - suppressAssistantErrorPersistence: params.suppressAssistantErrorPersistence, - onUserMessagePersisted, - onUserMessagePersistenceInvalidated: () => { - activePrompt.persisted = false; - }, - onAssistantErrorMessagePersisted: params.onAssistantErrorMessagePersisted, - }) - .catch((err: unknown): never => { - throw postCompactionAbortError ?? err; - }) - .finally(() => { - clearAttemptTimeoutRelease(); - stopLaneProgressHeartbeat(); - parentAbortSignal?.removeEventListener?.("abort", relayParentAbort); - if (postCompactionAbortController === attemptAbortController) { - postCompactionAbortController = undefined; - } - }); - if (postCompactionAbortError) { - throw postCompactionAbortError; - } - const attempt = normalizeEmbeddedRunAttemptResult(rawAttempt); - await waitForCurrentUserMessagePersistence(); - suppressNextUserMessagePersistence = activePrompt.persisted; - if (attemptCancellationRequested) { - throwIfAborted(); - throw createAgentRunDirectAbortError(); - } - - const { - aborted, - externalAbort, - promptError, - promptErrorSource, - preflightRecovery, - timedOut, - idleTimedOut, - timedOutDuringCompaction, - sessionIdUsed, - sessionFileUsed, - lastAssistant: sessionLastAssistant, - currentAttemptAssistant, - } = attempt; - const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false; - const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false; - // Transcript fallback can outlive a provider or alias transition. - // Reuse it only when it reports the effective model for this attempt. - const sessionAssistantForCandidate = - !currentAttemptAssistant && - !isAssistantForModelRef(sessionLastAssistant, { - provider: effectiveModel.provider, - model: effectiveModel.id, - }) - ? undefined - : sessionLastAssistant; - const attemptAssistant = currentAttemptAssistant ?? sessionAssistantForCandidate; - const terminalOutcome = resolveEmbeddedRunAttemptTerminalOutcome({ - attempt, - assistant: currentAttemptAssistant, - abortSignal: params.abortSignal, - }); - const terminalAborted = isEmbeddedRunTerminalAbort(terminalOutcome); - const terminalTimedOut = isEmbeddedRunTerminalTimeout(terminalOutcome); - const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalOutcome); - const signalOwnedInterruption = - terminalInterrupted && params.abortSignal?.aborted === true; - const terminalIdleTimedOut = terminalTimedOut && idleTimedOut; - const setTerminalLifecycleMeta: NonNullable = ( - meta, - ) => { - const { stopReason, ...remainingMeta } = meta; - const terminalStopReason = terminalInterrupted - ? terminalOutcome.stopReason - : stopReason; - attempt.setTerminalLifecycleMeta?.({ - ...remainingMeta, - ...(terminalStopReason ? { stopReason: terminalStopReason } : {}), - aborted: terminalAborted, - }); - }; - const previousActiveSessionId = activeSessionId; - const previousActiveSessionFile = activeSessionFile; - adoptActiveSessionId(sessionIdUsed); - if (sessionFileUsed && sessionFileUsed !== activeSessionFile) { - activeSessionFile = sessionFileUsed; - } - if ( - (sessionIdUsed && sessionIdUsed !== previousActiveSessionId) || - (sessionFileUsed && sessionFileUsed !== previousActiveSessionFile) - ) { - const activeSqliteMarker = parseSqliteSessionFileMarker(activeSessionFile); - activeSessionTarget = activeSqliteMarker - ? { - agentId: activeSqliteMarker.agentId, - sessionId: activeSqliteMarker.sessionId, - sessionKey: resolvedSessionKey, - storePath: activeSqliteMarker.storePath, - } - : undefined; - } - bootstrapPromptWarningSignaturesSeen = - attempt.bootstrapPromptWarningSignaturesSeen ?? - (attempt.bootstrapPromptWarningSignature - ? Array.from( - new Set([ - ...bootstrapPromptWarningSignaturesSeen, - attempt.bootstrapPromptWarningSignature, - ]), - ) - : bootstrapPromptWarningSignaturesSeen); - const lastAssistantUsage = normalizeUsage(sessionLastAssistant?.usage as UsageLike); - const currentAttemptAssistantUsage = normalizeUsage( - currentAttemptAssistant?.usage as UsageLike, - ); - const promptCacheLastCallUsage = normalizeUsage( - attempt.promptCache?.lastCallUsage as UsageLike, - ); - const callUsage = resolveLatestCallUsage({ - currentAttemptCandidates: [currentAttemptAssistantUsage, promptCacheLastCallUsage], - carriedCandidates: [lastRunPromptUsage, lastAssistantUsage], - }); - const attemptUsage = attempt.attemptUsage ?? callUsage.currentAttempt; - mergeUsageIntoAccumulator(usageAccumulator, attemptUsage); - // Keep prompt size from the latest model call so session totalTokens - // reflects current context usage, not accumulated tool-loop usage. - lastRunPromptUsage = callUsage.latest; - lastTurnTotal = callUsage.latest?.total; - // Idle-timeout cost-runaway breaker (#76293). Logic lives in the - // pure helper below so it stays unit-testable; the run loop just - // feeds it the latest attempt outcome and bails through the - // existing retry-limit exhaustion path when the cap is hit. - const breakerStep = stepIdleTimeoutBreaker(idleTimeoutBreakerState, { - idleTimedOut: terminalIdleTimedOut, - completedModelProgress: hasCompletedModelProgressForIdleBreaker(attempt), - outputTokens: attemptUsage?.output, - }); - if (breakerStep.tripped) { - const breakerMessage = - `Idle-timeout cost-runaway breaker tripped: ` + - `${breakerStep.consecutive} consecutive idle timeouts ` + - `without completed model progress ` + - `(cap=${MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT}). ` + - `Halting further attempts to bound paid model calls. ` + - `See issue #76293.`; - log.error( - `[idle-timeout-circuit-breaker-tripped] ` + - `sessionKey=${params.sessionKey ?? params.sessionId} ` + - `provider=${provider}/${modelId} ` + - `consecutive=${breakerStep.consecutive} ` + - `cap=${MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT}`, - ); - const breakerDecision = resolveRunFailoverDecision({ - stage: "retry_limit", - fallbackConfigured, - failoverReason: lastRetryFailoverReason, - }); - return handleRetryLimitExhaustion({ - message: breakerMessage, - decision: breakerDecision, - provider, - model: modelId, - profileId: lastProfileId, - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: activeSessionId, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastTurnTotal, - }), - replayInvalid: accumulatedReplayState.replayInvalid ? true : undefined, - livenessState: "blocked", - }); - } - const attemptCompactionCount = Math.max(0, attempt.compactionCount ?? 0); - autoCompactionCount += attemptCompactionCount; - if ( - typeof attempt.compactionTokensAfter === "number" && - Number.isFinite(attempt.compactionTokensAfter) && - attempt.compactionTokensAfter >= 0 - ) { - lastCompactionTokensAfter = Math.floor(attempt.compactionTokensAfter); - } - if (attempt.contextBudgetStatus) { - lastContextBudgetStatus = attempt.contextBudgetStatus; - } - const activeErrorContext = resolveActiveErrorContext({ - provider, - model: modelId, - assistant: attemptAssistant, - }); - const resolveReplayInvalidForAttempt = (incompleteTurnText?: string | null) => - accumulatedReplayState.replayInvalid || - resolveReplayInvalidFlag({ - attempt, - incompleteTurnText, - }); - if (resolveReplayInvalidForAttempt(null)) { - accumulatedReplayState.replayInvalid = true; - } - accumulatedReplayState = observeReplayMetadata( - accumulatedReplayState, - attempt.replayMetadata, - ); - const formattedAssistantErrorText = sessionAssistantForCandidate - ? formatAssistantErrorText(sessionAssistantForCandidate, { - cfg: params.config, - sessionKey: resolvedSessionKey ?? params.sessionId, - provider: activeErrorContext.provider, - model: activeErrorContext.model, - authMode: lastProfileId - ? attemptAuthProfileStore.profiles?.[lastProfileId]?.type - : undefined, - }) - : undefined; - const assistantErrorText = - sessionAssistantForCandidate?.stopReason === "error" - ? sessionAssistantForCandidate.errorMessage?.trim() || formattedAssistantErrorText - : undefined; - const canRestartForLiveSwitch = - !hasOutboundDeliveryEvidence(attempt) && - !attempt.didSendDeterministicApprovalPrompt && - !attempt.lastToolError && - (attempt.toolMetas?.length ?? 0) === 0 && - (attempt.assistantTexts?.length ?? 0) === 0; - if (!signalOwnedInterruption && !nativeModelOwned && preflightRecovery?.handled) { - const retryingFromTranscript = preflightRecovery.source === "mid-turn"; - log.info( - `[context-overflow-precheck] early recovery route=${preflightRecovery.route} ` + - `completed for ${provider}/${modelId}; ` + - (retryingFromTranscript ? "retrying from current transcript" : "retrying prompt"), - ); - if (retryingFromTranscript) { - continueFromCurrentTranscript(); - } - continue; - } - const requestedSelection = shouldSwitchToLiveModel({ - cfg: params.config, - sessionKey: resolvedSessionKey, - agentId: params.agentId, - defaultProvider: DEFAULT_PROVIDER, - defaultModel: DEFAULT_MODEL, - currentProvider: provider, - currentModel: modelId, - currentAgentRuntimeOverride: params.agentHarnessRuntimeOverride, - currentAuthProfileId: preferredProfileId, - currentAuthProfileIdSource: params.authProfileIdSource, - }); - if (!signalOwnedInterruption && requestedSelection && canRestartForLiveSwitch) { - await clearLiveModelSwitchPending({ - cfg: params.config, - sessionKey: resolvedSessionKey, - agentId: params.agentId, - }); - log.info( - `live session model switch requested during active attempt for ${params.sessionId}: ${provider}/${modelId} -> ${requestedSelection.provider}/${requestedSelection.model}`, - ); - throw new LiveSessionModelSwitchError(requestedSelection); - } - if ( - genericCompactionRecoveryAllowed && - contextTokenBudget !== undefined && - timedOut && - !signalOwnedInterruption && - !timedOutDuringCompaction && - !timedOutDuringToolExecution && - !timedOutByRunBudget - ) { - // Only consider prompt-side tokens here. API totals include output - // tokens, which can make a long generation look like high context - // pressure even when the prompt itself was small. - const lastTurnPromptTokens = deriveContextPromptTokens({ - lastCallUsage: lastRunPromptUsage, - }); - const tokenUsedRatio = - lastTurnPromptTokens != null && contextTokenBudget > 0 - ? lastTurnPromptTokens / contextTokenBudget - : 0; - if (timeoutCompactionAttempts >= MAX_TIMEOUT_COMPACTION_ATTEMPTS) { - log.warn( - `[timeout-compaction] already attempted timeout compaction ${timeoutCompactionAttempts} time(s); falling through to failover rotation`, - ); - } else if (tokenUsedRatio > 0.65) { - const timeoutDiagId = createCompactionDiagId(); - timeoutCompactionAttempts++; - log.warn( - `[timeout-compaction] LLM timed out with high prompt token usage (${Math.round(tokenUsedRatio * 100)}%); ` + - `attempting compaction before retry (attempt ${timeoutCompactionAttempts}/${MAX_TIMEOUT_COMPACTION_ATTEMPTS}) diagId=${timeoutDiagId}`, - ); - let timeoutCompactResult: Awaited>; - await runOwnsCompactionBeforeHook("timeout recovery"); - try { - const timeoutCompactionRuntimeContext = { - ...buildEmbeddedCompactionRuntimeContext({ - sessionKey: params.sessionKey, - messageChannel: params.messageChannel, - messageProvider: params.messageProvider, - clientCaps: params.clientCaps, - chatType: params.chatType, - agentAccountId: params.agentAccountId, - currentChannelId: params.currentChannelId, - currentThreadTs: params.currentThreadTs, - currentMessageId: params.currentMessageId, - authProfileId: lastProfileId, - authProfileIdSource: lockedProfileId ? "user" : "auto", - runtimeAuthPlan: runtimePlan.auth, - workspaceDir: resolvedWorkspace, - agentDir, - config: params.config, - skillsSnapshot: params.skillsSnapshot, - senderId: params.senderId, - provider, - modelId, - harnessRuntime: agentHarness.id, - modelSelectionLocked: params.modelSelectionLocked, - modelFallbacksOverride: params.modelFallbacksOverride, - thinkLevel, - reasoningLevel: params.reasoningLevel, - bashElevated: params.bashElevated, - extraSystemPrompt: params.extraSystemPrompt, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - ownerNumbers: params.ownerNumbers, - activeProcessSessions: listActiveProcessSessionReferences({ - scopeKey: resolveProcessToolScopeKey({ - sessionKey: params.sandboxSessionKey?.trim() || params.sessionKey, - sessionId: activeSessionId, - agentId: sessionAgentId, - }), - }), - }), - ...resolveContextEngineCapabilities({ - config: params.config, - sessionKey: params.sessionKey, - agentId: sessionAgentId, - contextEnginePluginId: resolveContextEnginePluginId(), - purpose: "context-engine.timeout-compaction", - }), - onCompactionHookMessages, - ...(attempt.promptCache ? { promptCache: attempt.promptCache } : {}), - runId: params.runId, - trigger: "timeout_recovery", - diagId: timeoutDiagId, - attempt: timeoutCompactionAttempts, - maxAttempts: MAX_TIMEOUT_COMPACTION_ATTEMPTS, - }; - // Bound plugin-owned compaction with the same finite safety - // timeout that protects native compaction, and thread the - // run-level abort signal through, so a hung plugin compact() - // cannot stall timeout recovery indefinitely. A timeout/abort - // surfaces as a thrown error handled by the catch below. - timeoutCompactResult = await compactContextEngineWithSafetyTimeout( - contextEngine, - { - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - agentId: sessionAgentId, - sessionTarget: buildContextEngineCompactionSessionTarget({ - agentId: sessionAgentId, - config: params.config, - sessionFile: activeSessionFile, - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - sessionTarget: activeSessionTarget, - }), - tokenBudget: contextTokenBudget, - force: true, - compactionTarget: "budget", - runtimeContext: timeoutCompactionRuntimeContext, - runtimeSettings: buildEmbeddedContextEngineRuntimeSettings({ - tokenBudget: contextTokenBudget, - }), - }, - resolveCompactionTimeoutMs(params.config), - params.abortSignal, - ); - } catch (compactErr) { - log.warn( - `[timeout-compaction] contextEngine.compact() threw during timeout recovery for ${provider}/${modelId}: ${String(compactErr)}`, - ); - timeoutCompactResult = { - ok: false, - compacted: false, - reason: String(compactErr), - }; - } - const previousSessionId = timeoutCompactResult.compacted - ? await adoptCompactionTranscript(timeoutCompactResult) - : undefined; - await runOwnsCompactionAfterHook( - "timeout recovery", - timeoutCompactResult, - previousSessionId, - ); - if (timeoutCompactResult.compacted) { - autoCompactionCount += 1; - if ( - typeof timeoutCompactResult.result?.tokensAfter === "number" && - Number.isFinite(timeoutCompactResult.result.tokensAfter) && - timeoutCompactResult.result.tokensAfter >= 0 - ) { - lastCompactionTokensAfter = Math.floor(timeoutCompactResult.result.tokensAfter); - } - if (contextEngine.info.ownsCompaction === true) { - await runPostCompactionSideEffects({ - config: params.config, - sessionKey: params.sessionKey, - sessionId: activeSessionId, - agentId: sessionAgentId, - sessionFile: activeSessionFile, - }); - } - log.info( - `[timeout-compaction] compaction succeeded for ${provider}/${modelId}; retrying prompt`, - ); - postCompactionGuard.armPostCompaction(); - continue; - } else { - log.warn( - `[timeout-compaction] compaction did not reduce context for ${provider}/${modelId}; falling through to normal handling`, - ); - } - } - } - - const contextOverflowError = - !aborted && !signalOwnedInterruption - ? (() => { - if (promptError) { - const errorText = formatErrorMessage(promptError); - if (isLikelyContextOverflowError(errorText)) { - return { text: errorText, source: "promptError" as const }; - } - // Prompt submission failed with a non-overflow error. Do not - // inspect prior assistant errors from history for this attempt. - return null; - } - if (assistantErrorText && isLikelyContextOverflowError(assistantErrorText)) { - return { - text: assistantErrorText, - source: "assistantError" as const, - }; - } - return null; - })() - : null; - - if ( - contextOverflowError && - genericCompactionRecoveryAllowed && - contextTokenBudget !== undefined - ) { - const overflowDiagId = createCompactionDiagId(); - const errorText = contextOverflowError.text; - const msgCount = attempt.messagesSnapshot?.length ?? 0; - const observedOverflowTokens = extractObservedOverflowTokenCount(errorText); - const preflightEstimatedPromptTokens = - typeof preflightRecovery?.estimatedPromptTokens === "number" && - Number.isFinite(preflightRecovery.estimatedPromptTokens) && - preflightRecovery.estimatedPromptTokens > 0 - ? Math.ceil(preflightRecovery.estimatedPromptTokens) - : undefined; - const overflowTokenCountForCompaction = - observedOverflowTokens ?? - preflightEstimatedPromptTokens ?? - (contextTokenBudget > 0 - ? // Confirmed overflow with an unparseable provider message still carries a - // minimally over-budget count for compaction engines and diagnostics. - contextTokenBudget + 1 - : undefined); - log.warn( - `[context-overflow-diag] sessionKey=${params.sessionKey ?? params.sessionId} ` + - `provider=${provider}/${modelId} source=${contextOverflowError.source} ` + - `messages=${msgCount} sessionFile=${activeSessionFile} ` + - `diagId=${overflowDiagId} compactionAttempts=${overflowCompactionAttempts} ` + - `observedTokens=${observedOverflowTokens ?? "unknown"} ` + - `preflightEstimatedTokens=${preflightEstimatedPromptTokens ?? "unknown"} ` + - `compactionTokens=${overflowTokenCountForCompaction ?? "unknown"} ` + - `error=${truncateUtf16Safe(errorText, 200)}`, - ); - const isCompactionFailure = isCompactionFailureError(errorText); - const hadAttemptLevelCompaction = attemptCompactionCount > 0; - // If this attempt already compacted (SDK auto-compaction), avoid immediately - // running another explicit compaction for the same overflow trigger. - if ( - !isCompactionFailure && - hadAttemptLevelCompaction && - overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS - ) { - overflowCompactionAttempts++; - log.warn( - `context overflow persisted after in-attempt compaction (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); retrying prompt without additional compaction for ${provider}/${modelId}`, - ); - if (preflightRecovery?.source === "mid-turn") { - continueFromCurrentTranscript(); - } - continue; - } - // Attempt explicit overflow compaction only when this attempt did not - // already auto-compact. - if ( - !isCompactionFailure && - !hadAttemptLevelCompaction && - overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS - ) { - if (log.isEnabled("debug")) { - log.debug( - `[compaction-diag] decision diagId=${overflowDiagId} branch=compact ` + - `isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` + - `attempt=${overflowCompactionAttempts + 1} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`, - ); - } - overflowCompactionAttempts++; - log.warn( - `context overflow detected (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); attempting auto-compaction for ${provider}/${modelId}`, - ); - let compactResult: Awaited>; - let previousSessionId: string | undefined; - await runOwnsCompactionBeforeHook("overflow recovery"); - try { - const overflowCompactionRuntimeContext = { - ...buildEmbeddedCompactionRuntimeContext({ - sessionKey: params.sessionKey, - messageChannel: params.messageChannel, - messageProvider: params.messageProvider, - clientCaps: params.clientCaps, - chatType: params.chatType, - agentAccountId: params.agentAccountId, - currentChannelId: params.currentChannelId, - currentThreadTs: params.currentThreadTs, - currentMessageId: params.currentMessageId, - authProfileId: lastProfileId, - authProfileIdSource: lockedProfileId ? "user" : "auto", - runtimeAuthPlan: runtimePlan.auth, - workspaceDir: resolvedWorkspace, - agentDir, - config: params.config, - skillsSnapshot: params.skillsSnapshot, - senderId: params.senderId, - provider, - modelId, - harnessRuntime: agentHarness.id, - modelSelectionLocked: params.modelSelectionLocked, - modelFallbacksOverride: params.modelFallbacksOverride, - thinkLevel, - reasoningLevel: params.reasoningLevel, - bashElevated: params.bashElevated, - extraSystemPrompt: params.extraSystemPrompt, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - ownerNumbers: params.ownerNumbers, - activeProcessSessions: listActiveProcessSessionReferences({ - scopeKey: resolveProcessToolScopeKey({ - sessionKey: params.sandboxSessionKey?.trim() || params.sessionKey, - sessionId: activeSessionId, - agentId: sessionAgentId, - }), - }), - }), - ...resolveContextEngineCapabilities({ - config: params.config, - sessionKey: params.sessionKey, - agentId: sessionAgentId, - contextEnginePluginId: resolveContextEnginePluginId(), - purpose: "context-engine.overflow-compaction", - }), - onCompactionHookMessages, - ...(attempt.promptCache ? { promptCache: attempt.promptCache } : {}), - runId: params.runId, - trigger: "overflow", - ...(overflowTokenCountForCompaction !== undefined - ? { currentTokenCount: overflowTokenCountForCompaction } - : {}), - diagId: overflowDiagId, - attempt: overflowCompactionAttempts, - maxAttempts: MAX_OVERFLOW_COMPACTION_ATTEMPTS, - }; - // Bound plugin-owned compaction with the same finite safety - // timeout that protects native compaction, and thread the - // run-level abort signal through, so a hung plugin compact() - // cannot stall overflow recovery indefinitely. A timeout/abort - // surfaces as a thrown error handled by the catch below. - const overflowCompactionRuntimeSettings = buildEmbeddedContextEngineRuntimeSettings( - { - tokenBudget: contextTokenBudget, - degradedReason: "context_overflow", - }, - ); - compactResult = await compactContextEngineWithSafetyTimeout( - contextEngine, - { - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - agentId: sessionAgentId, - sessionTarget: buildContextEngineCompactionSessionTarget({ - agentId: sessionAgentId, - config: params.config, - sessionFile: activeSessionFile, - sessionId: activeSessionId, - sessionKey: resolvedSessionKey, - sessionTarget: activeSessionTarget, - }), - tokenBudget: contextTokenBudget, - ...(overflowTokenCountForCompaction !== undefined - ? { currentTokenCount: overflowTokenCountForCompaction } - : {}), - force: true, - compactionTarget: "budget", - runtimeContext: overflowCompactionRuntimeContext, - runtimeSettings: overflowCompactionRuntimeSettings, - }, - resolveCompactionTimeoutMs(params.config), - params.abortSignal, - ); - if (compactResult.ok && compactResult.compacted) { - previousSessionId = await adoptCompactionTranscript(compactResult); - await runContextEngineMaintenance({ - contextEngine, - sessionId: activeSessionId, - sessionKey: params.sessionKey, - sessionTarget: activeSessionTarget, - sessionFile: activeSessionFile, - reason: "compaction", - runtimeContext: overflowCompactionRuntimeContext, - runtimeSettings: overflowCompactionRuntimeSettings, - config: params.config, - agentId: sessionAgentId, - }); - } - } catch (compactErr) { - log.warn( - `contextEngine.compact() threw during overflow recovery for ${provider}/${modelId}: ${String(compactErr)}`, - ); - compactResult = { - ok: false, - compacted: false, - reason: String(compactErr), - }; - } - await runOwnsCompactionAfterHook( - "overflow recovery", - compactResult, - previousSessionId, - ); - if (preflightRecovery && isNoRealConversationCompactionNoop(compactResult)) { - lastCompactionTokensAfter = undefined; - lastContextBudgetStatus = undefined; - await resetNoRealConversationTokenSnapshot({ - config: params.config, - sessionKey: params.sessionKey, - agentId: sessionAgentId, - }); - log.info( - `[context-overflow-precheck] stale token state had no real conversation messages for ` + - `${provider}/${modelId}; resetting the context snapshot and retrying prompt`, - ); - if (preflightRecovery.source === "mid-turn") { - continueFromCurrentTranscript(); - } - continue; - } - if (compactResult.compacted) { - await adoptCompactionTranscript(compactResult); - if ( - typeof compactResult.result?.tokensAfter === "number" && - Number.isFinite(compactResult.result.tokensAfter) && - compactResult.result.tokensAfter >= 0 - ) { - lastCompactionTokensAfter = Math.floor(compactResult.result.tokensAfter); - } - if (preflightRecovery?.route === "compact_then_truncate") { - const truncResult = await truncateOversizedToolResultsInActiveTarget({ - scope: { - sessionId: activeSessionId, - sessionKey: params.sessionKey ?? activeSessionId, - sessionFile: activeSessionFile, - agentId: sessionAgentId, - }, - contextWindowTokens: contextTokenBudget, - maxCharsOverride: resolveLiveToolResultMaxChars({ - contextWindowTokens: contextTokenBudget, - cfg: params.config, - agentId: sessionAgentId, - }), - config: params.config, - protectTrailingToolResults: true, - }); - if (truncResult.truncated) { - log.info( - `[context-overflow-precheck] post-compaction tool-result truncation succeeded for ` + - `${provider}/${modelId}; truncated ${truncResult.truncatedCount} tool result(s)`, - ); - } else { - log.warn( - `[context-overflow-precheck] post-compaction tool-result truncation did not help for ` + - `${provider}/${modelId}: ${truncResult.reason ?? "unknown"}`, - ); - } - } - autoCompactionCount += 1; - log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`); - postCompactionGuard.armPostCompaction(); - if (preflightRecovery?.source === "mid-turn") { - continueFromCurrentTranscript(); - } else { - await waitForCurrentUserMessagePersistence(); - if (activePrompt.internal) { - // Retry the same internal prompt and preserve its exact durability state. - suppressNextUserMessagePersistence = activePrompt.persisted; - } else if (activePrompt.persisted) { - // The first attempt reached the embedded agent far enough to persist this user turn. - // Retrying the original prompt would replay it, so resume from the - // compacted transcript and suppress the next user append. - activateInternalPrompt(MID_TURN_PRECHECK_CONTINUATION_PROMPT, true); - } - } - continue; - } - log.warn( - `auto-compaction failed for ${provider}/${modelId}: ${compactResult.reason ?? "nothing to compact"}`, - ); - } - if (!toolResultTruncationAttempted) { - const contextWindowTokens = contextTokenBudget; - const toolResultMaxChars = resolveLiveToolResultMaxChars({ - contextWindowTokens, - cfg: params.config, - agentId: sessionAgentId, - }); - const hasOversized = attempt.messagesSnapshot - ? sessionLikelyHasOversizedToolResults({ - messages: attempt.messagesSnapshot, - contextWindowTokens, - maxCharsOverride: toolResultMaxChars, - }) - : false; - - if (hasOversized) { - toolResultTruncationAttempted = true; - log.warn( - `[context-overflow-recovery] Attempting tool result truncation for ${provider}/${modelId} ` + - `(contextWindow=${contextWindowTokens} tokens)`, - ); - const truncResult = await truncateOversizedToolResultsInActiveTarget({ - scope: { - sessionId: activeSessionId, - sessionKey: params.sessionKey ?? activeSessionId, - sessionFile: activeSessionFile, - agentId: sessionAgentId, - }, - contextWindowTokens, - maxCharsOverride: toolResultMaxChars, - config: params.config, - protectTrailingToolResults: preflightRecovery?.route === "compact_then_truncate", - }); - if (truncResult.truncated) { - log.info( - `[context-overflow-recovery] Truncated ${truncResult.truncatedCount} tool result(s); retrying prompt`, - ); - if (preflightRecovery?.source === "mid-turn") { - continueFromCurrentTranscript(); - } - continue; - } - log.warn( - `[context-overflow-recovery] Tool result truncation did not help: ${truncResult.reason ?? "unknown"}`, - ); - } - } - if ( - (isCompactionFailure || - overflowCompactionAttempts >= MAX_OVERFLOW_COMPACTION_ATTEMPTS) && - log.isEnabled("debug") - ) { - log.debug( - `[compaction-diag] decision diagId=${overflowDiagId} branch=give_up ` + - `isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` + - `attempt=${overflowCompactionAttempts} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`, - ); - } - const kind = isCompactionFailure ? "compaction_failure" : "context_overflow"; - const overflowRecoveryText = - "Context overflow: prompt too large for the model. " + - "Try /reset (or /new) to start a fresh session, or use a larger-context model."; - log.warn( - `[context-overflow-recovery] exhausted provider overflow recovery for ${provider}/${modelId}; ` + - `livenessState=blocked suggestedAction=reset_or_new kind=${kind}`, - ); - setTerminalLifecycleMeta({ - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - }); - return { - payloads: [ - { - text: overflowRecoveryText, - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: sessionIdUsed, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastAssistant: attemptAssistant, - lastTurnTotal, - }), - systemPromptReport: attempt.systemPromptReport, - finalAssistantVisibleText: overflowRecoveryText, - finalAssistantRawText: overflowRecoveryText, - finalPromptText: attempt.finalPromptText, - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - error: { kind, message: errorText }, - }, - }; - } - - if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) { - const errorText = formatErrorMessage(promptError); - const replayInvalid = resolveReplayInvalidForAttempt(); - setTerminalLifecycleMeta({ - replayInvalid, - livenessState: "blocked", - }); - return { - payloads: [{ text: errorText, isError: true }], - meta: { - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: sessionIdUsed, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastAssistant: attemptAssistant, - lastTurnTotal, - }), - systemPromptReport: attempt.systemPromptReport, - finalAssistantVisibleText: errorText, - finalAssistantRawText: errorText, - finalPromptText: undefined, - replayInvalid, - livenessState: "blocked", - error: { kind: "hook_block", message: errorText }, - }, - }; - } - - const hasRecoverableCodexAppServerTimeoutOutcome = Boolean( - attempt.codexAppServerFailure && attempt.promptTimeoutOutcome, - ); - let shouldSurfaceCodexCompletionTimeout = false; - if (promptError && promptErrorSource !== "compaction" && attempt.codexAppServerFailure) { - // Retry replay-safe Codex app-server failures. - const codexAppServerRecoveryRetry = resolveCodexAppServerRecoveryRetry({ - attempt, - retryAvailable: codexAppServerRecoveryRetryAvailable, - }); - if (codexAppServerRecoveryRetry.retry) { - throwIfAborted(); - codexAppServerRecoveryRetries += 1; - suppressNextUserMessagePersistence = true; - log.warn( - `codex app-server replay-safe failure; retrying once ` + - `failureKind=${attempt.codexAppServerFailure?.kind} ` + - `runId=${params.runId} sessionId=${params.sessionId}`, - ); - continue; - } - // Completion-idle timeouts are timeout outcomes even when the - // app-server transport is not retryable, or the retry was exhausted. - shouldSurfaceCodexCompletionTimeout = - attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && - attempt.timedOut; - if ( - attempt.codexAppServerFailure && - !hasRecoverableCodexAppServerTimeoutOutcome && - !shouldSurfaceCodexCompletionTimeout - ) { - throw toErrorObject(promptError, "Prompt failed"); - } - } - - if ( - promptError && - !terminalInterrupted && - promptErrorSource !== "compaction" && - !hasRecoverableCodexAppServerTimeoutOutcome && - !shouldSurfaceCodexCompletionTimeout - ) { - // Normalize wrapped errors (e.g. abort-wrapped RESOURCE_EXHAUSTED) into - // FailoverError so rate-limit classification works even for nested shapes. - // - // promptErrorSource === "compaction" means the model call already completed and the - // abort happened only while waiting for compaction/retry cleanup. Retrying from here - // would replay that completed tool turn as a fresh prompt attempt. - const promptAuthMode = lastProfileId - ? attemptAuthProfileStore.profiles?.[lastProfileId]?.type - : undefined; - const normalizedPromptFailover = coerceToFailoverError(promptError, { - provider: activeErrorContext.provider, - model: activeErrorContext.model, - profileId: lastProfileId, - authMode: promptAuthMode, - sessionId: sessionIdUsed, - lane: globalLane, - }); - const promptErrorDetails = normalizedPromptFailover - ? describeFailoverError(normalizedPromptFailover) - : describeFailoverError(promptError); - if (normalizedPromptFailover?.suspend) { - suspendForFailure({ - cfg: params.config, - agentDir, - sessionId: activeSessionId ?? params.sessionId, - reason: resolveSessionSuspensionReason(normalizedPromptFailover.reason), - failedProvider: normalizedPromptFailover.provider ?? provider, - failedModel: normalizedPromptFailover.model ?? modelId, - }); - } - const errorText = promptErrorDetails.message || formatErrorMessage(promptError); - if (await maybeRefreshRuntimeAuthForAuthError(errorText, runtimeAuthRetry)) { - authRetryPending = true; - continue; - } - // Handle role ordering errors with a user-friendly message - if (/incorrect role information|roles must alternate/i.test(errorText)) { - setTerminalLifecycleMeta({ - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - }); - return { - payloads: [ - { - text: - "Message ordering conflict - please try again. " + - "If this persists, use /new to start a fresh session.", - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: sessionIdUsed, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastAssistant: attemptAssistant, - lastTurnTotal, - }), - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - error: { kind: "role_ordering", message: errorText }, - }, - }; - } - // Handle image size errors with a user-friendly message (no retry needed) - const imageSizeError = parseImageSizeError(errorText); - if (imageSizeError) { - const maxMb = imageSizeError.maxMb; - const maxMbLabel = - typeof maxMb === "number" && Number.isFinite(maxMb) ? `${maxMb}` : null; - const maxBytesHint = maxMbLabel ? ` (max ${maxMbLabel}MB)` : ""; - setTerminalLifecycleMeta({ - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - }); - return { - payloads: [ - { - text: - `Image too large for the model${maxBytesHint}. ` + - "Please compress or resize the image and try again.", - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: buildErrorAgentMeta({ - sessionId: sessionIdUsed, - sessionFile: activeSessionFile, - provider, - model: model.id, - ...outerContextTokenMeta, - usageAccumulator, - lastRunPromptUsage, - lastAssistant: attemptAssistant, - lastTurnTotal, - }), - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - replayInvalid: resolveReplayInvalidForAttempt(), - livenessState: "blocked", - error: { kind: "image_size", message: errorText }, - }, - }; - } - const promptFailoverReason = - promptErrorDetails.reason ?? classifyFailoverReason(errorText, { provider }); - const promptProfileFailureReason = resolveRunAuthProfileFailureReason( - promptFailoverReason, - { - providerStarted: promptErrorSource === "prompt", - transientRateLimit: - promptFailoverReason === "rate_limit" && isShortWindowRateLimitMessage(errorText), - }, - ); - const promptFailoverFailure = - promptFailoverReason !== null || isFailoverErrorMessage(errorText, { provider }); - const promptTimeoutFallbackSafe = - promptErrorSource === "prompt" && - promptFailoverReason === "timeout" && - !attempt.codexAppServerFailure && - attempt.promptTimeoutOutcome?.replayInvalid !== true && - attempt.replayMetadata.replaySafe; - // Capture the failing profile before auth-profile rotation mutates `lastProfileId`. - const failedPromptProfileId = lastProfileId; - const logPromptFailoverDecision = createFailoverDecisionLogger({ - stage: "prompt", - runId: params.runId, - rawError: errorText, - failoverReason: promptFailoverReason, - profileFailureReason: promptProfileFailureReason, - provider, - model: modelId, - sourceProvider: provider, - sourceModel: modelId, - profileId: failedPromptProfileId, - fallbackConfigured, - aborted, - }); - if (promptFailoverReason === "rate_limit") { - maybeEscalateRateLimitProfileFallback({ - failoverProvider: provider, - failoverModel: modelId, - logFallbackDecision: logPromptFailoverDecision, - }); - } - let promptFailoverDecision = resolveRunFailoverDecision({ - stage: "prompt", - aborted, - externalAbort, - fallbackConfigured, - failoverCode: promptErrorDetails.code, - failoverFailure: promptFailoverFailure, - failoverReason: promptFailoverReason, - harnessOwnsTransport: pluginHarnessOwnsTransport, - promptTimeoutFallbackSafe, - timedOutByRunBudget, - profileRotated: false, - }); - if ( - promptFailoverDecision.action === "rotate_profile" && - (await advanceAttemptAuthProfile()) - ) { - if (failedPromptProfileId && promptProfileFailureReason) { - void maybeMarkAuthProfileFailure({ - profileId: failedPromptProfileId, - reason: promptProfileFailureReason, - modelId, - }).catch((err: unknown) => { - log.warn(`prompt profile failure mark failed: ${String(err)}`); - }); - } - traceAttempts.push({ - provider, - model: modelId, - result: promptFailoverReason === "timeout" ? "timeout" : "rotate_profile", - ...(promptFailoverReason ? { reason: promptFailoverReason } : {}), - stage: "prompt", - }); - lastRetryFailoverReason = mergeRetryFailoverReason({ - previous: lastRetryFailoverReason, - failoverReason: promptFailoverReason, - }); - logPromptFailoverDecision("rotate_profile"); - await maybeBackoffBeforeOverloadFailover(promptFailoverReason); - continue; - } - if (promptFailoverDecision.action === "rotate_profile") { - promptFailoverDecision = resolveRunFailoverDecision({ - stage: "prompt", - aborted, - externalAbort, - fallbackConfigured, - failoverCode: promptErrorDetails.code, - failoverFailure: promptFailoverFailure, - failoverReason: promptFailoverReason, - harnessOwnsTransport: pluginHarnessOwnsTransport, - promptTimeoutFallbackSafe, - timedOutByRunBudget, - profileRotated: true, - }); - } - if (failedPromptProfileId && promptProfileFailureReason) { - try { - await maybeMarkAuthProfileFailure({ - profileId: failedPromptProfileId, - reason: promptProfileFailureReason, - modelId, - }); - } catch (err) { - log.warn(`prompt profile failure mark failed: ${String(err)}`); - } - } - const fallbackThinking = pickFallbackThinkingLevel({ - message: errorText, - attempted: attemptedThinking, - }); - if (fallbackThinking) { - log.warn( - `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, - ); - thinkLevel = fallbackThinking; - continue; - } - // Throw FailoverError for prompt-side failover reasons when fallbacks - // are configured so outer model fallback can continue on overload, - // rate-limit, auth, or billing failures. - if (promptFailoverDecision.action === "fallback_model") { - const fallbackReason = promptFailoverDecision.reason ?? "unknown"; - const status = resolveFailoverStatus(fallbackReason); - traceAttempts.push({ - provider, - model: modelId, - result: promptFailoverReason === "timeout" ? "timeout" : "fallback_model", - reason: fallbackReason, - stage: "prompt", - ...(typeof status === "number" ? { status } : {}), - }); - logPromptFailoverDecision("fallback_model", { status }); - await maybeBackoffBeforeOverloadFailover(promptFailoverReason); - throw ( - normalizedPromptFailover ?? - new FailoverError(errorText, { - reason: fallbackReason, - provider, - model: modelId, - profileId: lastProfileId, - authMode: promptAuthMode, - sessionId: sessionIdUsed, - lane: globalLane, - status, - }) - ); - } - if (promptFailoverDecision.action === "surface_error") { - traceAttempts.push({ - provider, - model: modelId, - result: promptFailoverReason === "timeout" ? "timeout" : "surface_error", - ...(promptFailoverReason ? { reason: promptFailoverReason } : {}), - stage: "prompt", - }); - logPromptFailoverDecision("surface_error"); - } - throw toErrorObject(promptError, "Prompt failed"); - } - - const fallbackThinking = pickFallbackThinkingLevel({ - message: attemptAssistant?.errorMessage, - attempted: attemptedThinking, - }); - if (fallbackThinking && !terminalInterrupted) { - log.warn( - `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, - ); - thinkLevel = fallbackThinking; - continue; - } - - const authFailure = isAuthAssistantError(attemptAssistant); - const rateLimitFailure = isRateLimitAssistantError(attemptAssistant); - const billingFailure = isBillingAssistantError(attemptAssistant); - const failoverFailure = isFailoverAssistantError(attemptAssistant); - const assistantFailoverReason = classifyAssistantFailoverReason(attemptAssistant); - const assistantProviderStarted = - Boolean(currentAttemptAssistant?.provider) || terminalOutcome.providerStarted === true; - const assistantProfileFailoverReason = - assistantFailoverReason ?? - (assistantProviderStarted && (timedOut || idleTimedOut) ? "timeout" : null); - const assistantProfileFailureReason = resolveRunAuthProfileFailureReason( - assistantProfileFailoverReason, - { - providerStarted: assistantProviderStarted, - transientRateLimit: - assistantProfileFailoverReason === "rate_limit" && - isShortWindowRateLimitMessage(attemptAssistant?.errorMessage), - }, - ); - const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError; - const imageDimensionError = parseImageDimensionError( - attemptAssistant?.errorMessage ?? "", - ); - // The shared runtime wraps interrupted streams as a timeout. Retry that - // wrapper only for reasoning-only output so ordinary timeouts keep failover. - const genericUnknownReasoningError = - assistantFailoverReason === "timeout" && - isGenericUnknownStreamErrorMessage(attemptAssistant?.errorMessage ?? "") && - Boolean(attemptAssistant && hasOnlyAssistantReasoningContent(attemptAssistant)); - const silentErrorRetryReason = - assistantFailoverReason === null || - genericUnknownReasoningError || - assistantFailoverReason === "no_error_details" || - assistantFailoverReason === "unclassified" || - assistantFailoverReason === "unknown" || - assistantFailoverReason === "server_error"; - // Retry replay-safe non-visible provider errors before assistant - // failover surfaces them as terminal provider failures. - if ( - !authFailure && - !rateLimitFailure && - !billingFailure && - !cloudCodeAssistFormatError && - !imageDimensionError && - !terminalInterrupted && - !promptError && - silentErrorRetryReason && - shouldRetrySilentErrorAssistantTurn({ attempt, assistant: attemptAssistant }) && - emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES - ) { - emptyErrorRetries += 1; - log.warn( - `[empty-error-retry] stopReason=error non-visible-output; resubmitting ` + - `attempt=${emptyErrorRetries}/${MAX_EMPTY_ERROR_RETRIES} ` + - `provider=${attemptAssistant?.provider ?? provider} ` + - `model=${attemptAssistant?.model ?? model.id} ` + - `sessionKey=${params.sessionKey ?? params.sessionId}`, - ); - continue; - } - // Capture the failing profile before auth-profile rotation mutates `lastProfileId`. - const failedAssistantProfileId = lastProfileId; - const logAssistantFailoverDecision = createFailoverDecisionLogger({ - stage: "assistant", - runId: params.runId, - rawError: attemptAssistant?.errorMessage?.trim(), - failoverReason: assistantFailoverReason, - profileFailureReason: assistantProfileFailureReason, - provider: activeErrorContext.provider, - model: activeErrorContext.model, - sourceProvider: attemptAssistant?.provider ?? provider, - sourceModel: attemptAssistant?.model ?? modelId, - profileId: failedAssistantProfileId, - fallbackConfigured, - timedOut, - aborted, - }); - - if ( - !signalOwnedInterruption && - authFailure && - (await maybeRefreshRuntimeAuthForAuthError( - attemptAssistant?.errorMessage ?? "", - runtimeAuthRetry, - )) - ) { - authRetryPending = true; - continue; - } - if (imageDimensionError && lastProfileId) { - const details = [ - imageDimensionError.messageIndex !== undefined - ? `message=${imageDimensionError.messageIndex}` - : null, - imageDimensionError.contentIndex !== undefined - ? `content=${imageDimensionError.contentIndex}` - : null, - imageDimensionError.maxDimensionPx !== undefined - ? `limit=${imageDimensionError.maxDimensionPx}px` - : null, - ] - .filter(Boolean) - .join(" "); - log.warn( - `Profile ${lastProfileId} rejected image payload${details ? ` (${details})` : ""}.`, - ); - } - - const assistantFailoverDecision = resolveRunFailoverDecision({ - stage: "assistant", - allowFormatRetry: cloudCodeAssistFormatError, - aborted, - externalAbort: externalAbort || signalOwnedInterruption, - fallbackConfigured, - failoverFailure, - failoverReason: assistantFailoverReason, - timedOut, - idleTimedOut, - timedOutDuringCompaction, - timedOutDuringToolExecution, - harnessOwnsTransport: pluginHarnessOwnsTransport, - timedOutByRunBudget, - profileRotated: false, - }); - const assistantFailoverOutcome = await handleAssistantFailover({ - initialDecision: assistantFailoverDecision, - aborted, - externalAbort: externalAbort || signalOwnedInterruption, - fallbackConfigured, - failoverFailure, - failoverReason: assistantFailoverReason, - timedOut, - idleTimedOut, - timedOutDuringCompaction, - timedOutDuringToolExecution, - timedOutByRunBudget, - allowSameModelIdleTimeoutRetry: - timedOut && - idleTimedOut && - !timedOutDuringCompaction && - !fallbackConfigured && - canRestartForLiveSwitch && - sameModelIdleTimeoutRetries < MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES, - allowSameModelRateLimitRetry: rateLimitProfileRotations < rateLimitProfileRotationLimit, - assistantProfileFailureReason, - lastProfileId, - modelId, - provider, - activeErrorContext, - lastAssistant: attemptAssistant, - config: params.config, - sessionKey: params.sessionKey ?? params.sessionId, - authFailure, - rateLimitFailure, - billingFailure, - authMode: lastProfileId - ? attemptAuthProfileStore.profiles?.[lastProfileId]?.type - : undefined, - cloudCodeAssistFormatError, - isProbeSession, - overloadProfileRotations, - overloadProfileRotationLimit, - previousRetryFailoverReason: lastRetryFailoverReason, - logAssistantFailoverDecision, - warn: (message) => log.warn(message), - maybeMarkAuthProfileFailure, - maybeEscalateRateLimitProfileFallback, - maybeRetrySameModelRateLimit, - maybeBackoffBeforeOverloadFailover, - advanceAuthProfile: advanceAttemptAuthProfile, - }); - overloadProfileRotations = assistantFailoverOutcome.overloadProfileRotations; - if (assistantFailoverOutcome.action === "retry") { - const retryTraceResult = - assistantFailoverOutcome.retryKind === "same_model_rate_limit" - ? "same_model_rate_limit" - : assistantFailoverOutcome.retryKind === "same_model_idle_timeout" || - assistantFailoverReason === "timeout" - ? "timeout" - : "rotate_profile"; - traceAttempts.push({ - provider: activeErrorContext.provider, - model: activeErrorContext.model, - result: retryTraceResult, - ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), - stage: "assistant", - }); - if (assistantFailoverOutcome.retryKind === "same_model_idle_timeout") { - sameModelIdleTimeoutRetries += 1; - } - if (assistantFailoverOutcome.retryKind !== "same_model_rate_limit") { - consecutiveSameModelRateLimitRetries = resolveNextSameModelRateLimitRetryCount({ - retriesSoFar: consecutiveSameModelRateLimitRetries, - retriedSameModelRateLimit: false, - }); - } - lastRetryFailoverReason = assistantFailoverOutcome.lastRetryFailoverReason; - continue; - } - consecutiveSameModelRateLimitRetries = resolveNextSameModelRateLimitRetryCount({ - retriesSoFar: consecutiveSameModelRateLimitRetries, - retriedSameModelRateLimit: false, - }); - if (assistantFailoverOutcome.action === "throw") { - traceAttempts.push({ - provider: activeErrorContext.provider, - model: activeErrorContext.model, - result: - assistantFailoverReason === "timeout" - ? "timeout" - : assistantFailoverDecision.action === "fallback_model" - ? "fallback_model" - : "error", - ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), - stage: "assistant", - ...(typeof assistantFailoverOutcome.error.status === "number" - ? { status: assistantFailoverOutcome.error.status } - : {}), - }); - if (assistantFailoverOutcome.error.suspend) { - suspendForFailure({ - cfg: params.config, - agentDir, - sessionId: activeSessionId ?? params.sessionId, - reason: resolveSessionSuspensionReason(assistantFailoverOutcome.error.reason), - failedProvider: assistantFailoverOutcome.error.provider ?? provider, - failedModel: assistantFailoverOutcome.error.model ?? modelId, - }); - } - throw assistantFailoverOutcome.error; - } - const usageMeta = buildUsageAgentMetaFields({ - usageAccumulator, - lastAssistantUsage: attemptAssistant?.usage as UsageLike | undefined, - lastRunPromptUsage, - lastTurnTotal, - }); - const reportedModelRef = resolveReportedModelRef({ - provider, - model: model.id, - assistant: attemptAssistant, - }); - const agentMeta: EmbeddedAgentMeta = { - sessionId: sessionIdUsed, - sessionFile: sessionFileUsed, - provider: reportedModelRef.provider, - model: reportedModelRef.model, - ...outerContextTokenMeta, - agentHarnessId: attempt.agentHarnessId, - usage: usageMeta.usage, - lastCallUsage: usageMeta.lastCallUsage, - promptTokens: usageMeta.promptTokens, - ...(lastContextBudgetStatus ? { contextBudgetStatus: lastContextBudgetStatus } : {}), - compactionCount: autoCompactionCount > 0 ? autoCompactionCount : undefined, - compactionTokensAfter: lastCompactionTokensAfter, - }; - const finalAssistantVisibleText = resolveFinalAssistantVisibleText(attemptAssistant); - const finalAssistantRawText = resolveFinalAssistantRawText(attemptAssistant); - const payloads = buildEmbeddedRunPayloads({ - assistantTexts: attempt.assistantTexts, - assistantMessageIndex: attempt.lastAssistantTextMessageIndex, - assistantTranscriptOwned: attempt.assistantTranscriptOwned, - toolMetas: attempt.toolMetas, - lastAssistant: attempt.lastAssistant, - currentAssistant: currentAttemptAssistant ?? null, - lastToolError: attempt.lastToolError, - config: params.config, - isCronTrigger: params.trigger === "cron", - isHeartbeatTrigger: params.trigger === "heartbeat", - sessionKey: params.sessionKey ?? params.sessionId, - provider: activeErrorContext.provider, - model: activeErrorContext.model, - authMode: lastProfileId - ? attemptAuthProfileStore.profiles?.[lastProfileId]?.type - : undefined, - verboseLevel: params.verboseLevel, - reasoningLevel: params.reasoningLevel, - thinkingLevel: params.thinkLevel, - toolResultFormat: resolvedToolResultFormat, - suppressToolErrorWarnings: params.suppressToolErrorWarnings, - inlineToolResultsAllowed: false, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - didDeliverSourceReplyViaMessageTool: - attempt.didDeliverSourceReplyViaMessageTool === true, - messagingToolSentTargets: attempt.messagingToolSentTargets, - messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - agentId: params.agentId, - runId: params.runId, - runAborted: terminalInterrupted, - didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, - heartbeatToolResponse: attempt.heartbeatToolResponse, - }); - const payloadsWithToolMedia = mergeAttemptToolMediaPayloads({ - payloads, - toolMediaUrls: attempt.toolMediaUrls, - toolAudioAsVoice: attempt.toolAudioAsVoice, - toolTrustedLocalMedia: attempt.toolTrustedLocalMedia, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - }); - const timedOutDuringPrompt = - terminalTimedOut && !timedOutDuringCompaction && !timedOutDuringToolExecution; - const finalAssistantStopReason = (attemptAssistant?.stopReason ?? "") - .trim() - .toLowerCase(); - const recoveredFinalAssistantTextAfterPromptTimeout = - timedOutDuringPrompt && - ["completed", "end_turn", "stop"].includes(finalAssistantStopReason) - ? (finalAssistantVisibleText ?? finalAssistantRawText)?.trim() - : undefined; - const payloadAlreadyContainsRecoveredFinalAssistant = - recoveredFinalAssistantTextAfterPromptTimeout - ? (payloadsWithToolMedia ?? []).some( - (payload) => - payload?.isError !== true && - payload?.isReasoning !== true && - typeof payload.text === "string" && - payload.text.trim() === recoveredFinalAssistantTextAfterPromptTimeout, - ) - : false; - const recoveredFinalAssistantPayloadsAfterPromptTimeout = - recoveredFinalAssistantTextAfterPromptTimeout && - !payloadAlreadyContainsRecoveredFinalAssistant - ? [{ text: recoveredFinalAssistantTextAfterPromptTimeout }] - : undefined; - const hasSuccessfulFinalAssistantAfterPromptTimeout = - timedOutDuringPrompt && - Boolean( - payloadAlreadyContainsRecoveredFinalAssistant || - recoveredFinalAssistantPayloadsAfterPromptTimeout?.length, - ); - const hasPartialAssistantTextAfterPromptTimeout = - timedOutDuringPrompt && - (attempt.assistantTexts ?? []).some((text) => text.trim().length > 0) && - !attempt.clientToolCalls && - !attempt.yieldDetected && - !attempt.didSendViaMessagingTool && - !attempt.didSendDeterministicApprovalPrompt && - !attempt.lastToolError && - (attempt.toolMetas?.length ?? 0) === 0; - const attemptToolSummary = buildTraceToolSummary({ - toolMetas: attempt.toolMetas, - fallbackHadFailure: Boolean(attempt.lastToolError), - }); - const failureSignal = resolveEmbeddedRunFailureSignal({ - trigger: params.trigger, - lastToolError: attempt.lastToolError, - }); - - // Timeout aborts can leave the run without payloads or with only a - // partial assistant fragment. Emit an explicit timeout error instead, - // preserving any tool payloads that succeeded before the timeout. - if ( - timedOutDuringPrompt && - !hasSuccessfulFinalAssistantAfterPromptTimeout && - (shouldSurfaceCodexCompletionTimeout || !hasMessagingToolDeliveryEvidence(attempt)) - ) { - const defaultTimeoutText = idleTimedOut - ? "The model did not produce a response before the model idle timeout. " + - "Please try again, or increase `models.providers..timeoutSeconds` for slow local or self-hosted providers. " + - "If `agents.defaults.timeoutSeconds` or a run-specific timeout is lower, raise that ceiling too; provider timeouts cannot extend the whole agent run." - : "Request timed out before a response was generated. " + - "Please try again, or increase `agents.defaults.timeoutSeconds` in your config."; - const promptTimeoutMessage = attempt.promptTimeoutOutcome?.message?.trim(); - const timeoutText = promptTimeoutMessage || defaultTimeoutText; - const replayInvalid = - attempt.promptTimeoutOutcome?.replayInvalid ?? resolveReplayInvalidForAttempt(null); - const livenessState = - attempt.promptTimeoutOutcome?.livenessState ?? - resolveRunLivenessState({ - payloadCount: hasPartialAssistantTextAfterPromptTimeout ? 0 : payloads.length, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - incompleteTurnText: null, - }); - const timeoutPhase = - attempt.promptTimeoutOutcome?.timeoutPhase ?? terminalOutcome.timeoutPhase; - const providerStarted = - attempt.promptTimeoutOutcome?.providerStarted ?? terminalOutcome.providerStarted; - const timeoutAttribution = { - ...(timeoutPhase ? { timeoutPhase } : {}), - ...(typeof providerStarted === "boolean" ? { providerStarted } : {}), - }; - setTerminalLifecycleMeta({ - replayInvalid, - livenessState, - ...timeoutAttribution, - }); - return { - payloads: [ - ...(hasPartialAssistantTextAfterPromptTimeout ? [] : payloadsWithToolMedia || []), - { - text: timeoutText, - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta, - aborted: terminalAborted, - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - finalAssistantVisibleText, - finalAssistantRawText, - replayInvalid, - livenessState, - ...timeoutAttribution, - // Completion-idle recovery is exhausted here. Keep this terminal so - // model fallback cannot replay a potentially still-active Codex turn. - ...(shouldSurfaceCodexCompletionTimeout - ? { - error: { - kind: "incomplete_turn" as const, - message: timeoutText, - fallbackSafe: false, - }, - } - : {}), - toolSummary: attemptToolSummary, - ...(failureSignal ? { failureSignal } : {}), - agentHarnessResultClassification: attempt.agentHarnessResultClassification, - }, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - didDeliverSourceReplyViaMessageTool: - attempt.didDeliverSourceReplyViaMessageTool === true, - didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, - messagingToolSentTexts: attempt.messagingToolSentTexts, - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls, - messagingToolSentTargets: attempt.messagingToolSentTargets, - messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, - heartbeatToolResponse: attempt.heartbeatToolResponse, - successfulCronAdds: attempt.successfulCronAdds, - acceptedSessionSpawns: attempt.acceptedSessionSpawns, - }; - } - - const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({ - isCronTrigger: params.trigger === "cron", - payloadCount: payloadsWithToolMedia?.length ?? 0, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - }); - const payloadsForTerminalPath = recoveredFinalAssistantPayloadsAfterPromptTimeout - ? recoveredFinalAssistantPayloadsAfterPromptTimeout - : payloadsWithToolMedia?.length - ? payloadsWithToolMedia - : silentToolResultReplyPayload - ? [silentToolResultReplyPayload] - : payloadsWithToolMedia; - const payloadCount = payloadsForTerminalPath?.length ?? 0; - const emptyAssistantReplyIsSilent = shouldTreatEmptyAssistantReplyAsSilent({ - allowEmptyAssistantReplyAsSilent: params.allowEmptyAssistantReplyAsSilent, - payloadCount, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - }); - const nextReasoningOnlyRetryInstruction = emptyAssistantReplyIsSilent - ? null - : resolveReasoningOnlyRetryInstruction({ - provider: activeErrorContext.provider, - modelId: activeErrorContext.model, - modelApi: effectiveModel.api, - executionContract, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - }); - const nextEmptyResponseRetryInstruction = emptyAssistantReplyIsSilent - ? null - : resolveEmptyResponseRetryInstruction({ - provider: activeErrorContext.provider, - modelId: activeErrorContext.model, - modelApi: effectiveModel.api, - executionContract, - payloadCount, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - }); - if ( - nextReasoningOnlyRetryInstruction && - reasoningOnlyRetryAttempts < maxReasoningOnlyRetryAttempts - ) { - reasoningOnlyRetryAttempts += 1; - // The assistant leaf is already durable, so persist a new user boundary. - activateInternalPrompt(nextReasoningOnlyRetryInstruction, false); - log.warn( - `reasoning-only assistant turn detected: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} — retrying ${reasoningOnlyRetryAttempts}/${maxReasoningOnlyRetryAttempts} ` + - `with visible-answer continuation`, - ); - continue; - } - const reasoningOnlyRetriesExhausted = - nextReasoningOnlyRetryInstruction && - reasoningOnlyRetryAttempts >= maxReasoningOnlyRetryAttempts; - if ( - !emptyAssistantReplyIsSilent && - shouldRetryMissingAssistantTurn({ - payloadCount, - aborted: terminalAborted, - promptError, - timedOut: terminalTimedOut, - attempt, - }) && - missingAssistantRetryAttempts < MAX_MISSING_ASSISTANT_RETRIES - ) { - missingAssistantRetryAttempts += 1; - // Same-prompt retries reuse the canonical user leaf only when it was written. - suppressNextUserMessagePersistence = activePrompt.persisted; - log.warn( - `missing assistant terminal message detected: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} — retrying ${missingAssistantRetryAttempts}/${MAX_MISSING_ASSISTANT_RETRIES} with same prompt`, - ); - continue; - } - if ( - !nextReasoningOnlyRetryInstruction && - nextEmptyResponseRetryInstruction && - emptyResponseRetryAttempts < maxEmptyResponseRetryAttempts - ) { - emptyResponseRetryAttempts += 1; - // The assistant leaf is already durable, so persist a new user boundary. - activateInternalPrompt(nextEmptyResponseRetryInstruction, false); - log.warn( - `empty response detected: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} — retrying ${emptyResponseRetryAttempts}/${maxEmptyResponseRetryAttempts} ` + - `with visible-answer continuation`, - ); - continue; - } - const incompleteTurnText = emptyAssistantReplyIsSilent - ? null - : resolveIncompleteTurnPayloadText({ - payloadCount, - aborted: terminalAborted, - externalAbort: externalAbort || signalOwnedInterruption, - timedOut: terminalTimedOut, - attempt, - }); - const incompleteTurnFallbackSafe = Boolean( - incompleteTurnText && - !terminalInterrupted && - !promptError && - !attempt.lastToolError && - !hasAttemptTerminalState(attempt) && - !accumulatedReplayState.hadPotentialSideEffects, - ); - const terminalToolPresentation = incompleteTurnFallbackSafe - ? readAttemptTerminalToolPresentation() - : undefined; - if ( - !emptyAssistantReplyIsSilent && - attemptCompactionCount > 0 && - payloadCount === 0 && - !terminalInterrupted && - !promptError && - !attempt.clientToolCalls && - !attempt.yieldDetected && - !attempt.didSendDeterministicApprovalPrompt && - !attempt.lastToolError && - !accumulatedReplayState.hadPotentialSideEffects && - compactionContinuationRetryAttempts < 1 - ) { - compactionContinuationRetryAttempts += 1; - compactionContinuationRetryInstruction = COMPACTION_CONTINUATION_RETRY_INSTRUCTION; - log.warn( - `compaction interrupted visible final answer: runId=${params.runId} sessionId=${params.sessionId} ` + - `compactions=${attemptCompactionCount} — retrying ${compactionContinuationRetryAttempts}/1 with compacted-transcript continuation`, - ); - postCompactionGuard.armPostCompaction(); - continue; - } - compactionContinuationRetryInstruction = null; - if (reasoningOnlyRetriesExhausted && !finalAssistantVisibleText) { - log.warn( - `reasoning-only retries exhausted: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} attempts=${reasoningOnlyRetryAttempts}/${maxReasoningOnlyRetryAttempts} — surfacing incomplete-turn error`, - ); - } - if (reasoningOnlyRetriesExhausted && !finalAssistantVisibleText) { - const incompletePayloadText = terminalToolPresentation - ? terminalToolPresentation.concat( - "\n\n", - "⚠️ Agent couldn't generate a response. Please try again.", - ) - : "⚠️ Agent couldn't generate a response. Please try again."; - const replayInvalid = resolveReplayInvalidForAttempt(incompletePayloadText); - const livenessState = resolveRunLivenessState({ - payloadCount: 0, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - incompleteTurnText: incompletePayloadText, - }); - setTerminalLifecycleMeta({ - replayInvalid, - livenessState, - }); - if (lastProfileId) { - await maybeMarkAuthProfileFailure({ - profileId: lastProfileId, - reason: assistantProfileFailureReason, - modelId, - }); - } - return { - payloads: [ - { - text: incompletePayloadText, - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta, - aborted: terminalAborted, - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - finalAssistantVisibleText, - finalAssistantRawText, - replayInvalid, - livenessState, - error: { - kind: "incomplete_turn", - message: "Agent couldn't generate a response.", - fallbackSafe: incompleteTurnFallbackSafe, - terminalPresentation: terminalToolPresentation !== undefined, - }, - toolSummary: attemptToolSummary, - ...(failureSignal ? { failureSignal } : {}), - agentHarnessResultClassification: attempt.agentHarnessResultClassification, - }, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - didDeliverSourceReplyViaMessageTool: - attempt.didDeliverSourceReplyViaMessageTool === true, - didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, - messagingToolSentTexts: attempt.messagingToolSentTexts, - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls, - messagingToolSentTargets: attempt.messagingToolSentTargets, - messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, - heartbeatToolResponse: attempt.heartbeatToolResponse, - successfulCronAdds: attempt.successfulCronAdds, - acceptedSessionSpawns: attempt.acceptedSessionSpawns, - }; - } - if ( - !nextReasoningOnlyRetryInstruction && - nextEmptyResponseRetryInstruction && - emptyResponseRetryAttempts >= maxEmptyResponseRetryAttempts - ) { - log.warn( - `empty response retries exhausted: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} attempts=${emptyResponseRetryAttempts}/${maxEmptyResponseRetryAttempts} — surfacing incomplete-turn error`, - ); - } - if (incompleteTurnText) { - const replayInvalid = resolveReplayInvalidForAttempt(incompleteTurnText); - const livenessState = resolveRunLivenessState({ - payloadCount, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - incompleteTurnText, - }); - setTerminalLifecycleMeta({ - replayInvalid, - livenessState, - }); - const incompleteStopReason = - attempt.currentAttemptAssistant?.stopReason ?? attempt.lastAssistant?.stopReason; - const replayMetadata = resolveAttemptReplayMetadata(attempt); - log.warn( - `incomplete turn detected: runId=${params.runId} sessionId=${params.sessionId} ` + - `provider=${activeErrorContext.provider}/${activeErrorContext.model} ` + - `stopReason=${incompleteStopReason ?? "missing"} hasLastAssistant=${attempt.lastAssistant ? "yes" : "no"} ` + - `hasCurrentAttemptAssistant=${attempt.currentAttemptAssistant ? "yes" : "no"} payloads=${payloadCount} ` + - `tools=${attempt.toolMetas?.length ?? 0} replaySafe=${replayMetadata.replaySafe ? "yes" : "no"} ` + - `compactions=${attemptCompactionCount} reasoningRetries=${reasoningOnlyRetryAttempts}/${maxReasoningOnlyRetryAttempts} ` + - `emptyRetries=${emptyResponseRetryAttempts}/${maxEmptyResponseRetryAttempts} ` + - `missingAssistantRetries=${missingAssistantRetryAttempts}/${MAX_MISSING_ASSISTANT_RETRIES} — ` + - (terminalToolPresentation - ? "surfacing tool-authored terminal presentation" - : "surfacing error to user"), - ); - - // Mark the failing profile for cooldown so multi-profile setups - // rotate away from the exhausted credential on the next turn. - if (lastProfileId) { - await maybeMarkAuthProfileFailure({ - profileId: lastProfileId, - reason: assistantProfileFailureReason, - modelId, - }); - } - - return { - payloads: [ - { - text: terminalToolPresentation - ? terminalToolPresentation.concat("\n\n", incompleteTurnText) - : incompleteTurnText, - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta, - aborted: terminalAborted, - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - finalAssistantVisibleText, - finalAssistantRawText, - replayInvalid, - livenessState, - error: { - kind: "incomplete_turn", - message: "Agent couldn't generate a response.", - fallbackSafe: incompleteTurnFallbackSafe, - terminalPresentation: terminalToolPresentation !== undefined, - }, - toolSummary: attemptToolSummary, - ...(failureSignal ? { failureSignal } : {}), - agentHarnessResultClassification: attempt.agentHarnessResultClassification, - }, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - didDeliverSourceReplyViaMessageTool: - attempt.didDeliverSourceReplyViaMessageTool === true, - didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, - messagingToolSentTexts: attempt.messagingToolSentTexts, - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls, - messagingToolSentTargets: attempt.messagingToolSentTargets, - messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, - heartbeatToolResponse: attempt.heartbeatToolResponse, - successfulCronAdds: attempt.successfulCronAdds, - acceptedSessionSpawns: attempt.acceptedSessionSpawns, - }; - } - - const beforeAgentFinalizeRevisionReason = attempt.beforeAgentFinalizeRevisionReason; - const shouldHonorBeforeAgentFinalizeRevision = - !terminalInterrupted && - !promptError && - !attempt.clientToolCalls && - !attempt.yieldDetected && - !emptyAssistantReplyIsSilent; - if (beforeAgentFinalizeRevisionReason && shouldHonorBeforeAgentFinalizeRevision) { - beforeAgentFinalizeRevisionAttempts += 1; - activateInternalPrompt( - buildBeforeAgentFinalizeRetryPrompt(beforeAgentFinalizeRevisionReason), - true, - ); - compactionContinuationRetryInstruction = null; - log.warn( - `before_agent_finalize requested one more pass: ` + - `runId=${params.runId} sessionId=${params.sessionId} ` + - `attempt=${beforeAgentFinalizeRevisionAttempts}/${MAX_BEFORE_AGENT_FINALIZE_REVISIONS}`, - ); - continue; - } - - log.debug( - `embedded run done: runId=${params.runId} sessionId=${params.sessionId} durationMs=${Date.now() - started} aborted=${aborted}`, - ); - markAuthProfileSuccessAfterRun(); - const successfulProfileId = lastProfileId; - const successfulCredential = successfulProfileId - ? attemptAuthProfileStore.profiles[successfulProfileId] - : undefined; - const successfulApiKeyInfo = getApiKeyInfo(); - const successfulPluginHarnessApiKeyInfo = (() => { - const apiKey = successfulApiKeyInfo?.apiKey; - if (!pluginHarnessOwnsTransport || !apiKey || !looksLikeSecretSentinel(apiKey)) { - return successfulApiKeyInfo; - } - const resolvedApiKey = resolveSecretSentinel(apiKey); - return resolvedApiKey ? { ...successfulApiKeyInfo, apiKey: resolvedApiKey } : null; - })(); - const authFingerprint = - successfulCredential?.type === "oauth" && successfulProfileId - ? fingerprintResolvedAuthProfileCredential({ - profileId: successfulProfileId, - credential: successfulCredential, - resolvedAuth: successfulApiKeyInfo, - }) - : successfulCredential && successfulProfileId && pluginHarnessOwnsAuthBootstrap - ? attempt.authBindingFingerprint - : successfulCredential && successfulProfileId && pluginHarnessOwnsTransport - ? fingerprintResolvedAuthProfileCredential({ - profileId: successfulProfileId, - credential: successfulCredential, - resolvedAuth: successfulPluginHarnessApiKeyInfo, - }) - : successfulApiKeyInfo - ? fingerprintResolvedProviderAuth(successfulApiKeyInfo) - : undefined; - const authProfileOwnerFingerprint = - successfulProfileId && successfulCredential !== undefined - ? fingerprintAuthProfileOwnerShape({ - profileId: successfulProfileId, - credential: successfulCredential, - }) - : undefined; - const runtimeArtifact = pluginHarnessOwnsTransport ? attempt.runtimeArtifact : undefined; - const runtimeOwnerFingerprint = authFingerprint - ? undefined - : successfulApiKeyInfo?.mode === "aws-sdk" - ? fingerprintAwsSdkRuntimeOwner({ - provider, - backendId: agentHarness.id, - auth: successfulApiKeyInfo, - }) - : pluginHarnessOwnsTransport - ? fingerprintOpaqueRuntimeOwner({ - kind: "plugin-harness", - runner: "embedded", - provider, - backendId: agentHarness.id, - ...(runtimeArtifact - ? { runtimeArtifactFingerprint: runtimeArtifact.fingerprint } - : {}), - ...(successfulProfileId ? { authProfileId: successfulProfileId } : {}), - ...(authProfileOwnerFingerprint ? { authProfileOwnerFingerprint } : {}), - }) - : undefined; - const opaqueRuntimeOwnerKind = - runtimeOwnerFingerprint && successfulApiKeyInfo?.mode === "aws-sdk" - ? ("aws-sdk" as const) - : runtimeOwnerFingerprint && pluginHarnessOwnsTransport - ? ("plugin-harness" as const) - : undefined; - const runtimeOwnerKind = - opaqueRuntimeOwnerKind ?? - (pluginHarnessOwnsTransport ? ("plugin-harness" as const) : undefined); - params.onSuccessfulAuthBinding?.({ - ...(successfulProfileId ? { authProfileId: successfulProfileId } : {}), - agentHarnessId: agentHarness.id, - ...(authFingerprint ? { authFingerprint } : {}), - ...(runtimeOwnerFingerprint ? { runtimeOwnerFingerprint } : {}), - ...(runtimeOwnerKind ? { runtimeOwnerKind } : {}), - ...(runtimeOwnerKind ? { runtimeOwnerId: agentHarness.id } : {}), - ...(runtimeArtifact - ? { - runtimeArtifactId: runtimeArtifact.id, - runtimeArtifactFingerprint: runtimeArtifact.fingerprint, - } - : {}), - }); - const replayInvalid = resolveReplayInvalidForAttempt(null); - const livenessState = attempt.yieldDetected - ? "paused" - : resolveRunLivenessState({ - payloadCount, - aborted: terminalAborted, - timedOut: terminalTimedOut, - attempt, - incompleteTurnText: null, - }); - const stopReason = attempt.clientToolCalls - ? "tool_calls" - : attempt.yieldDetected - ? "end_turn" - : (attemptAssistant?.stopReason as string | undefined); - const terminalPayloads = emptyAssistantReplyIsSilent - ? [{ text: SILENT_REPLY_TOKEN }] - : payloadsForTerminalPath; - setTerminalLifecycleMeta({ - replayInvalid, - livenessState, - stopReason, - yielded: attempt.yieldDetected === true, - }); - return { - payloads: terminalPayloads?.length ? terminalPayloads : undefined, - ...(attempt.diagnosticTrace - ? { diagnosticTrace: freezeDiagnosticTraceContext(attempt.diagnosticTrace) } - : {}), - meta: { - durationMs: Date.now() - started, - agentMeta, - aborted: terminalAborted, - systemPromptReport: attempt.systemPromptReport, - finalPromptText: attempt.finalPromptText, - finalAssistantVisibleText, - finalAssistantRawText, - replayInvalid, - livenessState, - agentHarnessResultClassification: attempt.agentHarnessResultClassification, - ...(attempt.yieldDetected ? { yielded: true } : {}), - ...(emptyAssistantReplyIsSilent - ? { terminalReplyKind: "silent-empty" as const } - : {}), - // Handle client tool calls (OpenResponses hosted tools) - // Propagate the LLM stop reason so callers (lifecycle events, - // ACP bridge) can distinguish end_turn from max_tokens. - stopReason, - pendingToolCalls: attempt.clientToolCalls?.map((call) => ({ - id: randomBytes(5).toString("hex").slice(0, 9), - name: call.name, - arguments: JSON.stringify(call.params), - })), - executionTrace: { - winnerProvider: reportedModelRef.provider, - winnerModel: reportedModelRef.model, - attempts: - traceAttempts.length > 0 || attemptAssistant?.provider || attemptAssistant?.model - ? [ - ...traceAttempts, - { - provider: reportedModelRef.provider, - model: reportedModelRef.model, - result: "success", - stage: "assistant", - }, - ] - : undefined, - fallbackUsed: traceAttempts.some(traceAttemptUsesFallback), - runner: "embedded", - }, - requestShaping: { - ...(lastProfileId ? { authMode: "auth-profile" } : {}), - ...(thinkLevel ? { thinking: thinkLevel } : {}), - ...(params.reasoningLevel ? { reasoning: params.reasoningLevel } : {}), - ...(params.verboseLevel ? { verbose: params.verboseLevel } : {}), - ...(params.blockReplyBreak ? { blockStreaming: params.blockReplyBreak } : {}), - }, - toolSummary: attemptToolSummary, - ...(failureSignal ? { failureSignal } : {}), - completion: { - ...(stopReason ? { stopReason } : {}), - ...(stopReason ? { finishReason: stopReason } : {}), - ...(stopReason?.toLowerCase().includes("refusal") ? { refusal: true } : {}), - }, - contextManagement: - autoCompactionCount > 0 ? { lastTurnCompactions: autoCompactionCount } : undefined, - }, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - didDeliverSourceReplyViaMessageTool: - attempt.didDeliverSourceReplyViaMessageTool === true, - didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, - messagingToolSentTexts: attempt.messagingToolSentTexts, - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls, - messagingToolSentTargets: attempt.messagingToolSentTargets, - messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, - heartbeatToolResponse: attempt.heartbeatToolResponse, - successfulCronAdds: attempt.successfulCronAdds, - acceptedSessionSpawns: attempt.acceptedSessionSpawns, - }; - } - } finally { - if (params.isFinalFallbackAttempt !== false) { - await maybeEmitFastModeAutoResetBestEffort(); - } - forgetPromptBuildDrainCacheForRun(params.runId); - stopRuntimeAuthRefreshTimer(); - await runAgentCleanupStep({ - runId: params.runId, - sessionId: params.sessionId, - step: "context-engine-dispose", - log, - cleanup: async () => { - await contextEngine.dispose?.(); - }, - }); - if (params.cleanupBundleMcpOnRunEnd === true) { - await runAgentCleanupStep({ - runId: params.runId, - sessionId: params.sessionId, - step: "bundle-mcp-retire", - log, - cleanup: async () => { - const onError = (errorLocal: unknown, sessionId: string) => { - log.warn( - `bundle-mcp cleanup failed after run for ${sessionId}: ${formatErrorMessage(errorLocal)}`, - ); - }; - const retiredBySessionKey = await retireSessionMcpRuntimeForSessionKey({ - sessionKey: params.sessionKey, - reason: "embedded-run-end", - // MCP App views hold bounded leases so their bridge can remain - // usable after a one-shot gateway run returns. - preserveActiveLeases: true, - onError, - }); - if (!retiredBySessionKey) { - await retireSessionMcpRuntime({ - sessionId: params.sessionId, - reason: "embedded-run-end", - preserveActiveLeases: true, - onError, - }); - } - }, - }); - } - } - }); - }); -} - -function resolveAuthProfileStateProvider( - store: AuthProfileStore, - profileId: string, - fallbackProvider: string, -): string { - const profileProvider = store.profiles?.[profileId]?.provider?.trim(); - if (profileProvider) { - return profileProvider; - } - const idProvider = profileId.split(":", 1)[0]?.trim(); - return idProvider || fallbackProvider; -} -export const testing = { - EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, - resolveEmbeddedRunLaneTimeoutMs, -}; -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ +export { runEmbeddedAgent, testing } from "./run-orchestrator.js"; diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts index d4b088bc07df..0bbc5e279c29 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.test.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.test.ts @@ -166,7 +166,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -197,7 +197,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -227,7 +227,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -257,7 +257,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -318,7 +318,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -351,7 +351,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); @@ -476,7 +476,7 @@ describe("handleAssistantFailover", () => { if (outcome.action !== "retry") { return; } - expect(outcome.retryKind).toBeUndefined(); + expect(outcome.retryKind).toBe("profile_rotation"); expect(maybeRetrySameModelRateLimit).toHaveBeenCalledTimes(1); expect(maybeEscalateRateLimitProfileFallback).toHaveBeenCalledTimes(1); expect(advanceAuthProfile).toHaveBeenCalledTimes(1); diff --git a/src/agents/embedded-agent-runner/run/assistant-failover.ts b/src/agents/embedded-agent-runner/run/assistant-failover.ts index 3a2a8c28bfb2..e16424ad31a3 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failover.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failover.ts @@ -28,7 +28,7 @@ type AssistantFailoverOutcome = action: "retry"; overloadProfileRotations: number; lastRetryFailoverReason: FailoverReason | null; - retryKind?: "same_model_idle_timeout" | "same_model_rate_limit"; + retryKind: "profile_rotation" | "same_model_idle_timeout" | "same_model_rate_limit"; } | { action: "throw"; @@ -227,6 +227,7 @@ export async function handleAssistantFailover(params: { return { action: "retry", overloadProfileRotations, + retryKind: "profile_rotation", lastRetryFailoverReason: mergeRetryFailoverReason({ previous: params.previousRetryFailoverReason, failoverReason: params.failoverReason, diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.ts b/src/agents/embedded-agent-runner/run/assistant-failure.ts new file mode 100644 index 000000000000..d6cc0bce682c --- /dev/null +++ b/src/agents/embedded-agent-runner/run/assistant-failure.ts @@ -0,0 +1,373 @@ +import type { ThinkLevel } from "../../../auto-reply/thinking.js"; +import type { AssistantMessage } from "../../../llm/types.js"; +import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js"; +import { + classifyAssistantFailoverReason, + type FailoverReason, + isAuthAssistantError, + isBillingAssistantError, + isFailoverAssistantError, + isGenericUnknownStreamErrorMessage, + isRateLimitAssistantError, + parseImageDimensionError, + pickFallbackThinkingLevel, +} from "../../embedded-agent-helpers.js"; +import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js"; +import { + resolveSessionSuspensionReason, + type SessionSuspensionParams, +} from "../../session-suspension.js"; +import { log } from "../logger.js"; +import type { TraceAttempt } from "../types.js"; +import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js"; +import { createFailoverDecisionLogger } from "./failover-observation.js"; +import { resolveRunFailoverDecision } from "./failover-policy.js"; +import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +const MAX_EMPTY_ERROR_RETRIES = 3; +const MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES = 1; + +export type EmbeddedRunAssistantFailureOutcome = { + action: "retry" | "proceed"; + thinkLevel: ThinkLevel; + authRetryPending: boolean; + emptyErrorRetries: number; + overloadProfileRotations: number; + sameModelIdleTimeoutRetries: number; + lastRetryFailoverReason: FailoverReason | null; + preserveSameModelRateLimitRetryCount: boolean; + assistantProfileFailureReason: AuthProfileFailureReason | null; +}; + +export async function handleEmbeddedAssistantFailure(input: { + runParams: RunEmbeddedAgentParams; + attempt: EmbeddedRunAttemptResult; + attemptAssistant?: AssistantMessage; + currentAttemptAssistant?: AssistantMessage; + terminalProviderStarted: boolean; + terminalInterrupted: boolean; + promptError: unknown; + activeErrorContext: { provider: string; model: string }; + provider: string; + modelId: string; + model: string; + thinkLevel: ThinkLevel; + // Profile rotation resets thinking inside the runtime; read it after advancing. + getThinkLevel: () => ThinkLevel; + attemptedThinking: Set; + timedOut: boolean; + idleTimedOut: boolean; + timedOutDuringCompaction: boolean; + timedOutDuringToolExecution: boolean; + timedOutByRunBudget: boolean; + signalOwnedInterruption: boolean; + externalAbort: boolean; + aborted: boolean; + fallbackConfigured: boolean; + pluginHarnessOwnsTransport: boolean; + canRestartForLiveSwitch: boolean; + authProfileId?: string; + authProfileStore: AuthProfileStore; + runtimeAuthRetry: boolean; + maybeRefreshRuntimeAuthForAuthError: (errorText: string, retry: boolean) => Promise; + resolveAuthProfileFailureReason: ( + reason: FailoverReason | null, + options?: { providerStarted?: boolean; transientRateLimit?: boolean }, + ) => AuthProfileFailureReason | null; + emptyErrorRetries: number; + overloadProfileRotations: number; + overloadProfileRotationLimit: number; + rateLimitProfileRotations: number; + rateLimitProfileRotationLimit: number; + sameModelIdleTimeoutRetries: number; + previousRetryFailoverReason: FailoverReason | null; + maybeMarkAuthProfileFailure: (failure: { + profileId?: string; + reason?: AuthProfileFailureReason | null; + modelId?: string; + }) => Promise; + maybeEscalateRateLimitProfileFallback: Parameters< + typeof handleAssistantFailover + >[0]["maybeEscalateRateLimitProfileFallback"]; + maybeRetrySameModelRateLimit: (retry?: { retryAfterSeconds?: number }) => Promise; + maybeBackoffBeforeOverloadFailover: (reason: FailoverReason | null) => Promise; + advanceAttemptAuthProfile: () => Promise; + traceAttempts: TraceAttempt[]; + suspendForFailure: (params: Omit) => void; + suspensionSessionId: string; + agentDir: string; + isProbeSession: boolean; +}): Promise { + const fallbackThinking = pickFallbackThinkingLevel({ + message: input.attemptAssistant?.errorMessage, + attempted: input.attemptedThinking, + }); + if (fallbackThinking && !input.terminalInterrupted) { + log.warn( + `unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`, + ); + return buildOutcome(input, { + action: "retry", + thinkLevel: fallbackThinking, + preserveSameModelRateLimitRetryCount: true, + assistantProfileFailureReason: null, + }); + } + + const authFailure = isAuthAssistantError(input.attemptAssistant); + const rateLimitFailure = isRateLimitAssistantError(input.attemptAssistant); + const billingFailure = isBillingAssistantError(input.attemptAssistant); + const failoverFailure = isFailoverAssistantError(input.attemptAssistant); + const assistantFailoverReason = classifyAssistantFailoverReason(input.attemptAssistant); + const assistantProviderStarted = + Boolean(input.currentAttemptAssistant?.provider) || input.terminalProviderStarted; + const assistantProfileFailoverReason = + assistantFailoverReason ?? + (assistantProviderStarted && (input.timedOut || input.idleTimedOut) ? "timeout" : null); + const assistantProfileFailureReason = input.resolveAuthProfileFailureReason( + assistantProfileFailoverReason, + { + providerStarted: assistantProviderStarted, + transientRateLimit: + assistantProfileFailoverReason === "rate_limit" && + isShortWindowRateLimitMessage(input.attemptAssistant?.errorMessage), + }, + ); + const cloudCodeAssistFormatError = input.attempt.cloudCodeAssistFormatError; + const imageDimensionError = parseImageDimensionError(input.attemptAssistant?.errorMessage ?? ""); + const genericUnknownReasoningError = + assistantFailoverReason === "timeout" && + isGenericUnknownStreamErrorMessage(input.attemptAssistant?.errorMessage ?? "") && + Boolean(input.attemptAssistant && hasOnlyAssistantReasoningContent(input.attemptAssistant)); + const silentErrorRetryReason = + assistantFailoverReason === null || + genericUnknownReasoningError || + assistantFailoverReason === "no_error_details" || + assistantFailoverReason === "unclassified" || + assistantFailoverReason === "unknown" || + assistantFailoverReason === "server_error"; + if ( + !authFailure && + !rateLimitFailure && + !billingFailure && + !cloudCodeAssistFormatError && + !imageDimensionError && + !input.terminalInterrupted && + !input.promptError && + silentErrorRetryReason && + shouldRetrySilentErrorAssistantTurn({ + attempt: input.attempt, + assistant: input.attemptAssistant, + }) && + input.emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES + ) { + const emptyErrorRetries = input.emptyErrorRetries + 1; + log.warn( + `[empty-error-retry] stopReason=error non-visible-output; resubmitting ` + + `attempt=${emptyErrorRetries}/${MAX_EMPTY_ERROR_RETRIES} ` + + `provider=${input.attemptAssistant?.provider ?? input.provider} ` + + `model=${input.attemptAssistant?.model ?? input.model} ` + + `sessionKey=${input.runParams.sessionKey ?? input.runParams.sessionId}`, + ); + return buildOutcome(input, { + action: "retry", + emptyErrorRetries, + preserveSameModelRateLimitRetryCount: true, + assistantProfileFailureReason, + }); + } + + const failedProfileId = input.authProfileId; + const logFailoverDecision = createFailoverDecisionLogger({ + stage: "assistant", + runId: input.runParams.runId, + rawError: input.attemptAssistant?.errorMessage?.trim(), + failoverReason: assistantFailoverReason, + profileFailureReason: assistantProfileFailureReason, + provider: input.activeErrorContext.provider, + model: input.activeErrorContext.model, + sourceProvider: input.attemptAssistant?.provider ?? input.provider, + sourceModel: input.attemptAssistant?.model ?? input.modelId, + profileId: failedProfileId, + fallbackConfigured: input.fallbackConfigured, + timedOut: input.timedOut, + aborted: input.aborted, + }); + if ( + !input.signalOwnedInterruption && + authFailure && + (await input.maybeRefreshRuntimeAuthForAuthError( + input.attemptAssistant?.errorMessage ?? "", + input.runtimeAuthRetry, + )) + ) { + return buildOutcome(input, { + action: "retry", + authRetryPending: true, + preserveSameModelRateLimitRetryCount: true, + assistantProfileFailureReason, + }); + } + if (imageDimensionError && input.authProfileId) { + const details = [ + imageDimensionError.messageIndex !== undefined + ? `message=${imageDimensionError.messageIndex}` + : null, + imageDimensionError.contentIndex !== undefined + ? `content=${imageDimensionError.contentIndex}` + : null, + imageDimensionError.maxDimensionPx !== undefined + ? `limit=${imageDimensionError.maxDimensionPx}px` + : null, + ] + .filter(Boolean) + .join(" "); + log.warn( + `Profile ${input.authProfileId} rejected image payload${details ? ` (${details})` : ""}.`, + ); + } + + const initialDecision = resolveRunFailoverDecision({ + stage: "assistant", + allowFormatRetry: cloudCodeAssistFormatError, + aborted: input.aborted, + externalAbort: input.externalAbort || input.signalOwnedInterruption, + fallbackConfigured: input.fallbackConfigured, + failoverFailure, + failoverReason: assistantFailoverReason, + timedOut: input.timedOut, + idleTimedOut: input.idleTimedOut, + timedOutDuringCompaction: input.timedOutDuringCompaction, + timedOutDuringToolExecution: input.timedOutDuringToolExecution, + harnessOwnsTransport: input.pluginHarnessOwnsTransport, + timedOutByRunBudget: input.timedOutByRunBudget, + profileRotated: false, + }); + const outcome = await handleAssistantFailover({ + initialDecision, + aborted: input.aborted, + externalAbort: input.externalAbort || input.signalOwnedInterruption, + fallbackConfigured: input.fallbackConfigured, + failoverFailure, + failoverReason: assistantFailoverReason, + timedOut: input.timedOut, + idleTimedOut: input.idleTimedOut, + timedOutDuringCompaction: input.timedOutDuringCompaction, + timedOutDuringToolExecution: input.timedOutDuringToolExecution, + timedOutByRunBudget: input.timedOutByRunBudget, + allowSameModelIdleTimeoutRetry: + input.timedOut && + input.idleTimedOut && + !input.timedOutDuringCompaction && + !input.fallbackConfigured && + input.canRestartForLiveSwitch && + input.sameModelIdleTimeoutRetries < MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES, + allowSameModelRateLimitRetry: + input.rateLimitProfileRotations < input.rateLimitProfileRotationLimit, + assistantProfileFailureReason, + lastProfileId: input.authProfileId, + modelId: input.modelId, + provider: input.provider, + activeErrorContext: input.activeErrorContext, + lastAssistant: input.attemptAssistant, + config: input.runParams.config, + sessionKey: input.runParams.sessionKey ?? input.runParams.sessionId, + authFailure, + rateLimitFailure, + billingFailure, + authMode: input.authProfileId + ? input.authProfileStore.profiles?.[input.authProfileId]?.type + : undefined, + cloudCodeAssistFormatError, + isProbeSession: input.isProbeSession, + overloadProfileRotations: input.overloadProfileRotations, + overloadProfileRotationLimit: input.overloadProfileRotationLimit, + previousRetryFailoverReason: input.previousRetryFailoverReason, + logAssistantFailoverDecision: logFailoverDecision, + warn: (message) => log.warn(message), + maybeMarkAuthProfileFailure: input.maybeMarkAuthProfileFailure, + maybeEscalateRateLimitProfileFallback: input.maybeEscalateRateLimitProfileFallback, + maybeRetrySameModelRateLimit: input.maybeRetrySameModelRateLimit, + maybeBackoffBeforeOverloadFailover: input.maybeBackoffBeforeOverloadFailover, + advanceAuthProfile: input.advanceAttemptAuthProfile, + }); + if (outcome.action === "retry") { + const retryTraceResult = + outcome.retryKind === "same_model_rate_limit" + ? "same_model_rate_limit" + : outcome.retryKind === "same_model_idle_timeout" || assistantFailoverReason === "timeout" + ? "timeout" + : "rotate_profile"; + input.traceAttempts.push({ + provider: input.activeErrorContext.provider, + model: input.activeErrorContext.model, + result: retryTraceResult, + ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), + stage: "assistant", + }); + return buildOutcome(input, { + action: "retry", + thinkLevel: + outcome.retryKind === "profile_rotation" ? input.getThinkLevel() : input.thinkLevel, + overloadProfileRotations: outcome.overloadProfileRotations, + sameModelIdleTimeoutRetries: + input.sameModelIdleTimeoutRetries + + (outcome.retryKind === "same_model_idle_timeout" ? 1 : 0), + lastRetryFailoverReason: outcome.lastRetryFailoverReason, + preserveSameModelRateLimitRetryCount: outcome.retryKind === "same_model_rate_limit", + assistantProfileFailureReason, + }); + } + if (outcome.action === "throw") { + input.traceAttempts.push({ + provider: input.activeErrorContext.provider, + model: input.activeErrorContext.model, + result: + assistantFailoverReason === "timeout" + ? "timeout" + : initialDecision.action === "fallback_model" + ? "fallback_model" + : "error", + ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), + stage: "assistant", + ...(typeof outcome.error.status === "number" ? { status: outcome.error.status } : {}), + }); + if (outcome.error.suspend) { + input.suspendForFailure({ + cfg: input.runParams.config, + agentDir: input.agentDir, + sessionId: input.suspensionSessionId, + reason: resolveSessionSuspensionReason(outcome.error.reason), + failedProvider: outcome.error.provider ?? input.provider, + failedModel: outcome.error.model ?? input.modelId, + }); + } + throw outcome.error; + } + return buildOutcome(input, { + action: "proceed", + overloadProfileRotations: outcome.overloadProfileRotations, + assistantProfileFailureReason, + }); +} + +function buildOutcome( + input: Parameters[0], + override: Partial & + Pick, +): EmbeddedRunAssistantFailureOutcome { + return { + action: override.action, + thinkLevel: override.thinkLevel ?? input.thinkLevel, + authRetryPending: override.authRetryPending ?? false, + emptyErrorRetries: override.emptyErrorRetries ?? input.emptyErrorRetries, + overloadProfileRotations: override.overloadProfileRotations ?? input.overloadProfileRotations, + sameModelIdleTimeoutRetries: + override.sameModelIdleTimeoutRetries ?? input.sameModelIdleTimeoutRetries, + lastRetryFailoverReason: override.lastRetryFailoverReason ?? input.previousRetryFailoverReason, + preserveSameModelRateLimitRetryCount: override.preserveSameModelRateLimitRetryCount ?? false, + assistantProfileFailureReason: override.assistantProfileFailureReason, + }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts b/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts new file mode 100644 index 000000000000..b4dd81e687a8 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts @@ -0,0 +1,241 @@ +import fs from "node:fs/promises"; +import { resolveStorePath } from "../../../config/sessions.js"; +import { resolveSessionTranscriptRuntimeReadTarget } from "../../../config/sessions/session-accessor.js"; +import type { resolveContextEngine } from "../../../context-engine/registry.js"; +import { createTrajectoryRuntimeRecorder } from "../../../trajectory/runtime.js"; +import { agentHarnessBuildsOpenClawTools } from "../../harness/selection.js"; +import { buildAgentRuntimePlan } from "../../runtime-plan/build.js"; +import { createEmbeddedRunReplayState } from "../replay-state.js"; +import { mapThinkingLevelForProvider } from "../utils.js"; +import { EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE } from "./attempt-stage-timing.js"; +import { resolveAttemptDispatchApiKey } from "./auth-store.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import { resolveEmbeddedAttemptBasePrompt } from "./helpers.js"; +import { dispatchEmbeddedRunAttempt } from "./run-attempt-dispatch.js"; +import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; +import { CODEX_HARNESS_ID, resolveAttemptTrajectoryAttribution } from "./runtime-resolution.js"; +import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js"; +import type { createEmbeddedRunTerminalRetryState } from "./terminal-retry-state.js"; +import { MAX_BEFORE_AGENT_FINALIZE_REVISIONS } from "./terminal-retry-state.js"; + +type PreparedRuntime = Awaited>; +type ContextEngine = Awaited>; +type SessionPromptState = ReturnType; +type TerminalRetryState = ReturnType; + +export async function prepareAndDispatchEmbeddedRunAttempt(input: { + runInput: PreparedEmbeddedRunInput; + preparedRuntime: PreparedRuntime; + contextEngine: ContextEngine; + sessionPromptState: SessionPromptState; + terminalRetryState: TerminalRetryState; + replayState: ReturnType; + provider: string; + modelId: string; + startupStagesEmitted: boolean; + bootstrapPromptWarningSignaturesSeen: string[]; + resolveRuntimeFallbackReason: () => string | null; + observeToolOutcome: Parameters[0]["control"]["onToolOutcome"]; + allocateToolOutcomeOrdinal: Parameters< + typeof dispatchEmbeddedRunAttempt + >[0]["control"]["allocateToolOutcomeOrdinal"]; + getPostCompactionAbortError: () => Error | undefined; + setPostCompactionAbortController: (controller: AbortController | undefined) => void; + clearPostCompactionAbortController: (controller: AbortController) => void; +}) { + const { + runInput, + preparedRuntime, + contextEngine, + sessionPromptState, + terminalRetryState, + provider, + modelId, + } = input; + const params = runInput.runParams; + const { + workspaceResolution, + workspaceDir, + isCanonicalWorkspace, + agentDir, + resolvedSessionKey, + resolvedToolResultFormat, + startupStages, + emitStartupStageSummary, + lifecycleGeneration, + } = runInput; + const { + fastModeAutoOnSeconds, + fastModeAutoProgressState, + fastModeStartedAtMs, + maybeAnnounceFastModeAutoOff, + notifyAgentEvent, + notifyExecutionPhase, + notifyRunProgress, + notifyToolResult, + resolveAttemptFastModeParam, + } = runInput.progressController; + const { laneTaskAbortController, laneTaskReleaseController, noteLaneTaskProgress } = + runInput.laneController; + const { + requestedModelId, + beforeAgentStartResult, + expectedHarnessArtifact, + nativeModelOwned, + authStorage, + modelRegistry, + attemptAuthProfileStore, + lockedProfileId, + resolveRunAttemptAuthProfileStore, + } = preparedRuntime; + const runtime = preparedRuntime.snapshot(); + + await fs.mkdir(workspaceDir, { recursive: true }); + if (!input.startupStagesEmitted) { + startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.workspace); + } + const basePrompt = + sessionPromptState.activePrompt.override ?? + resolveEmbeddedAttemptBasePrompt({ nativeModelOwned, provider, prompt: params.prompt }); + const prompt = terminalRetryState.compactionContinuationInstruction + ? `${basePrompt}\n\n${terminalRetryState.compactionContinuationInstruction}` + : basePrompt; + const resolvedStreamApiKey = resolveAttemptDispatchApiKey({ + apiKeyInfo: runtime.apiKeyInfo, + runtimeAuthState: runtime.runtimeAuthState, + }); + const attemptFastMode = resolveAttemptFastModeParam(); + const trajectorySessionFile = resolvedSessionKey + ? ( + await resolveSessionTranscriptRuntimeReadTarget({ + agentId: workspaceResolution.agentId, + sessionId: sessionPromptState.sessionId, + sessionKey: resolvedSessionKey, + storePath: resolveStorePath(params.config?.session?.store, { + agentId: workspaceResolution.agentId, + }), + }) + ).sessionFile + : sessionPromptState.sessionFile; + if (!input.startupStagesEmitted) { + startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.prompt); + } + const runtimePlan = buildAgentRuntimePlan({ + provider, + modelId, + model: runtime.effectiveModel, + modelApi: runtime.effectiveModel.api, + harnessId: runtime.agentHarness.id, + harnessRuntime: runtime.agentHarness.id, + preparedAuthPlan: runtime.activePreparedAuthPlan, + config: params.config, + workspaceDir, + agentDir, + agentId: workspaceResolution.agentId, + thinkingLevel: mapThinkingLevelForProvider(runtime.thinkLevel), + extraParamsOverride: { ...params.streamParams, fastMode: attemptFastMode }, + }); + const trajectoryAttribution = resolveAttemptTrajectoryAttribution({ + model: runtime.effectiveModel, + modelId, + provider, + runtimePlan, + }); + const trajectoryRecorder = + runtime.agentHarness.id === CODEX_HARNESS_ID && !params.disableTrajectory + ? createTrajectoryRuntimeRecorder({ + cfg: params.config, + env: process.env, + runId: params.runId, + sessionId: sessionPromptState.sessionId, + sessionKey: resolvedSessionKey, + sessionFile: trajectorySessionFile, + provider: trajectoryAttribution.provider, + modelId: trajectoryAttribution.modelId, + modelApi: trajectoryAttribution.modelApi, + workspaceDir, + }) + : undefined; + let startupStagesEmitted = input.startupStagesEmitted; + if (!startupStagesEmitted) { + startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.runtimePlan); + startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.dispatch); + notifyExecutionPhase("attempt_dispatch", { provider, model: modelId }); + emitStartupStageSummary(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.dispatch); + startupStagesEmitted = true; + } + const dispatchedAttempt = await dispatchEmbeddedRunAttempt({ + params, + runtime: { + sessionId: sessionPromptState.sessionId, + sessionFile: sessionPromptState.sessionFile, + sessionTarget: sessionPromptState.sessionTarget, + sessionKey: resolvedSessionKey, + trajectorySessionFile, + trajectoryRecorder: trajectoryRecorder ?? undefined, + workspaceDir, + isCanonicalWorkspace, + agentDir, + contextEngine: nativeModelOwned ? undefined : contextEngine, + contextTokenBudget: runtime.contextTokenBudget, + contextWindowInfo: runtime.contextWindowInfo, + prompt, + provider, + modelId, + requestedModelId, + fallbackActive: modelId !== requestedModelId || Boolean(input.resolveRuntimeFallbackReason()), + fallbackReason: input.resolveRuntimeFallbackReason(), + agentHarnessId: runtime.agentHarness.id, + expectedRuntimeArtifact: expectedHarnessArtifact?.artifact, + runtimePlan, + model: runtime.effectiveModel, + resolvedApiKey: resolvedStreamApiKey, + authProfileId: runtime.lastProfileId, + authProfileIdSource: lockedProfileId ? "user" : "auto", + initialReplayState: input.replayState, + authStorage, + authProfileStore: resolveRunAttemptAuthProfileStore(), + toolAuthProfileStore: agentHarnessBuildsOpenClawTools(runtime.agentHarness.id) + ? attemptAuthProfileStore + : undefined, + modelRegistry, + agentId: workspaceResolution.agentId, + beforeAgentStartResult, + thinkLevel: runtime.thinkLevel, + fastMode: attemptFastMode, + fastModeStartedAtMs, + fastModeAutoOnSeconds, + fastModeAutoProgressState, + toolResultFormat: resolvedToolResultFormat, + skipPreparedUserTurnMessage: sessionPromptState.activePrompt.internal, + apiKeyInfo: runtime.apiKeyInfo, + runtimeAuthActive: runtime.runtimeAuthState !== null, + captureRuntimeArtifact: Boolean(params.onSuccessfulAuthBinding || expectedHarnessArtifact), + }, + control: { + lifecycleGeneration, + pluginHarnessOwnsTransport: runtime.pluginHarnessOwnsTransport, + laneTaskAbortController, + laneTaskReleaseController, + noteLaneTaskProgress, + onToolOutcome: input.observeToolOutcome, + allocateToolOutcomeOrdinal: input.allocateToolOutcomeOrdinal, + onToolStreamBoundary: maybeAnnounceFastModeAutoOff, + onRunProgress: notifyRunProgress, + onToolResult: notifyToolResult, + onAgentEvent: notifyAgentEvent, + onUserMessagePersisted: sessionPromptState.onUserMessagePersisted, + onUserMessagePersistenceInvalidated: () => { + sessionPromptState.activePrompt.persisted = false; + }, + getPostCompactionAbortError: input.getPostCompactionAbortError, + setPostCompactionAbortController: input.setPostCompactionAbortController, + clearPostCompactionAbortController: input.clearPostCompactionAbortController, + }, + bootstrapPromptWarningSignaturesSeen: input.bootstrapPromptWarningSignaturesSeen, + suppressNextUserMessagePersistence: sessionPromptState.suppressNextUserMessagePersistence, + beforeAgentFinalizeRevisionAttempts: terminalRetryState.beforeFinalizeRevisionAttempts, + maxBeforeAgentFinalizeRevisions: MAX_BEFORE_AGENT_FINALIZE_REVISIONS, + }); + return { dispatchedAttempt, runtimePlan, startupStagesEmitted }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-normalization.ts new file mode 100644 index 000000000000..8ef286eb459e --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-normalization.ts @@ -0,0 +1,341 @@ +import { parseSqliteSessionFileMarker } from "../../../config/sessions/sqlite-marker.js"; +import { formatAssistantErrorText } from "../../embedded-agent-helpers.js"; +import { createAgentRunDirectAbortError } from "../../run-termination.js"; +import { normalizeUsage, type UsageLike } from "../../usage.js"; +import { hasOutboundDeliveryEvidence } from "../delivery-evidence.js"; +import { log } from "../logger.js"; +import { createEmbeddedRunReplayState, observeReplayMetadata } from "../replay-state.js"; +import type { EmbeddedAgentRunResult } from "../types.js"; +import type { createUsageAccumulator } from "../usage-accumulator.js"; +import { mergeUsageIntoAccumulator } from "../usage-accumulator.js"; +import type { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import { resolveRunFailoverDecision } from "./failover-policy.js"; +import { + buildErrorAgentMeta, + isAssistantForModelRef, + resolveActiveErrorContext, + resolveLatestCallUsage, +} from "./helpers.js"; +import { + MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT, + stepIdleTimeoutBreaker, + type createIdleTimeoutBreakerState, +} from "./idle-timeout-breaker.js"; +import { resolveReplayInvalidFlag } from "./incomplete-turn.js"; +import { handleRetryLimitExhaustion } from "./retry-limit.js"; +import type { dispatchEmbeddedRunAttempt } from "./run-attempt-dispatch.js"; +import { + hasCompletedModelProgressForIdleBreaker, + normalizeEmbeddedRunAttemptResult, +} from "./run-attempt-result.js"; +import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; +import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js"; +import { + isEmbeddedRunTerminalAbort, + isEmbeddedRunTerminalInterrupted, + isEmbeddedRunTerminalTimeout, + resolveEmbeddedRunAttemptTerminalOutcome, +} from "./terminal-outcome.js"; + +type PreparedRuntime = Awaited>; +type SessionPromptState = ReturnType; +type ReplayState = ReturnType; + +export async function normalizeEmbeddedRunAttempt(input: { + runInput: PreparedEmbeddedRunInput; + preparedRuntime: PreparedRuntime; + dispatchedAttempt: Awaited>; + sessionPromptState: SessionPromptState; + provider: string; + modelId: string; + bootstrapPromptWarningSignaturesSeen: string[]; + usageAccumulator: ReturnType; + lastRunPromptUsage: ReturnType | undefined; + lastTurnTotal: number | undefined; + idleTimeoutBreakerState: ReturnType; + contextRecoveryState: ReturnType; + replayState: ReplayState; + lastRetryFailoverReason: Parameters[0]["failoverReason"]; +}): Promise< + | { action: "complete"; result: EmbeddedAgentRunResult } + | { + action: "retry"; + bootstrapPromptWarningSignaturesSeen: string[]; + lastRunPromptUsage: ReturnType | undefined; + lastTurnTotal: number | undefined; + replayState: ReplayState; + } + | { + action: "proceed"; + bootstrapPromptWarningSignaturesSeen: string[]; + lastRunPromptUsage: ReturnType | undefined; + lastTurnTotal: number | undefined; + replayState: ReplayState; + attempt: ReturnType; + aborted: boolean; + externalAbort: boolean; + promptError: unknown; + promptErrorSource: ReturnType["promptErrorSource"]; + timedOut: boolean; + idleTimedOut: boolean; + timedOutDuringCompaction: boolean; + timedOutDuringToolExecution: boolean; + timedOutByRunBudget: boolean; + sessionIdUsed: string; + sessionFileUsed: string | undefined; + currentAttemptAssistant: ReturnType< + typeof normalizeEmbeddedRunAttemptResult + >["currentAttemptAssistant"]; + attemptAssistant: ReturnType< + typeof normalizeEmbeddedRunAttemptResult + >["currentAttemptAssistant"]; + terminalOutcome: ReturnType; + terminalAborted: boolean; + terminalTimedOut: boolean; + terminalInterrupted: boolean; + signalOwnedInterruption: boolean; + setTerminalLifecycleMeta: NonNullable< + ReturnType["setTerminalLifecycleMeta"] + >; + attemptCompactionCount: number; + activeErrorContext: ReturnType; + resolveReplayInvalidForAttempt: (incompleteTurnText?: string | null) => boolean; + assistantErrorText: string | undefined; + canRestartForLiveSwitch: boolean; + } +> { + const { runInput, preparedRuntime, dispatchedAttempt, sessionPromptState, provider, modelId } = + input; + const params = runInput.runParams; + const runtime = preparedRuntime.snapshot(); + const attempt = normalizeEmbeddedRunAttemptResult(dispatchedAttempt.rawAttempt); + await sessionPromptState.waitForCurrentUserMessagePersistence(); + sessionPromptState.suppressNextUserMessagePersistence = sessionPromptState.activePrompt.persisted; + if (dispatchedAttempt.cancellationRequested) { + runInput.laneController.throwIfAborted(); + throw createAgentRunDirectAbortError(); + } + const { + aborted, + externalAbort, + promptError, + promptErrorSource, + preflightRecovery, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + sessionIdUsed, + sessionFileUsed, + lastAssistant: sessionLastAssistant, + currentAttemptAssistant, + } = attempt; + const timedOutDuringToolExecution = attempt.timedOutDuringToolExecution ?? false; + const timedOutByRunBudget = attempt.timedOutByRunBudget ?? false; + const sessionAssistantForCandidate = + !currentAttemptAssistant && + !isAssistantForModelRef(sessionLastAssistant, { + provider: runtime.effectiveModel.provider, + model: runtime.effectiveModel.id, + }) + ? undefined + : sessionLastAssistant; + const attemptAssistant = currentAttemptAssistant ?? sessionAssistantForCandidate; + const terminalOutcome = resolveEmbeddedRunAttemptTerminalOutcome({ + attempt, + assistant: currentAttemptAssistant, + abortSignal: params.abortSignal, + }); + const terminalAborted = isEmbeddedRunTerminalAbort(terminalOutcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(terminalOutcome); + const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalOutcome); + const signalOwnedInterruption = terminalInterrupted && params.abortSignal?.aborted === true; + const setTerminalLifecycleMeta: NonNullable = (meta) => { + const { stopReason, ...remainingMeta } = meta; + const terminalStopReason = terminalInterrupted ? terminalOutcome.stopReason : stopReason; + attempt.setTerminalLifecycleMeta?.({ + ...remainingMeta, + ...(terminalStopReason ? { stopReason: terminalStopReason } : {}), + aborted: terminalAborted, + }); + }; + const previousSessionId = sessionPromptState.sessionId; + const previousSessionFile = sessionPromptState.sessionFile; + sessionPromptState.adoptSessionId(sessionIdUsed); + if (sessionFileUsed && sessionFileUsed !== sessionPromptState.sessionFile) { + sessionPromptState.sessionFile = sessionFileUsed; + } + if ( + (sessionIdUsed && sessionIdUsed !== previousSessionId) || + (sessionFileUsed && sessionFileUsed !== previousSessionFile) + ) { + const marker = parseSqliteSessionFileMarker(sessionPromptState.sessionFile); + sessionPromptState.sessionTarget = marker + ? { + agentId: marker.agentId, + sessionId: marker.sessionId, + sessionKey: runInput.resolvedSessionKey, + storePath: marker.storePath, + } + : undefined; + } + const bootstrapPromptWarningSignaturesSeen = + attempt.bootstrapPromptWarningSignaturesSeen ?? + (attempt.bootstrapPromptWarningSignature + ? Array.from( + new Set([ + ...input.bootstrapPromptWarningSignaturesSeen, + attempt.bootstrapPromptWarningSignature, + ]), + ) + : input.bootstrapPromptWarningSignaturesSeen); + const lastAssistantUsage = normalizeUsage(sessionLastAssistant?.usage as UsageLike); + const currentAttemptAssistantUsage = normalizeUsage(currentAttemptAssistant?.usage as UsageLike); + const promptCacheLastCallUsage = normalizeUsage(attempt.promptCache?.lastCallUsage as UsageLike); + const callUsage = resolveLatestCallUsage({ + currentAttemptCandidates: [currentAttemptAssistantUsage, promptCacheLastCallUsage], + carriedCandidates: [input.lastRunPromptUsage, lastAssistantUsage], + }); + const attemptUsage = attempt.attemptUsage ?? callUsage.currentAttempt; + mergeUsageIntoAccumulator(input.usageAccumulator, attemptUsage); + const lastRunPromptUsage = callUsage.latest; + const lastTurnTotal = callUsage.latest?.total; + const breakerStep = stepIdleTimeoutBreaker(input.idleTimeoutBreakerState, { + idleTimedOut: terminalTimedOut && idleTimedOut, + completedModelProgress: hasCompletedModelProgressForIdleBreaker(attempt), + outputTokens: attemptUsage?.output, + }); + if (breakerStep.tripped) { + const message = + `Idle-timeout cost-runaway breaker tripped: ${breakerStep.consecutive} consecutive idle timeouts ` + + `without completed model progress (cap=${MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT}). ` + + "Halting further attempts to bound paid model calls. See issue #76293."; + log.error( + `[idle-timeout-circuit-breaker-tripped] sessionKey=${params.sessionKey ?? params.sessionId} ` + + `provider=${provider}/${modelId} consecutive=${breakerStep.consecutive} ` + + `cap=${MAX_CONSECUTIVE_IDLE_TIMEOUTS_BEFORE_OUTPUT}`, + ); + return { + action: "complete", + result: handleRetryLimitExhaustion({ + message, + decision: resolveRunFailoverDecision({ + stage: "retry_limit", + fallbackConfigured: runInput.fallbackConfigured, + failoverReason: input.lastRetryFailoverReason, + }), + provider, + model: modelId, + profileId: runtime.lastProfileId, + durationMs: Date.now() - runInput.startedAtMs, + agentMeta: buildErrorAgentMeta({ + sessionId: sessionPromptState.sessionId, + sessionFile: sessionPromptState.sessionFile, + provider, + model: preparedRuntime.model.id, + ...runtime.outerContextTokenMeta, + usageAccumulator: input.usageAccumulator, + lastRunPromptUsage, + lastTurnTotal, + }), + replayInvalid: input.replayState.replayInvalid ? true : undefined, + livenessState: "blocked", + }), + }; + } + const attemptCompactionCount = Math.max(0, attempt.compactionCount ?? 0); + input.contextRecoveryState.autoCompactionCount += attemptCompactionCount; + if ( + typeof attempt.compactionTokensAfter === "number" && + Number.isFinite(attempt.compactionTokensAfter) && + attempt.compactionTokensAfter >= 0 + ) { + input.contextRecoveryState.lastCompactionTokensAfter = Math.floor( + attempt.compactionTokensAfter, + ); + } + if (attempt.contextBudgetStatus) { + input.contextRecoveryState.lastContextBudgetStatus = attempt.contextBudgetStatus; + } + const activeErrorContext = resolveActiveErrorContext({ + provider, + model: modelId, + assistant: attemptAssistant, + }); + let replayState = input.replayState; + const resolveReplayInvalidForAttempt = (incompleteTurnText?: string | null) => + replayState.replayInvalid || resolveReplayInvalidFlag({ attempt, incompleteTurnText }); + if (resolveReplayInvalidForAttempt(null)) { + replayState.replayInvalid = true; + } + replayState = observeReplayMetadata(replayState, attempt.replayMetadata); + const formattedAssistantErrorText = sessionAssistantForCandidate + ? formatAssistantErrorText(sessionAssistantForCandidate, { + cfg: params.config, + sessionKey: runInput.resolvedSessionKey ?? params.sessionId, + provider: activeErrorContext.provider, + model: activeErrorContext.model, + authMode: runtime.lastProfileId + ? preparedRuntime.attemptAuthProfileStore.profiles?.[runtime.lastProfileId]?.type + : undefined, + }) + : undefined; + const assistantErrorText = + sessionAssistantForCandidate?.stopReason === "error" + ? sessionAssistantForCandidate.errorMessage?.trim() || formattedAssistantErrorText + : undefined; + if (!signalOwnedInterruption && !preparedRuntime.nativeModelOwned && preflightRecovery?.handled) { + const retryingFromTranscript = preflightRecovery.source === "mid-turn"; + log.info( + `[context-overflow-precheck] early recovery route=${preflightRecovery.route} completed for ${provider}/${modelId}; ` + + (retryingFromTranscript ? "retrying from current transcript" : "retrying prompt"), + ); + if (retryingFromTranscript) { + sessionPromptState.continueFromCurrentTranscript(); + } + return { + action: "retry", + bootstrapPromptWarningSignaturesSeen, + lastRunPromptUsage, + lastTurnTotal, + replayState, + }; + } + return { + action: "proceed", + bootstrapPromptWarningSignaturesSeen, + lastRunPromptUsage, + lastTurnTotal, + replayState, + attempt, + aborted, + externalAbort, + promptError, + promptErrorSource, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + sessionIdUsed, + sessionFileUsed, + currentAttemptAssistant, + attemptAssistant, + terminalOutcome, + terminalAborted, + terminalTimedOut, + terminalInterrupted, + signalOwnedInterruption, + setTerminalLifecycleMeta, + attemptCompactionCount, + activeErrorContext, + resolveReplayInvalidForAttempt, + assistantErrorText, + canRestartForLiveSwitch: + !hasOutboundDeliveryEvidence(attempt) && + !attempt.didSendDeterministicApprovalPrompt && + !attempt.lastToolError && + (attempt.toolMetas?.length ?? 0) === 0 && + (attempt.assistantTexts?.length ?? 0) === 0, + }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt-recovery.ts b/src/agents/embedded-agent-runner/run/attempt-recovery.ts new file mode 100644 index 000000000000..2ec6cd8503c9 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-recovery.ts @@ -0,0 +1,351 @@ +import { formatErrorMessage, toErrorObject } from "../../../infra/errors.js"; +import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../../defaults.js"; +import type { FailoverReason } from "../../embedded-agent-helpers.js"; +import { LiveSessionModelSwitchError } from "../../live-model-switch-error.js"; +import { shouldSwitchToLiveModel, clearLiveModelSwitchPending } from "../../live-model-switch.js"; +import type { normalizeUsage } from "../../usage.js"; +import { log } from "../logger.js"; +import type { EmbeddedAgentRunResult, TraceAttempt } from "../types.js"; +import type { createUsageAccumulator } from "../usage-accumulator.js"; +import type { prepareAndDispatchEmbeddedRunAttempt } from "./attempt-dispatch-preparation.js"; +import type { normalizeEmbeddedRunAttempt } from "./attempt-normalization.js"; +import { buildEmbeddedRunBlockedResult } from "./blocked-run-result.js"; +import { resolveCodexAppServerRecoveryRetry } from "./codex-app-server-recovery.js"; +import type { createEmbeddedRunCompactionRuntime } from "./compaction-runtime.js"; +import type { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import type { createEmbeddedRunFailoverRetryController } from "./failover-retry-controller.js"; +import { buildErrorAgentMeta } from "./helpers.js"; +import { recoverEmbeddedRunOverflow } from "./overflow-context-recovery.js"; +import { handleEmbeddedPromptFailure } from "./prompt-failure.js"; +import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; +import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js"; +import { recoverEmbeddedRunTimeout } from "./timeout-context-recovery.js"; + +type PreparedRuntime = Awaited>; +type NormalizedAttempt = Extract< + Awaited>, + { action: "proceed" } +>; +type Dispatch = Awaited>; +type SessionPromptState = ReturnType; +type FailoverRetryController = ReturnType; +type CompactionRuntime = ReturnType; + +export async function recoverEmbeddedRunAttempt(input: { + runInput: PreparedEmbeddedRunInput; + preparedRuntime: PreparedRuntime; + normalizedAttempt: NormalizedAttempt; + runtimePlan: Dispatch["runtimePlan"]; + sessionPromptState: SessionPromptState; + failoverRetryController: FailoverRetryController; + compactionRuntime: CompactionRuntime; + contextEngine: Parameters[0]["contextEngine"]; + contextRecoveryState: ReturnType; + resolveContextEnginePluginId: Parameters< + typeof recoverEmbeddedRunTimeout + >[0]["resolveContextEnginePluginId"]; + buildRuntimeSettings: Parameters[0]["buildRuntimeSettings"]; + armPostCompactionGuard: () => void; + usageAccumulator: ReturnType; + lastRunPromptUsage: ReturnType | undefined; + lastTurnTotal: number | undefined; + runtimeAuthRetry: boolean; + codexAppServerRecoveryRetryAvailable: boolean; + codexAppServerRecoveryRetries: number; + lastRetryFailoverReason: FailoverReason | null; + traceAttempts: TraceAttempt[]; + sessionAgentId: string; +}): Promise< + | { action: "complete"; result: EmbeddedAgentRunResult } + | { + action: "retry"; + authRetryPending: boolean; + codexAppServerRecoveryRetries: number; + lastRetryFailoverReason: FailoverReason | null; + thinkLevel: PreparedRuntime["snapshot"] extends () => infer Snapshot + ? Snapshot extends { thinkLevel: infer ThinkLevel } + ? ThinkLevel + : never + : never; + } + | { action: "proceed"; shouldSurfaceCodexCompletionTimeout: boolean } +> { + const { + runInput, + preparedRuntime, + normalizedAttempt, + runtimePlan, + sessionPromptState, + failoverRetryController, + compactionRuntime, + } = input; + const params = runInput.runParams; + const runtime = preparedRuntime.snapshot(); + const { + attempt, + aborted, + externalAbort, + promptError, + promptErrorSource, + timedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + sessionIdUsed, + attemptAssistant, + terminalInterrupted, + signalOwnedInterruption, + setTerminalLifecycleMeta, + attemptCompactionCount, + activeErrorContext, + resolveReplayInvalidForAttempt, + assistantErrorText, + canRestartForLiveSwitch, + } = normalizedAttempt; + const retry = (updates?: { + authRetryPending?: boolean; + codexAppServerRecoveryRetries?: number; + lastRetryFailoverReason?: FailoverReason | null; + thinkLevel?: typeof runtime.thinkLevel; + }) => ({ + action: "retry" as const, + authRetryPending: updates?.authRetryPending ?? false, + codexAppServerRecoveryRetries: + updates?.codexAppServerRecoveryRetries ?? input.codexAppServerRecoveryRetries, + lastRetryFailoverReason: + updates?.lastRetryFailoverReason === undefined + ? input.lastRetryFailoverReason + : updates.lastRetryFailoverReason, + thinkLevel: updates?.thinkLevel ?? runtime.thinkLevel, + }); + + const requestedSelection = shouldSwitchToLiveModel({ + cfg: params.config, + sessionKey: runInput.resolvedSessionKey, + agentId: params.agentId, + defaultProvider: DEFAULT_PROVIDER, + defaultModel: DEFAULT_MODEL, + currentProvider: preparedRuntime.provider, + currentModel: preparedRuntime.modelId, + currentAgentRuntimeOverride: params.agentHarnessRuntimeOverride, + currentAuthProfileId: preparedRuntime.preferredProfileId, + currentAuthProfileIdSource: params.authProfileIdSource, + }); + if (!signalOwnedInterruption && requestedSelection && canRestartForLiveSwitch) { + await clearLiveModelSwitchPending({ + cfg: params.config, + sessionKey: runInput.resolvedSessionKey, + agentId: params.agentId, + }); + log.info( + `live session model switch requested during active attempt for ${params.sessionId}: ` + + `${preparedRuntime.provider}/${preparedRuntime.modelId} -> ${requestedSelection.provider}/${requestedSelection.model}`, + ); + throw new LiveSessionModelSwitchError(requestedSelection); + } + const commonRecoveryInput = { + runParams: params, + state: input.contextRecoveryState, + contextEngine: input.contextEngine, + contextTokenBudget: runtime.contextTokenBudget, + genericCompactionRecoveryAllowed: preparedRuntime.genericCompactionRecoveryAllowed, + attempt, + runtimeAuthPlan: runtimePlan.auth, + resolvedSessionKey: runInput.resolvedSessionKey, + sessionAgentId: input.sessionAgentId, + agentDir: runInput.agentDir, + workspaceDir: runInput.workspaceDir, + provider: preparedRuntime.provider, + modelId: preparedRuntime.modelId, + harnessRuntime: runtime.agentHarness.id, + thinkLevel: runtime.thinkLevel, + authProfileId: runtime.lastProfileId, + authProfileIdSource: preparedRuntime.lockedProfileId ? ("user" as const) : ("auto" as const), + resolveContextEnginePluginId: input.resolveContextEnginePluginId, + buildRuntimeSettings: input.buildRuntimeSettings, + ...compactionRuntime, + getActiveSession: () => ({ + id: sessionPromptState.sessionId, + file: sessionPromptState.sessionFile, + target: sessionPromptState.sessionTarget, + }), + armPostCompactionGuard: input.armPostCompactionGuard, + }; + if ( + await recoverEmbeddedRunTimeout({ + ...commonRecoveryInput, + timedOut, + signalOwnedInterruption, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + lastRunPromptUsage: input.lastRunPromptUsage, + }) + ) { + return retry(); + } + const overflowRecovery = await recoverEmbeddedRunOverflow({ + ...commonRecoveryInput, + aborted, + signalOwnedInterruption, + promptError, + assistantErrorText, + attemptCompactionCount, + prepareCurrentTranscriptRetry: sessionPromptState.continueFromCurrentTranscript, + prepareCompactedTranscriptRetry: sessionPromptState.prepareCompactedTranscriptRetry, + }); + if (overflowRecovery.action === "retry") { + return retry(); + } + if (overflowRecovery.action === "surface") { + const replayInvalid = resolveReplayInvalidForAttempt(); + setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" }); + return { + action: "complete", + result: buildEmbeddedRunBlockedResult({ + text: overflowRecovery.userText, + errorKind: overflowRecovery.kind, + errorMessage: overflowRecovery.errorText, + durationMs: Date.now() - runInput.startedAtMs, + agentMeta: buildErrorAgentMeta({ + sessionId: sessionIdUsed, + sessionFile: sessionPromptState.sessionFile, + provider: preparedRuntime.provider, + model: preparedRuntime.model.id, + ...runtime.outerContextTokenMeta, + usageAccumulator: input.usageAccumulator, + lastRunPromptUsage: input.lastRunPromptUsage, + lastAssistant: attemptAssistant, + lastTurnTotal: input.lastTurnTotal, + }), + attempt, + replayInvalid, + finalPromptText: attempt.finalPromptText, + }), + }; + } + if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) { + const errorText = formatErrorMessage(promptError); + const replayInvalid = resolveReplayInvalidForAttempt(); + setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" }); + return { + action: "complete", + result: buildEmbeddedRunBlockedResult({ + text: errorText, + errorKind: "hook_block", + errorMessage: errorText, + durationMs: Date.now() - runInput.startedAtMs, + agentMeta: buildErrorAgentMeta({ + sessionId: sessionIdUsed, + sessionFile: sessionPromptState.sessionFile, + provider: preparedRuntime.provider, + model: preparedRuntime.model.id, + ...runtime.outerContextTokenMeta, + usageAccumulator: input.usageAccumulator, + lastRunPromptUsage: input.lastRunPromptUsage, + lastAssistant: attemptAssistant, + lastTurnTotal: input.lastTurnTotal, + }), + attempt, + replayInvalid, + }), + }; + } + const hasRecoverableCodexAppServerTimeoutOutcome = Boolean( + attempt.codexAppServerFailure && attempt.promptTimeoutOutcome, + ); + let shouldSurfaceCodexCompletionTimeout = false; + if (promptError && promptErrorSource !== "compaction" && attempt.codexAppServerFailure) { + const recoveryRetry = resolveCodexAppServerRecoveryRetry({ + attempt, + retryAvailable: input.codexAppServerRecoveryRetryAvailable, + }); + if (recoveryRetry.retry) { + runInput.laneController.throwIfAborted(); + sessionPromptState.suppressNextUserMessagePersistence = true; + log.warn( + `codex app-server replay-safe failure; retrying once failureKind=${attempt.codexAppServerFailure?.kind} ` + + `runId=${params.runId} sessionId=${params.sessionId}`, + ); + return retry({ codexAppServerRecoveryRetries: input.codexAppServerRecoveryRetries + 1 }); + } + shouldSurfaceCodexCompletionTimeout = + attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && attempt.timedOut; + if ( + attempt.codexAppServerFailure && + !hasRecoverableCodexAppServerTimeoutOutcome && + !shouldSurfaceCodexCompletionTimeout + ) { + throw toErrorObject(promptError, "Prompt failed"); + } + } + if ( + promptError && + !terminalInterrupted && + promptErrorSource !== "compaction" && + !hasRecoverableCodexAppServerTimeoutOutcome && + !shouldSurfaceCodexCompletionTimeout + ) { + const promptFailureOutcome = await handleEmbeddedPromptFailure({ + runParams: params, + attempt, + promptError, + promptErrorSource, + activeErrorContext, + provider: preparedRuntime.provider, + modelId: preparedRuntime.modelId, + authProfileId: runtime.lastProfileId, + authProfileStore: preparedRuntime.attemptAuthProfileStore, + sessionIdUsed, + lane: runInput.globalLane, + agentDir: runInput.agentDir, + suspensionSessionId: sessionPromptState.sessionId ?? params.sessionId, + runtimeAuthRetry: input.runtimeAuthRetry, + maybeRefreshRuntimeAuthForAuthError: preparedRuntime.maybeRefreshRuntimeAuthForAuthError, + suspendForFailure: runInput.suspendForFailure, + resolveReplayInvalid: resolveReplayInvalidForAttempt, + setTerminalLifecycleMeta, + buildErrorAgentMeta: () => + buildErrorAgentMeta({ + sessionId: sessionIdUsed, + sessionFile: sessionPromptState.sessionFile, + provider: preparedRuntime.provider, + model: preparedRuntime.model.id, + ...runtime.outerContextTokenMeta, + usageAccumulator: input.usageAccumulator, + lastRunPromptUsage: input.lastRunPromptUsage, + lastAssistant: attemptAssistant, + lastTurnTotal: input.lastTurnTotal, + }), + startedAtMs: runInput.startedAtMs, + fallbackConfigured: runInput.fallbackConfigured, + aborted, + externalAbort, + pluginHarnessOwnsTransport: runtime.pluginHarnessOwnsTransport, + timedOutByRunBudget, + resolveAuthProfileFailureReason: failoverRetryController.resolveAuthProfileFailureReason, + maybeEscalateRateLimitProfileFallback: + failoverRetryController.maybeEscalateRateLimitProfileFallback, + advanceAttemptAuthProfile: preparedRuntime.advanceAttemptAuthProfile, + maybeMarkAuthProfileFailure: failoverRetryController.maybeMarkAuthProfileFailure, + maybeBackoffBeforeOverloadFailover: + failoverRetryController.maybeBackoffBeforeOverloadFailover, + attemptedThinking: preparedRuntime.attemptedThinking, + thinkLevel: runtime.thinkLevel, + getThinkLevel: () => preparedRuntime.snapshot().thinkLevel, + traceAttempts: input.traceAttempts, + previousRetryFailoverReason: input.lastRetryFailoverReason, + }); + if (promptFailureOutcome.action === "complete") { + return { action: "complete", result: promptFailureOutcome.result }; + } + preparedRuntime.setThinkLevel(promptFailureOutcome.thinkLevel); + return retry({ + authRetryPending: promptFailureOutcome.authRetryPending, + lastRetryFailoverReason: promptFailureOutcome.lastRetryFailoverReason, + thinkLevel: promptFailureOutcome.thinkLevel, + }); + } + return { action: "proceed", shouldSurfaceCodexCompletionTimeout }; +} diff --git a/src/agents/embedded-agent-runner/run/auth-profile-success.ts b/src/agents/embedded-agent-runner/run/auth-profile-success.ts new file mode 100644 index 000000000000..b9e6e34ddf04 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/auth-profile-success.ts @@ -0,0 +1,185 @@ +import { sanitizeForLog } from "../../../../packages/terminal-core/src/ansi.js"; +import { formatErrorMessage } from "../../../infra/errors.js"; +import { redactIdentifier } from "../../../logging/redact-identifier.js"; +import { looksLikeSecretSentinel, resolveSecretSentinel } from "../../../secrets/sentinel.js"; +import type { AuthProfileStore } from "../../auth-profiles.js"; +import { markAuthProfileSuccess } from "../../auth-profiles.js"; +import { + fingerprintAuthProfileOwnerShape, + fingerprintAwsSdkRuntimeOwner, + fingerprintOpaqueRuntimeOwner, + fingerprintResolvedAuthProfileCredential, + fingerprintResolvedProviderAuth, + type AgentExecutionAuthBinding, +} from "../../execution-auth-binding.js"; +import type { ResolvedProviderAuth } from "../../model-auth.js"; +import { log } from "../logger.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +const POST_RUN_AUTH_PROFILE_SUCCESS_SLOW_MS = 1_000; + +export function markEmbeddedRunAuthProfileSuccess(input: { + authProfileStateMode?: "read-write" | "read-only"; + profileId?: string; + profileStore: AuthProfileStore; + provider: string; + agentDir?: string; + runId: string; + sessionId: string; +}): void { + if (input.authProfileStateMode === "read-only" || !input.profileId) { + return; + } + const successProfileId = input.profileId; + const safeSuccessProfileId = redactIdentifier(successProfileId, { len: 12 }); + const successProvider = resolveAuthProfileStateProvider( + input.profileStore, + successProfileId, + input.provider, + ); + const successStarted = Date.now(); + void markAuthProfileSuccess({ + store: input.profileStore, + provider: successProvider, + profileId: successProfileId, + agentDir: input.agentDir, + }) + .then(() => { + const durationMs = Date.now() - successStarted; + if (durationMs >= POST_RUN_AUTH_PROFILE_SUCCESS_SLOW_MS) { + log.warn( + `post-run auth-profile success bookkeeping completed after ${durationMs}ms: ` + + `runId=${input.runId} sessionId=${input.sessionId} ` + + `provider=${sanitizeForLog(successProvider)} profileId=${safeSuccessProfileId}`, + ); + } else if (log.isEnabled("trace")) { + log.trace( + `post-run auth-profile success bookkeeping completed: ` + + `runId=${input.runId} sessionId=${input.sessionId} durationMs=${durationMs}`, + ); + } + }) + .catch((error: unknown) => { + log.warn( + `post-run auth-profile success bookkeeping failed: ` + + `runId=${input.runId} sessionId=${input.sessionId} ` + + `provider=${sanitizeForLog(successProvider)} profileId=${safeSuccessProfileId} ` + + `error=${formatErrorMessage(error)}`, + ); + }); +} + +export function reportEmbeddedRunSuccessfulAuthBinding(input: { + profileId?: string; + profileStore: AuthProfileStore; + apiKeyInfo: ResolvedProviderAuth | null; + attempt: EmbeddedRunAttemptResult; + provider: string; + agentHarnessId: string; + pluginHarnessOwnsTransport: boolean; + pluginHarnessOwnsAuthBootstrap: boolean; + onSuccessfulAuthBinding?: (binding: AgentExecutionAuthBinding) => void; +}): void { + const credential = input.profileId ? input.profileStore.profiles[input.profileId] : undefined; + const pluginHarnessApiKeyInfo = resolvePluginHarnessApiKeyInfo({ + apiKeyInfo: input.apiKeyInfo, + pluginHarnessOwnsTransport: input.pluginHarnessOwnsTransport, + }); + const authFingerprint = + credential?.type === "oauth" && input.profileId + ? fingerprintResolvedAuthProfileCredential({ + profileId: input.profileId, + credential, + resolvedAuth: input.apiKeyInfo, + }) + : credential && input.profileId && input.pluginHarnessOwnsAuthBootstrap + ? input.attempt.authBindingFingerprint + : credential && input.profileId && input.pluginHarnessOwnsTransport + ? fingerprintResolvedAuthProfileCredential({ + profileId: input.profileId, + credential, + resolvedAuth: pluginHarnessApiKeyInfo, + }) + : input.apiKeyInfo + ? fingerprintResolvedProviderAuth(input.apiKeyInfo) + : undefined; + const authProfileOwnerFingerprint = + input.profileId && credential !== undefined + ? fingerprintAuthProfileOwnerShape({ profileId: input.profileId, credential }) + : undefined; + const runtimeArtifact = input.pluginHarnessOwnsTransport + ? input.attempt.runtimeArtifact + : undefined; + const runtimeOwnerFingerprint = authFingerprint + ? undefined + : input.apiKeyInfo?.mode === "aws-sdk" + ? fingerprintAwsSdkRuntimeOwner({ + provider: input.provider, + backendId: input.agentHarnessId, + auth: input.apiKeyInfo, + }) + : input.pluginHarnessOwnsTransport + ? fingerprintOpaqueRuntimeOwner({ + kind: "plugin-harness", + runner: "embedded", + provider: input.provider, + backendId: input.agentHarnessId, + ...(runtimeArtifact ? { runtimeArtifactFingerprint: runtimeArtifact.fingerprint } : {}), + ...(input.profileId ? { authProfileId: input.profileId } : {}), + ...(authProfileOwnerFingerprint ? { authProfileOwnerFingerprint } : {}), + }) + : undefined; + const runtimeOwnerKind = runtimeOwnerFingerprint + ? input.apiKeyInfo?.mode === "aws-sdk" + ? ("aws-sdk" as const) + : input.pluginHarnessOwnsTransport + ? ("plugin-harness" as const) + : undefined + : input.pluginHarnessOwnsTransport + ? ("plugin-harness" as const) + : undefined; + input.onSuccessfulAuthBinding?.({ + ...(input.profileId ? { authProfileId: input.profileId } : {}), + agentHarnessId: input.agentHarnessId, + ...(authFingerprint ? { authFingerprint } : {}), + ...(runtimeOwnerFingerprint ? { runtimeOwnerFingerprint } : {}), + ...(runtimeOwnerKind ? { runtimeOwnerKind } : {}), + ...(runtimeOwnerKind ? { runtimeOwnerId: input.agentHarnessId } : {}), + ...(runtimeArtifact + ? { + runtimeArtifactId: runtimeArtifact.id, + runtimeArtifactFingerprint: runtimeArtifact.fingerprint, + } + : {}), + }); +} + +function resolvePluginHarnessApiKeyInfo(input: { + apiKeyInfo: ResolvedProviderAuth | null; + pluginHarnessOwnsTransport: boolean; +}): ResolvedProviderAuth | null { + const apiKeyInfo = input.apiKeyInfo; + const apiKey = apiKeyInfo?.apiKey; + if ( + !input.pluginHarnessOwnsTransport || + !apiKeyInfo || + !apiKey || + !looksLikeSecretSentinel(apiKey) + ) { + return apiKeyInfo; + } + const resolvedApiKey = resolveSecretSentinel(apiKey); + return resolvedApiKey ? { ...apiKeyInfo, apiKey: resolvedApiKey } : null; +} + +function resolveAuthProfileStateProvider( + store: AuthProfileStore, + profileId: string, + fallbackProvider: string, +): string { + const profileProvider = store.profiles?.[profileId]?.provider?.trim(); + if (profileProvider) { + return profileProvider; + } + return profileId.split(":", 1)[0]?.trim() || fallbackProvider; +} diff --git a/src/agents/embedded-agent-runner/run/blocked-run-result.ts b/src/agents/embedded-agent-runner/run/blocked-run-result.ts new file mode 100644 index 000000000000..3cdc4934a2d6 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/blocked-run-result.ts @@ -0,0 +1,28 @@ +import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +export function buildEmbeddedRunBlockedResult(input: { + text: string; + errorKind: NonNullable["kind"]; + errorMessage: string; + durationMs: number; + agentMeta: EmbeddedAgentMeta; + attempt: EmbeddedRunAttemptResult; + replayInvalid: boolean; + finalPromptText?: string; +}): EmbeddedAgentRunResult { + return { + payloads: [{ text: input.text, isError: true }], + meta: { + durationMs: input.durationMs, + agentMeta: input.agentMeta, + systemPromptReport: input.attempt.systemPromptReport, + finalAssistantVisibleText: input.text, + finalAssistantRawText: input.text, + finalPromptText: input.finalPromptText, + replayInvalid: input.replayInvalid, + livenessState: "blocked", + error: { kind: input.errorKind, message: input.errorMessage }, + }, + }; +} diff --git a/src/agents/embedded-agent-runner/run/compaction-runtime.ts b/src/agents/embedded-agent-runner/run/compaction-runtime.ts new file mode 100644 index 000000000000..571d7932dc33 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/compaction-runtime.ts @@ -0,0 +1,116 @@ +import type { resolveContextEngine } from "../../../context-engine/registry.js"; +import { resolveCompactionSuccessorTranscript } from "../../../context-engine/types.js"; +import { log } from "../logger.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js"; + +type ContextEngine = Awaited>; +type SessionPromptState = ReturnType; + +export function createEmbeddedRunCompactionRuntime(input: { + runParams: PreparedEmbeddedRunInput["runParams"]; + contextEngine: ContextEngine; + hookRunner: PreparedEmbeddedRunInput["hookRunner"]; + hookContext: PreparedEmbeddedRunInput["hookContext"]; + sessionPromptState: SessionPromptState; +}) { + const { runParams: params, contextEngine, hookRunner, hookContext, sessionPromptState } = input; + const resolveActiveHookContext = () => ({ + ...hookContext, + sessionId: sessionPromptState.sessionId, + }); + const adoptCompactionTranscript = async ( + compactResult: Awaited>, + ): Promise => { + const previousSessionId = sessionPromptState.sessionId; + const nextSessionTarget = compactResult.result?.sessionTarget; + const successor = resolveCompactionSuccessorTranscript(compactResult); + await sessionPromptState.adoptSessionTarget( + nextSessionTarget && successor.sessionId + ? { + ...nextSessionTarget, + sessionId: nextSessionTarget.sessionId ?? successor.sessionId, + } + : nextSessionTarget, + ); + if ( + !nextSessionTarget && + successor.sessionFile && + successor.sessionFile !== sessionPromptState.sessionFile + ) { + sessionPromptState.sessionFile = successor.sessionFile; + } + sessionPromptState.adoptSessionId(successor.sessionId); + return successor.sessionId && successor.sessionId !== previousSessionId + ? previousSessionId + : undefined; + }; + const onCompactionHookMessages = async (payload: { + phase: "before" | "after"; + messages: string[]; + }) => { + const messages = payload.messages.filter((message) => message.trim().length > 0); + if (messages.length === 0) { + return; + } + await params.onAgentEvent?.({ + stream: "compaction", + data: { + phase: payload.phase === "before" ? "start" : "end", + ...(payload.phase === "after" ? { completed: true } : {}), + messages, + }, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + }); + }; + const runOwnsCompactionBeforeHook = async (reason: string) => { + if (contextEngine.info.ownsCompaction !== true || !hookRunner?.hasHooks("before_compaction")) { + return; + } + try { + await hookRunner.runBeforeCompaction( + { messageCount: -1, sessionFile: sessionPromptState.sessionFile }, + resolveActiveHookContext(), + ); + } catch (error) { + log.warn(`before_compaction hook failed during ${reason}: ${String(error)}`); + } + }; + const runOwnsCompactionAfterHook = async ( + reason: string, + compactResult: Awaited>, + previousSessionId?: string, + ) => { + if ( + contextEngine.info.ownsCompaction !== true || + !compactResult.ok || + !compactResult.compacted || + !hookRunner?.hasHooks("after_compaction") + ) { + return; + } + try { + await hookRunner.runAfterCompaction( + { + messageCount: -1, + compactedCount: -1, + tokenCount: compactResult.result?.tokensAfter, + sessionFile: + resolveCompactionSuccessorTranscript(compactResult).sessionFile ?? + sessionPromptState.sessionFile, + ...(previousSessionId ? { previousSessionId } : {}), + }, + resolveActiveHookContext(), + ); + } catch (error) { + log.warn(`after_compaction hook failed during ${reason}: ${String(error)}`); + } + }; + + return { + adoptCompactionTranscript, + onCompactionHookMessages, + runOwnsCompactionBeforeHook, + runOwnsCompactionAfterHook, + }; +} diff --git a/src/agents/embedded-agent-runner/run/context-recovery-state.ts b/src/agents/embedded-agent-runner/run/context-recovery-state.ts new file mode 100644 index 000000000000..c285b4cff206 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/context-recovery-state.ts @@ -0,0 +1,16 @@ +import type { EmbeddedAgentMeta } from "../types.js"; + +export function createEmbeddedRunContextRecoveryState() { + return { + autoCompactionCount: 0, + lastCompactionTokensAfter: undefined as number | undefined, + lastContextBudgetStatus: undefined as EmbeddedAgentMeta["contextBudgetStatus"], + overflowCompactionAttempts: 0, + timeoutCompactionAttempts: 0, + toolResultTruncationAttempted: false, + }; +} + +export type EmbeddedRunContextRecoveryState = ReturnType< + typeof createEmbeddedRunContextRecoveryState +>; diff --git a/src/agents/embedded-agent-runner/run/execution-context.ts b/src/agents/embedded-agent-runner/run/execution-context.ts new file mode 100644 index 000000000000..191ed015576d --- /dev/null +++ b/src/agents/embedded-agent-runner/run/execution-context.ts @@ -0,0 +1,33 @@ +import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; +import type { SessionSuspensionParams } from "../../session-suspension.js"; +import { resolveRunWorkspaceDir } from "../../workspace-run.js"; +import { createEmbeddedRunStageTracker } from "./attempt-stage-timing.js"; +import type { RunEmbeddedAgentParamsWithSessionFile } from "./internal-params.js"; +import { createEmbeddedRunLaneController } from "./lane-controller.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { createEmbeddedRunProgressController } from "./progress-controller.js"; +import { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; + +export type PreparedEmbeddedRunInput = { + runParams: RunEmbeddedAgentParamsWithSessionFile; + provider: string; + modelId: string; + agentDir: string; + workspaceResolution: ReturnType; + workspaceDir: string; + isCanonicalWorkspace: boolean; + globalLane: string; + hookRunner: ReturnType; + hookContext: Parameters[0]["hookContext"]; + fallbackConfigured: boolean; + isProbeSession: boolean; + resolvedSessionKey: string; + resolvedToolResultFormat: NonNullable; + startedAtMs: number; + startupStages: ReturnType; + emitStartupStageSummary: (phase: string) => void; + progressController: ReturnType; + laneController: ReturnType; + lifecycleGeneration: NonNullable; + suspendForFailure: (params: Omit) => void; +}; diff --git a/src/agents/embedded-agent-runner/run/failover-retry-controller.ts b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts new file mode 100644 index 000000000000..bb14d4e707ea --- /dev/null +++ b/src/agents/embedded-agent-runner/run/failover-retry-controller.ts @@ -0,0 +1,167 @@ +import { sanitizeForLog } from "../../../../packages/terminal-core/src/ansi.js"; +import { sleepWithAbort } from "../../../infra/backoff.js"; +import { type AuthProfileFailureReason, markAuthProfileFailure } from "../../auth-profiles.js"; +import type { FailoverReason } from "../../embedded-agent-helpers.js"; +import { FailoverError, resolveFailoverStatus } from "../../failover-error.js"; +import { log } from "../logger.js"; +import { resolveAuthProfileFailureReason } from "./auth-profile-failure-policy.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import { + MAX_SAME_MODEL_RATE_LIMIT_RETRIES, + resolveNextSameModelRateLimitRetryCount, + resolveOverloadFailoverBackoffMs, + resolveOverloadProfileRotationLimit, + resolveRateLimitProfileRotationLimit, + resolveSameModelRateLimitRetryDelayMs, +} from "./helpers.js"; +import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; + +type PreparedRuntime = Awaited>; + +export function createEmbeddedRunFailoverRetryController(input: { + runParams: PreparedEmbeddedRunInput["runParams"]; + provider: string; + modelId: string; + globalLane: string; + agentDir: string; + fallbackConfigured: boolean; + profileFailureStore: PreparedRuntime["profileFailureStore"]; + getLastProfileId: () => string | undefined; + getSessionId: () => string; + harnessOwnsTransport: () => boolean; +}) { + const { + runParams: params, + provider, + modelId, + globalLane, + agentDir, + fallbackConfigured, + profileFailureStore, + } = input; + const overloadFailoverBackoffMs = resolveOverloadFailoverBackoffMs(params.config); + const overloadProfileRotationLimit = resolveOverloadProfileRotationLimit(params.config); + const rateLimitProfileRotationLimit = resolveRateLimitProfileRotationLimit(params.config); + let rateLimitProfileRotations = 0; + let consecutiveSameModelRateLimitRetries = 0; + + const sleepForRetry = async (delayMs: number) => { + try { + await sleepWithAbort(delayMs, params.abortSignal); + } catch (error) { + if (!params.abortSignal?.aborted) { + throw error; + } + const abortError = new Error("Operation aborted", { cause: error }); + abortError.name = "AbortError"; + throw abortError; + } + }; + + return { + overloadProfileRotationLimit, + rateLimitProfileRotationLimit, + get rateLimitProfileRotations() { + return rateLimitProfileRotations; + }, + get consecutiveSameModelRateLimitRetries() { + return consecutiveSameModelRateLimitRetries; + }, + resetSameModelRateLimitRetries() { + consecutiveSameModelRateLimitRetries = resolveNextSameModelRateLimitRetryCount({ + retriesSoFar: consecutiveSameModelRateLimitRetries, + retriedSameModelRateLimit: false, + }); + }, + maybeEscalateRateLimitProfileFallback(paramsLocal: { + failoverProvider: string; + failoverModel: string; + logFallbackDecision: (decision: "fallback_model", extra?: { status?: number }) => void; + }) { + rateLimitProfileRotations += 1; + if (rateLimitProfileRotations <= rateLimitProfileRotationLimit || !fallbackConfigured) { + return; + } + const status = resolveFailoverStatus("rate_limit"); + log.warn( + `rate-limit profile rotation cap reached for ${sanitizeForLog(provider)}/${sanitizeForLog(modelId)} after ${rateLimitProfileRotations} rotations; escalating to model fallback`, + ); + paramsLocal.logFallbackDecision("fallback_model", { status }); + throw new FailoverError( + "The AI service is temporarily rate-limited. Please try again in a moment.", + { + reason: "rate_limit", + provider: paramsLocal.failoverProvider, + model: paramsLocal.failoverModel, + profileId: input.getLastProfileId(), + sessionId: input.getSessionId(), + lane: globalLane, + status, + }, + ); + }, + async maybeMarkAuthProfileFailure(failure: { + profileId?: string; + reason?: AuthProfileFailureReason | null; + modelId?: string; + }) { + if (params.authProfileStateMode === "read-only") { + return; + } + const { profileId, reason } = failure; + if (!profileId || !reason) { + return; + } + if (input.harnessOwnsTransport() && reason === "timeout") { + return; + } + await markAuthProfileFailure({ + store: profileFailureStore, + profileId, + reason, + cfg: params.config, + agentDir, + runId: params.runId, + modelId: failure.modelId, + }); + }, + resolveAuthProfileFailureReason( + failoverReason: FailoverReason | null, + opts?: { providerStarted?: boolean; transientRateLimit?: boolean }, + ) { + return resolveAuthProfileFailureReason({ + failoverReason, + providerStarted: opts?.providerStarted, + transientRateLimit: opts?.transientRateLimit, + policy: params.authProfileFailurePolicy, + }); + }, + async maybeBackoffBeforeOverloadFailover(reason: FailoverReason | null) { + if (reason !== "overloaded" || overloadFailoverBackoffMs <= 0) { + return; + } + log.warn( + `overload backoff before failover for ${provider}/${modelId}: delayMs=${overloadFailoverBackoffMs}`, + ); + await sleepForRetry(overloadFailoverBackoffMs); + }, + async maybeRetrySameModelRateLimit(retry?: { retryAfterSeconds?: number }): Promise { + if (consecutiveSameModelRateLimitRetries >= MAX_SAME_MODEL_RATE_LIMIT_RETRIES) { + return false; + } + const delayMs = resolveSameModelRateLimitRetryDelayMs({ + retriesSoFar: consecutiveSameModelRateLimitRetries, + retryAfterSeconds: retry?.retryAfterSeconds, + }); + log.warn( + `rate-limit same-model retry ${consecutiveSameModelRateLimitRetries + 1}/${MAX_SAME_MODEL_RATE_LIMIT_RETRIES} for ${sanitizeForLog(provider)}/${sanitizeForLog(modelId)}: delayMs=${delayMs}`, + ); + await sleepForRetry(delayMs); + consecutiveSameModelRateLimitRetries = resolveNextSameModelRateLimitRetryCount({ + retriesSoFar: consecutiveSameModelRateLimitRetries, + retriedSameModelRateLimit: true, + }); + return true; + }, + }; +} diff --git a/src/agents/embedded-agent-runner/run/internal-params.ts b/src/agents/embedded-agent-runner/run/internal-params.ts new file mode 100644 index 000000000000..c08fd555c3c7 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/internal-params.ts @@ -0,0 +1,14 @@ +import type { AgentExecutionAuthBinding } from "../../execution-auth-binding.js"; +import type { SystemAgentToolOptions } from "../../tools/system-agent-tool.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; + +export type RunEmbeddedAgentInternalParams = RunEmbeddedAgentParams & { + onSuccessfulAuthBinding?: (binding: AgentExecutionAuthBinding) => void; + authProfileStateMode?: "read-write" | "read-only"; + /** Ring-zero tool override, supplied only by the OpenClaw orchestrator. */ + systemAgentTool?: SystemAgentToolOptions; +}; + +export type RunEmbeddedAgentParamsWithSessionFile = RunEmbeddedAgentInternalParams & { + sessionFile: string; +}; diff --git a/src/agents/embedded-agent-runner/run/overflow-context-recovery.ts b/src/agents/embedded-agent-runner/run/overflow-context-recovery.ts new file mode 100644 index 000000000000..e39abe322905 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/overflow-context-recovery.ts @@ -0,0 +1,433 @@ +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js"; +import type { ContextEngine, ContextEngineSessionTarget } from "../../../context-engine/types.js"; +import { formatErrorMessage } from "../../../infra/errors.js"; +import { resolveProcessToolScopeKey } from "../../agent-tools.js"; +import { listActiveProcessSessionReferences } from "../../bash-process-references.js"; +import { + extractObservedOverflowTokenCount, + isCompactionFailureError, + isLikelyContextOverflowError, +} from "../../embedded-agent-helpers.js"; +import { buildEmbeddedCompactionRuntimeContext } from "../compaction-runtime-context.js"; +import { + compactContextEngineWithSafetyTimeout, + resolveCompactionTimeoutMs, +} from "../compaction-safety-timeout.js"; +import { resolveContextEngineCapabilities } from "../context-engine-capabilities.js"; +import { runContextEngineMaintenance } from "../context-engine-maintenance.js"; +import { log } from "../logger.js"; +import { + resolveLiveToolResultMaxChars, + sessionLikelyHasOversizedToolResults, + truncateOversizedToolResultsInActiveTarget, +} from "../tool-result-truncation.js"; +import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import { createCompactionDiagId } from "./helpers.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { + buildContextEngineCompactionSessionTarget, + isNoRealConversationCompactionNoop, + resetNoRealConversationTokenSnapshot, +} from "./session-bootstrap.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3; + +type CompactResult = Awaited>; + +type ActiveSession = { + id: string; + file: string; + target?: ContextEngineSessionTarget; +}; + +export type EmbeddedRunOverflowRecoveryOutcome = + | { action: "none" } + | { action: "retry" } + | { + action: "surface"; + kind: "compaction_failure" | "context_overflow"; + errorText: string; + userText: string; + }; + +export async function recoverEmbeddedRunOverflow(input: { + runParams: RunEmbeddedAgentParams; + state: EmbeddedRunContextRecoveryState; + contextEngine: ContextEngine; + contextTokenBudget?: number; + genericCompactionRecoveryAllowed: boolean; + aborted: boolean; + signalOwnedInterruption: boolean; + promptError: unknown; + assistantErrorText?: string; + attempt: EmbeddedRunAttemptResult; + attemptCompactionCount: number; + runtimeAuthPlan: Parameters[0]["runtimeAuthPlan"]; + resolvedSessionKey: string; + sessionAgentId: string; + agentDir: string; + workspaceDir: string; + provider: string; + modelId: string; + harnessRuntime: string; + thinkLevel: Parameters[0]["thinkLevel"]; + authProfileId?: string; + authProfileIdSource: "auto" | "user"; + resolveContextEnginePluginId: () => string | undefined; + buildRuntimeSettings: (settings: { + tokenBudget?: number | null; + degradedReason?: string | null; + }) => ReturnType; + onCompactionHookMessages: (payload: { + phase: "before" | "after"; + messages: string[]; + }) => Promise; + runOwnsCompactionBeforeHook: (reason: string) => Promise; + runOwnsCompactionAfterHook: ( + reason: string, + result: CompactResult, + previousSessionId?: string, + ) => Promise; + adoptCompactionTranscript: (result: CompactResult) => Promise; + getActiveSession: () => ActiveSession; + prepareCurrentTranscriptRetry: () => void; + prepareCompactedTranscriptRetry: () => Promise; + armPostCompactionGuard: () => void; +}): Promise { + const contextOverflowError = + !input.aborted && !input.signalOwnedInterruption + ? (() => { + if (input.promptError) { + const errorText = formatErrorMessage(input.promptError); + if (isLikelyContextOverflowError(errorText)) { + return { text: errorText, source: "promptError" as const }; + } + // A non-overflow prompt failure must not inherit a stale assistant + // error from the previous transcript leaf. + return null; + } + if (input.assistantErrorText && isLikelyContextOverflowError(input.assistantErrorText)) { + return { text: input.assistantErrorText, source: "assistantError" as const }; + } + return null; + })() + : null; + if ( + !contextOverflowError || + !input.genericCompactionRecoveryAllowed || + input.contextTokenBudget === undefined + ) { + return { action: "none" }; + } + + const runParams = input.runParams; + const overflowDiagId = createCompactionDiagId(); + const errorText = contextOverflowError.text; + const observedOverflowTokens = extractObservedOverflowTokenCount(errorText); + const preflightRecovery = input.attempt.preflightRecovery; + const preflightEstimatedPromptTokens = + typeof preflightRecovery?.estimatedPromptTokens === "number" && + Number.isFinite(preflightRecovery.estimatedPromptTokens) && + preflightRecovery.estimatedPromptTokens > 0 + ? Math.ceil(preflightRecovery.estimatedPromptTokens) + : undefined; + const overflowTokenCountForCompaction = + observedOverflowTokens ?? + preflightEstimatedPromptTokens ?? + (input.contextTokenBudget > 0 ? input.contextTokenBudget + 1 : undefined); + const activeSession = input.getActiveSession(); + log.warn( + `[context-overflow-diag] sessionKey=${runParams.sessionKey ?? runParams.sessionId} ` + + `provider=${input.provider}/${input.modelId} source=${contextOverflowError.source} ` + + `messages=${input.attempt.messagesSnapshot?.length ?? 0} sessionFile=${activeSession.file} ` + + `diagId=${overflowDiagId} compactionAttempts=${input.state.overflowCompactionAttempts} ` + + `observedTokens=${observedOverflowTokens ?? "unknown"} ` + + `preflightEstimatedTokens=${preflightEstimatedPromptTokens ?? "unknown"} ` + + `compactionTokens=${overflowTokenCountForCompaction ?? "unknown"} ` + + `error=${truncateUtf16Safe(errorText, 200)}`, + ); + + const isCompactionFailure = isCompactionFailureError(errorText); + if ( + !isCompactionFailure && + input.attemptCompactionCount > 0 && + input.state.overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS + ) { + input.state.overflowCompactionAttempts += 1; + log.warn( + `context overflow persisted after in-attempt compaction (attempt ${input.state.overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); retrying prompt without additional compaction for ${input.provider}/${input.modelId}`, + ); + if (preflightRecovery?.source === "mid-turn") { + input.prepareCurrentTranscriptRetry(); + } + return { action: "retry" }; + } + + if ( + !isCompactionFailure && + input.attemptCompactionCount === 0 && + input.state.overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS + ) { + if (log.isEnabled("debug")) { + log.debug( + `[compaction-diag] decision diagId=${overflowDiagId} branch=compact ` + + `isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` + + `attempt=${input.state.overflowCompactionAttempts + 1} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`, + ); + } + input.state.overflowCompactionAttempts += 1; + log.warn( + `context overflow detected (attempt ${input.state.overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); attempting auto-compaction for ${input.provider}/${input.modelId}`, + ); + let compactResult: CompactResult; + let previousSessionId: string | undefined; + await input.runOwnsCompactionBeforeHook("overflow recovery"); + try { + const sessionBeforeCompaction = input.getActiveSession(); + const overflowCompactionRuntimeContext = { + ...buildEmbeddedCompactionRuntimeContext({ + sessionKey: runParams.sessionKey, + messageChannel: runParams.messageChannel, + messageProvider: runParams.messageProvider, + clientCaps: runParams.clientCaps, + chatType: runParams.chatType, + agentAccountId: runParams.agentAccountId, + currentChannelId: runParams.currentChannelId, + currentThreadTs: runParams.currentThreadTs, + currentMessageId: runParams.currentMessageId, + authProfileId: input.authProfileId, + authProfileIdSource: input.authProfileIdSource, + runtimeAuthPlan: input.runtimeAuthPlan, + workspaceDir: input.workspaceDir, + agentDir: input.agentDir, + config: runParams.config, + skillsSnapshot: runParams.skillsSnapshot, + senderId: runParams.senderId, + provider: input.provider, + modelId: input.modelId, + harnessRuntime: input.harnessRuntime, + modelSelectionLocked: runParams.modelSelectionLocked, + modelFallbacksOverride: runParams.modelFallbacksOverride, + thinkLevel: input.thinkLevel, + reasoningLevel: runParams.reasoningLevel, + bashElevated: runParams.bashElevated, + extraSystemPrompt: runParams.extraSystemPrompt, + sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode, + ownerNumbers: runParams.ownerNumbers, + activeProcessSessions: listActiveProcessSessionReferences({ + scopeKey: resolveProcessToolScopeKey({ + sessionKey: runParams.sandboxSessionKey?.trim() || runParams.sessionKey, + sessionId: sessionBeforeCompaction.id, + agentId: input.sessionAgentId, + }), + }), + }), + ...resolveContextEngineCapabilities({ + config: runParams.config, + sessionKey: runParams.sessionKey, + agentId: input.sessionAgentId, + contextEnginePluginId: input.resolveContextEnginePluginId(), + purpose: "context-engine.overflow-compaction", + }), + onCompactionHookMessages: input.onCompactionHookMessages, + ...(input.attempt.promptCache ? { promptCache: input.attempt.promptCache } : {}), + runId: runParams.runId, + trigger: "overflow", + ...(overflowTokenCountForCompaction !== undefined + ? { currentTokenCount: overflowTokenCountForCompaction } + : {}), + diagId: overflowDiagId, + attempt: input.state.overflowCompactionAttempts, + maxAttempts: MAX_OVERFLOW_COMPACTION_ATTEMPTS, + }; + const overflowCompactionRuntimeSettings = input.buildRuntimeSettings({ + tokenBudget: input.contextTokenBudget, + degradedReason: "context_overflow", + }); + compactResult = await compactContextEngineWithSafetyTimeout( + input.contextEngine, + { + sessionId: sessionBeforeCompaction.id, + sessionKey: input.resolvedSessionKey, + agentId: input.sessionAgentId, + sessionTarget: buildContextEngineCompactionSessionTarget({ + agentId: input.sessionAgentId, + config: runParams.config, + sessionFile: sessionBeforeCompaction.file, + sessionId: sessionBeforeCompaction.id, + sessionKey: input.resolvedSessionKey, + sessionTarget: sessionBeforeCompaction.target, + }), + tokenBudget: input.contextTokenBudget, + ...(overflowTokenCountForCompaction !== undefined + ? { currentTokenCount: overflowTokenCountForCompaction } + : {}), + force: true, + compactionTarget: "budget", + runtimeContext: overflowCompactionRuntimeContext, + runtimeSettings: overflowCompactionRuntimeSettings, + }, + resolveCompactionTimeoutMs(runParams.config), + runParams.abortSignal, + ); + if (compactResult.ok && compactResult.compacted) { + previousSessionId = await input.adoptCompactionTranscript(compactResult); + const sessionAfterCompaction = input.getActiveSession(); + await runContextEngineMaintenance({ + contextEngine: input.contextEngine, + sessionId: sessionAfterCompaction.id, + sessionKey: runParams.sessionKey, + sessionTarget: sessionAfterCompaction.target, + sessionFile: sessionAfterCompaction.file, + reason: "compaction", + runtimeContext: overflowCompactionRuntimeContext, + runtimeSettings: overflowCompactionRuntimeSettings, + config: runParams.config, + agentId: input.sessionAgentId, + }); + } + } catch (compactErr) { + log.warn( + `contextEngine.compact() threw during overflow recovery for ${input.provider}/${input.modelId}: ${String(compactErr)}`, + ); + compactResult = { ok: false, compacted: false, reason: String(compactErr) }; + } + await input.runOwnsCompactionAfterHook("overflow recovery", compactResult, previousSessionId); + + if (preflightRecovery && isNoRealConversationCompactionNoop(compactResult)) { + input.state.lastCompactionTokensAfter = undefined; + input.state.lastContextBudgetStatus = undefined; + await resetNoRealConversationTokenSnapshot({ + config: runParams.config, + sessionKey: runParams.sessionKey, + agentId: input.sessionAgentId, + }); + log.info( + `[context-overflow-precheck] stale token state had no real conversation messages for ` + + `${input.provider}/${input.modelId}; resetting the context snapshot and retrying prompt`, + ); + if (preflightRecovery.source === "mid-turn") { + input.prepareCurrentTranscriptRetry(); + } + return { action: "retry" }; + } + + if (compactResult.compacted) { + await input.adoptCompactionTranscript(compactResult); + const tokensAfter = compactResult.result?.tokensAfter; + if (typeof tokensAfter === "number" && Number.isFinite(tokensAfter) && tokensAfter >= 0) { + input.state.lastCompactionTokensAfter = Math.floor(tokensAfter); + } + if (preflightRecovery?.route === "compact_then_truncate") { + const sessionAfterCompaction = input.getActiveSession(); + const truncResult = await truncateOversizedToolResultsInActiveTarget({ + scope: { + sessionId: sessionAfterCompaction.id, + sessionKey: runParams.sessionKey ?? sessionAfterCompaction.id, + sessionFile: sessionAfterCompaction.file, + agentId: input.sessionAgentId, + }, + contextWindowTokens: input.contextTokenBudget, + maxCharsOverride: resolveLiveToolResultMaxChars({ + contextWindowTokens: input.contextTokenBudget, + cfg: runParams.config, + agentId: input.sessionAgentId, + }), + config: runParams.config, + protectTrailingToolResults: true, + }); + if (truncResult.truncated) { + log.info( + `[context-overflow-precheck] post-compaction tool-result truncation succeeded for ${input.provider}/${input.modelId}; truncated ${truncResult.truncatedCount} tool result(s)`, + ); + } else { + log.warn( + `[context-overflow-precheck] post-compaction tool-result truncation did not help for ${input.provider}/${input.modelId}: ${truncResult.reason ?? "unknown"}`, + ); + } + } + input.state.autoCompactionCount += 1; + log.info(`auto-compaction succeeded for ${input.provider}/${input.modelId}; retrying prompt`); + input.armPostCompactionGuard(); + if (preflightRecovery?.source === "mid-turn") { + input.prepareCurrentTranscriptRetry(); + } else { + await input.prepareCompactedTranscriptRetry(); + } + return { action: "retry" }; + } + log.warn( + `auto-compaction failed for ${input.provider}/${input.modelId}: ${compactResult.reason ?? "nothing to compact"}`, + ); + } + + if (!input.state.toolResultTruncationAttempted) { + const toolResultMaxChars = resolveLiveToolResultMaxChars({ + contextWindowTokens: input.contextTokenBudget, + cfg: runParams.config, + agentId: input.sessionAgentId, + }); + const hasOversized = input.attempt.messagesSnapshot + ? sessionLikelyHasOversizedToolResults({ + messages: input.attempt.messagesSnapshot, + contextWindowTokens: input.contextTokenBudget, + maxCharsOverride: toolResultMaxChars, + }) + : false; + if (hasOversized) { + input.state.toolResultTruncationAttempted = true; + log.warn( + `[context-overflow-recovery] Attempting tool result truncation for ${input.provider}/${input.modelId} ` + + `(contextWindow=${input.contextTokenBudget} tokens)`, + ); + const session = input.getActiveSession(); + const truncResult = await truncateOversizedToolResultsInActiveTarget({ + scope: { + sessionId: session.id, + sessionKey: runParams.sessionKey ?? session.id, + sessionFile: session.file, + agentId: input.sessionAgentId, + }, + contextWindowTokens: input.contextTokenBudget, + maxCharsOverride: toolResultMaxChars, + config: runParams.config, + protectTrailingToolResults: preflightRecovery?.route === "compact_then_truncate", + }); + if (truncResult.truncated) { + log.info( + `[context-overflow-recovery] Truncated ${truncResult.truncatedCount} tool result(s); retrying prompt`, + ); + if (preflightRecovery?.source === "mid-turn") { + input.prepareCurrentTranscriptRetry(); + } + return { action: "retry" }; + } + log.warn( + `[context-overflow-recovery] Tool result truncation did not help: ${truncResult.reason ?? "unknown"}`, + ); + } + } + + if ( + (isCompactionFailure || + input.state.overflowCompactionAttempts >= MAX_OVERFLOW_COMPACTION_ATTEMPTS) && + log.isEnabled("debug") + ) { + log.debug( + `[compaction-diag] decision diagId=${overflowDiagId} branch=give_up ` + + `isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` + + `attempt=${input.state.overflowCompactionAttempts} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`, + ); + } + const kind = isCompactionFailure ? "compaction_failure" : "context_overflow"; + const userText = + "Context overflow: prompt too large for the model. " + + "Try /reset (or /new) to start a fresh session, or use a larger-context model."; + log.warn( + `[context-overflow-recovery] exhausted provider overflow recovery for ${input.provider}/${input.modelId}; ` + + `livenessState=blocked suggestedAction=reset_or_new kind=${kind}`, + ); + return { action: "surface", kind, errorText, userText }; +} diff --git a/src/agents/embedded-agent-runner/run/prompt-failure.ts b/src/agents/embedded-agent-runner/run/prompt-failure.ts new file mode 100644 index 000000000000..4d685a5f0f8f --- /dev/null +++ b/src/agents/embedded-agent-runner/run/prompt-failure.ts @@ -0,0 +1,327 @@ +import type { ThinkLevel } from "../../../auto-reply/thinking.js"; +import { formatErrorMessage, toErrorObject } from "../../../infra/errors.js"; +import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js"; +import { + classifyFailoverReason, + type FailoverReason, + isFailoverErrorMessage, + parseImageSizeError, + pickFallbackThinkingLevel, +} from "../../embedded-agent-helpers.js"; +import { + coerceToFailoverError, + describeFailoverError, + FailoverError, + resolveFailoverStatus, +} from "../../failover-error.js"; +import { + resolveSessionSuspensionReason, + type SessionSuspensionParams, +} from "../../session-suspension.js"; +import { log } from "../logger.js"; +import type { EmbeddedAgentMeta, EmbeddedAgentRunResult, TraceAttempt } from "../types.js"; +import { isShortWindowRateLimitMessage } from "./assistant-failover.js"; +import { buildEmbeddedRunBlockedResult } from "./blocked-run-result.js"; +import { createFailoverDecisionLogger } from "./failover-observation.js"; +import { mergeRetryFailoverReason, resolveRunFailoverDecision } from "./failover-policy.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +type PromptFailureOutcome = + | { + action: "retry"; + thinkLevel: ThinkLevel; + authRetryPending: boolean; + lastRetryFailoverReason: FailoverReason | null; + } + | { action: "complete"; result: EmbeddedAgentRunResult }; + +export async function handleEmbeddedPromptFailure(input: { + runParams: RunEmbeddedAgentParams; + attempt: EmbeddedRunAttemptResult; + promptError: unknown; + promptErrorSource: EmbeddedRunAttemptResult["promptErrorSource"]; + activeErrorContext: { provider: string; model: string }; + provider: string; + modelId: string; + authProfileId?: string; + authProfileStore: AuthProfileStore; + sessionIdUsed: string; + lane: string; + agentDir: string; + suspensionSessionId: string; + runtimeAuthRetry: boolean; + maybeRefreshRuntimeAuthForAuthError: (errorText: string, retry: boolean) => Promise; + suspendForFailure: (params: Omit) => void; + resolveReplayInvalid: () => boolean; + setTerminalLifecycleMeta: NonNullable; + buildErrorAgentMeta: () => EmbeddedAgentMeta; + startedAtMs: number; + fallbackConfigured: boolean; + aborted: boolean; + externalAbort: boolean; + pluginHarnessOwnsTransport: boolean; + timedOutByRunBudget: boolean; + resolveAuthProfileFailureReason: ( + reason: FailoverReason | null, + options?: { providerStarted?: boolean; transientRateLimit?: boolean }, + ) => AuthProfileFailureReason | null; + maybeEscalateRateLimitProfileFallback: (params: { + failoverProvider: string; + failoverModel: string; + logFallbackDecision: ReturnType; + }) => void; + advanceAttemptAuthProfile: () => Promise; + maybeMarkAuthProfileFailure: (failure: { + profileId?: string; + reason?: AuthProfileFailureReason | null; + modelId?: string; + }) => Promise; + maybeBackoffBeforeOverloadFailover: (reason: FailoverReason | null) => Promise; + attemptedThinking: Set; + thinkLevel: ThinkLevel; + // Profile rotation resets thinking inside the runtime; read it after advancing. + getThinkLevel: () => ThinkLevel; + traceAttempts: TraceAttempt[]; + previousRetryFailoverReason: FailoverReason | null; +}): Promise { + const promptAuthMode = input.authProfileId + ? input.authProfileStore.profiles?.[input.authProfileId]?.type + : undefined; + const normalizedPromptFailover = coerceToFailoverError(input.promptError, { + provider: input.activeErrorContext.provider, + model: input.activeErrorContext.model, + profileId: input.authProfileId, + authMode: promptAuthMode, + sessionId: input.sessionIdUsed, + lane: input.lane, + }); + const promptErrorDetails = normalizedPromptFailover + ? describeFailoverError(normalizedPromptFailover) + : describeFailoverError(input.promptError); + if (normalizedPromptFailover?.suspend) { + input.suspendForFailure({ + cfg: input.runParams.config, + agentDir: input.agentDir, + sessionId: input.suspensionSessionId, + reason: resolveSessionSuspensionReason(normalizedPromptFailover.reason), + failedProvider: normalizedPromptFailover.provider ?? input.provider, + failedModel: normalizedPromptFailover.model ?? input.modelId, + }); + } + const errorText = promptErrorDetails.message || formatErrorMessage(input.promptError); + if (await input.maybeRefreshRuntimeAuthForAuthError(errorText, input.runtimeAuthRetry)) { + return { + action: "retry", + thinkLevel: input.thinkLevel, + authRetryPending: true, + lastRetryFailoverReason: input.previousRetryFailoverReason, + }; + } + + const blockedResult = resolveBlockedPromptResult(input, errorText); + if (blockedResult) { + return { action: "complete", result: blockedResult }; + } + + const promptFailoverReason = + promptErrorDetails.reason ?? classifyFailoverReason(errorText, { provider: input.provider }); + const promptProfileFailureReason = input.resolveAuthProfileFailureReason(promptFailoverReason, { + providerStarted: input.promptErrorSource === "prompt", + transientRateLimit: + promptFailoverReason === "rate_limit" && isShortWindowRateLimitMessage(errorText), + }); + const promptFailoverFailure = + promptFailoverReason !== null || + isFailoverErrorMessage(errorText, { provider: input.provider }); + const promptTimeoutFallbackSafe = + input.promptErrorSource === "prompt" && + promptFailoverReason === "timeout" && + !input.attempt.codexAppServerFailure && + input.attempt.promptTimeoutOutcome?.replayInvalid !== true && + input.attempt.replayMetadata.replaySafe; + const failedProfileId = input.authProfileId; + const logFailoverDecision = createFailoverDecisionLogger({ + stage: "prompt", + runId: input.runParams.runId, + rawError: errorText, + failoverReason: promptFailoverReason, + profileFailureReason: promptProfileFailureReason, + provider: input.provider, + model: input.modelId, + sourceProvider: input.provider, + sourceModel: input.modelId, + profileId: failedProfileId, + fallbackConfigured: input.fallbackConfigured, + aborted: input.aborted, + }); + if (promptFailoverReason === "rate_limit") { + input.maybeEscalateRateLimitProfileFallback({ + failoverProvider: input.provider, + failoverModel: input.modelId, + logFallbackDecision: logFailoverDecision, + }); + } + let failoverDecision = resolveRunFailoverDecision({ + stage: "prompt", + aborted: input.aborted, + externalAbort: input.externalAbort, + fallbackConfigured: input.fallbackConfigured, + failoverCode: promptErrorDetails.code, + failoverFailure: promptFailoverFailure, + failoverReason: promptFailoverReason, + harnessOwnsTransport: input.pluginHarnessOwnsTransport, + promptTimeoutFallbackSafe, + timedOutByRunBudget: input.timedOutByRunBudget, + profileRotated: false, + }); + if (failoverDecision.action === "rotate_profile" && (await input.advanceAttemptAuthProfile())) { + if (failedProfileId && promptProfileFailureReason) { + void input + .maybeMarkAuthProfileFailure({ + profileId: failedProfileId, + reason: promptProfileFailureReason, + modelId: input.modelId, + }) + .catch((error: unknown) => { + log.warn(`prompt profile failure mark failed: ${String(error)}`); + }); + } + input.traceAttempts.push({ + provider: input.provider, + model: input.modelId, + result: promptFailoverReason === "timeout" ? "timeout" : "rotate_profile", + ...(promptFailoverReason ? { reason: promptFailoverReason } : {}), + stage: "prompt", + }); + const lastRetryFailoverReason = mergeRetryFailoverReason({ + previous: input.previousRetryFailoverReason, + failoverReason: promptFailoverReason, + }); + logFailoverDecision("rotate_profile"); + await input.maybeBackoffBeforeOverloadFailover(promptFailoverReason); + return { + action: "retry", + thinkLevel: input.getThinkLevel(), + authRetryPending: false, + lastRetryFailoverReason, + }; + } + if (failoverDecision.action === "rotate_profile") { + failoverDecision = resolveRunFailoverDecision({ + stage: "prompt", + aborted: input.aborted, + externalAbort: input.externalAbort, + fallbackConfigured: input.fallbackConfigured, + failoverCode: promptErrorDetails.code, + failoverFailure: promptFailoverFailure, + failoverReason: promptFailoverReason, + harnessOwnsTransport: input.pluginHarnessOwnsTransport, + promptTimeoutFallbackSafe, + timedOutByRunBudget: input.timedOutByRunBudget, + profileRotated: true, + }); + } + if (failedProfileId && promptProfileFailureReason) { + try { + await input.maybeMarkAuthProfileFailure({ + profileId: failedProfileId, + reason: promptProfileFailureReason, + modelId: input.modelId, + }); + } catch (error) { + log.warn(`prompt profile failure mark failed: ${String(error)}`); + } + } + const fallbackThinking = pickFallbackThinkingLevel({ + message: errorText, + attempted: input.attemptedThinking, + }); + if (fallbackThinking) { + log.warn( + `unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`, + ); + return { + action: "retry", + thinkLevel: fallbackThinking, + authRetryPending: false, + lastRetryFailoverReason: input.previousRetryFailoverReason, + }; + } + if (failoverDecision.action === "fallback_model") { + const fallbackReason = failoverDecision.reason ?? "unknown"; + const status = resolveFailoverStatus(fallbackReason); + input.traceAttempts.push({ + provider: input.provider, + model: input.modelId, + result: promptFailoverReason === "timeout" ? "timeout" : "fallback_model", + reason: fallbackReason, + stage: "prompt", + ...(typeof status === "number" ? { status } : {}), + }); + logFailoverDecision("fallback_model", { status }); + await input.maybeBackoffBeforeOverloadFailover(promptFailoverReason); + throw ( + normalizedPromptFailover ?? + new FailoverError(errorText, { + reason: fallbackReason, + provider: input.provider, + model: input.modelId, + profileId: input.authProfileId, + authMode: promptAuthMode, + sessionId: input.sessionIdUsed, + lane: input.lane, + status, + }) + ); + } + if (failoverDecision.action === "surface_error") { + input.traceAttempts.push({ + provider: input.provider, + model: input.modelId, + result: promptFailoverReason === "timeout" ? "timeout" : "surface_error", + ...(promptFailoverReason ? { reason: promptFailoverReason } : {}), + stage: "prompt", + }); + logFailoverDecision("surface_error"); + } + throw toErrorObject(input.promptError, "Prompt failed"); +} + +function resolveBlockedPromptResult( + input: Parameters[0], + errorText: string, +): EmbeddedAgentRunResult | undefined { + let text: string; + let errorKind: "role_ordering" | "image_size"; + if (/incorrect role information|roles must alternate/i.test(errorText)) { + text = + "Message ordering conflict - please try again. " + + "If this persists, use /new to start a fresh session."; + errorKind = "role_ordering"; + } else { + const imageSizeError = parseImageSizeError(errorText); + if (!imageSizeError) { + return undefined; + } + const maxMb = imageSizeError.maxMb; + const maxMbLabel = typeof maxMb === "number" && Number.isFinite(maxMb) ? `${maxMb}` : null; + const maxBytesHint = maxMbLabel ? ` (max ${maxMbLabel}MB)` : ""; + text = + `Image too large for the model${maxBytesHint}. ` + + "Please compress or resize the image and try again."; + errorKind = "image_size"; + } + const replayInvalid = input.resolveReplayInvalid(); + input.setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" }); + return buildEmbeddedRunBlockedResult({ + text, + errorKind, + errorMessage: errorText, + durationMs: Date.now() - input.startedAtMs, + agentMeta: input.buildErrorAgentMeta(), + attempt: input.attempt, + replayInvalid, + finalPromptText: input.attempt.finalPromptText, + }); +} diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts new file mode 100644 index 000000000000..02f3923ef6d7 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts @@ -0,0 +1,371 @@ +import type { ContextEngineSessionTarget } from "../../../context-engine/types.js"; +import { createAgentHarnessTaskRuntimeScope } from "../../../tasks/agent-harness-task-runtime-scope.js"; +import type { ToolOutcomeObserver } from "../../agent-tools.before-tool-call.js"; +import type { AuthProfileStore } from "../../auth-profiles.js"; +import type { AgentHarnessRuntimeArtifactBinding } from "../../harness/runtime-artifact.types.js"; +import { applyAuthHeaderOverride, applyLocalNoAuthHeaderOverride } from "../../model-auth.js"; +import type { AgentRuntimePlan } from "../../runtime-plan/types.js"; +import type { SystemAgentToolOptions } from "../../tools/system-agent-tool.js"; +import { runEmbeddedAttemptWithBackend } from "./backend.js"; +import { + EMBEDDED_RUN_LANE_HEARTBEAT_MS, + EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, +} from "./lane-runtime.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js"; +import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js"; + +type InternalRunParams = RunEmbeddedAgentParams & { + sessionFile: string; + systemAgentTool?: SystemAgentToolOptions; +}; + +type AttemptRuntime = { + sessionId: string; + sessionFile: string; + sessionTarget?: ContextEngineSessionTarget; + sessionKey?: string; + trajectorySessionFile: string; + trajectoryRecorder?: EmbeddedRunAttemptTrajectoryRecorder; + workspaceDir: string; + isCanonicalWorkspace: boolean; + agentDir: string; + contextEngine?: EmbeddedRunAttemptParams["contextEngine"]; + contextTokenBudget?: number; + contextWindowInfo?: EmbeddedRunAttemptParams["contextWindowInfo"]; + prompt: string; + provider: string; + modelId: string; + requestedModelId: string; + fallbackActive: boolean; + fallbackReason: string | null; + agentHarnessId: string; + expectedRuntimeArtifact?: AgentHarnessRuntimeArtifactBinding; + runtimePlan: AgentRuntimePlan; + model: EmbeddedRunAttemptParams["model"]; + resolvedApiKey?: string; + authProfileId?: string; + authProfileIdSource: "auto" | "user"; + initialReplayState: NonNullable; + authStorage: EmbeddedRunAttemptParams["authStorage"]; + authProfileStore: AuthProfileStore; + toolAuthProfileStore?: AuthProfileStore; + modelRegistry: EmbeddedRunAttemptParams["modelRegistry"]; + agentId: string; + beforeAgentStartResult: EmbeddedRunAttemptParams["beforeAgentStartResult"]; + thinkLevel: EmbeddedRunAttemptParams["thinkLevel"]; + fastMode: EmbeddedRunAttemptParams["fastMode"]; + fastModeStartedAtMs?: number; + fastModeAutoOnSeconds?: number; + fastModeAutoProgressState?: EmbeddedRunAttemptParams["fastModeAutoProgressState"]; + toolResultFormat: EmbeddedRunAttemptParams["toolResultFormat"]; + skipPreparedUserTurnMessage: boolean; + apiKeyInfo: Parameters[1]; + runtimeAuthActive: boolean; + captureRuntimeArtifact: boolean; +}; + +type AttemptControl = { + lifecycleGeneration: string; + pluginHarnessOwnsTransport: boolean; + laneTaskAbortController: AbortController; + laneTaskReleaseController: AbortController; + noteLaneTaskProgress: () => void; + onToolOutcome: ToolOutcomeObserver; + allocateToolOutcomeOrdinal: (toolCallId?: string) => number; + onToolStreamBoundary: () => void; + onRunProgress: NonNullable; + onToolResult: NonNullable; + onAgentEvent: NonNullable; + onUserMessagePersisted: NonNullable; + onUserMessagePersistenceInvalidated: NonNullable< + EmbeddedRunAttemptParams["onUserMessagePersistenceInvalidated"] + >; + getPostCompactionAbortError: () => Error | undefined; + setPostCompactionAbortController: (controller: AbortController | undefined) => void; + clearPostCompactionAbortController: (controller: AbortController) => void; +}; + +export async function dispatchEmbeddedRunAttempt(input: { + params: InternalRunParams; + runtime: AttemptRuntime; + control: AttemptControl; + bootstrapPromptWarningSignaturesSeen: string[]; + suppressNextUserMessagePersistence: boolean; + beforeAgentFinalizeRevisionAttempts: number; + maxBeforeAgentFinalizeRevisions: number; +}): Promise<{ + rawAttempt: Awaited>; + cancellationRequested: boolean; +}> { + const { params, runtime, control } = input; + const attemptAbortController = new AbortController(); + control.setPostCompactionAbortController(attemptAbortController); + const parentAbortSignal = params.abortSignal; + const relayParentAbort = (): void => { + control.laneTaskAbortController.abort(parentAbortSignal?.reason); + attemptAbortController.abort(parentAbortSignal?.reason); + }; + if (parentAbortSignal?.aborted) { + relayParentAbort(); + } else { + parentAbortSignal?.addEventListener("abort", relayParentAbort, { once: true }); + } + + // Native attempts start the heartbeat only after their own timeout watchdog + // is armed, keeping preflight inside the requested deadline. + let progressInterval: ReturnType | undefined; + const stopLaneProgressHeartbeat = () => { + if (progressInterval) { + clearInterval(progressInterval); + progressInterval = undefined; + } + attemptAbortController.signal.removeEventListener("abort", stopLaneProgressHeartbeat); + }; + const startLaneProgressHeartbeat = () => { + if (progressInterval || attemptAbortController.signal.aborted) { + return; + } + progressInterval = setInterval( + () => control.noteLaneTaskProgress(), + EMBEDDED_RUN_LANE_HEARTBEAT_MS, + ); + progressInterval.unref?.(); + attemptAbortController.signal.addEventListener("abort", stopLaneProgressHeartbeat, { + once: true, + }); + }; + + // Timeout recovery can continue after an attempt returns, but a native + // transport that ignores its timeout releases the lane after one grace. + let timeoutReleaseTimer: ReturnType | undefined; + const clearAttemptTimeoutRelease = () => { + if (timeoutReleaseTimer) { + clearTimeout(timeoutReleaseTimer); + timeoutReleaseTimer = undefined; + } + }; + const armAttemptTimeoutRelease = (reason: Error) => { + if (timeoutReleaseTimer) { + return; + } + timeoutReleaseTimer = setTimeout( + () => control.laneTaskReleaseController.abort(reason), + EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, + ); + timeoutReleaseTimer.unref?.(); + }; + + let cancellationRequested = false; + const rawAttempt = await runEmbeddedAttemptWithBackend({ + sessionId: runtime.sessionId, + sessionKey: runtime.sessionKey, + promptCacheKey: params.promptCacheKey, + sandboxSessionKey: params.sandboxSessionKey, + trigger: params.trigger, + memoryFlushWritePath: params.memoryFlushWritePath, + messageChannel: params.messageChannel, + messageProvider: params.messageProvider, + clientCaps: params.clientCaps, + chatType: params.chatType, + agentAccountId: params.agentAccountId, + messageTo: params.messageTo, + messageThreadId: params.messageThreadId, + messageActionTurnCapability: params.messageActionTurnCapability, + groupId: params.groupId, + groupChannel: params.groupChannel, + groupSpace: params.groupSpace, + memberRoleIds: params.memberRoleIds, + spawnedBy: params.spawnedBy, + isCanonicalWorkspace: runtime.isCanonicalWorkspace, + senderId: params.senderId, + senderName: params.senderName, + senderUsername: params.senderUsername, + senderE164: params.senderE164, + senderIsOwner: params.senderIsOwner, + approvalReviewerDeviceId: params.approvalReviewerDeviceId, + currentChannelId: params.currentChannelId, + chatId: params.chatId, + channelContext: params.channelContext, + currentMessagingTarget: params.currentMessagingTarget, + currentThreadTs: params.currentThreadTs, + currentMessageId: params.currentMessageId, + currentInboundAudio: params.currentInboundAudio, + replyToMode: params.replyToMode, + hasRepliedRef: params.hasRepliedRef, + sessionFile: runtime.sessionFile, + sessionTarget: runtime.sessionTarget, + trajectorySessionFile: runtime.trajectorySessionFile, + trajectoryRecorder: runtime.trajectoryRecorder, + workspaceDir: runtime.workspaceDir, + cwd: params.cwd, + agentDir: runtime.agentDir, + config: params.config, + allowGatewaySubagentBinding: params.allowGatewaySubagentBinding, + ...(runtime.contextEngine + ? { + contextEngine: runtime.contextEngine, + contextTokenBudget: runtime.contextTokenBudget, + contextWindowInfo: runtime.contextWindowInfo, + } + : {}), + skillsSnapshot: params.skillsSnapshot, + prompt: runtime.prompt, + transcriptPrompt: params.transcriptPrompt, + userTurnTranscriptRecorder: params.userTurnTranscriptRecorder, + skipPreparedUserTurnMessage: runtime.skipPreparedUserTurnMessage, + currentInboundEventKind: params.currentInboundEventKind, + currentInboundContext: params.currentInboundContext, + images: params.images, + imageOrder: params.imageOrder, + clientTools: params.clientTools, + disableTools: params.disableTools, + provider: runtime.provider, + modelId: runtime.modelId, + requestedModelId: runtime.requestedModelId, + fallbackActive: runtime.fallbackActive, + fallbackReason: runtime.fallbackReason, + isFinalFallbackAttempt: params.isFinalFallbackAttempt, + agentHarnessId: runtime.agentHarnessId, + agentHarnessRuntimeOverride: runtime.agentHarnessId, + modelSelectionLocked: params.modelSelectionLocked, + ...(runtime.captureRuntimeArtifact ? { captureRuntimeArtifact: true } : {}), + ...(runtime.expectedRuntimeArtifact + ? { expectedRuntimeArtifact: runtime.expectedRuntimeArtifact } + : {}), + ...(params.sessionKey + ? { + agentHarnessTaskRuntimeScope: createAgentHarnessTaskRuntimeScope({ + requesterSessionKey: params.sessionKey, + }), + } + : {}), + runtimePlan: runtime.runtimePlan, + model: applyAuthHeaderOverride( + applyLocalNoAuthHeaderOverride(runtime.model, runtime.apiKeyInfo), + runtime.runtimeAuthActive ? null : runtime.apiKeyInfo, + params.config, + ), + resolvedApiKey: runtime.resolvedApiKey, + authProfileId: runtime.authProfileId, + authProfileIdSource: runtime.authProfileIdSource, + initialReplayState: runtime.initialReplayState, + authStorage: runtime.authStorage, + authProfileStore: runtime.authProfileStore, + toolAuthProfileStore: runtime.toolAuthProfileStore, + modelRegistry: runtime.modelRegistry, + agentId: runtime.agentId, + beforeAgentStartResult: runtime.beforeAgentStartResult, + thinkLevel: runtime.thinkLevel, + onToolOutcome: control.onToolOutcome, + allocateToolOutcomeOrdinal: control.allocateToolOutcomeOrdinal, + onToolStreamBoundary: control.onToolStreamBoundary, + onRunProgress: control.onRunProgress, + fastMode: runtime.fastMode, + fastModeAuto: params.fastMode === "auto", + ...(params.fastMode === "auto" + ? { + fastModeStartedAtMs: runtime.fastModeStartedAtMs, + fastModeAutoOnSeconds: runtime.fastModeAutoOnSeconds, + fastModeAutoProgressState: runtime.fastModeAutoProgressState, + } + : {}), + verboseLevel: params.verboseLevel, + reasoningLevel: params.reasoningLevel, + toolResultFormat: runtime.toolResultFormat, + toolProgressDetail: params.toolProgressDetail, + execOverrides: params.execOverrides, + bashElevated: params.bashElevated, + timeoutMs: params.timeoutMs, + runTimeoutOverrideMs: params.runTimeoutOverrideMs, + runId: params.runId, + lifecycleGeneration: control.lifecycleGeneration, + abortSignal: attemptAbortController.signal, + onAttemptTimeoutArmed: control.pluginHarnessOwnsTransport + ? undefined + : startLaneProgressHeartbeat, + onAttemptTimeout: control.pluginHarnessOwnsTransport ? undefined : armAttemptTimeoutRelease, + onAttemptAbort: () => { + cancellationRequested = true; + if (!params.abortSignal?.aborted) { + params.replyOperation?.abortByUser(); + } + if (!control.pluginHarnessOwnsTransport) { + stopLaneProgressHeartbeat(); + control.laneTaskAbortController.abort(); + } + }, + replyOperation: params.replyOperation, + shouldEmitToolResult: params.shouldEmitToolResult, + shouldEmitToolOutput: params.shouldEmitToolOutput, + onPartialReply: params.onPartialReply, + onAssistantMessageStart: params.onAssistantMessageStart, + onBlockReply: params.onBlockReply, + onBlockReplyFlush: params.onBlockReplyFlush, + blockReplyBreak: params.blockReplyBreak, + blockReplyChunking: params.blockReplyChunking, + onReasoningStream: params.onReasoningStream, + streamReasoningInNonStreamModes: params.streamReasoningInNonStreamModes, + onReasoningEnd: params.onReasoningEnd, + onToolResult: control.onToolResult, + onAgentToolResult: params.onAgentToolResult, + onAgentEvent: control.onAgentEvent, + deferTerminalLifecycle: params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd, + deferTerminalLifecycleEnd: params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd, + onExecutionPhase: params.onExecutionPhase, + extraSystemPrompt: params.extraSystemPrompt, + sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, + taskSuggestionDeliveryMode: params.taskSuggestionDeliveryMode, + inputProvenance: params.inputProvenance, + streamParams: params.streamParams, + modelRun: params.modelRun, + disableTrajectory: params.disableTrajectory, + ...resolveSkillWorkshopAttemptParams(params), + promptMode: params.promptMode, + ownerNumbers: params.ownerNumbers, + enforceFinalTag: params.enforceFinalTag, + silentExpected: params.silentExpected, + suppressLiveStreamOutput: params.suppressLiveStreamOutput, + bootstrapContextMode: params.bootstrapContextMode, + bootstrapContextRunKind: params.bootstrapContextRunKind, + jobId: params.jobId, + toolsAllow: params.toolsAllow, + ...(params.systemAgentTool ? { systemAgentTool: params.systemAgentTool } : {}), + cleanupBundleMcpOnRunEnd: params.cleanupBundleMcpOnRunEnd, + disableMessageTool: params.disableMessageTool, + forceRestartSafeTools: params.forceRestartSafeTools, + forceMessageTool: params.forceMessageTool, + enableHeartbeatTool: params.enableHeartbeatTool, + forceHeartbeatTool: params.forceHeartbeatTool, + requireExplicitMessageTarget: params.requireExplicitMessageTarget, + internalEvents: params.internalEvents, + bootstrapPromptWarningSignaturesSeen: input.bootstrapPromptWarningSignaturesSeen, + bootstrapPromptWarningSignature: + input.bootstrapPromptWarningSignaturesSeen[ + input.bootstrapPromptWarningSignaturesSeen.length - 1 + ], + suppressNextUserMessagePersistence: input.suppressNextUserMessagePersistence, + beforeAgentFinalizeRevisionAttempts: input.beforeAgentFinalizeRevisionAttempts, + maxBeforeAgentFinalizeRevisions: input.maxBeforeAgentFinalizeRevisions, + suppressTranscriptOnlyAssistantPersistence: params.suppressTranscriptOnlyAssistantPersistence, + suppressAssistantErrorPersistence: params.suppressAssistantErrorPersistence, + onUserMessagePersisted: control.onUserMessagePersisted, + onUserMessagePersistenceInvalidated: control.onUserMessagePersistenceInvalidated, + onAssistantErrorMessagePersisted: params.onAssistantErrorMessagePersisted, + }) + .catch((err: unknown): never => { + throw control.getPostCompactionAbortError() ?? err; + }) + .finally(() => { + clearAttemptTimeoutRelease(); + stopLaneProgressHeartbeat(); + parentAbortSignal?.removeEventListener?.("abort", relayParentAbort); + control.clearPostCompactionAbortController(attemptAbortController); + }); + + const postCompactionAbortError = control.getPostCompactionAbortError(); + if (postCompactionAbortError) { + throw postCompactionAbortError; + } + return { rawAttempt, cancellationRequested }; +} diff --git a/src/agents/embedded-agent-runner/run/runtime-preparation.ts b/src/agents/embedded-agent-runner/run/runtime-preparation.ts new file mode 100644 index 000000000000..7c814a1cef9d --- /dev/null +++ b/src/agents/embedded-agent-runner/run/runtime-preparation.ts @@ -0,0 +1,518 @@ +import type { ThinkLevel } from "../../../auto-reply/thinking.js"; +import type { AuthProfileStore } from "../../auth-profiles.js"; +import { isProfileInCooldown } from "../../auth-profiles.js"; +import type { ResolvedProviderAuth } from "../../model-auth.js"; +import { + hasPreparedAuthAttemptModelMetadata, + resolveCredentialScopedAuthAttemptModelDecision, +} from "../../runtime-plan/credential-scoped-model.js"; +import { + canRunPreparedAgentRuntimeAuthAttempt, + type PreparedAgentRuntimeAuthAttempt, +} from "../../runtime-plan/prepare-auth.js"; +import type { AgentRuntimeAuthPlan } from "../../runtime-plan/types.js"; +import { resolveCandidateThinkingLevel } from "../../thinking-runtime.js"; +import { log } from "../logger.js"; +import { + createEmbeddedRunStageTracker, + formatEmbeddedRunStageSummary, +} from "./attempt-stage-timing.js"; +import { + createEmbeddedRunAuthController, + resolveEmbeddedAuthCooldownProbePolicy, +} from "./auth-controller.js"; +import { prepareEmbeddedRunAuthPlan } from "./auth-plan.js"; +import { createScopedAuthProfileStore } from "./auth-store.js"; +import type { RuntimeAuthState } from "./helpers.js"; +import { + resolveEmbeddedRunEffectiveModel, + selectEmbeddedRunHarness, + selectEmbeddedRunHarnessForPreparedAttempts, +} from "./model-harness.js"; +import { resolveEmbeddedRunModelSetup } from "./model-setup.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { resolveInitialThinkLevel } from "./runtime-resolution.js"; + +type ApiKeyInfo = ResolvedProviderAuth; + +export async function prepareEmbeddedRunRuntime(input: { + runParams: RunEmbeddedAgentParams; + provider: string; + modelId: string; + agentDir: string; + workspaceDir: string; + globalLane: string; + hookRunner: Parameters[0]["hookRunner"]; + hookContext: Parameters[0]["hookContext"]; + markStartupStage: (stage: string) => void; + notifyExecutionPhase: ( + phase: Parameters>[0]["phase"], + context?: Omit>[0], "phase">, + ) => void; + fallbackConfigured: boolean; +}) { + const params = input.runParams; + let provider = input.provider; + let modelId = input.modelId; + const modelSetup = await resolveEmbeddedRunModelSetup({ + runParams: params, + provider, + modelId, + agentDir: input.agentDir, + workspaceDir: input.workspaceDir, + globalLane: input.globalLane, + hookRunner: input.hookRunner, + hookContext: input.hookContext, + onHooksResolved: () => input.markStartupStage("hooks"), + }); + provider = modelSetup.provider; + modelId = modelSetup.modelId; + const { + requestedModelId, + modelSelectionChangedByHook, + beforeAgentStartResult, + requestStreamTransportOverrides, + expectedHarnessArtifact, + nativeModelOwnedHarnessId, + nativeModelOwned, + modelConfigProvider, + model, + authStorage, + modelRegistry, + } = modelSetup; + let agentHarness = modelSetup.agentHarness; + let pluginHarnessOwnsTransport = modelSetup.pluginHarnessOwnsTransport; + let runtimeModel = model; + const resolveEffectiveModel = (candidate: typeof runtimeModel) => + resolveEmbeddedRunEffectiveModel({ + runParams: params, + provider, + modelConfigProvider, + modelId, + agentHarnessId: agentHarness.id, + runtimeModel: candidate, + nativeModelOwned, + requestStreamTransportOverrides, + nativeModelOwnedHarnessId, + }); + const initialResolvedRuntimeModel = resolveEffectiveModel(runtimeModel); + let contextTokenBudget = initialResolvedRuntimeModel.contextTokenBudget; + let contextWindowInfo = initialResolvedRuntimeModel.contextWindowInfo; + let outerContextTokenMeta: { contextTokens?: number } = + contextTokenBudget === undefined ? {} : { contextTokens: contextTokenBudget }; + let effectiveModel = initialResolvedRuntimeModel.effectiveModel; + const applyResolvedRuntimeModel = ( + candidate: typeof runtimeModel, + resolved = resolveEffectiveModel(candidate), + ) => { + runtimeModel = candidate; + effectiveModel = resolved.effectiveModel; + contextTokenBudget = resolved.contextTokenBudget; + contextWindowInfo = resolved.contextWindowInfo; + outerContextTokenMeta = + contextTokenBudget === undefined ? {} : { contextTokens: contextTokenBudget }; + }; + const selectHarnessForModel = ( + candidate: typeof effectiveModel, + plan?: AgentRuntimeAuthPlan, + preparedAuthAttempt?: PreparedAgentRuntimeAuthAttempt, + ) => + selectEmbeddedRunHarness({ + runParams: params, + provider, + modelId, + model: candidate, + plan, + preparedAuthAttempt, + requestStreamTransportOverrides, + nativeModelOwnedHarnessId, + }); + const selectHarnessForPreparedAttempts = ( + candidate: typeof effectiveModel, + attempts: readonly PreparedAgentRuntimeAuthAttempt[], + ) => + selectEmbeddedRunHarnessForPreparedAttempts({ + runParams: params, + provider, + modelId, + model: candidate, + attempts, + requestStreamTransportOverrides, + nativeModelOwnedHarnessId, + }); + input.markStartupStage("model-resolution"); + input.notifyExecutionPhase("model_resolution", { provider, model: modelId }); + + agentHarness = selectHarnessForModel(effectiveModel); + pluginHarnessOwnsTransport = agentHarness.id !== "openclaw"; + const authStages = log.isEnabled("trace") ? createEmbeddedRunStageTracker() : undefined; + const preparedAuthPlan = await prepareEmbeddedRunAuthPlan({ + runParams: params, + provider, + modelId, + model, + agentDir: input.agentDir, + workspaceDir: input.workspaceDir, + requestStreamTransportOverrides, + nativeModelOwned, + authStorage, + modelRegistry, + getAgentHarness: () => agentHarness, + setAgentHarness: (nextHarness) => { + agentHarness = nextHarness; + pluginHarnessOwnsTransport = agentHarness.id !== "openclaw"; + }, + getRuntimeModel: () => runtimeModel, + getEffectiveModel: () => effectiveModel, + applyResolvedRuntimeModel, + selectHarnessForPreparedAttempts, + markStage: (stage) => authStages?.mark(stage), + }); + const { + usesOpenAIAuthRouting, + attemptAuthProfileStore, + lockedProfileId, + preferredProfileId, + providerUsesProfileScopedModelMetadata, + materializeAuthPlan, + materializeAuthPlanUncached, + preparedAuthAttempts, + } = preparedAuthPlan; + let { activePreparedAuthPlan } = preparedAuthPlan; + const genericCompactionRecoveryAllowed = !pluginHarnessOwnsTransport; + const profileCandidates = preparedAuthAttempts.map((attempt) => attempt.profileId); + const forwardedPluginHarnessProfileId = pluginHarnessOwnsTransport + ? activePreparedAuthPlan.forwardedAuthProfileId + : undefined; + let profileIndex = 0; + const requestedThinkLevel = resolveInitialThinkLevel({ + requested: params.thinkLevel, + config: params.config, + provider, + modelId, + model: effectiveModel, + }); + const initialThinkLevel = modelSelectionChangedByHook + ? (resolveCandidateThinkingLevel({ + cfg: params.config, + provider, + modelId, + level: requestedThinkLevel, + catalog: [ + { + provider, + id: modelId, + api: effectiveModel.api, + reasoning: effectiveModel.reasoning, + params: effectiveModel.params, + compat: effectiveModel.compat, + }, + ], + agentId: params.agentId, + sessionKey: params.sessionKey, + agentRuntime: agentHarness.id, + }) ?? requestedThinkLevel) + : requestedThinkLevel; + let thinkLevel = initialThinkLevel; + const attemptedThinking = new Set(); + let apiKeyInfo: ApiKeyInfo | null = null; + let lastProfileId: string | undefined; + let runtimeAuthState: RuntimeAuthState | null = null; + let runtimeAuthRefreshCancelled = false; + const pluginHarnessOwnsAuthBootstrap = + pluginHarnessOwnsTransport && agentHarness.authBootstrap === "harness"; + const preparedApiKeyRoute = activePreparedAuthPlan.modelRoute?.authRequirement === "api-key"; + const pluginHarnessHasPreparedApiKeyAttempt = preparedAuthAttempts.some( + (attempt) => attempt.plan.modelRoute?.authRequirement === "api-key", + ); + const pluginHarnessNeedsOpenClawAuthBootstrap = + pluginHarnessOwnsTransport && + usesOpenAIAuthRouting && + (preparedApiKeyRoute || + (!pluginHarnessOwnsAuthBootstrap && + profileCandidates.some((profileId) => Boolean(profileId)))); + const findPreparedAuthAttempt = (profileId: string | undefined, attemptIndex?: number) => { + const attempt = + attemptIndex === undefined + ? preparedAuthAttempts.find((candidate) => candidate.profileId === profileId) + : preparedAuthAttempts[attemptIndex]; + return attempt?.profileId === profileId ? attempt : undefined; + }; + let preparedProfileAttempted = false; + const prepareAuthAttempt = async (attempt: (typeof preparedAuthAttempts)[number]) => { + if ( + !canRunPreparedAgentRuntimeAuthAttempt({ + attempt, + priorProfileAttempted: preparedProfileAttempted, + }) + ) { + throw new Error( + `Prepared direct auth fallback cannot bypass unavailable profiles for ${provider}/${modelId}.`, + ); + } + const modelDecision = resolveCredentialScopedAuthAttemptModelDecision({ + attempt, + priorProfileAttempted: preparedProfileAttempted, + requestedProfileId: params.authProfileId, + providerUsesProfileScopedModelMetadata, + }); + const nextRuntimeModel = modelDecision.shouldMaterialize + ? modelDecision.forceResolve + ? await materializeAuthPlanUncached(attempt.plan, true) + : await materializeAuthPlan(attempt.plan) + : runtimeModel; + const nextResolvedModel = resolveEffectiveModel(nextRuntimeModel); + const nextHarness = selectHarnessForPreparedAttempts( + nextResolvedModel.effectiveModel, + preparedAuthAttempts, + ); + if (nextHarness.id !== agentHarness.id) { + throw new Error( + `Prepared auth retry changed the selected agent harness for ${provider}/${modelId}.`, + ); + } + preparedProfileAttempted ||= attempt.kind === "profile"; + return { + runtimeModel: nextRuntimeModel, + authRequirement: modelDecision.authRequirement, + allowAuthProfileFallback: attempt.allowAuthProfileFallback, + commit() { + applyResolvedRuntimeModel(nextRuntimeModel, nextResolvedModel); + activePreparedAuthPlan = attempt.plan; + }, + }; + }; + const hasPreparedAuthAttemptMetadata = hasPreparedAuthAttemptModelMetadata({ + attempts: preparedAuthAttempts, + providerUsesProfileScopedModelMetadata, + }); + const prepareModelForAuthProfile = + hasPreparedAuthAttemptMetadata && + (!pluginHarnessOwnsAuthBootstrap || pluginHarnessHasPreparedApiKeyAttempt) + ? async (profileId: string | undefined, attemptIndex?: number) => { + const attempt = findPreparedAuthAttempt(profileId, attemptIndex); + if (!attempt) { + throw new Error( + `Auth profile "${profileId ?? "(none)"}" is outside the prepared attempts for ${provider}/${modelId}.`, + ); + } + const prepared = await prepareAuthAttempt(attempt); + if (attempt.plan.modelRoute && !prepared.authRequirement) { + throw new Error(`Prepared route metadata is missing for ${provider}/${modelId}.`); + } + return { + runtimeModel: prepared.runtimeModel, + authRequirement: prepared.authRequirement, + allowAuthProfileFallback: prepared.allowAuthProfileFallback, + commit: () => prepared.commit(), + }; + } + : undefined; + const authController = createEmbeddedRunAuthController({ + config: params.config, + agentDir: input.agentDir, + workspaceDir: input.workspaceDir, + authStore: attemptAuthProfileStore, + authStorage, + profileCandidates, + lockedProfileId, + initialThinkLevel, + attemptedThinking, + fallbackConfigured: input.fallbackConfigured, + allowTransientCooldownProbe: params.allowTransientCooldownProbe === true, + getProvider: () => provider, + getModelId: () => modelId, + getRuntimeModel: () => runtimeModel, + setRuntimeModel: (next) => { + runtimeModel = next; + }, + getEffectiveModel: () => effectiveModel, + setEffectiveModel: (next) => { + effectiveModel = next; + }, + getApiKeyInfo: () => apiKeyInfo, + setApiKeyInfo: (next) => { + apiKeyInfo = next; + }, + getLastProfileId: () => lastProfileId, + setLastProfileId: (next) => { + lastProfileId = next; + }, + getRuntimeAuthState: () => runtimeAuthState, + setRuntimeAuthState: (next) => { + runtimeAuthState = next; + }, + getRuntimeAuthRefreshCancelled: () => runtimeAuthRefreshCancelled, + setRuntimeAuthRefreshCancelled: (next) => { + runtimeAuthRefreshCancelled = next; + }, + getProfileIndex: () => profileIndex, + setProfileIndex: (next) => { + profileIndex = next; + }, + ...(prepareModelForAuthProfile ? { prepareModelForAuthProfile } : {}), + setThinkLevel: (next) => { + thinkLevel = next; + }, + log, + }); + authStages?.mark("controller"); + const advancePluginHarnessAuthAttempt = async (): Promise => { + if (!pluginHarnessOwnsTransport || lockedProfileId) { + return false; + } + let nextIndex = profileIndex + 1; + while (nextIndex < preparedAuthAttempts.length) { + const candidateAttempt = preparedAuthAttempts[nextIndex]; + if (!candidateAttempt) { + nextIndex += 1; + continue; + } + const candidate = candidateAttempt.profileId; + if ( + candidate && + isProfileInCooldown(attemptAuthProfileStore, candidate, undefined, modelId) + ) { + nextIndex += 1; + continue; + } + if ( + !canRunPreparedAgentRuntimeAuthAttempt({ + attempt: candidateAttempt, + priorProfileAttempted: preparedProfileAttempted, + }) + ) { + return false; + } + if (candidateAttempt.plan.modelRoute?.authRequirement === "api-key") { + try { + await authController.applyAuthProfileCandidate(candidate, nextIndex); + profileIndex = nextIndex; + thinkLevel = initialThinkLevel; + attemptedThinking.clear(); + return true; + } catch { + nextIndex += 1; + continue; + } + } + if (!candidate || candidateAttempt.plan.forwardedAuthProfileId !== candidate) { + nextIndex += 1; + continue; + } + const prepared = await prepareAuthAttempt(candidateAttempt); + authController.stopRuntimeAuthRefreshTimer(); + apiKeyInfo = null; + runtimeAuthState = null; + prepared.commit(); + profileIndex = nextIndex; + lastProfileId = candidate; + thinkLevel = initialThinkLevel; + attemptedThinking.clear(); + return true; + } + return false; + }; + const advanceAttemptAuthProfile = pluginHarnessOwnsAuthBootstrap + ? advancePluginHarnessAuthAttempt + : authController.advanceAuthProfile; + + if (!pluginHarnessOwnsTransport || pluginHarnessNeedsOpenClawAuthBootstrap) { + await authController.initializeAuthProfile(); + } else if (lockedProfileId) { + lastProfileId = lockedProfileId; + } else if (forwardedPluginHarnessProfileId) { + const initialAttempt = preparedAuthAttempts[profileIndex]; + const initialProfileInCooldown = + initialAttempt?.kind === "profile" && + isProfileInCooldown(attemptAuthProfileStore, initialAttempt.profileId, undefined, modelId); + const cooldownProbePolicy = resolveEmbeddedAuthCooldownProbePolicy({ + authStore: attemptAuthProfileStore, + profileCandidates, + lockedProfileId, + modelId, + allowTransientCooldownProbe: params.allowTransientCooldownProbe === true, + }); + if (initialProfileInCooldown && !cooldownProbePolicy.allowProbe) { + if (!(await advancePluginHarnessAuthAttempt())) { + throw new Error( + `Prepared auth profiles are temporarily unavailable for ${provider}/${modelId}.`, + ); + } + } else { + if (initialProfileInCooldown) { + log.warn( + `probing cooldowned auth profile for ${provider}/${modelId} due to ${cooldownProbePolicy.unavailableReason ?? "transient"} unavailability`, + ); + } + preparedProfileAttempted = initialAttempt?.kind === "profile"; + lastProfileId = forwardedPluginHarnessProfileId; + } + } + authStages?.mark("initialize"); + if (authStages) { + log.trace( + formatEmbeddedRunStageSummary( + `[trace:embedded-run] auth stages: runId=${params.runId} sessionId=${params.sessionId} phase=auth`, + authStages.snapshot(), + ), + ); + } + input.markStartupStage("auth"); + input.notifyExecutionPhase("auth", { provider, model: modelId }); + + return { + provider, + modelId, + requestedModelId, + beforeAgentStartResult, + expectedHarnessArtifact, + nativeModelOwned, + model, + authStorage, + modelRegistry, + attemptAuthProfileStore, + lockedProfileId, + preferredProfileId, + profileCandidates, + profileFailureStore: attemptAuthProfileStore, + genericCompactionRecoveryAllowed, + pluginHarnessOwnsAuthBootstrap, + attemptedThinking, + advanceAttemptAuthProfile, + maybeRefreshRuntimeAuthForAuthError: authController.maybeRefreshRuntimeAuthForAuthError, + stopRuntimeAuthRefreshTimer: authController.stopRuntimeAuthRefreshTimer, + getApiKeyInfo: () => apiKeyInfo, + setThinkLevel: (next: ThinkLevel) => { + thinkLevel = next; + }, + resolveRunAttemptAuthProfileStore: (): AuthProfileStore => { + if (!pluginHarnessOwnsTransport) { + return attemptAuthProfileStore; + } + const activeProfileIds = activePreparedAuthPlan.modelRoute + ? [ + activePreparedAuthPlan.forwardedAuthProfileId, + ...(activePreparedAuthPlan.forwardedAuthProfileCandidateIds ?? []), + ] + : [lastProfileId]; + return createScopedAuthProfileStore( + attemptAuthProfileStore, + activeProfileIds.filter((profileId): profileId is string => Boolean(profileId)), + ); + }, + snapshot: () => ({ + agentHarness, + pluginHarnessOwnsTransport, + effectiveModel, + contextTokenBudget, + contextWindowInfo, + outerContextTokenMeta, + activePreparedAuthPlan, + thinkLevel, + apiKeyInfo, + lastProfileId, + runtimeAuthState, + }), + }; +} diff --git a/src/agents/embedded-agent-runner/run/session-prompt-state.ts b/src/agents/embedded-agent-runner/run/session-prompt-state.ts new file mode 100644 index 000000000000..5fe6569580a9 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/session-prompt-state.ts @@ -0,0 +1,156 @@ +import type { ContextEngineSessionTarget } from "../../../context-engine/types.js"; +import { registerAgentRunContext } from "../../../infra/agent-events.js"; +import { formatErrorMessage } from "../../../infra/errors.js"; +import { resolveAgentRunSessionTarget } from "../../run-session-target.js"; +import { log } from "../logger.js"; +import type { PreparedEmbeddedRunInput } from "./execution-context.js"; +import { buildContextEngineCompactionSessionTarget } from "./session-bootstrap.js"; + +const MID_TURN_PRECHECK_CONTINUATION_PROMPT = + "Continue from the current transcript after the latest tool result. Do not repeat the original user request, and do not rerun completed tools unless the transcript shows they are still needed."; + +type ActivePrompt = { + override?: string; + persisted: boolean; + internal: boolean; +}; + +export function createEmbeddedRunSessionPromptState(input: { + runParams: PreparedEmbeddedRunInput["runParams"]; + sessionAgentId: string; + resolvedSessionKey: string; + lifecycleGeneration: PreparedEmbeddedRunInput["lifecycleGeneration"]; +}) { + const { runParams: params, sessionAgentId, resolvedSessionKey, lifecycleGeneration } = input; + let activeSessionId = params.sessionId; + let activeSessionFile = params.sessionFile; + let activeSessionTarget: ContextEngineSessionTarget | undefined = + buildContextEngineCompactionSessionTarget({ + agentId: params.agentId ?? sessionAgentId, + config: params.config, + sessionFile: activeSessionFile, + sessionId: activeSessionId, + sessionKey: resolvedSessionKey, + sessionTarget: params.sessionTarget, + }); + let suppressNextUserMessagePersistence = params.suppressNextUserMessagePersistence ?? false; + let activePrompt: ActivePrompt = { + persisted: suppressNextUserMessagePersistence, + internal: false, + }; + + const adoptSessionId = (nextSessionId: string | undefined) => { + if (!nextSessionId || nextSessionId === activeSessionId) { + return; + } + activeSessionId = nextSessionId; + // Keep every active-run owner on the rotated identity. Restart recovery + // uses the reply registry while lifecycle persistence uses run context. + params.replyOperation?.updateSessionId(activeSessionId); + params.onSessionIdChanged?.(activeSessionId); + registerAgentRunContext(params.runId, { + sessionId: activeSessionId, + lifecycleGeneration, + }); + }; + const adoptSessionTarget = async (nextSessionTarget: ContextEngineSessionTarget | undefined) => { + if (!nextSessionTarget) { + return; + } + const resolvedTarget = await resolveAgentRunSessionTarget({ + agentId: nextSessionTarget.agentId ?? sessionAgentId, + config: params.config, + sessionId: nextSessionTarget.sessionId ?? activeSessionId, + sessionKey: nextSessionTarget.sessionKey ?? resolvedSessionKey, + sessionTarget: nextSessionTarget, + }); + activeSessionTarget = nextSessionTarget; + activeSessionFile = resolvedTarget.sessionFile; + adoptSessionId(resolvedTarget.sessionId); + }; + const activateInternalPrompt = (prompt: string, persisted: boolean) => { + activePrompt = { override: prompt, persisted, internal: true }; + suppressNextUserMessagePersistence = persisted; + }; + const onUserMessagePersisted: NonNullable< + PreparedEmbeddedRunInput["runParams"]["onUserMessagePersisted"] + > = (message) => { + const messageMetadata = message as { + __openclaw?: { beforeAgentRunBlocked?: unknown }; + }; + const blockedBeforeAgentRun = messageMetadata["__openclaw"]?.beforeAgentRunBlocked; + const markCurrentUserMessagePersisted = () => { + activePrompt.persisted = true; + params.onUserMessagePersisted?.(message); + }; + const recorder = params.userTurnTranscriptRecorder; + if (!recorder) { + markCurrentUserMessagePersisted(); + return; + } + const markWhenPersisted = (persisted: { message?: unknown } | undefined) => { + if (persisted?.message || recorder.hasPersisted()) { + markCurrentUserMessagePersisted(); + } + }; + const canonicalPersistence = + blockedBeforeAgentRun !== undefined + ? recorder.persistBlocked(message) + : recorder.persistApproved(); + const observedPersistence = canonicalPersistence + .then(markWhenPersisted) + .catch((persistError: unknown) => { + log.warn( + `failed to persist canonical ${blockedBeforeAgentRun !== undefined ? "blocked " : ""}embedded user turn transcript: ${formatErrorMessage(persistError)}`, + ); + }); + recorder.markRuntimePersistencePending(observedPersistence); + }; + const waitForCurrentUserMessagePersistence = async () => { + if (params.userTurnTranscriptRecorder?.hasRuntimePersistencePending() === true) { + await params.userTurnTranscriptRecorder.waitForRuntimePersistence(); + } + }; + + return { + get sessionId() { + return activeSessionId; + }, + get sessionFile() { + return activeSessionFile; + }, + set sessionFile(value: string) { + activeSessionFile = value; + }, + get sessionTarget() { + return activeSessionTarget; + }, + set sessionTarget(value: ContextEngineSessionTarget | undefined) { + activeSessionTarget = value; + }, + get activePrompt() { + return activePrompt; + }, + get suppressNextUserMessagePersistence() { + return suppressNextUserMessagePersistence; + }, + set suppressNextUserMessagePersistence(value: boolean) { + suppressNextUserMessagePersistence = value; + }, + adoptSessionId, + adoptSessionTarget, + activateInternalPrompt, + continueFromCurrentTranscript: () => + activateInternalPrompt(MID_TURN_PRECHECK_CONTINUATION_PROMPT, true), + onUserMessagePersisted, + waitForCurrentUserMessagePersistence, + async prepareCompactedTranscriptRetry() { + await waitForCurrentUserMessagePersistence(); + if (activePrompt.internal) { + suppressNextUserMessagePersistence = activePrompt.persisted; + } else if (activePrompt.persisted) { + activateInternalPrompt(MID_TURN_PRECHECK_CONTINUATION_PROMPT, true); + } + }, + }; +} diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.ts new file mode 100644 index 000000000000..40459a786030 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.ts @@ -0,0 +1,228 @@ +import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js"; +import type { AssistantMessage } from "../../../llm/types.js"; +import type { AuthProfileStore } from "../../auth-profiles.js"; +import type { NormalizedUsage, UsageLike } from "../../usage.js"; +import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js"; +import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; +import type { UsageAccumulator } from "../usage-accumulator.js"; +import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import { + buildUsageAgentMetaFields, + resolveFinalAssistantRawText, + resolveFinalAssistantVisibleText, + resolveReportedModelRef, +} from "./helpers.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { buildEmbeddedRunPayloads } from "./payloads.js"; +import { buildTraceToolSummary } from "./run-attempt-result.js"; +import { mergeAttemptToolMediaPayloads } from "./tool-media-payloads.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +export function prepareEmbeddedRunTerminal(input: { + runParams: RunEmbeddedAgentParams; + attempt: EmbeddedRunAttemptResult; + attemptAssistant?: AssistantMessage; + currentAttemptAssistant?: AssistantMessage; + provider: string; + model: string; + activeErrorContext: { provider: string; model: string }; + authProfileStore: AuthProfileStore; + authProfileId?: string; + sessionIdUsed: string; + sessionFileUsed?: string; + outerContextTokenMeta: { contextTokens?: number }; + usageAccumulator: UsageAccumulator; + lastRunPromptUsage?: NormalizedUsage; + lastTurnTotal?: number; + contextRecoveryState: EmbeddedRunContextRecoveryState; + resolvedToolResultFormat: NonNullable; + terminalInterrupted: boolean; + terminalTimedOut: boolean; + timedOutDuringCompaction: boolean; + timedOutDuringToolExecution: boolean; +}): { + agentMeta: EmbeddedAgentMeta; + reportedModelRef: { provider: string; model: string }; + finalAssistantVisibleText: string | undefined; + finalAssistantRawText: string | undefined; + payloads: ReturnType; + payloadsWithToolMedia: ReturnType; + timedOutDuringPrompt: boolean; + recoveredFinalAssistantPayloadsAfterPromptTimeout: EmbeddedAgentRunResult["payloads"]; + hasSuccessfulFinalAssistantAfterPromptTimeout: boolean; + hasPartialAssistantTextAfterPromptTimeout: boolean; + attemptToolSummary: ReturnType; + failureSignal: ReturnType; +} { + const { runParams, attempt, attemptAssistant } = input; + const timedOutDuringPrompt = + input.terminalTimedOut && !input.timedOutDuringCompaction && !input.timedOutDuringToolExecution; + // A prior same-model assistant can remain in the session snapshot. Timeout + // recovery must project only output owned by the prompt that just timed out. + const terminalAssistant = timedOutDuringPrompt ? input.currentAttemptAssistant : attemptAssistant; + const usageMeta = buildUsageAgentMetaFields({ + usageAccumulator: input.usageAccumulator, + lastAssistantUsage: terminalAssistant?.usage as UsageLike | undefined, + lastRunPromptUsage: input.lastRunPromptUsage, + lastTurnTotal: input.lastTurnTotal, + }); + const reportedModelRef = resolveReportedModelRef({ + provider: input.provider, + model: input.model, + assistant: terminalAssistant, + }); + const agentMeta: EmbeddedAgentMeta = { + sessionId: input.sessionIdUsed, + sessionFile: input.sessionFileUsed, + provider: reportedModelRef.provider, + model: reportedModelRef.model, + ...input.outerContextTokenMeta, + agentHarnessId: attempt.agentHarnessId, + usage: usageMeta.usage, + lastCallUsage: usageMeta.lastCallUsage, + promptTokens: usageMeta.promptTokens, + ...(input.contextRecoveryState.lastContextBudgetStatus + ? { contextBudgetStatus: input.contextRecoveryState.lastContextBudgetStatus } + : {}), + compactionCount: + input.contextRecoveryState.autoCompactionCount > 0 + ? input.contextRecoveryState.autoCompactionCount + : undefined, + compactionTokensAfter: input.contextRecoveryState.lastCompactionTokensAfter, + }; + const finalAssistantVisibleText = resolveFinalAssistantVisibleText(terminalAssistant); + const finalAssistantRawText = resolveFinalAssistantRawText(terminalAssistant); + const payloads = buildEmbeddedRunPayloads({ + assistantTexts: attempt.assistantTexts, + assistantMessageIndex: attempt.lastAssistantTextMessageIndex, + assistantTranscriptOwned: attempt.assistantTranscriptOwned, + toolMetas: attempt.toolMetas, + lastAssistant: timedOutDuringPrompt ? input.currentAttemptAssistant : attempt.lastAssistant, + currentAssistant: input.currentAttemptAssistant ?? null, + lastToolError: attempt.lastToolError, + config: runParams.config, + isCronTrigger: runParams.trigger === "cron", + isHeartbeatTrigger: runParams.trigger === "heartbeat", + sessionKey: runParams.sessionKey ?? runParams.sessionId, + provider: input.activeErrorContext.provider, + model: input.activeErrorContext.model, + authMode: input.authProfileId + ? input.authProfileStore.profiles?.[input.authProfileId]?.type + : undefined, + verboseLevel: runParams.verboseLevel, + reasoningLevel: runParams.reasoningLevel, + thinkingLevel: runParams.thinkLevel, + toolResultFormat: input.resolvedToolResultFormat, + suppressToolErrorWarnings: runParams.suppressToolErrorWarnings, + inlineToolResultsAllowed: false, + didSendViaMessagingTool: attempt.didSendViaMessagingTool, + didDeliverSourceReplyViaMessageTool: attempt.didDeliverSourceReplyViaMessageTool === true, + messagingToolSentTargets: attempt.messagingToolSentTargets, + messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, + sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode, + agentId: runParams.agentId, + runId: runParams.runId, + runAborted: input.terminalInterrupted, + didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, + heartbeatToolResponse: attempt.heartbeatToolResponse, + }); + const payloadsWithToolMedia = mergeAttemptToolMediaPayloads({ + payloads, + toolMediaUrls: attempt.toolMediaUrls, + toolAudioAsVoice: attempt.toolAudioAsVoice, + toolTrustedLocalMedia: attempt.toolTrustedLocalMedia, + sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode, + }); + const finalAssistantStopReason = (terminalAssistant?.stopReason ?? "").trim().toLowerCase(); + const recoveredFinalAssistantTextAfterPromptTimeout = + timedOutDuringPrompt && ["completed", "end_turn", "stop"].includes(finalAssistantStopReason) + ? (finalAssistantVisibleText ?? finalAssistantRawText)?.trim() + : undefined; + const payloadAlreadyContainsRecoveredFinalAssistant = + recoveredFinalAssistantTextAfterPromptTimeout + ? (payloadsWithToolMedia ?? []).some( + (payload) => + payload?.isError !== true && + payload?.isReasoning !== true && + typeof payload.text === "string" && + payload.text.trim() === recoveredFinalAssistantTextAfterPromptTimeout, + ) + : false; + const recoveredFinalAssistantPayloadsAfterPromptTimeout = + recoveredFinalAssistantTextAfterPromptTimeout && !payloadAlreadyContainsRecoveredFinalAssistant + ? replacePartialAssistantPayload({ + payloads: payloadsWithToolMedia, + assistantTexts: attempt.assistantTexts, + recoveredText: recoveredFinalAssistantTextAfterPromptTimeout, + }) + : undefined; + const hasSuccessfulFinalAssistantAfterPromptTimeout = + timedOutDuringPrompt && + Boolean( + payloadAlreadyContainsRecoveredFinalAssistant || + recoveredFinalAssistantPayloadsAfterPromptTimeout?.length, + ); + const hasPartialAssistantTextAfterPromptTimeout = + timedOutDuringPrompt && + (attempt.assistantTexts ?? []).some((text) => text.trim().length > 0) && + !attempt.clientToolCalls && + !attempt.yieldDetected && + !attempt.didSendViaMessagingTool && + !attempt.didSendDeterministicApprovalPrompt && + !attempt.lastToolError && + (attempt.toolMetas?.length ?? 0) === 0; + const attemptToolSummary = buildTraceToolSummary({ + toolMetas: attempt.toolMetas, + fallbackHadFailure: Boolean(attempt.lastToolError), + }); + const failureSignal = resolveEmbeddedRunFailureSignal({ + trigger: runParams.trigger, + lastToolError: attempt.lastToolError, + }); + return { + agentMeta, + reportedModelRef, + finalAssistantVisibleText, + finalAssistantRawText, + payloads, + payloadsWithToolMedia, + timedOutDuringPrompt, + recoveredFinalAssistantPayloadsAfterPromptTimeout, + hasSuccessfulFinalAssistantAfterPromptTimeout, + hasPartialAssistantTextAfterPromptTimeout, + attemptToolSummary, + failureSignal, + }; +} + +function replacePartialAssistantPayload(input: { + payloads: EmbeddedAgentRunResult["payloads"]; + assistantTexts?: string[]; + recoveredText: string; +}): NonNullable { + const payloads = input.payloads ? [...input.payloads] : []; + const assistantTextSignatures = new Set( + (input.assistantTexts ?? []).map((text) => text.trim()).filter((text) => text.length > 0), + ); + // The attempt can contain completed assistant blocks before its partial tail. + // Recover the latest matching payload or we can overwrite already-delivered text. + const partialPayloadIndex = payloads.findLastIndex( + (payload) => + payload.isError !== true && + payload.isReasoning !== true && + typeof payload.text === "string" && + assistantTextSignatures.has(payload.text.trim()), + ); + if (partialPayloadIndex < 0) { + return [...payloads, { text: input.recoveredText }]; + } + const partialPayload = payloads[partialPayloadIndex]; + if (!partialPayload) { + return [...payloads, { text: input.recoveredText }]; + } + payloads[partialPayloadIndex] = copyReplyPayloadMetadata(partialPayload, { + ...partialPayload, + text: input.recoveredText, + }); + return payloads; +} diff --git a/src/agents/embedded-agent-runner/run/terminal-resolution.ts b/src/agents/embedded-agent-runner/run/terminal-resolution.ts new file mode 100644 index 000000000000..e1cc5603dd22 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/terminal-resolution.ts @@ -0,0 +1,531 @@ +import { randomBytes } from "node:crypto"; +import { SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js"; +import { freezeDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js"; +import type { AssistantMessage } from "../../../llm/types.js"; +import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js"; +import type { AgentExecutionAuthBinding } from "../../execution-auth-binding.js"; +import type { ResolvedProviderAuth } from "../../model-auth.js"; +import { log } from "../logger.js"; +import type { EmbeddedRunReplayState } from "../replay-state.js"; +import type { + EmbeddedAgentMeta, + EmbeddedAgentRunResult, + EmbeddedRunFailureSignal, + TraceAttempt, +} from "../types.js"; +import { + markEmbeddedRunAuthProfileSuccess, + reportEmbeddedRunSuccessfulAuthBinding, +} from "./auth-profile-success.js"; +import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import { + hasAttemptTerminalState, + resolveAttemptReplayMetadata, + resolveEmptyResponseRetryInstruction, + resolveIncompleteTurnPayloadText, + resolveReasoningOnlyRetryInstruction, + resolveRunLivenessState, + resolveSilentToolResultReplyPayload, + shouldRetryMissingAssistantTurn, + shouldTreatEmptyAssistantReplyAsSilent, +} from "./incomplete-turn.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { + MAX_BEFORE_AGENT_FINALIZE_REVISIONS, + type EmbeddedRunTerminalRetryState, +} from "./terminal-retry-state.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +const MAX_MISSING_ASSISTANT_RETRIES = 1; +const COMPACTION_CONTINUATION_RETRY_INSTRUCTION = + "The previous attempt compacted the conversation context before producing a final user-visible answer. Continue from the compacted transcript and produce the final answer now. Do not restart from scratch, do not repeat completed work, and do not rerun tools unless the transcript clearly lacks required evidence."; +const BEFORE_AGENT_FINALIZE_RETRY_PROMPT_PREFIX = + "Before accepting the previous final answer, apply this revision request and produce the revised final answer. Do not repeat completed work or rerun tools unless the request explicitly requires it."; + +type TerminalRunParams = RunEmbeddedAgentParams & { + authProfileStateMode?: "read-write" | "read-only"; + onSuccessfulAuthBinding?: (binding: AgentExecutionAuthBinding) => void; +}; + +type TerminalResolution = + | { action: "retry" } + | { action: "complete"; result: EmbeddedAgentRunResult }; + +export async function resolveEmbeddedRunTerminal(input: { + runParams: TerminalRunParams; + retryState: EmbeddedRunTerminalRetryState; + attempt: EmbeddedRunAttemptResult; + attemptAssistant?: AssistantMessage; + activeErrorContext: { provider: string; model: string }; + modelApi: Parameters[0]["modelApi"]; + executionContract: Parameters< + typeof resolveReasoningOnlyRetryInstruction + >[0]["executionContract"]; + terminalAborted: boolean; + terminalTimedOut: boolean; + terminalInterrupted: boolean; + externalAbort: boolean; + signalOwnedInterruption: boolean; + promptError: unknown; + payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"]; + recoveredFinalAssistantPayloadsAfterPromptTimeout?: EmbeddedAgentRunResult["payloads"]; + finalAssistantVisibleText?: string; + finalAssistantRawText?: string; + agentMeta: EmbeddedAgentMeta; + attemptToolSummary: EmbeddedAgentRunResult["meta"]["toolSummary"]; + failureSignal?: EmbeddedRunFailureSignal; + maxReasoningOnlyRetryAttempts: number; + maxEmptyResponseRetryAttempts: number; + attemptCompactionCount: number; + replayState: EmbeddedRunReplayState; + activePromptPersisted: boolean; + activateInternalPrompt: (prompt: string, persisted: boolean) => void; + setSuppressNextUserMessagePersistence: (value: boolean) => void; + armPostCompactionGuard: () => void; + readTerminalToolPresentation: () => string | undefined; + resolveReplayInvalid: (incompleteTurnText?: string | null) => boolean; + setTerminalLifecycleMeta: NonNullable; + maybeMarkAuthProfileFailure: (failure: { + profileId?: string; + reason?: AuthProfileFailureReason | null; + modelId?: string; + }) => Promise; + assistantProfileFailureReason?: AuthProfileFailureReason | null; + startedAtMs: number; + provider: string; + modelId: string; + authProfileId?: string; + profileFailureStore: AuthProfileStore; + attemptAuthProfileStore: AuthProfileStore; + apiKeyInfo: ResolvedProviderAuth | null; + agentHarnessId: string; + pluginHarnessOwnsTransport: boolean; + pluginHarnessOwnsAuthBootstrap: boolean; + reportedModelRef: { provider: string; model: string }; + traceAttempts: TraceAttempt[]; + traceAttemptUsesFallback: (attempt: TraceAttempt) => boolean; + thinkLevel?: string; + contextRecoveryState: EmbeddedRunContextRecoveryState; +}): Promise { + const { runParams, attempt, retryState } = input; + const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({ + isCronTrigger: runParams.trigger === "cron", + payloadCount: input.payloadsWithToolMedia?.length ?? 0, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); + const payloadsForTerminalPath = input.recoveredFinalAssistantPayloadsAfterPromptTimeout + ? input.recoveredFinalAssistantPayloadsAfterPromptTimeout + : input.payloadsWithToolMedia?.length + ? input.payloadsWithToolMedia + : silentToolResultReplyPayload + ? [silentToolResultReplyPayload] + : input.payloadsWithToolMedia; + const payloadCount = payloadsForTerminalPath?.length ?? 0; + const emptyAssistantReplyIsSilent = shouldTreatEmptyAssistantReplyAsSilent({ + allowEmptyAssistantReplyAsSilent: runParams.allowEmptyAssistantReplyAsSilent, + payloadCount, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); + const nextReasoningOnlyRetryInstruction = emptyAssistantReplyIsSilent + ? null + : resolveReasoningOnlyRetryInstruction({ + provider: input.activeErrorContext.provider, + modelId: input.activeErrorContext.model, + modelApi: input.modelApi, + executionContract: input.executionContract, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); + const nextEmptyResponseRetryInstruction = emptyAssistantReplyIsSilent + ? null + : resolveEmptyResponseRetryInstruction({ + provider: input.activeErrorContext.provider, + modelId: input.activeErrorContext.model, + modelApi: input.modelApi, + executionContract: input.executionContract, + payloadCount, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); + if ( + nextReasoningOnlyRetryInstruction && + retryState.reasoningOnlyAttempts < input.maxReasoningOnlyRetryAttempts + ) { + retryState.reasoningOnlyAttempts += 1; + input.activateInternalPrompt(nextReasoningOnlyRetryInstruction, false); + log.warn( + `reasoning-only assistant turn detected: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — retrying ${retryState.reasoningOnlyAttempts}/${input.maxReasoningOnlyRetryAttempts} ` + + `with visible-answer continuation`, + ); + return { action: "retry" }; + } + const reasoningOnlyRetriesExhausted = + nextReasoningOnlyRetryInstruction && + retryState.reasoningOnlyAttempts >= input.maxReasoningOnlyRetryAttempts; + if ( + !emptyAssistantReplyIsSilent && + shouldRetryMissingAssistantTurn({ + payloadCount, + aborted: input.terminalAborted, + promptError: input.promptError, + timedOut: input.terminalTimedOut, + attempt, + }) && + retryState.missingAssistantAttempts < MAX_MISSING_ASSISTANT_RETRIES + ) { + retryState.missingAssistantAttempts += 1; + input.setSuppressNextUserMessagePersistence(input.activePromptPersisted); + log.warn( + `missing assistant terminal message detected: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — retrying ${retryState.missingAssistantAttempts}/${MAX_MISSING_ASSISTANT_RETRIES} with same prompt`, + ); + return { action: "retry" }; + } + if ( + !nextReasoningOnlyRetryInstruction && + nextEmptyResponseRetryInstruction && + retryState.emptyResponseAttempts < input.maxEmptyResponseRetryAttempts + ) { + retryState.emptyResponseAttempts += 1; + input.activateInternalPrompt(nextEmptyResponseRetryInstruction, false); + log.warn( + `empty response detected: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — retrying ${retryState.emptyResponseAttempts}/${input.maxEmptyResponseRetryAttempts} ` + + `with visible-answer continuation`, + ); + return { action: "retry" }; + } + const incompleteTurnText = emptyAssistantReplyIsSilent + ? null + : resolveIncompleteTurnPayloadText({ + payloadCount, + aborted: input.terminalAborted, + externalAbort: input.externalAbort || input.signalOwnedInterruption, + timedOut: input.terminalTimedOut, + attempt, + }); + const incompleteTurnFallbackSafe = Boolean( + incompleteTurnText && + !input.terminalInterrupted && + !input.promptError && + !attempt.lastToolError && + !hasAttemptTerminalState(attempt) && + !input.replayState.hadPotentialSideEffects, + ); + const terminalToolPresentation = incompleteTurnFallbackSafe + ? input.readTerminalToolPresentation() + : undefined; + if ( + !emptyAssistantReplyIsSilent && + input.attemptCompactionCount > 0 && + payloadCount === 0 && + !input.terminalInterrupted && + !input.promptError && + !attempt.clientToolCalls && + !attempt.yieldDetected && + !attempt.didSendDeterministicApprovalPrompt && + !attempt.lastToolError && + !input.replayState.hadPotentialSideEffects && + retryState.compactionContinuationAttempts < 1 + ) { + retryState.compactionContinuationAttempts += 1; + retryState.compactionContinuationInstruction = COMPACTION_CONTINUATION_RETRY_INSTRUCTION; + log.warn( + `compaction interrupted visible final answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `compactions=${input.attemptCompactionCount} — retrying ${retryState.compactionContinuationAttempts}/1 with compacted-transcript continuation`, + ); + input.armPostCompactionGuard(); + return { action: "retry" }; + } + retryState.compactionContinuationInstruction = null; + + if (reasoningOnlyRetriesExhausted && !input.finalAssistantVisibleText) { + const incompletePayloadText = "⚠️ Agent couldn't generate a response. Please try again."; + log.warn( + `reasoning-only retries exhausted: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} attempts=${retryState.reasoningOnlyAttempts}/${input.maxReasoningOnlyRetryAttempts} — surfacing incomplete-turn error`, + ); + return surfaceIncompleteTurn({ + ...input, + text: incompletePayloadText, + payloadCount: 0, + incompleteTurnFallbackSafe, + terminalToolPresentation, + }); + } + if ( + !nextReasoningOnlyRetryInstruction && + nextEmptyResponseRetryInstruction && + retryState.emptyResponseAttempts >= input.maxEmptyResponseRetryAttempts + ) { + log.warn( + `empty response retries exhausted: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} attempts=${retryState.emptyResponseAttempts}/${input.maxEmptyResponseRetryAttempts} — surfacing incomplete-turn error`, + ); + } + if (incompleteTurnText) { + const replayMetadata = resolveAttemptReplayMetadata(attempt); + const incompleteStopReason = + attempt.currentAttemptAssistant?.stopReason ?? attempt.lastAssistant?.stopReason; + log.warn( + `incomplete turn detected: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} ` + + `stopReason=${incompleteStopReason ?? "missing"} hasLastAssistant=${attempt.lastAssistant ? "yes" : "no"} ` + + `hasCurrentAttemptAssistant=${attempt.currentAttemptAssistant ? "yes" : "no"} payloads=${payloadCount} ` + + `tools=${attempt.toolMetas?.length ?? 0} replaySafe=${replayMetadata.replaySafe ? "yes" : "no"} ` + + `compactions=${input.attemptCompactionCount} reasoningRetries=${retryState.reasoningOnlyAttempts}/${input.maxReasoningOnlyRetryAttempts} ` + + `emptyRetries=${retryState.emptyResponseAttempts}/${input.maxEmptyResponseRetryAttempts} ` + + `missingAssistantRetries=${retryState.missingAssistantAttempts}/${MAX_MISSING_ASSISTANT_RETRIES} — ` + + (terminalToolPresentation + ? "surfacing tool-authored terminal presentation" + : "surfacing error to user"), + ); + return surfaceIncompleteTurn({ + ...input, + text: incompleteTurnText, + payloadCount, + incompleteTurnFallbackSafe, + terminalToolPresentation, + }); + } + + const beforeFinalizeRevisionReason = attempt.beforeAgentFinalizeRevisionReason; + if ( + beforeFinalizeRevisionReason && + !input.terminalInterrupted && + !input.promptError && + !attempt.clientToolCalls && + !attempt.yieldDetected && + !emptyAssistantReplyIsSilent + ) { + retryState.beforeFinalizeRevisionAttempts += 1; + input.activateInternalPrompt( + `${BEFORE_AGENT_FINALIZE_RETRY_PROMPT_PREFIX}\n\n${beforeFinalizeRevisionReason}`, + true, + ); + retryState.compactionContinuationInstruction = null; + log.warn( + `before_agent_finalize requested one more pass: ` + + `runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `attempt=${retryState.beforeFinalizeRevisionAttempts}/${MAX_BEFORE_AGENT_FINALIZE_REVISIONS}`, + ); + return { action: "retry" }; + } + + return completeEmbeddedRun({ + ...input, + payloadCount, + payloadsForTerminalPath, + emptyAssistantReplyIsSilent, + }); +} + +async function surfaceIncompleteTurn( + input: Parameters[0] & { + text: string; + payloadCount: number; + incompleteTurnFallbackSafe: boolean; + terminalToolPresentation?: string; + }, +): Promise { + const replayInvalid = input.resolveReplayInvalid(input.text); + const livenessState = resolveRunLivenessState({ + payloadCount: input.payloadCount, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt: input.attempt, + incompleteTurnText: input.text, + }); + input.setTerminalLifecycleMeta({ replayInvalid, livenessState }); + if (input.authProfileId) { + await input.maybeMarkAuthProfileFailure({ + profileId: input.authProfileId, + reason: input.assistantProfileFailureReason, + modelId: input.modelId, + }); + } + return { + action: "complete", + result: { + payloads: [ + { + text: input.terminalToolPresentation + ? input.terminalToolPresentation.concat("\n\n", input.text) + : input.text, + isError: true, + }, + ], + meta: { + durationMs: Date.now() - input.startedAtMs, + agentMeta: input.agentMeta, + aborted: input.terminalAborted, + systemPromptReport: input.attempt.systemPromptReport, + finalPromptText: input.attempt.finalPromptText, + finalAssistantVisibleText: input.finalAssistantVisibleText, + finalAssistantRawText: input.finalAssistantRawText, + replayInvalid, + livenessState, + error: { + kind: "incomplete_turn", + message: "Agent couldn't generate a response.", + fallbackSafe: input.incompleteTurnFallbackSafe, + terminalPresentation: input.terminalToolPresentation !== undefined, + }, + toolSummary: input.attemptToolSummary, + ...(input.failureSignal ? { failureSignal: input.failureSignal } : {}), + agentHarnessResultClassification: input.attempt.agentHarnessResultClassification, + }, + ...copyAttemptDeliveryState(input.attempt), + }, + }; +} + +function completeEmbeddedRun( + input: Parameters[0] & { + payloadCount: number; + payloadsForTerminalPath: EmbeddedAgentRunResult["payloads"]; + emptyAssistantReplyIsSilent: boolean; + }, +): TerminalResolution { + log.debug( + `embedded run done: runId=${input.runParams.runId} sessionId=${input.runParams.sessionId} durationMs=${Date.now() - input.startedAtMs} aborted=${input.terminalAborted}`, + ); + markEmbeddedRunAuthProfileSuccess({ + authProfileStateMode: input.runParams.authProfileStateMode, + profileId: input.authProfileId, + profileStore: input.profileFailureStore, + provider: input.provider, + agentDir: input.runParams.agentDir, + runId: input.runParams.runId, + sessionId: input.runParams.sessionId, + }); + reportEmbeddedRunSuccessfulAuthBinding({ + profileId: input.authProfileId, + profileStore: input.attemptAuthProfileStore, + apiKeyInfo: input.apiKeyInfo, + attempt: input.attempt, + provider: input.provider, + agentHarnessId: input.agentHarnessId, + pluginHarnessOwnsTransport: input.pluginHarnessOwnsTransport, + pluginHarnessOwnsAuthBootstrap: input.pluginHarnessOwnsAuthBootstrap, + onSuccessfulAuthBinding: input.runParams.onSuccessfulAuthBinding, + }); + const replayInvalid = input.resolveReplayInvalid(null); + const livenessState = input.attempt.yieldDetected + ? "paused" + : resolveRunLivenessState({ + payloadCount: input.payloadCount, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt: input.attempt, + incompleteTurnText: null, + }); + const stopReason = input.attempt.clientToolCalls + ? "tool_calls" + : input.attempt.yieldDetected + ? "end_turn" + : (input.attemptAssistant?.stopReason as string | undefined); + const terminalPayloads = input.emptyAssistantReplyIsSilent + ? [{ text: SILENT_REPLY_TOKEN }] + : input.payloadsForTerminalPath; + input.setTerminalLifecycleMeta({ + replayInvalid, + livenessState, + stopReason, + yielded: input.attempt.yieldDetected === true, + }); + return { + action: "complete", + result: { + payloads: terminalPayloads?.length ? terminalPayloads : undefined, + ...(input.attempt.diagnosticTrace + ? { diagnosticTrace: freezeDiagnosticTraceContext(input.attempt.diagnosticTrace) } + : {}), + meta: { + durationMs: Date.now() - input.startedAtMs, + agentMeta: input.agentMeta, + aborted: input.terminalAborted, + systemPromptReport: input.attempt.systemPromptReport, + finalPromptText: input.attempt.finalPromptText, + finalAssistantVisibleText: input.finalAssistantVisibleText, + finalAssistantRawText: input.finalAssistantRawText, + replayInvalid, + livenessState, + agentHarnessResultClassification: input.attempt.agentHarnessResultClassification, + ...(input.attempt.yieldDetected ? { yielded: true } : {}), + ...(input.emptyAssistantReplyIsSilent + ? { terminalReplyKind: "silent-empty" as const } + : {}), + stopReason, + pendingToolCalls: input.attempt.clientToolCalls?.map((call) => ({ + id: randomBytes(5).toString("hex").slice(0, 9), + name: call.name, + arguments: JSON.stringify(call.params), + })), + executionTrace: { + winnerProvider: input.reportedModelRef.provider, + winnerModel: input.reportedModelRef.model, + attempts: + input.traceAttempts.length > 0 || + input.attemptAssistant?.provider || + input.attemptAssistant?.model + ? [ + ...input.traceAttempts, + { + provider: input.reportedModelRef.provider, + model: input.reportedModelRef.model, + result: "success", + stage: "assistant", + }, + ] + : undefined, + fallbackUsed: input.traceAttempts.some(input.traceAttemptUsesFallback), + runner: "embedded", + }, + requestShaping: { + ...(input.authProfileId ? { authMode: "auth-profile" } : {}), + ...(input.thinkLevel ? { thinking: input.thinkLevel } : {}), + ...(input.runParams.reasoningLevel ? { reasoning: input.runParams.reasoningLevel } : {}), + ...(input.runParams.verboseLevel ? { verbose: input.runParams.verboseLevel } : {}), + ...(input.runParams.blockReplyBreak + ? { blockStreaming: input.runParams.blockReplyBreak } + : {}), + }, + toolSummary: input.attemptToolSummary, + ...(input.failureSignal ? { failureSignal: input.failureSignal } : {}), + completion: { + ...(stopReason ? { stopReason } : {}), + ...(stopReason ? { finishReason: stopReason } : {}), + ...(stopReason?.toLowerCase().includes("refusal") ? { refusal: true } : {}), + }, + contextManagement: + input.contextRecoveryState.autoCompactionCount > 0 + ? { lastTurnCompactions: input.contextRecoveryState.autoCompactionCount } + : undefined, + }, + ...copyAttemptDeliveryState(input.attempt), + }, + }; +} + +export function copyAttemptDeliveryState(attempt: EmbeddedRunAttemptResult) { + return { + didSendViaMessagingTool: attempt.didSendViaMessagingTool, + didDeliverSourceReplyViaMessageTool: attempt.didDeliverSourceReplyViaMessageTool === true, + didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, + messagingToolSentTexts: attempt.messagingToolSentTexts, + messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls, + messagingToolSentTargets: attempt.messagingToolSentTargets, + messagingToolSourceReplyPayloads: attempt.messagingToolSourceReplyPayloads, + heartbeatToolResponse: attempt.heartbeatToolResponse, + successfulCronAdds: attempt.successfulCronAdds, + acceptedSessionSpawns: attempt.acceptedSessionSpawns, + }; +} diff --git a/src/agents/embedded-agent-runner/run/terminal-retry-state.ts b/src/agents/embedded-agent-runner/run/terminal-retry-state.ts new file mode 100644 index 000000000000..d14193ccf0b9 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/terminal-retry-state.ts @@ -0,0 +1,21 @@ +export const MAX_BEFORE_AGENT_FINALIZE_REVISIONS = 3; + +export type EmbeddedRunTerminalRetryState = { + reasoningOnlyAttempts: number; + emptyResponseAttempts: number; + missingAssistantAttempts: number; + compactionContinuationAttempts: number; + compactionContinuationInstruction: string | null; + beforeFinalizeRevisionAttempts: number; +}; + +export function createEmbeddedRunTerminalRetryState(): EmbeddedRunTerminalRetryState { + return { + reasoningOnlyAttempts: 0, + emptyResponseAttempts: 0, + missingAssistantAttempts: 0, + compactionContinuationAttempts: 0, + compactionContinuationInstruction: null, + beforeFinalizeRevisionAttempts: 0, + }; +} diff --git a/src/agents/embedded-agent-runner/run/terminal-timeout.ts b/src/agents/embedded-agent-runner/run/terminal-timeout.ts new file mode 100644 index 000000000000..8aaf57fa81b9 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/terminal-timeout.ts @@ -0,0 +1,98 @@ +import { hasMessagingToolDeliveryEvidence } from "../delivery-evidence.js"; +import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; +import { resolveRunLivenessState } from "./incomplete-turn.js"; +import { copyAttemptDeliveryState } from "./terminal-resolution.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +export function resolveEmbeddedRunTerminalTimeout(input: { + timedOutDuringPrompt: boolean; + hasSuccessfulFinalAssistantAfterPromptTimeout: boolean; + shouldSurfaceCodexCompletionTimeout: boolean; + idleTimedOut: boolean; + attempt: EmbeddedRunAttemptResult; + hasPartialAssistantTextAfterPromptTimeout: boolean; + payloads: EmbeddedAgentRunResult["payloads"]; + payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"]; + terminalAborted: boolean; + terminalTimedOut: boolean; + terminalOutcome: { + timeoutPhase?: EmbeddedAgentRunResult["meta"]["timeoutPhase"]; + providerStarted?: boolean; + }; + resolveReplayInvalid: (incompleteTurnText?: string | null) => boolean; + setTerminalLifecycleMeta: NonNullable; + startedAtMs: number; + agentMeta: EmbeddedAgentMeta; + finalAssistantVisibleText?: string; + finalAssistantRawText?: string; + attemptToolSummary: EmbeddedAgentRunResult["meta"]["toolSummary"]; + failureSignal: EmbeddedAgentRunResult["meta"]["failureSignal"]; +}): EmbeddedAgentRunResult | undefined { + if ( + !input.timedOutDuringPrompt || + input.hasSuccessfulFinalAssistantAfterPromptTimeout || + (!input.shouldSurfaceCodexCompletionTimeout && hasMessagingToolDeliveryEvidence(input.attempt)) + ) { + return undefined; + } + const defaultTimeoutText = input.idleTimedOut + ? "The model did not produce a response before the model idle timeout. " + + "Please try again, or increase `models.providers..timeoutSeconds` for slow local or self-hosted providers. " + + "If `agents.defaults.timeoutSeconds` or a run-specific timeout is lower, raise that ceiling too; provider timeouts cannot extend the whole agent run." + : "Request timed out before a response was generated. " + + "Please try again, or increase `agents.defaults.timeoutSeconds` in your config."; + const timeoutText = input.attempt.promptTimeoutOutcome?.message?.trim() || defaultTimeoutText; + const replayInvalid = + input.attempt.promptTimeoutOutcome?.replayInvalid ?? input.resolveReplayInvalid(null); + const livenessState = + input.attempt.promptTimeoutOutcome?.livenessState ?? + resolveRunLivenessState({ + payloadCount: input.hasPartialAssistantTextAfterPromptTimeout + ? 0 + : (input.payloads?.length ?? 0), + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt: input.attempt, + incompleteTurnText: null, + }); + const timeoutPhase = + input.attempt.promptTimeoutOutcome?.timeoutPhase ?? input.terminalOutcome.timeoutPhase; + const providerStarted = + input.attempt.promptTimeoutOutcome?.providerStarted ?? input.terminalOutcome.providerStarted; + const timeoutAttribution = { + ...(timeoutPhase ? { timeoutPhase } : {}), + ...(typeof providerStarted === "boolean" ? { providerStarted } : {}), + }; + input.setTerminalLifecycleMeta({ replayInvalid, livenessState, ...timeoutAttribution }); + return { + payloads: [ + ...(input.hasPartialAssistantTextAfterPromptTimeout ? [] : input.payloadsWithToolMedia || []), + { text: timeoutText, isError: true }, + ], + meta: { + durationMs: Date.now() - input.startedAtMs, + agentMeta: input.agentMeta, + aborted: input.terminalAborted, + systemPromptReport: input.attempt.systemPromptReport, + finalPromptText: input.attempt.finalPromptText, + finalAssistantVisibleText: input.finalAssistantVisibleText, + finalAssistantRawText: input.finalAssistantRawText, + replayInvalid, + livenessState, + ...timeoutAttribution, + ...(input.shouldSurfaceCodexCompletionTimeout + ? { + error: { + kind: "incomplete_turn" as const, + message: timeoutText, + fallbackSafe: false, + }, + } + : {}), + toolSummary: input.attemptToolSummary, + ...(input.failureSignal ? { failureSignal: input.failureSignal } : {}), + agentHarnessResultClassification: input.attempt.agentHarnessResultClassification, + }, + ...copyAttemptDeliveryState(input.attempt), + }; +} diff --git a/src/agents/embedded-agent-runner/run/timeout-context-recovery.ts b/src/agents/embedded-agent-runner/run/timeout-context-recovery.ts new file mode 100644 index 000000000000..41a4f6b1a805 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/timeout-context-recovery.ts @@ -0,0 +1,236 @@ +import { buildContextEngineRuntimeSettings } from "../../../context-engine/runtime-settings.js"; +import type { ContextEngine, ContextEngineSessionTarget } from "../../../context-engine/types.js"; +import { resolveProcessToolScopeKey } from "../../agent-tools.js"; +import { listActiveProcessSessionReferences } from "../../bash-process-references.js"; +import { deriveContextPromptTokens, normalizeUsage } from "../../usage.js"; +import { runPostCompactionSideEffects } from "../compaction-hooks.js"; +import { buildEmbeddedCompactionRuntimeContext } from "../compaction-runtime-context.js"; +import { + compactContextEngineWithSafetyTimeout, + resolveCompactionTimeoutMs, +} from "../compaction-safety-timeout.js"; +import { resolveContextEngineCapabilities } from "../context-engine-capabilities.js"; +import { log } from "../logger.js"; +import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import { createCompactionDiagId } from "./helpers.js"; +import type { RunEmbeddedAgentParams } from "./params.js"; +import { buildContextEngineCompactionSessionTarget } from "./session-bootstrap.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +const MAX_TIMEOUT_COMPACTION_ATTEMPTS = 2; + +type CompactResult = Awaited>; + +type ActiveSession = { + id: string; + file: string; + target?: ContextEngineSessionTarget; +}; + +export async function recoverEmbeddedRunTimeout(input: { + runParams: RunEmbeddedAgentParams; + state: EmbeddedRunContextRecoveryState; + contextEngine: ContextEngine; + contextTokenBudget?: number; + genericCompactionRecoveryAllowed: boolean; + timedOut: boolean; + signalOwnedInterruption: boolean; + timedOutDuringCompaction: boolean; + timedOutDuringToolExecution: boolean; + timedOutByRunBudget: boolean; + lastRunPromptUsage?: ReturnType; + attempt: EmbeddedRunAttemptResult; + runtimeAuthPlan: Parameters[0]["runtimeAuthPlan"]; + resolvedSessionKey: string; + sessionAgentId: string; + agentDir: string; + workspaceDir: string; + provider: string; + modelId: string; + harnessRuntime: string; + thinkLevel: Parameters[0]["thinkLevel"]; + authProfileId?: string; + authProfileIdSource: "auto" | "user"; + resolveContextEnginePluginId: () => string | undefined; + buildRuntimeSettings: (settings: { + tokenBudget?: number | null; + }) => ReturnType; + onCompactionHookMessages: (payload: { + phase: "before" | "after"; + messages: string[]; + }) => Promise; + runOwnsCompactionBeforeHook: (reason: string) => Promise; + runOwnsCompactionAfterHook: ( + reason: string, + result: CompactResult, + previousSessionId?: string, + ) => Promise; + adoptCompactionTranscript: (result: CompactResult) => Promise; + getActiveSession: () => ActiveSession; + armPostCompactionGuard: () => void; +}): Promise { + if ( + !input.genericCompactionRecoveryAllowed || + input.contextTokenBudget === undefined || + !input.timedOut || + input.signalOwnedInterruption || + input.timedOutDuringCompaction || + input.timedOutDuringToolExecution || + input.timedOutByRunBudget + ) { + return false; + } + + // API totals include output tokens. Timeout compaction only considers the + // prompt-side pressure that can actually be reduced by compaction. + const lastTurnPromptTokens = deriveContextPromptTokens({ + lastCallUsage: input.lastRunPromptUsage, + }); + const tokenUsedRatio = + lastTurnPromptTokens != null && input.contextTokenBudget > 0 + ? lastTurnPromptTokens / input.contextTokenBudget + : 0; + if (input.state.timeoutCompactionAttempts >= MAX_TIMEOUT_COMPACTION_ATTEMPTS) { + log.warn( + `[timeout-compaction] already attempted timeout compaction ${input.state.timeoutCompactionAttempts} time(s); falling through to failover rotation`, + ); + return false; + } + if (tokenUsedRatio <= 0.65) { + return false; + } + + const timeoutDiagId = createCompactionDiagId(); + input.state.timeoutCompactionAttempts += 1; + log.warn( + `[timeout-compaction] LLM timed out with high prompt token usage (${Math.round(tokenUsedRatio * 100)}%); ` + + `attempting compaction before retry (attempt ${input.state.timeoutCompactionAttempts}/${MAX_TIMEOUT_COMPACTION_ATTEMPTS}) diagId=${timeoutDiagId}`, + ); + let timeoutCompactResult: CompactResult; + await input.runOwnsCompactionBeforeHook("timeout recovery"); + try { + const activeSession = input.getActiveSession(); + const runParams = input.runParams; + const timeoutCompactionRuntimeContext = { + ...buildEmbeddedCompactionRuntimeContext({ + sessionKey: runParams.sessionKey, + messageChannel: runParams.messageChannel, + messageProvider: runParams.messageProvider, + clientCaps: runParams.clientCaps, + chatType: runParams.chatType, + agentAccountId: runParams.agentAccountId, + currentChannelId: runParams.currentChannelId, + currentThreadTs: runParams.currentThreadTs, + currentMessageId: runParams.currentMessageId, + authProfileId: input.authProfileId, + authProfileIdSource: input.authProfileIdSource, + runtimeAuthPlan: input.runtimeAuthPlan, + workspaceDir: input.workspaceDir, + agentDir: input.agentDir, + config: runParams.config, + skillsSnapshot: runParams.skillsSnapshot, + senderId: runParams.senderId, + provider: input.provider, + modelId: input.modelId, + harnessRuntime: input.harnessRuntime, + modelSelectionLocked: runParams.modelSelectionLocked, + modelFallbacksOverride: runParams.modelFallbacksOverride, + thinkLevel: input.thinkLevel, + reasoningLevel: runParams.reasoningLevel, + bashElevated: runParams.bashElevated, + extraSystemPrompt: runParams.extraSystemPrompt, + sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode, + ownerNumbers: runParams.ownerNumbers, + activeProcessSessions: listActiveProcessSessionReferences({ + scopeKey: resolveProcessToolScopeKey({ + sessionKey: runParams.sandboxSessionKey?.trim() || runParams.sessionKey, + sessionId: activeSession.id, + agentId: input.sessionAgentId, + }), + }), + }), + ...resolveContextEngineCapabilities({ + config: runParams.config, + sessionKey: runParams.sessionKey, + agentId: input.sessionAgentId, + contextEnginePluginId: input.resolveContextEnginePluginId(), + purpose: "context-engine.timeout-compaction", + }), + onCompactionHookMessages: input.onCompactionHookMessages, + ...(input.attempt.promptCache ? { promptCache: input.attempt.promptCache } : {}), + runId: runParams.runId, + trigger: "timeout_recovery", + diagId: timeoutDiagId, + attempt: input.state.timeoutCompactionAttempts, + maxAttempts: MAX_TIMEOUT_COMPACTION_ATTEMPTS, + }; + timeoutCompactResult = await compactContextEngineWithSafetyTimeout( + input.contextEngine, + { + sessionId: activeSession.id, + sessionKey: input.resolvedSessionKey, + agentId: input.sessionAgentId, + sessionTarget: buildContextEngineCompactionSessionTarget({ + agentId: input.sessionAgentId, + config: runParams.config, + sessionFile: activeSession.file, + sessionId: activeSession.id, + sessionKey: input.resolvedSessionKey, + sessionTarget: activeSession.target, + }), + tokenBudget: input.contextTokenBudget, + force: true, + compactionTarget: "budget", + runtimeContext: timeoutCompactionRuntimeContext, + runtimeSettings: input.buildRuntimeSettings({ tokenBudget: input.contextTokenBudget }), + }, + resolveCompactionTimeoutMs(runParams.config), + runParams.abortSignal, + ); + } catch (compactErr) { + log.warn( + `[timeout-compaction] contextEngine.compact() threw during timeout recovery for ${input.provider}/${input.modelId}: ${String(compactErr)}`, + ); + timeoutCompactResult = { + ok: false, + compacted: false, + reason: String(compactErr), + }; + } + + const previousSessionId = timeoutCompactResult.compacted + ? await input.adoptCompactionTranscript(timeoutCompactResult) + : undefined; + await input.runOwnsCompactionAfterHook( + "timeout recovery", + timeoutCompactResult, + previousSessionId, + ); + if (!timeoutCompactResult.compacted) { + log.warn( + `[timeout-compaction] compaction did not reduce context for ${input.provider}/${input.modelId}; falling through to normal handling`, + ); + return false; + } + + input.state.autoCompactionCount += 1; + const tokensAfter = timeoutCompactResult.result?.tokensAfter; + if (typeof tokensAfter === "number" && Number.isFinite(tokensAfter) && tokensAfter >= 0) { + input.state.lastCompactionTokensAfter = Math.floor(tokensAfter); + } + if (input.contextEngine.info.ownsCompaction === true) { + const activeSession = input.getActiveSession(); + await runPostCompactionSideEffects({ + config: input.runParams.config, + sessionKey: input.runParams.sessionKey, + sessionId: activeSession.id, + agentId: input.sessionAgentId, + sessionFile: activeSession.file, + }); + } + log.info( + `[timeout-compaction] compaction succeeded for ${input.provider}/${input.modelId}; retrying prompt`, + ); + input.armPostCompactionGuard(); + return true; +}