diff --git a/docs/help/testing-live.md b/docs/help/testing-live.md index 0d60b34f1d7f..c4ab7b437911 100644 --- a/docs/help/testing-live.md +++ b/docs/help/testing-live.md @@ -320,14 +320,26 @@ Docker notes: - Default thinking: `low` - Model override: `OPENCLAW_LIVE_CODEX_HARNESS_MODEL=openai/` - Thinking override: `OPENCLAW_LIVE_CODEX_HARNESS_THINKING=` +- Non-default model effort assertion: + `OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT=` - Matrix override: `OPENCLAW_LIVE_CODEX_HARNESS_TARGETS==,...` - Auth mode: `OPENCLAW_LIVE_CODEX_HARNESS_AUTH=codex-auth` (default) uses the copied Codex login; `api-key` uses `OPENAI_API_KEY` through Codex app-server. - Optional image probe: `OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE=1` - Optional MCP/tool probe: `OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE=1` - Optional Guardian probe: `OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE=1` +- Optional resume stress: `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1` adds + four history turns, then closes and restarts the Gateway and Codex app-server + three times while requiring the same native thread id and conversation + history. Override the bounded counts with + `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS` (1-20) and + `OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS` (1-10). - Optional loop-relay opt-out probe: `OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY=1` +- The requested thinking preference may map to the nearest effort advertised + by Codex for that model. For example, Luna maps `minimal` to `low`. +- Known Codex catalog models derive that exact native effort automatically. + Unknown model overrides must state the expected mapped effort. - The smoke forces provider/model `agentRuntime.id: "codex"` so a broken Codex harness cannot pass by silently falling back to OpenClaw. - Auth: Codex app-server auth from the local Codex subscription login, or @@ -351,6 +363,13 @@ Docker recipe: pnpm test:docker:live-codex-harness ``` +Restart and history stress: + +```bash +OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS=1 \ + pnpm test:docker:live-codex-harness +``` + GPT-5.6 native Codex matrix: ```bash diff --git a/scripts/test-live-codex-harness-docker.sh b/scripts/test-live-codex-harness-docker.sh index bea27cafb017..0f06a4ea7ca4 100644 --- a/scripts/test-live-codex-harness-docker.sh +++ b/scripts/test-live-codex-harness-docker.sh @@ -359,6 +359,7 @@ fi echo "==> Run Codex harness live test in Docker" echo "==> Model: ${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" echo "==> Thinking: ${OPENCLAW_LIVE_CODEX_HARNESS_THINKING:-low}" +echo "==> Expected native effort: ${OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT:-auto}" echo "==> Targets: ${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-single model}" echo "==> Target count: $CODEX_HARNESS_TARGET_COUNT" echo "==> Docker run timeout: $CODEX_HARNESS_DOCKER_RUN_TIMEOUT" @@ -368,6 +369,9 @@ echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" echo "==> Subagent probe: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" echo "==> Subagent-only fast path: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-auto}" echo "==> Guardian probe: ${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" +echo "==> Resume stress: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" +echo "==> Resume stress history turns: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" +echo "==> Resume stress restarts: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE" echo "==> Profile file: $PROFILE_STATUS" echo "==> CI-safe Codex config: ${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" @@ -404,8 +408,12 @@ DOCKER_RUN_ARGS+=(--rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_TARGETS="${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_THINKING="${OPENCLAW_LIVE_CODEX_HARNESS_THINKING:-low}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT="${OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT:-}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS="${OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS:-1}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS="${OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$CODEX_HARNESS_SETUP_TIMEOUT_SECONDS" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" \ diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index d4fe04880cf0..a781904dbe1c 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -73,6 +73,21 @@ const CODEX_HARNESS_SUBAGENT_ONLY = const CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS, ); +const CODEX_HARNESS_RESUME_STRESS = isTruthyEnvValue( + process.env.OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS, +); +const CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS, + 4, + 20, +); +const CODEX_HARNESS_RESUME_STRESS_RESTARTS = resolveBoundedPositiveIntEnv( + "OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS", + process.env.OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS, + 3, + 10, +); const CODEX_HARNESS_REQUEST_TIMEOUT_MS = resolveLiveTimeoutMs( process.env.OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS, 300_000, @@ -88,9 +103,27 @@ const CODEX_HARNESS_THINKING = resolveCodexHarnessThinkingLevel( ); const describeLive = LIVE && CODEX_HARNESS_LIVE ? describe : describe.skip; const describeDisabled = LIVE && !CODEX_HARNESS_LIVE ? describe : describe.skip; -const CODEX_HARNESS_TIMEOUT_MS = 900_000; +const CODEX_HARNESS_TIMEOUT_MS = CODEX_HARNESS_RESUME_STRESS ? 1_800_000 : 900_000; const DEFAULT_CODEX_MODEL = "openai/gpt-5.6-luna"; const GATEWAY_CONNECT_TIMEOUT_MS = 60_000; +const CODEX_HARNESS_REASONING_EFFORTS = [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + "ultra", +] as const; +const CODEX_HARNESS_SUPPORTED_EFFORTS = new Map([ + ["gpt-5.6-sol", ["low", "medium", "high", "xhigh", "max", "ultra"]], + ["gpt-5.6-terra", ["low", "medium", "high", "xhigh", "max", "ultra"]], + ["gpt-5.6-luna", ["low", "medium", "high", "xhigh", "max"]], + ["gpt-5.5", ["low", "medium", "high", "xhigh"]], + ["gpt-5.4", ["low", "medium", "high", "xhigh"]], + ["gpt-5.4-mini", ["low", "medium", "high", "xhigh"]], + ["gpt-5.2", ["low", "medium", "high", "xhigh"]], +]); type CapturedAgentEvent = { stream: string; @@ -98,6 +131,8 @@ type CapturedAgentEvent = { sessionKey?: string; }; +const observedCodexThreadIds = new Map(); + type GuardianPluginApprovalDecision = "allow-once" | "deny"; type CodexHarnessThinkingLevel = | "off" @@ -114,6 +149,25 @@ function resolveLiveTimeoutMs(raw: string | undefined, fallback: number): number return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; } +function resolveBoundedPositiveIntEnv( + name: string, + raw: string | undefined, + fallback: number, + max: number, +): number { + if (raw === undefined || raw.trim() === "") { + return fallback; + } + if (!/^\d+$/u.test(raw)) { + throw new Error(`invalid ${name}: ${raw}`); + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > max) { + throw new Error(`${name} must be between 1 and ${max}`); + } + return parsed; +} + function resolveCodexHarnessThinkingLevel(raw: string | undefined): CodexHarnessThinkingLevel { const normalized = raw?.trim().toLowerCase() || "low"; if (!["off", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"].includes(normalized)) { @@ -122,6 +176,38 @@ function resolveCodexHarnessThinkingLevel(raw: string | undefined): CodexHarness return normalized as CodexHarnessThinkingLevel; } +function resolveCodexHarnessExpectedEffort(modelId: string): string | null { + const configured = process.env.OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT; + if (configured?.trim()) { + const expected = resolveCodexHarnessThinkingLevel(configured); + return expected === "off" ? null : expected; + } + const supported = CODEX_HARNESS_SUPPORTED_EFFORTS.get(modelId); + if (!supported) { + throw new Error(`set OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT for unknown model ${modelId}`); + } + if (CODEX_HARNESS_THINKING === "off") { + return null; + } + // Independent oracle for the pinned Codex model catalog. Lower requested + // levels choose the nearest advertised effort; Ultra remains explicit. + const candidates = + CODEX_HARNESS_THINKING === "ultra" + ? supported + : supported.filter((effort) => effort !== "ultra"); + const requestedRank = CODEX_HARNESS_REASONING_EFFORTS.indexOf(CODEX_HARNESS_THINKING); + return ( + candidates.find( + (effort) => + CODEX_HARNESS_REASONING_EFFORTS.indexOf( + effort as (typeof CODEX_HARNESS_REASONING_EFFORTS)[number], + ) >= requestedRank, + ) ?? + candidates.at(-1) ?? + null + ); +} + function logCodexLiveStep(step: string, details?: Record): void { if (!CODEX_HARNESS_DEBUG) { return; @@ -437,11 +523,22 @@ async function requestAgentText(params: { const expectedModel = parseModelKey( process.env.OPENCLAW_LIVE_CODEX_HARNESS_MODEL ?? DEFAULT_CODEX_MODEL, ).modelId; - expect(turnStarting?.data).toMatchObject({ - model: expectedModel, - effort: CODEX_HARNESS_THINKING, - collaborationEffort: CODEX_HARNESS_THINKING, - }); + expect(turnStarting?.data).toMatchObject({ model: expectedModel }); + const actualEffort = turnStarting?.data?.effort; + const actualCollaborationEffort = turnStarting?.data?.collaborationEffort; + const expectedEffort = resolveCodexHarnessExpectedEffort(expectedModel); + expect(actualEffort ?? null).toBe(expectedEffort); + expect(actualCollaborationEffort ?? null).toBe(actualEffort ?? null); + const threadReady = events.find( + (event) => + event.stream === "codex_app_server.lifecycle" && event.data?.phase === "thread_ready", + ); + const threadId = threadReady?.data?.threadId; + expect( + typeof threadId === "string" && threadId.trim().length > 0, + `expected Codex thread_ready identity for ${params.sessionKey}; events=${JSON.stringify(events)}`, + ).toBe(true); + observedCodexThreadIds.set(params.sessionKey, threadId as string); return text; } @@ -1220,6 +1317,9 @@ describeLive("gateway live (Codex harness)", () => { const resolvedGuardianPluginApprovalIds = new Set(); let guardianPluginApprovalDecision: GuardianPluginApprovalDecision | undefined; let activeApprovalClient: GatewayClient | undefined; + let resumeStressState: + | { lastMarker: string; sessionKey: string; threadId: string } + | undefined; const maybeResolveGuardianPluginApproval = (event: EventFrame): void => { const decision = guardianPluginApprovalDecision; const approvalClient = activeApprovalClient; @@ -1244,6 +1344,7 @@ describeLive("gateway live (Codex harness)", () => { }); }; logCodexLiveStep("config-written", { configPath, modelKey, port }); + observedCodexThreadIds.clear(); try { server = await startGatewayServer(port, { @@ -1428,6 +1529,34 @@ describeLive("gateway live (Codex harness)", () => { }); logCodexLiveStep("guardian-probe:done"); } + if (CODEX_HARNESS_RESUME_STRESS) { + const threadId = observedCodexThreadIds.get(sessionKey); + if (!threadId) { + throw new Error("Codex resume stress did not observe a thread identity"); + } + let lastMarker = ""; + for ( + let historyTurn = 1; + historyTurn <= CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS; + historyTurn += 1 + ) { + lastMarker = `CODEX-HISTORY-${historyTurn}-${randomBytes(3) + .toString("hex") + .toUpperCase()}`; + await requestAgentText({ + client: activeClient, + sessionKey, + expectedToken: lastMarker, + message: `Reply with exactly ${lastMarker} and nothing else.`, + }); + expect(observedCodexThreadIds.get(sessionKey)).toBe(threadId); + } + resumeStressState = { lastMarker, sessionKey, threadId }; + logCodexLiveStep("resume-stress:history-ready", { + historyTurns: CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS + 2, + threadId, + }); + } break; } catch (error) { if (isCodexAccountTokenError(error)) { @@ -1454,6 +1583,65 @@ describeLive("gateway live (Codex harness)", () => { } } } + if (CODEX_HARNESS_RESUME_STRESS) { + if (!resumeStressState) { + throw new Error("Codex resume stress did not seed a thread"); + } + for (let restart = 1; restart <= CODEX_HARNESS_RESUME_STRESS_RESTARTS; restart += 1) { + activeApprovalClient = undefined; + await client?.stopAndWait(); + client = undefined; + await server?.close(); + server = undefined; + clearRuntimeConfigSnapshot(); + gatewayEvents.length = 0; + + server = await startGatewayServer(port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + }); + client = await connectTestGatewayClient({ + url: `ws://127.0.0.1:${port}`, + token, + deviceIdentity, + timeoutMs: GATEWAY_CONNECT_TIMEOUT_MS, + requestTimeoutMs: CODEX_HARNESS_REQUEST_TIMEOUT_MS, + clientDisplayName: `vitest-codex-resume-stress-${restart}`, + onEvent: (event) => { + gatewayEvents.push(event); + maybeResolveGuardianPluginApproval(event); + }, + }); + activeApprovalClient = client; + await assertCodexHarnessSessionSelection({ + client, + modelKey, + sessionKey: resumeStressState.sessionKey, + }); + const nextMarker = `CODEX-RESTART-${restart}-${randomBytes(3) + .toString("hex") + .toUpperCase()}`; + const resumedText = await requestAgentText({ + client, + sessionKey: resumeStressState.sessionKey, + expectedToken: nextMarker, + message: [ + "Reply with exactly two lines and nothing else.", + "First line: the final marker from the immediately preceding assistant message.", + `Second line: ${nextMarker}`, + ].join(" "), + }); + expect(resumedText.trim()).toBe(`${resumeStressState.lastMarker}\n${nextMarker}`); + const resumedThreadId = observedCodexThreadIds.get(resumeStressState.sessionKey); + expect(resumedThreadId).toBe(resumeStressState.threadId); + resumeStressState.lastMarker = nextMarker; + logCodexLiveStep("resume-stress:restart-complete", { + restart, + threadId: resumedThreadId, + }); + } + } } finally { try { clearRuntimeConfigSnapshot(); diff --git a/test/scripts/test-live-codex-harness-docker.test.ts b/test/scripts/test-live-codex-harness-docker.test.ts index 281a101e3acc..9e3a45758ed3 100644 --- a/test/scripts/test-live-codex-harness-docker.test.ts +++ b/test/scripts/test-live-codex-harness-docker.test.ts @@ -95,6 +95,23 @@ describe("scripts/test-live-codex-harness-docker.sh", () => { ); }); + it("forwards bounded resume stress controls into Docker", () => { + const script = fs.readFileSync(SCRIPT_PATH, "utf8"); + + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT="${OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT:-}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}"', + ); + expect(script).toContain( + '-e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}"', + ); + }); + it("installs the plugin-pinned Codex CLI package for app-server proof", () => { const script = fs.readFileSync(SCRIPT_PATH, "utf8");