From 397fe272a0d1cb1fc759a8fd484642f181981bab Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 31 Jul 2026 14:36:59 +0800 Subject: [PATCH] fix(qa): stop waiting on yielded fanout runs (#116440) * fix(qa): observe resumed fanout completions * test(qa): keep runtime watchdog assertions in runtime suite --- .../src/scenario-catalog-channels.test.ts | 7 ++++-- .../src/scenario-catalog.runtime-pair.test.ts | 7 ++++++ .../agents/subagent-fanout-synthesis.yaml | 25 +++++++++++-------- .../long-context-progress-watchdog.yaml | 6 ++++- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/extensions/qa-lab/src/scenario-catalog-channels.test.ts b/extensions/qa-lab/src/scenario-catalog-channels.test.ts index 743967809edc..e9c6fac77586 100644 --- a/extensions/qa-lab/src/scenario-catalog-channels.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-channels.test.ts @@ -77,7 +77,10 @@ describe("qa scenario catalog channel contracts", () => { const completionWait = flow.indexOf('"saveAs":"completedFanout"'); const storeReads = [...flow.matchAll(/readRawQaSessionStore/gu)].map((match) => match.index); - expect(flow).toContain("readSessionTranscriptSummary(env, sessionKey)"); + expect(flow).toContain('"call":"startAgentRun"'); + expect(flow).not.toContain('"call":"runAgentPrompt"'); + expect(flow).toContain('"saveAs":"parentOutbound"'); + expect(flow).toContain("messages.slice(parentOutboundStartIndex)"); expect(flow).not.toContain("waitForAgentHistoryReply"); expect(flow).not.toContain('"call":"waitForOutboundMessage"'); expect(flow).not.toContain("childCompletionMarker"); @@ -87,7 +90,7 @@ describe("qa scenario catalog channel contracts", () => { ); expect(flow).toContain("Boolean(env.mock) ? config.expectedChildCompletionMarkers[0] : 'ok'"); expect(flow).toContain('saveAs":"timeoutEvidence'); - expect(flow).toContain('saveAs":"recoveredParentTranscript'); + expect(flow).toContain('saveAs":"recoveredParentOutbound'); expect(flow).not.toContain('"value":"subagent-1: ok\\nsubagent-2: ok"'); expect(flow).toContain("Promise.all([readSessionTranscriptSummary"); expect(completionWait).toBeGreaterThan(-1); diff --git a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts index 2709939b6b98..f8a4c80561c4 100644 --- a/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.runtime-pair.test.ts @@ -76,5 +76,12 @@ describe("QA runtime-pair scenario catalog", () => { requiredProviderMode: "live-frontier", harnessRuntime: "codex", }); + const longContextFlow = JSON.stringify( + readQaScenarioById("long-context-progress-watchdog").execution.flow, + ); + expect(longContextFlow).toContain("agentRuntime: { id: config.harnessRuntime }, params: null"); + expect(longContextFlow).toContain( + "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.params === undefined", + ); }); }); diff --git a/qa/scenarios/agents/subagent-fanout-synthesis.yaml b/qa/scenarios/agents/subagent-fanout-synthesis.yaml index 3011a271345e..bf90728c933d 100644 --- a/qa/scenarios/agents/subagent-fanout-synthesis.yaml +++ b/qa/scenarios/agents/subagent-fanout-synthesis.yaml @@ -93,7 +93,12 @@ flow: - set: sessionKey value: expr: "`agent:qa:fanout:${attempt}:${randomUUID().slice(0, 8)}`" - - call: runAgentPrompt + - set: parentOutboundStartIndex + value: + expr: "state.getSnapshot().messages.length" + # sessions_yield ends the original run. Observe the resumed + # announce turn through channel delivery instead of agent.wait. + - call: startAgentRun args: - ref: env - sessionKey: @@ -101,14 +106,13 @@ flow: message: ref: prompt timeoutMs: - expr: liveTurnTimeoutMs(env, 90000) + expr: liveTurnTimeoutMs(env, 30000) - call: waitForCondition - saveAs: parentTranscript + saveAs: parentOutbound args: - lambda: - async: true - expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(summary.finalText).includes(needle))) ? summary : undefined).catch(() => undefined)" - - expr: "30000" + expr: "state.getSnapshot().messages.slice(parentOutboundStartIndex).find((message) => message.direction === 'outbound' && message.conversation.id === 'qa-operator' && config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(message.text).includes(needle))))" + - expr: liveTurnTimeoutMs(env, 90000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" # Native child completions are private. Wait for this parent's # actual SQLite rows and both settled final transcripts. @@ -151,7 +155,7 @@ flow: expr: "`expected at least two sessions_spawn tool calls during subagent fanout scenario, saw ${fanoutSpawnRequests.length}`" - set: details value: - expr: "parentTranscript.finalText" + expr: "parentOutbound.text" - set: lastError value: __done__ catchAs: attemptError @@ -218,16 +222,15 @@ flow: expr: "timeoutSawAlpha && timeoutSawBeta && timeoutAlphaOk && timeoutBetaOk && (!env.mock || timeoutSpawnRequests.length >= 2)" then: - call: waitForCondition - saveAs: recoveredParentTranscript + saveAs: recoveredParentOutbound args: - lambda: - async: true - expr: "readSessionTranscriptSummary(env, sessionKey).then((summary) => config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(summary.finalText).includes(needle))) ? summary : undefined).catch(() => undefined)" + expr: "state.getSnapshot().messages.slice(parentOutboundStartIndex).find((message) => message.direction === 'outbound' && message.conversation.id === 'qa-operator' && config.expectedReplyGroups.every((group) => group.some((needle) => normalizeLowercaseStringOrEmpty(message.text).includes(needle))))" - expr: liveTurnTimeoutMs(env, 30000) - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - set: details value: - expr: recoveredParentTranscript.finalText + expr: recoveredParentOutbound.text - set: lastError value: __done__ catchAs: recoveryError diff --git a/qa/scenarios/runtime/long-context-progress-watchdog.yaml b/qa/scenarios/runtime/long-context-progress-watchdog.yaml index 939094c84936..7245551c756d 100644 --- a/qa/scenarios/runtime/long-context-progress-watchdog.yaml +++ b/qa/scenarios/runtime/long-context-progress-watchdog.yaml @@ -74,7 +74,7 @@ flow: agents: defaults: models: - expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime } } })" + expr: "({ [env.primaryModel]: { agentRuntime: { id: config.harnessRuntime }, params: null } })" - call: waitForGatewayHealthy args: - ref: env @@ -91,6 +91,10 @@ flow: expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime?.id === config.harnessRuntime" message: expr: "`expected ${env.primaryModel} agentRuntime.id=${config.harnessRuntime}, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.agentRuntime)}`" + - assert: + expr: "snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.params === undefined" + message: + expr: "`expected ${env.primaryModel} provider request params to be absent for the Codex harness, got ${JSON.stringify(snapshot.config.agents?.defaults?.models?.[env.primaryModel]?.params)}`" - call: reset - set: logCursor value: