From c2d059dc294bcd8c038f781e46840c5bce459faa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 27 May 2026 08:07:34 +0200 Subject: [PATCH] fix(qa): scope mock image prompts to latest turn --- .../src/providers/mock-openai/server.test.ts | 47 ++++++++++++++++ .../src/providers/mock-openai/server.ts | 53 ++++++++++++++++--- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index de08b44818f..eb25300da1a 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -3129,6 +3129,53 @@ describe("qa mock openai server", () => { expect(text).not.toBe("ui bridge armed"); }); + it("keeps stale image prompts from overriding later marker turns", async () => { + const server = await startMockServer(); + + const response = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + stream: false, + model: "mock-openai/gpt-5.5", + input: [ + { + role: "user", + content: [ + { + type: "input_text", + text: "Image understanding check: describe the top and bottom colors.", + }, + { + type: "input_image", + source: { + type: "base64", + mime_type: "image/png", + data: QA_IMAGE_PNG_BASE64, + }, + }, + ], + }, + { + role: "assistant", + content: [ + { + type: "output_text", + text: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.", + }, + ], + }, + makeUserInput("Marker exact marker: `fresh-marker-ok`"), + ], + }), + }); + expect(response.status).toBe(200); + const payload = (await response.json()) as { + output?: Array<{ content?: Array<{ text?: string }> }>; + }; + expect(payload.output?.[0]?.content?.[0]?.text).toBe("fresh-marker-ok"); + }); + it("handles deeply nested image input shapes without recursive traversal failure", async () => { const server = await startQaMockOpenAiServer({ host: "127.0.0.1", diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 13625ca9d9e..4fedea4f184 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -555,6 +555,35 @@ function countImageInputs(value: unknown): number { return count; } +function extractLatestImageUserTurn(input: ResponsesInputItem[]) { + const latestUserIndex = findLastUserIndex(input); + if (latestUserIndex < 0) { + return { text: "", imageInputCount: 0 }; + } + + let startIndex = latestUserIndex; + while ( + startIndex > 0 && + input[startIndex - 1]?.role === "user" && + Array.isArray(input[startIndex - 1]?.content) + ) { + startIndex -= 1; + } + + const imageTurnItems = input.slice(startIndex, latestUserIndex + 1); + const imageInputCount = countImageInputs(imageTurnItems.map((item) => item.content)); + if (imageInputCount === 0) { + return { text: "", imageInputCount: 0 }; + } + return { + text: imageTurnItems + .map((item) => extractInputText(item.content as unknown[])) + .filter(Boolean) + .join("\n"), + imageInputCount, + }; +} + function parseToolOutputJson(toolOutput: string): Record | null { if (!toolOutput.trim()) { return null; @@ -1009,7 +1038,7 @@ function buildAssistantText( extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); const finishExactlyDirective = extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText); - const imageInputCount = countImageInputs(input); + const latestImageUserTurn = extractLatestImageUserTurn(input); const activeMemorySummary = extractActiveMemorySummary(allInputText); const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet); const sessionsSpawnError = extractToolErrorForNamedCall({ @@ -1036,10 +1065,16 @@ function buildAssistantText( if (isHeartbeatPrompt(prompt)) { return "HEARTBEAT_OK"; } - if (/roundtrip image inspection check/i.test(allInputText) && imageInputCount > 0) { + if ( + /roundtrip image inspection check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { return "Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step."; } - if (/image understanding check/i.test(allInputText) && imageInputCount > 0) { + if ( + /image understanding check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom."; } if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) { @@ -1565,7 +1600,7 @@ async function buildResponsesPayload( extractExactReplyDirective(prompt) ?? extractExactReplyDirective(allInputText); const exactMarkerDirective = extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText); - const imageInputCount = countImageInputs(input); + const latestImageUserTurn = extractLatestImageUserTurn(input); const firstExactMarkerDirective = extractLabeledMarkerDirective( allInputText, "first exact marker", @@ -1652,12 +1687,18 @@ async function buildResponsesPayload( if (/fanout worker beta/i.test(prompt)) { return buildAssistantEvents("BETA-OK"); } - if (/roundtrip image inspection check/i.test(allInputText) && imageInputCount > 0) { + if ( + /roundtrip image inspection check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { return buildAssistantEvents( "Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step.", ); } - if (/image understanding check/i.test(allInputText) && imageInputCount > 0) { + if ( + /image understanding check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { return buildAssistantEvents( "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.", );