fix(qa): scope mock image prompts to latest turn

This commit is contained in:
Vincent Koc
2026-05-27 08:07:34 +02:00
parent 14198a1c66
commit c2d059dc29
2 changed files with 94 additions and 6 deletions

View File

@@ -3129,6 +3129,53 @@ describe("qa mock openai server", () => {
expect(text).not.toBe("ui bridge armed");
});
it("keeps stale image prompts from overriding later marker turns", async () => {
const server = await startMockServer();
const response = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
stream: false,
model: "mock-openai/gpt-5.5",
input: [
{
role: "user",
content: [
{
type: "input_text",
text: "Image understanding check: describe the top and bottom colors.",
},
{
type: "input_image",
source: {
type: "base64",
mime_type: "image/png",
data: QA_IMAGE_PNG_BASE64,
},
},
],
},
{
role: "assistant",
content: [
{
type: "output_text",
text: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.",
},
],
},
makeUserInput("Marker exact marker: `fresh-marker-ok`"),
],
}),
});
expect(response.status).toBe(200);
const payload = (await response.json()) as {
output?: Array<{ content?: Array<{ text?: string }> }>;
};
expect(payload.output?.[0]?.content?.[0]?.text).toBe("fresh-marker-ok");
});
it("handles deeply nested image input shapes without recursive traversal failure", async () => {
const server = await startQaMockOpenAiServer({
host: "127.0.0.1",

View File

@@ -555,6 +555,35 @@ function countImageInputs(value: unknown): number {
return count;
}
function extractLatestImageUserTurn(input: ResponsesInputItem[]) {
const latestUserIndex = findLastUserIndex(input);
if (latestUserIndex < 0) {
return { text: "", imageInputCount: 0 };
}
let startIndex = latestUserIndex;
while (
startIndex > 0 &&
input[startIndex - 1]?.role === "user" &&
Array.isArray(input[startIndex - 1]?.content)
) {
startIndex -= 1;
}
const imageTurnItems = input.slice(startIndex, latestUserIndex + 1);
const imageInputCount = countImageInputs(imageTurnItems.map((item) => item.content));
if (imageInputCount === 0) {
return { text: "", imageInputCount: 0 };
}
return {
text: imageTurnItems
.map((item) => extractInputText(item.content as unknown[]))
.filter(Boolean)
.join("\n"),
imageInputCount,
};
}
function parseToolOutputJson(toolOutput: string): Record<string, unknown> | null {
if (!toolOutput.trim()) {
return null;
@@ -1009,7 +1038,7 @@ function buildAssistantText(
extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText);
const finishExactlyDirective =
extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText);
const imageInputCount = countImageInputs(input);
const latestImageUserTurn = extractLatestImageUserTurn(input);
const activeMemorySummary = extractActiveMemorySummary(allInputText);
const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet);
const sessionsSpawnError = extractToolErrorForNamedCall({
@@ -1036,10 +1065,16 @@ function buildAssistantText(
if (isHeartbeatPrompt(prompt)) {
return "HEARTBEAT_OK";
}
if (/roundtrip image inspection check/i.test(allInputText) && imageInputCount > 0) {
if (
/roundtrip image inspection check/i.test(latestImageUserTurn.text) &&
latestImageUserTurn.imageInputCount > 0
) {
return "Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step.";
}
if (/image understanding check/i.test(allInputText) && imageInputCount > 0) {
if (
/image understanding check/i.test(latestImageUserTurn.text) &&
latestImageUserTurn.imageInputCount > 0
) {
return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.";
}
if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) {
@@ -1565,7 +1600,7 @@ async function buildResponsesPayload(
extractExactReplyDirective(prompt) ?? extractExactReplyDirective(allInputText);
const exactMarkerDirective =
extractExactMarkerDirective(prompt) ?? extractExactMarkerDirective(allInputText);
const imageInputCount = countImageInputs(input);
const latestImageUserTurn = extractLatestImageUserTurn(input);
const firstExactMarkerDirective = extractLabeledMarkerDirective(
allInputText,
"first exact marker",
@@ -1652,12 +1687,18 @@ async function buildResponsesPayload(
if (/fanout worker beta/i.test(prompt)) {
return buildAssistantEvents("BETA-OK");
}
if (/roundtrip image inspection check/i.test(allInputText) && imageInputCount > 0) {
if (
/roundtrip image inspection check/i.test(latestImageUserTurn.text) &&
latestImageUserTurn.imageInputCount > 0
) {
return buildAssistantEvents(
"Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step.",
);
}
if (/image understanding check/i.test(allInputText) && imageInputCount > 0) {
if (
/image understanding check/i.test(latestImageUserTurn.text) &&
latestImageUserTurn.imageInputCount > 0
) {
return buildAssistantEvents(
"Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.",
);