diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index fb73b8da8ce..53cb28629b5 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -39,6 +39,9 @@ type QaSuiteEnvironment = { alternateModel: string; }; +const QA_IMAGE_UNDERSTANDING_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3RQQkAMAzAwPg33Wnos+wgBo40dboAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANYADwAAAAAAAAAAAAAAAAAAAAAAAAAAAAC+Azy47PDiI4pA2wAAAABJRU5ErkJggg=="; + type QaSkillStatusEntry = { name?: string; eligible?: boolean; @@ -538,6 +541,11 @@ async function runAgentPrompt( provider?: string; model?: string; timeoutMs?: number; + attachments?: Array<{ + mimeType: string; + fileName: string; + content: string; + }>; }, ) { const target = params.to ?? "dm:qa-operator"; @@ -556,6 +564,7 @@ async function runAgentPrompt( ...(params.threadId ? { threadId: params.threadId } : {}), ...(params.provider ? { provider: params.provider } : {}), ...(params.model ? { model: params.model } : {}), + ...(params.attachments ? { attachments: params.attachments } : {}), }, { timeoutMs: params.timeoutMs ?? 30_000, @@ -1485,6 +1494,55 @@ When the user asks for the hot install marker exactly, reply with exactly: HOT-I }, ]), ], + [ + "image-understanding-attachment", + async () => + await runScenario("Image understanding from attachment", [ + { + name: "describes an attached image in one short sentence", + run: async () => { + await reset(); + await runAgentPrompt(env, { + sessionKey: "agent:qa:image-understanding", + message: + "Image understanding check: describe the attached image in one short sentence.", + attachments: [ + { + mimeType: "image/png", + fileName: "red-top-blue-bottom.png", + content: QA_IMAGE_UNDERSTANDING_PNG_BASE64, + }, + ], + timeoutMs: liveTurnTimeoutMs(env, 45_000), + }); + const outbound = await waitForOutboundMessage( + state, + (candidate) => candidate.conversation.id === "qa-operator", + liveTurnTimeoutMs(env, 45_000), + ); + const lower = outbound.text.toLowerCase(); + if (!lower.includes("red") || !lower.includes("blue")) { + throw new Error(`missing expected colors in image description: ${outbound.text}`); + } + if (env.mock) { + const mockBaseUrl = env.mock.baseUrl; + const requests = await fetchJson< + Array<{ prompt?: string; imageInputCount?: number; model?: string }> + >(`${mockBaseUrl}/debug/requests`); + const imageRequest = requests.find((request) => + String(request.prompt ?? "").includes("Image understanding check"), + ); + if ((imageRequest?.imageInputCount ?? 0) < 1) { + throw new Error( + `expected at least one input image, got ${String(imageRequest?.imageInputCount ?? 0)}`, + ); + } + } + return outbound.text; + }, + }, + ]), + ], [ "config-patch-hot-apply", async () => diff --git a/qa/seed-scenarios.json b/qa/seed-scenarios.json index 3bdbf22a314..df12582ceb4 100644 --- a/qa/seed-scenarios.json +++ b/qa/seed-scenarios.json @@ -230,6 +230,23 @@ "extensions/qa-lab/src/mock-openai-server.ts" ] }, + { + "id": "image-understanding-attachment", + "title": "Image understanding from attachment", + "surface": "image-understanding", + "objective": "Verify an attached image reaches the agent model and the agent can describe what it sees.", + "successCriteria": [ + "Agent receives at least one image attachment.", + "Final answer describes the visible image content in one short sentence.", + "The description mentions the expected red and blue regions." + ], + "docsRefs": ["docs/help/testing.md", "docs/tools/index.md"], + "codeRefs": [ + "src/gateway/server-methods/agent.ts", + "extensions/qa-lab/src/suite.ts", + "extensions/qa-lab/src/mock-openai-server.ts" + ] + }, { "id": "config-patch-hot-apply", "title": "Config patch skill disable",