From dd4e2abfb55103f068bb96a71e66e43e780b5489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?frond=20scribe=20=F0=9F=8C=BF?= Date: Sun, 28 Jun 2026 10:47:31 -0700 Subject: [PATCH] fix: use no-output placeholder for empty OpenAI tool results (#97423) Co-authored-by: scribe-dandelion-cult Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agents/openai-transport-stream.test.ts | 116 ++++++++++++++++++ src/agents/openai-transport-stream.ts | 12 +- src/llm/providers/openai-completions.test.ts | 115 +++++++++++++++++ src/llm/providers/openai-completions.ts | 15 ++- .../providers/openai-responses-shared.test.ts | 94 ++++++++++++++ src/llm/providers/openai-responses-shared.ts | 18 ++- 6 files changed, 361 insertions(+), 9 deletions(-) diff --git a/src/agents/openai-transport-stream.test.ts b/src/agents/openai-transport-stream.test.ts index 7093f81adf9..003dc624d3c 100644 --- a/src/agents/openai-transport-stream.test.ts +++ b/src/agents/openai-transport-stream.test.ts @@ -5075,6 +5075,122 @@ describe("openai transport stream", () => { } }); + it("replays update_plan-style empty non-image Responses tool results as no output", () => { + const params = buildOpenAIResponsesParams( + { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 8192, + } satisfies Model<"openai-responses">, + { + systemPrompt: "system", + messages: [ + { + role: "assistant", + api: "openai-responses", + provider: "openai", + model: "gpt-5.5", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 1, + content: [{ type: "toolCall", id: "call_plan", name: "update_plan", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "call_plan", + toolName: "update_plan", + content: [], + isError: false, + timestamp: 2, + }, + ], + tools: [], + } as never, + { sessionId: "session-123" }, + ) as { + input?: Array<{ type?: string; call_id?: string; output?: unknown }>; + }; + + expect(params.input?.find((item) => item.type === "function_call_output")).toMatchObject({ + type: "function_call_output", + call_id: "call_plan", + output: "(no output)", + }); + }); + + it("preserves image-bearing Responses tool results as image input parts", () => { + const params = buildOpenAIResponsesParams( + { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 8192, + } satisfies Model<"openai-responses">, + { + systemPrompt: "system", + messages: [ + { + role: "assistant", + api: "openai-responses", + provider: "openai", + model: "gpt-5.5", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 1, + content: [{ type: "toolCall", id: "call_shot", name: "screenshot", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "call_shot", + toolName: "screenshot", + content: [{ type: "image", mimeType: "image/png", data: "aW1n" }], + isError: false, + timestamp: 2, + }, + ], + tools: [], + } as never, + { sessionId: "session-123" }, + ) as { + input?: Array<{ type?: string; output?: unknown }>; + }; + + expect(params.input?.find((item) => item.type === "function_call_output")?.output).toEqual([ + { + type: "input_image", + detail: "auto", + image_url: "data:image/png;base64,aW1n", + }, + ]); + }); + it("omits distinct overlong Copilot Responses replay item ids when store is disabled", () => { const sharedToolItemPrefix = "iVec" + "A".repeat(160); const firstToolCallId = `call_first|${sharedToolItemPrefix}Aa`; diff --git a/src/agents/openai-transport-stream.ts b/src/agents/openai-transport-stream.ts index 7957f6f7d72..efdb4155dcd 100644 --- a/src/agents/openai-transport-stream.ts +++ b/src/agents/openai-transport-stream.ts @@ -101,6 +101,7 @@ import { failTransportStream, finalizeTransportStream, mergeTransportMetadata, + sanitizeNonEmptyTransportPayloadText, sanitizeTransportPayloadText, } from "./transport-stream-shared.js"; @@ -1277,6 +1278,8 @@ function convertResponsesMessages( .filter((item) => item.type === "text") .map((item) => item.text) .join("\n"); + const sanitizedTextResult = sanitizeTransportPayloadText(textResult); + const hasText = sanitizedTextResult.trim().length > 0; const hasImages = msg.content.some((item) => item.type === "image"); const [callId] = msg.toolCallId.split("|"); messages.push({ @@ -1285,9 +1288,7 @@ function convertResponsesMessages( output: hasImages && model.input.includes("image") ? ([ - ...(textResult - ? [{ type: "input_text", text: sanitizeTransportPayloadText(textResult) }] - : []), + ...(hasText ? [{ type: "input_text", text: sanitizedTextResult }] : []), ...msg.content .filter((item) => item.type === "image") .map((item) => ({ @@ -1296,7 +1297,10 @@ function convertResponsesMessages( image_url: `data:${item.mimeType};base64,${item.data}`, })), ] as ResponseFunctionCallOutputItemList) - : sanitizeTransportPayloadText(textResult || "(see attached image)"), + : sanitizeNonEmptyTransportPayloadText( + textResult, + hasImages ? "(see attached image)" : "(no output)", + ), }); } msgIndex += 1; diff --git a/src/llm/providers/openai-completions.test.ts b/src/llm/providers/openai-completions.test.ts index ede1f9ed320..86f89323492 100644 --- a/src/llm/providers/openai-completions.test.ts +++ b/src/llm/providers/openai-completions.test.ts @@ -248,6 +248,121 @@ describe("OpenAI-compatible completions params", () => { expect(capturedPayload?.tools).toEqual([]); }); + it("replays update_plan-style empty non-image tool results as no output", async () => { + let capturedMessages: + | Array<{ role?: string; content?: unknown; tool_call_id?: string }> + | undefined; + const stream = streamOpenAICompletions( + model, + { + messages: [ + { + role: "assistant", + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + content: [{ type: "toolCall", id: "call_plan", name: "update_plan", arguments: {} }], + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call_plan", + toolName: "update_plan", + content: [], + isError: false, + timestamp: 2, + }, + ], + } as never, + { + apiKey: "sk-test", + onPayload(payload) { + capturedMessages = (payload as { messages?: typeof capturedMessages }).messages; + throw new Error("stop before network"); + }, + }, + ); + + const result = await stream.result(); + + expect(result.stopReason).toBe("error"); + expect(capturedMessages?.find((message) => message.role === "tool")).toMatchObject({ + role: "tool", + content: "(no output)", + tool_call_id: "call_plan", + }); + }); + + it("preserves image-bearing tool results with image placeholders and attachments", async () => { + let capturedMessages: + | Array<{ role?: string; content?: unknown; tool_call_id?: string }> + | undefined; + const stream = streamOpenAICompletions( + { ...model, input: ["text", "image"] }, + { + messages: [ + { + role: "assistant", + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + content: [{ type: "toolCall", id: "call_shot", name: "screenshot", arguments: {} }], + timestamp: 1, + }, + { + role: "toolResult", + toolCallId: "call_shot", + toolName: "screenshot", + content: [{ type: "image", mimeType: "image/png", data: "aW1n" }], + isError: false, + timestamp: 2, + }, + ], + } as never, + { + apiKey: "sk-test", + onPayload(payload) { + capturedMessages = (payload as { messages?: typeof capturedMessages }).messages; + throw new Error("stop before network"); + }, + }, + ); + + const result = await stream.result(); + + expect(result.stopReason).toBe("error"); + expect(capturedMessages?.find((message) => message.role === "tool")).toMatchObject({ + role: "tool", + content: "(see attached image)", + tool_call_id: "call_shot", + }); + expect(capturedMessages?.find((message) => Array.isArray(message.content))).toMatchObject({ + role: "user", + content: [ + { type: "text", text: "Attached image(s) from tool result:" }, + { type: "image_url", image_url: { url: "data:image/png;base64,aW1n" } }, + ], + }); + }); + it("does not reread an unreadable tool inventory length", async () => { let capturedPayload: Record | undefined; const tools = new Proxy([], { diff --git a/src/llm/providers/openai-completions.ts b/src/llm/providers/openai-completions.ts index 7e8cbd6ee97..cb585785611 100644 --- a/src/llm/providers/openai-completions.ts +++ b/src/llm/providers/openai-completions.ts @@ -90,6 +90,14 @@ function isImageContentBlock(block: { type: string }): block is ImageContent { return block.type === "image"; } +const EMPTY_TOOL_RESULT_TEXT = "(no output)"; +const IMAGE_TOOL_RESULT_TEXT = "(see attached image)"; + +function sanitizeToolResultText(text: string, fallback: string): string { + const sanitized = sanitizeSurrogates(text); + return sanitized.trim().length > 0 ? sanitized : fallback; +} + export interface OpenAICompletionsOptions extends StreamOptions { toolChoice?: OpenAICompletionsToolChoice; reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh"; @@ -1135,11 +1143,14 @@ export function convertMessages( const hasImages = toolMsg.content.some((c) => c.type === "image"); // Always send tool result with text (or placeholder if only images) - const hasText = textResult.length > 0; + const content = sanitizeToolResultText( + textResult, + hasImages ? IMAGE_TOOL_RESULT_TEXT : EMPTY_TOOL_RESULT_TEXT, + ); // Some providers require the 'name' field in tool results const toolResultMsg: ChatCompletionToolMessageParam = { role: "tool", - content: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + content, tool_call_id: toolMsg.toolCallId, }; if (compat.requiresToolResultName && toolMsg.toolName) { diff --git a/src/llm/providers/openai-responses-shared.test.ts b/src/llm/providers/openai-responses-shared.test.ts index 99142070c96..9f23fdeeb33 100644 --- a/src/llm/providers/openai-responses-shared.test.ts +++ b/src/llm/providers/openai-responses-shared.test.ts @@ -481,6 +481,100 @@ describe("convertResponsesMessages", () => { expect(functionCall).not.toHaveProperty("id"); }); + it("replays update_plan-style empty non-image tool results as no output", () => { + const input = convertResponsesMessages( + nativeOpenAIModel, + { + systemPrompt: "system", + messages: [ + { + role: "assistant", + api: nativeOpenAIModel.api, + provider: nativeOpenAIModel.provider, + model: nativeOpenAIModel.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 1, + content: [{ type: "toolCall", id: "call_plan", name: "update_plan", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "call_plan", + toolName: "update_plan", + content: [], + isError: false, + timestamp: 2, + }, + ], + } satisfies Context, + allowedToolCallProviders, + { includeSystemPrompt: false }, + ) as unknown as Array>; + + const functionOutput = input.find((item) => item.type === "function_call_output"); + expect(functionOutput).toMatchObject({ + type: "function_call_output", + call_id: "call_plan", + output: "(no output)", + }); + }); + + it("preserves image-bearing tool results instead of using no-output text", () => { + const input = convertResponsesMessages( + { ...nativeOpenAIModel, input: ["text", "image"] }, + { + systemPrompt: "system", + messages: [ + { + role: "assistant", + api: nativeOpenAIModel.api, + provider: nativeOpenAIModel.provider, + model: nativeOpenAIModel.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "toolUse", + timestamp: 1, + content: [ + { type: "toolCall", id: "call_screenshot", name: "screenshot", arguments: {} }, + ], + }, + { + role: "toolResult", + toolCallId: "call_screenshot", + toolName: "screenshot", + content: [{ type: "image", mimeType: "image/png", data: "aW1n" }], + isError: false, + timestamp: 2, + }, + ], + } satisfies Context, + allowedToolCallProviders, + { includeSystemPrompt: false }, + ) as unknown as Array<{ type?: string; output?: unknown }>; + + const functionOutput = input.find((item) => item.type === "function_call_output"); + expect(functionOutput?.output).toEqual([ + { + type: "input_image", + detail: "auto", + image_url: "data:image/png;base64,aW1n", + }, + ]); + }); + it("keeps encrypted reasoning replay item ids when requested", () => { const input = convertResponsesMessages( nativeOpenAIModel, diff --git a/src/llm/providers/openai-responses-shared.ts b/src/llm/providers/openai-responses-shared.ts index 060d8d5f6bf..aae3cc1036b 100644 --- a/src/llm/providers/openai-responses-shared.ts +++ b/src/llm/providers/openai-responses-shared.ts @@ -51,6 +51,14 @@ import { transformMessages } from "./transform-messages.js"; // Utilities // ============================================================================= +const EMPTY_TOOL_RESULT_TEXT = "(no output)"; +const IMAGE_TOOL_RESULT_TEXT = "(see attached image)"; + +function sanitizeToolResultText(text: string, fallback: string): string { + const sanitized = sanitizeSurrogates(text); + return sanitized.trim().length > 0 ? sanitized : fallback; +} + type ReplayableResponseOutputMessage = Omit & { id?: string }; type ReplayableResponseReasoningItem = Omit & { id?: string }; type ResponsesTextContentPart = @@ -376,8 +384,9 @@ export function convertResponsesMessages( .filter((c): c is TextContent => c.type === "text") .map((c) => c.text) .join("\n"); + const sanitizedTextResult = sanitizeSurrogates(textResult); const hasImages = msg.content.some((c): c is ImageContent => c.type === "image"); - const hasText = textResult.length > 0; + const hasText = sanitizedTextResult.trim().length > 0; const [callId] = msg.toolCallId.split("|"); let output: string | ResponseFunctionCallOutputItemList; @@ -387,7 +396,7 @@ export function convertResponsesMessages( if (hasText) { contentParts.push({ type: "input_text", - text: sanitizeSurrogates(textResult), + text: sanitizedTextResult, }); } @@ -403,7 +412,10 @@ export function convertResponsesMessages( output = contentParts; } else { - output = sanitizeSurrogates(hasText ? textResult : "(see attached image)"); + output = sanitizeToolResultText( + textResult, + hasImages ? IMAGE_TOOL_RESULT_TEXT : EMPTY_TOOL_RESULT_TEXT, + ); } messages.push({