diff --git a/CHANGELOG.md b/CHANGELOG.md index 3161d793c79..0b5167ac306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ Docs: https://docs.openclaw.ai - Telegram: preserve replied-to bot messages, captions, and media metadata in group reply chains so follow-up replies understand what the user is reacting to. (#82863) - Providers/Together: update PI runtime packages to 0.74.1 and emit Together-style `reasoning.enabled`/`max_tokens` controls for reasoning-capable OpenAI-completions models. - Agents/diagnostics: split slow embedded-run `attempt-dispatch` startup summaries into workspace, prompt, runtime-plan, and final dispatch subspans so traces identify the delayed setup phase. Fixes #82782. (#82783) Thanks @galiniliev. +- Agents/Codex: flatten nested tool-result middleware blocks into bounded text so successful message sends are no longer replaced with `Tool output unavailable due to post-processing error`. Fixes #82912. Thanks @joeykrug. - CLI/media: accept HTTP(S) URLs in `openclaw infer image describe --file`, fetching remote images through the guarded media path instead of treating URLs as local files. Fixes #82837. (#82854) Thanks @neeravmakwana. - Agents/subagents: keep session-backed parent runs active when the child wait call times out before the child session has actually settled, so late subagent completions are reconciled instead of being lost. Fixes #82787. Thanks @ramitrkar-hash. - Control UI: advertise shared Gateway protocol constants in browser connect frames, fixing protocol mismatch handshakes after protocol constant drift. Fixes #82882. Thanks @galiniliev. diff --git a/extensions/codex/src/app-server/dynamic-tools.test.ts b/extensions/codex/src/app-server/dynamic-tools.test.ts index 3abf0ef9e33..2bec50df6e3 100644 --- a/extensions/codex/src/app-server/dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/dynamic-tools.test.ts @@ -495,6 +495,43 @@ describe("createCodexDynamicToolBridge", () => { expectContextFields(callArg(handler, 0, 1, "middleware context"), { runtime: "codex" }); }); + it("preserves nested toolResult content after no-op middleware", async () => { + const registry = createEmptyPluginRegistry(); + const handler = vi.fn(async () => undefined); + registry.agentToolResultMiddlewares.push({ + pluginId: "tokenjuice", + pluginName: "Tokenjuice", + rawHandler: handler, + handler, + runtimes: ["codex"], + source: "test", + }); + setActivePluginRegistry(registry); + + const bridge = createBridgeWithToolResult("message", { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [{ type: "text", text: "message sent: msg_123" }], + } as never, + ], + details: { messageId: "msg_123" }, + }); + + const result = await bridge.handleToolCall({ + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + namespace: null, + tool: "message", + arguments: { text: "hello" }, + }); + + expect(result).toEqual(expectInputText("message sent: msg_123")); + expect(handler).toHaveBeenCalledTimes(1); + }); + it("passes raw tool failure state into agent tool result middleware", async () => { const registry = createEmptyPluginRegistry(); const handler = vi.fn(async (_event: { isError?: boolean }) => undefined); diff --git a/src/agents/harness/tool-result-middleware.test.ts b/src/agents/harness/tool-result-middleware.test.ts index c0698ebed1a..7a61eba62d9 100644 --- a/src/agents/harness/tool-result-middleware.test.ts +++ b/src/agents/harness/tool-result-middleware.test.ts @@ -167,6 +167,291 @@ describe("createAgentToolResultMiddlewareRunner", () => { }); }); + it("sanitizes incoming details before failing closed on uncoercible content", async () => { + const details: Record = { + ok: true, + callback: () => 1, + }; + details.self = details; + let observedDetails: unknown; + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [ + (event) => { + observedDetails = event.result.details; + return undefined; + }, + ]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { + content: [{ type: "unknown", payload: "raw" } as never], + details, + }, + }); + + expect(result.details).toEqual({ status: "error", middlewareError: true }); + expect(observedDetails).toEqual({ ok: true }); + }); + + it("coerces incoming nested toolResult content before middleware validation", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [ + { type: "text", text: "sent message id msg_123" }, + { type: "text", text: "status delivered" }, + ], + } as never, + ], + details: { status: "sent", messageId: "msg_123" }, + }, + }); + + expect(result.content).toEqual([ + { + type: "text", + text: "sent message id msg_123\nstatus delivered", + }, + ]); + expect(result.details).toEqual({ status: "sent", messageId: "msg_123" }); + }); + + it("coerces nested tool_result blocks returned by middleware", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [ + () => ({ + result: { + content: [ + { + type: "tool_result", + content: { + message: "message delivered", + id: "msg_456", + }, + } as never, + ], + details: { status: "sent" }, + }, + }), + ]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { content: [{ type: "text", text: "raw" }], details: {} }, + }); + + expect(result.content).toEqual([{ type: "text", text: "message delivered" }]); + expect(result.details).toEqual({ status: "sent" }); + }); + + it("does not coerce tool/function call blocks as middleware results", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [ + () => ({ + result: { + content: [ + { + type: "function", + name: "send_message", + arguments: { text: "raw" }, + } as never, + ], + details: {}, + }, + }), + ]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { content: [{ type: "text", text: "raw" }], details: {} }, + }); + + expect(result.details).toEqual({ status: "error", middlewareError: true }); + }); + + it("bounds nested toolResult content before flattening", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [ + ...Array.from({ length: 200 }, () => ({ + type: "text", + text: "x".repeat(600), + })), + { type: "text", text: "late chunk" }, + ], + } as never, + ], + details: {}, + }, + }); + + const content = result.content[0]; + if (content?.type !== "text") { + throw new Error("expected flattened text content"); + } + expect(content.text.length).toBeLessThanOrEqual(100_000); + expect(content.text).not.toContain("late chunk"); + }); + + it("preserves nested image toolResult content without stringifying data", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "vision", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [{ type: "image", mimeType: "image/png", data: "base64-image" }], + } as never, + ], + details: {}, + }, + }); + + expect(result.content).toEqual([ + { type: "image", mimeType: "image/png", data: "base64-image" }, + ]); + }); + + it("preserves mixed nested text and image toolResult content", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "screenshot", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [ + { type: "text", text: "captured screenshot" }, + { type: "image", mimeType: "image/png", data: "base64-image" }, + ], + } as never, + ], + details: {}, + }, + }); + + expect(result.content).toEqual([ + { type: "text", text: "captured screenshot" }, + { type: "image", mimeType: "image/png", data: "base64-image" }, + ]); + }); + + it("preserves images from deeper nested toolResult content", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "screenshot", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [ + { + type: "tool_result", + content: [ + { type: "text", text: "captured screenshot" }, + { type: "image", mimeType: "image/png", data: "base64-image" }, + ], + }, + ], + } as never, + ], + details: {}, + }, + }); + + expect(result.content).toEqual([ + { type: "text", text: "captured screenshot" }, + { type: "image", mimeType: "image/png", data: "base64-image" }, + ]); + }); + + it("preserves interleaved nested text and image order", async () => { + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "screenshot", + args: {}, + result: { + content: [ + { + type: "toolResult", + toolUseId: "call-1", + content: [ + { type: "text", text: "first caption" }, + { type: "image", mimeType: "image/png", data: "image-one" }, + { type: "text", text: "second caption" }, + { type: "image", mimeType: "image/png", data: "image-two" }, + ], + } as never, + ], + details: {}, + }, + }); + + expect(result.content).toEqual([ + { type: "text", text: "first caption" }, + { type: "image", mimeType: "image/png", data: "image-one" }, + { type: "text", text: "second caption" }, + { type: "image", mimeType: "image/png", data: "image-two" }, + ]); + }); + + it("fails closed instead of recursing forever on cyclic nested content", async () => { + const nested: Record = { + type: "toolResult", + content: [], + }; + nested.content = [nested]; + const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); + + const result = await runner.applyToolResultMiddleware({ + toolCallId: "call-1", + toolName: "message", + args: {}, + result: { + content: [nested as never], + details: {}, + }, + }); + + expect(result.details).toEqual({ status: "error", middlewareError: true }); + }); + it("sanitizes incoming function/symbol/bigint values in details", async () => { const runner = createAgentToolResultMiddlewareRunner({ runtime: "codex" }, [() => undefined]); diff --git a/src/agents/harness/tool-result-middleware.ts b/src/agents/harness/tool-result-middleware.ts index 8a5febf3ae3..d464dced603 100644 --- a/src/agents/harness/tool-result-middleware.ts +++ b/src/agents/harness/tool-result-middleware.ts @@ -12,9 +12,14 @@ const log = createSubsystemLogger("agents/harness"); const MAX_MIDDLEWARE_CONTENT_BLOCKS = 200; const MAX_MIDDLEWARE_TEXT_CHARS = 100_000; const MAX_MIDDLEWARE_IMAGE_DATA_CHARS = 5_000_000; +const MAX_MIDDLEWARE_CONTENT_DEPTH = 20; const MAX_MIDDLEWARE_DETAILS_BYTES = 100_000; const MAX_MIDDLEWARE_DETAILS_DEPTH = 20; const MAX_MIDDLEWARE_DETAILS_KEYS = 1_000; +const NESTED_TOOL_RESULT_BLOCK_TYPES = new Set(["toolresult", "tool_result"]); + +type MiddlewareContentBlock = OpenClawAgentToolResult["content"][number]; +type MiddlewareContentCoerceState = { depth: number; seen: Set }; function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); @@ -105,6 +110,230 @@ function isValidMiddlewareToolResult(value: unknown): value is OpenClawAgentTool ); } +function createMiddlewareContentCoerceState(): MiddlewareContentCoerceState { + return { depth: 0, seen: new Set() }; +} + +function descendMiddlewareContentCoerceState( + value: unknown, + state: MiddlewareContentCoerceState, +): MiddlewareContentCoerceState | undefined { + if (state.depth >= MAX_MIDDLEWARE_CONTENT_DEPTH) { + return undefined; + } + if (value !== null && typeof value === "object") { + if (state.seen.has(value)) { + return undefined; + } + const seen = new Set(state.seen); + seen.add(value); + return { depth: state.depth + 1, seen }; + } + return { depth: state.depth + 1, seen: state.seen }; +} + +function stringifyMiddlewareTextPayload(value: unknown): string | undefined { + const seen = new WeakSet(); + try { + return JSON.stringify(value, (_key, val) => { + if (typeof val === "bigint") { + return val.toString(); + } + if (typeof val === "function" || typeof val === "symbol" || val === undefined) { + return undefined; + } + if (val !== null && typeof val === "object") { + if (seen.has(val)) { + return undefined; + } + seen.add(val); + } + return val; + }); + } catch { + return undefined; + } +} + +function coerceMiddlewareText( + value: unknown, + state: MiddlewareContentCoerceState = createMiddlewareContentCoerceState(), +): string | undefined { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") { + return String(value); + } + if (!isRecord(value)) { + return undefined; + } + const nextState = descendMiddlewareContentCoerceState(value, state); + if (!nextState) { + return undefined; + } + for (const key of ["text", "output", "result", "message"]) { + const text = coerceMiddlewareText(value[key], nextState); + if (text !== undefined) { + return text; + } + } + const content = value.content; + if (Array.isArray(content)) { + const chunks = coerceMiddlewareContentArray(content, nextState) + .filter( + (block): block is Extract => + block.type === "text", + ) + .map((block) => block.text) + .filter((text) => text.length > 0); + return chunks.length > 0 ? chunks.join("\n") : undefined; + } + return stringifyMiddlewareTextPayload(value); +} + +function appendMiddlewareContentBlock( + blocks: MiddlewareContentBlock[], + block: MiddlewareContentBlock, +): void { + if (blocks.length >= MAX_MIDDLEWARE_CONTENT_BLOCKS) { + return; + } + if (block.type !== "text") { + blocks.push(block); + return; + } + if (!block.text) { + return; + } + const previous = blocks.at(-1); + if (previous?.type !== "text") { + blocks.push({ + type: "text", + text: truncateUtf16Safe(block.text, MAX_MIDDLEWARE_TEXT_CHARS), + }); + return; + } + const remainingChars = MAX_MIDDLEWARE_TEXT_CHARS - previous.text.length - 1; + if (remainingChars <= 0) { + return; + } + previous.text = `${previous.text}\n${truncateUtf16Safe(block.text, remainingChars)}`; +} + +function coerceMiddlewareContentArray( + content: unknown[], + state: MiddlewareContentCoerceState, +): MiddlewareContentBlock[] { + const blocks: MiddlewareContentBlock[] = []; + let inspectedBlocks = 0; + for (const entry of content) { + inspectedBlocks += 1; + if ( + inspectedBlocks > MAX_MIDDLEWARE_CONTENT_BLOCKS || + blocks.length >= MAX_MIDDLEWARE_CONTENT_BLOCKS + ) { + break; + } + const coercedBlocks = coerceMiddlewareContentBlocks(entry, state); + if (coercedBlocks.length > 0) { + for (const block of coercedBlocks) { + appendMiddlewareContentBlock(blocks, block); + if (blocks.length >= MAX_MIDDLEWARE_CONTENT_BLOCKS) { + break; + } + } + continue; + } + const text = coerceMiddlewareText(entry, state); + if (text) { + appendMiddlewareContentBlock(blocks, { + type: "text", + text: truncateUtf16Safe(text, MAX_MIDDLEWARE_TEXT_CHARS), + }); + } + } + return blocks; +} + +function coerceMiddlewareContentBlocks( + value: unknown, + state: MiddlewareContentCoerceState = createMiddlewareContentCoerceState(), +): MiddlewareContentBlock[] { + if (isValidMiddlewareContentBlock(value)) { + return [value as MiddlewareContentBlock]; + } + if (!isRecord(value) || typeof value.type !== "string") { + return []; + } + const normalizedType = value.type.toLowerCase(); + if (!NESTED_TOOL_RESULT_BLOCK_TYPES.has(normalizedType)) { + return []; + } + const content = value.content; + if (Array.isArray(content) && content.length > 0) { + const nextState = descendMiddlewareContentCoerceState(value, state); + return nextState ? coerceMiddlewareContentArray(content, nextState) : []; + } + const text = coerceMiddlewareText(content, state) ?? coerceMiddlewareText(value, state); + if (!text) { + return []; + } + return [ + { + type: "text", + text: truncateUtf16Safe(text, MAX_MIDDLEWARE_TEXT_CHARS), + }, + ]; +} + +function coerceMiddlewareToolResult( + value: unknown, + options: { sanitizeDetails?: boolean } = {}, +): OpenClawAgentToolResult | undefined { + if (isValidMiddlewareToolResult(value)) { + return value; + } + if (!isRecord(value) || !Array.isArray(value.content)) { + return undefined; + } + const content: OpenClawAgentToolResult["content"] = []; + const state = createMiddlewareContentCoerceState(); + let inspectedBlocks = 0; + for (const block of value.content) { + inspectedBlocks += 1; + if (inspectedBlocks > MAX_MIDDLEWARE_CONTENT_BLOCKS) { + break; + } + for (const coerced of coerceMiddlewareContentBlocks(block, state)) { + content.push(coerced); + if (content.length >= MAX_MIDDLEWARE_CONTENT_BLOCKS) { + break; + } + } + if (content.length >= MAX_MIDDLEWARE_CONTENT_BLOCKS) { + break; + } + } + if (content.length === 0) { + return undefined; + } + const details = isValidMiddlewareDetails(value.details) + ? value.details + : options.sanitizeDetails === true + ? sanitizeMiddlewareDetailsValue(value.details) + : undefined; + if (details === undefined && !isValidMiddlewareDetails(value.details)) { + return undefined; + } + const result = { + ...value, + content, + details, + }; + return isValidMiddlewareToolResult(result) ? result : undefined; +} + /** * Coerce an arbitrary value into a JSON-safe shape that satisfies * `isValidMiddlewareDetails`. Round-trips through `JSON.stringify` with a @@ -150,6 +379,10 @@ function sanitizeMiddlewareDetailsValue(value: unknown): unknown { * subsequent middleware-side mutations are still validated strictly. */ function sanitizeToolResultForMiddleware(result: OpenClawAgentToolResult): OpenClawAgentToolResult { + const coerced = coerceMiddlewareToolResult(result, { sanitizeDetails: true }); + if (coerced) { + return coerced; + } if (result.details === undefined || result.details === null) { return result; } @@ -214,8 +447,9 @@ export function createAgentToolResultMiddlewareRunner( // Validate the current object after every handler so in-place writes // cannot bypass the same shape and size bounds as returned results. const candidate = next?.result ?? current; - if (isValidMiddlewareToolResult(candidate)) { - current = candidate; + const coercedCandidate = coerceMiddlewareToolResult(candidate); + if (coercedCandidate) { + current = coercedCandidate; } else { log.warn( `[${ctx.runtime}] discarded invalid tool result middleware output for ${truncateUtf16Safe(