diff --git a/CHANGELOG.md b/CHANGELOG.md index fff858ba5fc..7d0a735cb18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai - ACP/bridge mode: reject unsupported per-session MCP server setup and propagate rejected session-mode changes so IDE clients see explicit bridge limitations instead of silent success. (#41424) Thanks @mbelinky. - ACP/session UX: replay stored user and assistant text on `loadSession`, expose Gateway-backed session controls and metadata, and emit approximate session usage updates so IDE clients restore context more faithfully. (#41425) Thanks @mbelinky. - ACP/tool streaming: enrich `tool_call` and `tool_call_update` events with best-effort text content and file-location hints so IDE clients can follow bridge tool activity more naturally. (#41442) Thanks @mbelinky. +- ACP/runtime attachments: forward normalized inbound image attachments into ACP runtime turns so ACPX sessions can preserve image prompt content on the runtime path. (#41427) Thanks @mbelinky. ## 2026.3.8 diff --git a/extensions/acpx/src/runtime.ts b/extensions/acpx/src/runtime.ts index 5fa56d109e5..7e310638699 100644 --- a/extensions/acpx/src/runtime.ts +++ b/extensions/acpx/src/runtime.ts @@ -310,7 +310,20 @@ export class AcpxRuntime implements AcpRuntime { // Ignore EPIPE when the child exits before stdin flush completes. }); - child.stdin.end(input.text); + if (input.attachments && input.attachments.length > 0) { + const blocks: unknown[] = []; + if (input.text) { + blocks.push({ type: "text", text: input.text }); + } + for (const attachment of input.attachments) { + if (attachment.mediaType.startsWith("image/")) { + blocks.push({ type: "image", mimeType: attachment.mediaType, data: attachment.data }); + } + } + child.stdin.end(blocks.length > 0 ? JSON.stringify(blocks) : input.text); + } else { + child.stdin.end(input.text); + } let stderr = ""; child.stderr.on("data", (chunk) => { diff --git a/src/acp/control-plane/manager.core.ts b/src/acp/control-plane/manager.core.ts index a64b1fae7eb..f511355ae87 100644 --- a/src/acp/control-plane/manager.core.ts +++ b/src/acp/control-plane/manager.core.ts @@ -655,6 +655,7 @@ export class AcpSessionManager { for await (const event of runtime.runTurn({ handle, text: input.text, + attachments: input.attachments, mode: input.mode, requestId: input.requestId, signal: combinedSignal, diff --git a/src/acp/control-plane/manager.types.ts b/src/acp/control-plane/manager.types.ts index 7337e8063f9..33c2355305c 100644 --- a/src/acp/control-plane/manager.types.ts +++ b/src/acp/control-plane/manager.types.ts @@ -47,10 +47,16 @@ export type AcpInitializeSessionInput = { backendId?: string; }; +export type AcpTurnAttachment = { + mediaType: string; + data: string; +}; + export type AcpRunTurnInput = { cfg: OpenClawConfig; sessionKey: string; text: string; + attachments?: AcpTurnAttachment[]; mode: AcpRuntimePromptMode; requestId: string; signal?: AbortSignal; diff --git a/src/acp/runtime/types.ts b/src/acp/runtime/types.ts index 6a3d3bb3f8e..2d4b10ccf2c 100644 --- a/src/acp/runtime/types.ts +++ b/src/acp/runtime/types.ts @@ -39,9 +39,15 @@ export type AcpRuntimeEnsureInput = { env?: Record; }; +export type AcpRuntimeTurnAttachment = { + mediaType: string; + data: string; +}; + export type AcpRuntimeTurnInput = { handle: AcpRuntimeHandle; text: string; + attachments?: AcpRuntimeTurnAttachment[]; mode: AcpRuntimePromptMode; requestId: string; signal?: AbortSignal; diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index 286b73a7ceb..290846a6075 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -1,3 +1,6 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { AcpRuntimeError } from "../../acp/runtime/errors.js"; import type { AcpSessionStoreEntry } from "../../acp/runtime/session-meta.js"; @@ -131,6 +134,7 @@ async function runDispatch(params: { dispatcher?: ReplyDispatcher; shouldRouteToOriginating?: boolean; onReplyStart?: () => void; + ctxOverrides?: Record; }) { return tryDispatchAcpReply({ ctx: buildTestCtx({ @@ -138,6 +142,7 @@ async function runDispatch(params: { Surface: "discord", SessionKey: sessionKey, BodyForAgent: params.bodyForAgent, + ...params.ctxOverrides, }), cfg: params.cfg ?? createAcpTestConfig(), dispatcher: params.dispatcher ?? createDispatcher().dispatcher, @@ -353,6 +358,34 @@ describe("tryDispatchAcpReply", () => { expect(onReplyStart).not.toHaveBeenCalled(); }); + it("forwards normalized image attachments into ACP turns", async () => { + setReadyAcpResolution(); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-")); + const imagePath = path.join(tempDir, "inbound.png"); + await fs.writeFile(imagePath, "image-bytes"); + managerMocks.runTurn.mockResolvedValue(undefined); + + await runDispatch({ + bodyForAgent: " ", + ctxOverrides: { + MediaPath: imagePath, + MediaType: "image/png", + }, + }); + + expect(managerMocks.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: "", + attachments: [ + { + mediaType: "image/png", + data: Buffer.from("image-bytes").toString("base64"), + }, + ], + }), + ); + }); + it("surfaces ACP policy errors as final error replies", async () => { setReadyAcpResolution(); policyMocks.resolveAcpDispatchPolicyError.mockReturnValue( diff --git a/src/auto-reply/reply/dispatch-acp.ts b/src/auto-reply/reply/dispatch-acp.ts index 33990cb20d6..3b89feaae13 100644 --- a/src/auto-reply/reply/dispatch-acp.ts +++ b/src/auto-reply/reply/dispatch-acp.ts @@ -1,4 +1,6 @@ +import fs from "node:fs/promises"; import { getAcpSessionManager } from "../../acp/control-plane/manager.js"; +import type { AcpTurnAttachment } from "../../acp/control-plane/manager.types.js"; import { resolveAcpAgentPolicyError, resolveAcpDispatchPolicyError } from "../../acp/policy.js"; import { formatAcpRuntimeErrorText } from "../../acp/runtime/error-text.js"; import { toAcpRuntimeError } from "../../acp/runtime/errors.js"; @@ -14,6 +16,10 @@ import { logVerbose } from "../../globals.js"; import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js"; import { generateSecureUuid } from "../../infra/secure-random.js"; import { prefixSystemMessage } from "../../infra/system-message.js"; +import { + normalizeAttachmentPath, + normalizeAttachments, +} from "../../media-understanding/attachments.normalize.js"; import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { maybeApplyTtsToPayload, resolveTtsConfig } from "../../tts/tts.js"; import { @@ -57,6 +63,36 @@ function resolveAcpPromptText(ctx: FinalizedMsgContext): string { ]).trim(); } +const ACP_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024; + +async function resolveAcpAttachments(ctx: FinalizedMsgContext): Promise { + const mediaAttachments = normalizeAttachments(ctx); + const results: AcpTurnAttachment[] = []; + for (const attachment of mediaAttachments) { + const filePath = normalizeAttachmentPath(attachment.path); + if (!filePath) { + continue; + } + try { + const stat = await fs.stat(filePath); + if (stat.size > ACP_ATTACHMENT_MAX_BYTES) { + logVerbose( + `dispatch-acp: skipping attachment ${filePath} (${stat.size} bytes exceeds ${ACP_ATTACHMENT_MAX_BYTES} byte limit)`, + ); + continue; + } + const buf = await fs.readFile(filePath); + results.push({ + mediaType: attachment.mime ?? "application/octet-stream", + data: buf.toString("base64"), + }); + } catch { + // Skip unreadable files. Text content should still be delivered. + } + } + return results; +} + function resolveCommandCandidateText(ctx: FinalizedMsgContext): string { return resolveFirstContextText(ctx, ["CommandBody", "BodyForCommands", "RawBody", "Body"]).trim(); } @@ -189,7 +225,8 @@ export async function tryDispatchAcpReply(params: { }); const promptText = resolveAcpPromptText(params.ctx); - if (!promptText) { + const attachments = await resolveAcpAttachments(params.ctx); + if (!promptText && attachments.length === 0) { const counts = params.dispatcher.getQueuedCounts(); delivery.applyRoutedCounts(counts); params.recordProcessed("completed", { reason: "acp_empty_prompt" }); @@ -251,6 +288,7 @@ export async function tryDispatchAcpReply(params: { cfg: params.cfg, sessionKey, text: promptText, + attachments: attachments.length > 0 ? attachments : undefined, mode: "prompt", requestId: resolveAcpRequestId(params.ctx), onEvent: async (event) => await projector.onEvent(event),