acp: forward attachments into ACP runtime sessions (#41427)

Merged via squash.

Prepared head SHA: f2ac51df2c
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Reviewed-by: @mbelinky
This commit is contained in:
Mariano
2026-03-09 22:32:32 +01:00
committed by GitHub
parent 8e3f3bc3cf
commit 4aebff78bc
7 changed files with 100 additions and 2 deletions

View File

@@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai
- ACP/bridge mode: reject unsupported per-session MCP server setup and propagate rejected session-mode changes so IDE clients see explicit bridge limitations instead of silent success. (#41424) Thanks @mbelinky.
- ACP/session UX: replay stored user and assistant text on `loadSession`, expose Gateway-backed session controls and metadata, and emit approximate session usage updates so IDE clients restore context more faithfully. (#41425) Thanks @mbelinky.
- ACP/tool streaming: enrich `tool_call` and `tool_call_update` events with best-effort text content and file-location hints so IDE clients can follow bridge tool activity more naturally. (#41442) Thanks @mbelinky.
- ACP/runtime attachments: forward normalized inbound image attachments into ACP runtime turns so ACPX sessions can preserve image prompt content on the runtime path. (#41427) Thanks @mbelinky.
## 2026.3.8

View File

@@ -310,7 +310,20 @@ export class AcpxRuntime implements AcpRuntime {
// Ignore EPIPE when the child exits before stdin flush completes.
});
child.stdin.end(input.text);
if (input.attachments && input.attachments.length > 0) {
const blocks: unknown[] = [];
if (input.text) {
blocks.push({ type: "text", text: input.text });
}
for (const attachment of input.attachments) {
if (attachment.mediaType.startsWith("image/")) {
blocks.push({ type: "image", mimeType: attachment.mediaType, data: attachment.data });
}
}
child.stdin.end(blocks.length > 0 ? JSON.stringify(blocks) : input.text);
} else {
child.stdin.end(input.text);
}
let stderr = "";
child.stderr.on("data", (chunk) => {

View File

@@ -655,6 +655,7 @@ export class AcpSessionManager {
for await (const event of runtime.runTurn({
handle,
text: input.text,
attachments: input.attachments,
mode: input.mode,
requestId: input.requestId,
signal: combinedSignal,

View File

@@ -47,10 +47,16 @@ export type AcpInitializeSessionInput = {
backendId?: string;
};
export type AcpTurnAttachment = {
mediaType: string;
data: string;
};
export type AcpRunTurnInput = {
cfg: OpenClawConfig;
sessionKey: string;
text: string;
attachments?: AcpTurnAttachment[];
mode: AcpRuntimePromptMode;
requestId: string;
signal?: AbortSignal;

View File

@@ -39,9 +39,15 @@ export type AcpRuntimeEnsureInput = {
env?: Record<string, string>;
};
export type AcpRuntimeTurnAttachment = {
mediaType: string;
data: string;
};
export type AcpRuntimeTurnInput = {
handle: AcpRuntimeHandle;
text: string;
attachments?: AcpRuntimeTurnAttachment[];
mode: AcpRuntimePromptMode;
requestId: string;
signal?: AbortSignal;

View File

@@ -1,3 +1,6 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { AcpRuntimeError } from "../../acp/runtime/errors.js";
import type { AcpSessionStoreEntry } from "../../acp/runtime/session-meta.js";
@@ -131,6 +134,7 @@ async function runDispatch(params: {
dispatcher?: ReplyDispatcher;
shouldRouteToOriginating?: boolean;
onReplyStart?: () => void;
ctxOverrides?: Record<string, unknown>;
}) {
return tryDispatchAcpReply({
ctx: buildTestCtx({
@@ -138,6 +142,7 @@ async function runDispatch(params: {
Surface: "discord",
SessionKey: sessionKey,
BodyForAgent: params.bodyForAgent,
...params.ctxOverrides,
}),
cfg: params.cfg ?? createAcpTestConfig(),
dispatcher: params.dispatcher ?? createDispatcher().dispatcher,
@@ -353,6 +358,34 @@ describe("tryDispatchAcpReply", () => {
expect(onReplyStart).not.toHaveBeenCalled();
});
it("forwards normalized image attachments into ACP turns", async () => {
setReadyAcpResolution();
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-"));
const imagePath = path.join(tempDir, "inbound.png");
await fs.writeFile(imagePath, "image-bytes");
managerMocks.runTurn.mockResolvedValue(undefined);
await runDispatch({
bodyForAgent: " ",
ctxOverrides: {
MediaPath: imagePath,
MediaType: "image/png",
},
});
expect(managerMocks.runTurn).toHaveBeenCalledWith(
expect.objectContaining({
text: "",
attachments: [
{
mediaType: "image/png",
data: Buffer.from("image-bytes").toString("base64"),
},
],
}),
);
});
it("surfaces ACP policy errors as final error replies", async () => {
setReadyAcpResolution();
policyMocks.resolveAcpDispatchPolicyError.mockReturnValue(

View File

@@ -1,4 +1,6 @@
import fs from "node:fs/promises";
import { getAcpSessionManager } from "../../acp/control-plane/manager.js";
import type { AcpTurnAttachment } from "../../acp/control-plane/manager.types.js";
import { resolveAcpAgentPolicyError, resolveAcpDispatchPolicyError } from "../../acp/policy.js";
import { formatAcpRuntimeErrorText } from "../../acp/runtime/error-text.js";
import { toAcpRuntimeError } from "../../acp/runtime/errors.js";
@@ -14,6 +16,10 @@ import { logVerbose } from "../../globals.js";
import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js";
import { generateSecureUuid } from "../../infra/secure-random.js";
import { prefixSystemMessage } from "../../infra/system-message.js";
import {
normalizeAttachmentPath,
normalizeAttachments,
} from "../../media-understanding/attachments.normalize.js";
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
import { maybeApplyTtsToPayload, resolveTtsConfig } from "../../tts/tts.js";
import {
@@ -57,6 +63,36 @@ function resolveAcpPromptText(ctx: FinalizedMsgContext): string {
]).trim();
}
const ACP_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024;
async function resolveAcpAttachments(ctx: FinalizedMsgContext): Promise<AcpTurnAttachment[]> {
const mediaAttachments = normalizeAttachments(ctx);
const results: AcpTurnAttachment[] = [];
for (const attachment of mediaAttachments) {
const filePath = normalizeAttachmentPath(attachment.path);
if (!filePath) {
continue;
}
try {
const stat = await fs.stat(filePath);
if (stat.size > ACP_ATTACHMENT_MAX_BYTES) {
logVerbose(
`dispatch-acp: skipping attachment ${filePath} (${stat.size} bytes exceeds ${ACP_ATTACHMENT_MAX_BYTES} byte limit)`,
);
continue;
}
const buf = await fs.readFile(filePath);
results.push({
mediaType: attachment.mime ?? "application/octet-stream",
data: buf.toString("base64"),
});
} catch {
// Skip unreadable files. Text content should still be delivered.
}
}
return results;
}
function resolveCommandCandidateText(ctx: FinalizedMsgContext): string {
return resolveFirstContextText(ctx, ["CommandBody", "BodyForCommands", "RawBody", "Body"]).trim();
}
@@ -189,7 +225,8 @@ export async function tryDispatchAcpReply(params: {
});
const promptText = resolveAcpPromptText(params.ctx);
if (!promptText) {
const attachments = await resolveAcpAttachments(params.ctx);
if (!promptText && attachments.length === 0) {
const counts = params.dispatcher.getQueuedCounts();
delivery.applyRoutedCounts(counts);
params.recordProcessed("completed", { reason: "acp_empty_prompt" });
@@ -251,6 +288,7 @@ export async function tryDispatchAcpReply(params: {
cfg: params.cfg,
sessionKey,
text: promptText,
attachments: attachments.length > 0 ? attachments : undefined,
mode: "prompt",
requestId: resolveAcpRequestId(params.ctx),
onEvent: async (event) => await projector.onEvent(event),