mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-12 07:20:45 +00:00
acp: forward attachments into ACP runtime sessions (#41427)
Merged via squash.
Prepared head SHA: f2ac51df2c
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Co-authored-by: mbelinky <132747814+mbelinky@users.noreply.github.com>
Reviewed-by: @mbelinky
This commit is contained in:
@@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai
|
|||||||
- ACP/bridge mode: reject unsupported per-session MCP server setup and propagate rejected session-mode changes so IDE clients see explicit bridge limitations instead of silent success. (#41424) Thanks @mbelinky.
|
- ACP/bridge mode: reject unsupported per-session MCP server setup and propagate rejected session-mode changes so IDE clients see explicit bridge limitations instead of silent success. (#41424) Thanks @mbelinky.
|
||||||
- ACP/session UX: replay stored user and assistant text on `loadSession`, expose Gateway-backed session controls and metadata, and emit approximate session usage updates so IDE clients restore context more faithfully. (#41425) Thanks @mbelinky.
|
- ACP/session UX: replay stored user and assistant text on `loadSession`, expose Gateway-backed session controls and metadata, and emit approximate session usage updates so IDE clients restore context more faithfully. (#41425) Thanks @mbelinky.
|
||||||
- ACP/tool streaming: enrich `tool_call` and `tool_call_update` events with best-effort text content and file-location hints so IDE clients can follow bridge tool activity more naturally. (#41442) Thanks @mbelinky.
|
- ACP/tool streaming: enrich `tool_call` and `tool_call_update` events with best-effort text content and file-location hints so IDE clients can follow bridge tool activity more naturally. (#41442) Thanks @mbelinky.
|
||||||
|
- ACP/runtime attachments: forward normalized inbound image attachments into ACP runtime turns so ACPX sessions can preserve image prompt content on the runtime path. (#41427) Thanks @mbelinky.
|
||||||
|
|
||||||
## 2026.3.8
|
## 2026.3.8
|
||||||
|
|
||||||
|
|||||||
@@ -310,7 +310,20 @@ export class AcpxRuntime implements AcpRuntime {
|
|||||||
// Ignore EPIPE when the child exits before stdin flush completes.
|
// Ignore EPIPE when the child exits before stdin flush completes.
|
||||||
});
|
});
|
||||||
|
|
||||||
child.stdin.end(input.text);
|
if (input.attachments && input.attachments.length > 0) {
|
||||||
|
const blocks: unknown[] = [];
|
||||||
|
if (input.text) {
|
||||||
|
blocks.push({ type: "text", text: input.text });
|
||||||
|
}
|
||||||
|
for (const attachment of input.attachments) {
|
||||||
|
if (attachment.mediaType.startsWith("image/")) {
|
||||||
|
blocks.push({ type: "image", mimeType: attachment.mediaType, data: attachment.data });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
child.stdin.end(blocks.length > 0 ? JSON.stringify(blocks) : input.text);
|
||||||
|
} else {
|
||||||
|
child.stdin.end(input.text);
|
||||||
|
}
|
||||||
|
|
||||||
let stderr = "";
|
let stderr = "";
|
||||||
child.stderr.on("data", (chunk) => {
|
child.stderr.on("data", (chunk) => {
|
||||||
|
|||||||
@@ -655,6 +655,7 @@ export class AcpSessionManager {
|
|||||||
for await (const event of runtime.runTurn({
|
for await (const event of runtime.runTurn({
|
||||||
handle,
|
handle,
|
||||||
text: input.text,
|
text: input.text,
|
||||||
|
attachments: input.attachments,
|
||||||
mode: input.mode,
|
mode: input.mode,
|
||||||
requestId: input.requestId,
|
requestId: input.requestId,
|
||||||
signal: combinedSignal,
|
signal: combinedSignal,
|
||||||
|
|||||||
@@ -47,10 +47,16 @@ export type AcpInitializeSessionInput = {
|
|||||||
backendId?: string;
|
backendId?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type AcpTurnAttachment = {
|
||||||
|
mediaType: string;
|
||||||
|
data: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type AcpRunTurnInput = {
|
export type AcpRunTurnInput = {
|
||||||
cfg: OpenClawConfig;
|
cfg: OpenClawConfig;
|
||||||
sessionKey: string;
|
sessionKey: string;
|
||||||
text: string;
|
text: string;
|
||||||
|
attachments?: AcpTurnAttachment[];
|
||||||
mode: AcpRuntimePromptMode;
|
mode: AcpRuntimePromptMode;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
|||||||
@@ -39,9 +39,15 @@ export type AcpRuntimeEnsureInput = {
|
|||||||
env?: Record<string, string>;
|
env?: Record<string, string>;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type AcpRuntimeTurnAttachment = {
|
||||||
|
mediaType: string;
|
||||||
|
data: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type AcpRuntimeTurnInput = {
|
export type AcpRuntimeTurnInput = {
|
||||||
handle: AcpRuntimeHandle;
|
handle: AcpRuntimeHandle;
|
||||||
text: string;
|
text: string;
|
||||||
|
attachments?: AcpRuntimeTurnAttachment[];
|
||||||
mode: AcpRuntimePromptMode;
|
mode: AcpRuntimePromptMode;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { AcpRuntimeError } from "../../acp/runtime/errors.js";
|
import { AcpRuntimeError } from "../../acp/runtime/errors.js";
|
||||||
import type { AcpSessionStoreEntry } from "../../acp/runtime/session-meta.js";
|
import type { AcpSessionStoreEntry } from "../../acp/runtime/session-meta.js";
|
||||||
@@ -131,6 +134,7 @@ async function runDispatch(params: {
|
|||||||
dispatcher?: ReplyDispatcher;
|
dispatcher?: ReplyDispatcher;
|
||||||
shouldRouteToOriginating?: boolean;
|
shouldRouteToOriginating?: boolean;
|
||||||
onReplyStart?: () => void;
|
onReplyStart?: () => void;
|
||||||
|
ctxOverrides?: Record<string, unknown>;
|
||||||
}) {
|
}) {
|
||||||
return tryDispatchAcpReply({
|
return tryDispatchAcpReply({
|
||||||
ctx: buildTestCtx({
|
ctx: buildTestCtx({
|
||||||
@@ -138,6 +142,7 @@ async function runDispatch(params: {
|
|||||||
Surface: "discord",
|
Surface: "discord",
|
||||||
SessionKey: sessionKey,
|
SessionKey: sessionKey,
|
||||||
BodyForAgent: params.bodyForAgent,
|
BodyForAgent: params.bodyForAgent,
|
||||||
|
...params.ctxOverrides,
|
||||||
}),
|
}),
|
||||||
cfg: params.cfg ?? createAcpTestConfig(),
|
cfg: params.cfg ?? createAcpTestConfig(),
|
||||||
dispatcher: params.dispatcher ?? createDispatcher().dispatcher,
|
dispatcher: params.dispatcher ?? createDispatcher().dispatcher,
|
||||||
@@ -353,6 +358,34 @@ describe("tryDispatchAcpReply", () => {
|
|||||||
expect(onReplyStart).not.toHaveBeenCalled();
|
expect(onReplyStart).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("forwards normalized image attachments into ACP turns", async () => {
|
||||||
|
setReadyAcpResolution();
|
||||||
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-"));
|
||||||
|
const imagePath = path.join(tempDir, "inbound.png");
|
||||||
|
await fs.writeFile(imagePath, "image-bytes");
|
||||||
|
managerMocks.runTurn.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
await runDispatch({
|
||||||
|
bodyForAgent: " ",
|
||||||
|
ctxOverrides: {
|
||||||
|
MediaPath: imagePath,
|
||||||
|
MediaType: "image/png",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(managerMocks.runTurn).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
text: "",
|
||||||
|
attachments: [
|
||||||
|
{
|
||||||
|
mediaType: "image/png",
|
||||||
|
data: Buffer.from("image-bytes").toString("base64"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("surfaces ACP policy errors as final error replies", async () => {
|
it("surfaces ACP policy errors as final error replies", async () => {
|
||||||
setReadyAcpResolution();
|
setReadyAcpResolution();
|
||||||
policyMocks.resolveAcpDispatchPolicyError.mockReturnValue(
|
policyMocks.resolveAcpDispatchPolicyError.mockReturnValue(
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
|
import fs from "node:fs/promises";
|
||||||
import { getAcpSessionManager } from "../../acp/control-plane/manager.js";
|
import { getAcpSessionManager } from "../../acp/control-plane/manager.js";
|
||||||
|
import type { AcpTurnAttachment } from "../../acp/control-plane/manager.types.js";
|
||||||
import { resolveAcpAgentPolicyError, resolveAcpDispatchPolicyError } from "../../acp/policy.js";
|
import { resolveAcpAgentPolicyError, resolveAcpDispatchPolicyError } from "../../acp/policy.js";
|
||||||
import { formatAcpRuntimeErrorText } from "../../acp/runtime/error-text.js";
|
import { formatAcpRuntimeErrorText } from "../../acp/runtime/error-text.js";
|
||||||
import { toAcpRuntimeError } from "../../acp/runtime/errors.js";
|
import { toAcpRuntimeError } from "../../acp/runtime/errors.js";
|
||||||
@@ -14,6 +16,10 @@ import { logVerbose } from "../../globals.js";
|
|||||||
import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js";
|
import { getSessionBindingService } from "../../infra/outbound/session-binding-service.js";
|
||||||
import { generateSecureUuid } from "../../infra/secure-random.js";
|
import { generateSecureUuid } from "../../infra/secure-random.js";
|
||||||
import { prefixSystemMessage } from "../../infra/system-message.js";
|
import { prefixSystemMessage } from "../../infra/system-message.js";
|
||||||
|
import {
|
||||||
|
normalizeAttachmentPath,
|
||||||
|
normalizeAttachments,
|
||||||
|
} from "../../media-understanding/attachments.normalize.js";
|
||||||
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||||
import { maybeApplyTtsToPayload, resolveTtsConfig } from "../../tts/tts.js";
|
import { maybeApplyTtsToPayload, resolveTtsConfig } from "../../tts/tts.js";
|
||||||
import {
|
import {
|
||||||
@@ -57,6 +63,36 @@ function resolveAcpPromptText(ctx: FinalizedMsgContext): string {
|
|||||||
]).trim();
|
]).trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ACP_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024;
|
||||||
|
|
||||||
|
async function resolveAcpAttachments(ctx: FinalizedMsgContext): Promise<AcpTurnAttachment[]> {
|
||||||
|
const mediaAttachments = normalizeAttachments(ctx);
|
||||||
|
const results: AcpTurnAttachment[] = [];
|
||||||
|
for (const attachment of mediaAttachments) {
|
||||||
|
const filePath = normalizeAttachmentPath(attachment.path);
|
||||||
|
if (!filePath) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const stat = await fs.stat(filePath);
|
||||||
|
if (stat.size > ACP_ATTACHMENT_MAX_BYTES) {
|
||||||
|
logVerbose(
|
||||||
|
`dispatch-acp: skipping attachment ${filePath} (${stat.size} bytes exceeds ${ACP_ATTACHMENT_MAX_BYTES} byte limit)`,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const buf = await fs.readFile(filePath);
|
||||||
|
results.push({
|
||||||
|
mediaType: attachment.mime ?? "application/octet-stream",
|
||||||
|
data: buf.toString("base64"),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Skip unreadable files. Text content should still be delivered.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
function resolveCommandCandidateText(ctx: FinalizedMsgContext): string {
|
function resolveCommandCandidateText(ctx: FinalizedMsgContext): string {
|
||||||
return resolveFirstContextText(ctx, ["CommandBody", "BodyForCommands", "RawBody", "Body"]).trim();
|
return resolveFirstContextText(ctx, ["CommandBody", "BodyForCommands", "RawBody", "Body"]).trim();
|
||||||
}
|
}
|
||||||
@@ -189,7 +225,8 @@ export async function tryDispatchAcpReply(params: {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const promptText = resolveAcpPromptText(params.ctx);
|
const promptText = resolveAcpPromptText(params.ctx);
|
||||||
if (!promptText) {
|
const attachments = await resolveAcpAttachments(params.ctx);
|
||||||
|
if (!promptText && attachments.length === 0) {
|
||||||
const counts = params.dispatcher.getQueuedCounts();
|
const counts = params.dispatcher.getQueuedCounts();
|
||||||
delivery.applyRoutedCounts(counts);
|
delivery.applyRoutedCounts(counts);
|
||||||
params.recordProcessed("completed", { reason: "acp_empty_prompt" });
|
params.recordProcessed("completed", { reason: "acp_empty_prompt" });
|
||||||
@@ -251,6 +288,7 @@ export async function tryDispatchAcpReply(params: {
|
|||||||
cfg: params.cfg,
|
cfg: params.cfg,
|
||||||
sessionKey,
|
sessionKey,
|
||||||
text: promptText,
|
text: promptText,
|
||||||
|
attachments: attachments.length > 0 ? attachments : undefined,
|
||||||
mode: "prompt",
|
mode: "prompt",
|
||||||
requestId: resolveAcpRequestId(params.ctx),
|
requestId: resolveAcpRequestId(params.ctx),
|
||||||
onEvent: async (event) => await projector.onEvent(event),
|
onEvent: async (event) => await projector.onEvent(event),
|
||||||
|
|||||||
Reference in New Issue
Block a user