fix(agents): repair discord image generation delivery

This commit is contained in:
Peter Steinberger
2026-04-05 17:29:15 +01:00
parent aee1f0b453
commit 9b89fa3937
6 changed files with 151 additions and 4 deletions

View File

@@ -9,6 +9,7 @@ import type { EmbeddedPiSubscribeContext } from "./pi-embedded-subscribe.handler
function createMockContext(overrides?: {
shouldEmitToolOutput?: boolean;
onToolResult?: ReturnType<typeof vi.fn>;
toolResultFormat?: "markdown" | "plain";
}): EmbeddedPiSubscribeContext {
const onToolResult = overrides?.onToolResult ?? vi.fn();
return {
@@ -16,6 +17,7 @@ function createMockContext(overrides?: {
runId: "test-run",
onToolResult,
onAgentEvent: vi.fn(),
toolResultFormat: overrides?.toolResultFormat,
},
state: {
toolMetaById: new Map(),
@@ -217,6 +219,68 @@ describe("handleToolExecutionEnd media emission", () => {
expect(ctx.state.pendingToolAudioAsVoice).toBe(true);
});
it("does not queue structured media already emitted in plain verbose output", async () => {
const ctx = createMockContext({
shouldEmitToolOutput: true,
onToolResult: vi.fn(),
toolResultFormat: "plain",
});
await handleToolExecutionEnd(ctx, {
type: "tool_execution_end",
toolName: "image_generate",
toolCallId: "tc-1",
isError: false,
result: {
content: [
{
type: "text",
text: "Generated 1 image with google/gemini-3.1-flash-image-preview.\nMEDIA:/tmp/generated.png",
},
],
details: {
media: {
mediaUrls: ["/tmp/generated.png"],
},
},
},
});
expect(ctx.emitToolOutput).toHaveBeenCalled();
expect(ctx.state.pendingToolMediaUrls).toEqual([]);
});
it("still queues structured media for markdown verbose output", async () => {
const ctx = createMockContext({
shouldEmitToolOutput: true,
onToolResult: vi.fn(),
toolResultFormat: "markdown",
});
await handleToolExecutionEnd(ctx, {
type: "tool_execution_end",
toolName: "image_generate",
toolCallId: "tc-1",
isError: false,
result: {
content: [
{
type: "text",
text: "Generated 1 image with google/gemini-3.1-flash-image-preview.\nMEDIA:/tmp/generated.png",
},
],
details: {
media: {
mediaUrls: ["/tmp/generated.png"],
},
},
},
});
expect(ctx.emitToolOutput).toHaveBeenCalled();
expect(ctx.state.pendingToolMediaUrls).toEqual(["/tmp/generated.png"]);
});
it("does NOT emit media for error results", async () => {
const onToolResult = vi.fn();
const ctx = createMockContext({ shouldEmitToolOutput: false, onToolResult });

View File

@@ -17,6 +17,7 @@ import {
buildExecApprovalUnavailableReplyPayload,
} from "../infra/exec-approval-reply.js";
import type { ExecApprovalDecision } from "../infra/exec-approvals.js";
import { splitMediaFromOutput } from "../media/parse.js";
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
import type { PluginHookAfterToolCallEvent } from "../plugins/types.js";
import type { ApplyPatchSummary } from "./apply-patch.js";
@@ -282,6 +283,18 @@ function queuePendingToolMedia(
}
}
function collectEmittedToolOutputMediaUrls(
toolName: string,
outputText: string,
result: unknown,
): string[] {
const mediaUrls = splitMediaFromOutput(outputText).mediaUrls ?? [];
if (mediaUrls.length === 0) {
return [];
}
return filterToolResultMediaUrls(toolName, mediaUrls, result);
}
function readExecApprovalPendingDetails(result: unknown): {
approvalId: string;
approvalSlug: string;
@@ -382,6 +395,7 @@ async function emitToolResultOutput(params: {
"object" &&
!Array.isArray((result as { details?: { media?: unknown } }).details?.media);
const approvalPending = readExecApprovalPendingDetails(result);
let emittedToolOutputMediaUrls: string[] = [];
if (!isToolError && approvalPending) {
if (!ctx.params.onToolResult) {
return;
@@ -431,6 +445,13 @@ async function emitToolResultOutput(params: {
if (ctx.shouldEmitToolOutput()) {
const outputText = extractToolResultText(sanitizedResult);
if (outputText) {
if (ctx.params.toolResultFormat === "plain") {
emittedToolOutputMediaUrls = collectEmittedToolOutputMediaUrls(
toolName,
outputText,
result,
);
}
ctx.emitToolOutput(toolName, meta, outputText, result);
}
if (!hasStructuredMedia) {
@@ -447,11 +468,15 @@ async function emitToolResultOutput(params: {
return;
}
const mediaUrls = filterToolResultMediaUrls(toolName, mediaReply.mediaUrls, result);
if (mediaUrls.length === 0) {
const pendingMediaUrls =
mediaReply.audioAsVoice || emittedToolOutputMediaUrls.length === 0
? mediaUrls
: mediaUrls.filter((url) => !emittedToolOutputMediaUrls.includes(url));
if (pendingMediaUrls.length === 0) {
return;
}
queuePendingToolMedia(ctx, {
mediaUrls,
mediaUrls: pendingMediaUrls,
...(mediaReply.audioAsVoice ? { audioAsVoice: true } : {}),
});
}

View File

@@ -140,6 +140,7 @@ export type ToolHandlerParams = Pick<
| "sessionKey"
| "sessionId"
| "agentId"
| "toolResultFormat"
>;
export type ToolHandlerState = Pick<

View File

@@ -365,7 +365,57 @@ describe("createImageGenerateTool", () => {
},
});
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).not.toContain("MEDIA:");
expect(text).toContain("MEDIA:/tmp/generated-1.png");
expect(text).toContain("MEDIA:/tmp/generated-2.png");
});
it("includes MEDIA paths in content text so follow-up replies use the real saved file", async () => {
vi.spyOn(imageGenerationRuntime, "generateImage").mockResolvedValue({
provider: "google",
model: "gemini-3.1-flash-image-preview",
attempts: [],
images: [
{
buffer: Buffer.from("jpg-data"),
mimeType: "image/jpeg",
fileName: "kodo_sawaki_zazen.jpg",
},
],
});
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: "/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
id: "kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
size: 8,
contentType: "image/jpeg",
});
const tool = createImageGenerateTool({
config: {
agents: {
defaults: {
imageGenerationModel: { primary: "google/gemini-3.1-flash-image-preview" },
},
},
},
});
expect(tool).not.toBeNull();
if (!tool) {
throw new Error("expected image_generate tool");
}
const result = await tool.execute("call-regression", { prompt: "kodo sawaki zazen" });
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).toContain(
"MEDIA:/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
);
expect(result.details).toMatchObject({
media: {
mediaUrls: [
"/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
],
},
});
});
it("rejects counts outside the supported range", async () => {

View File

@@ -670,6 +670,9 @@ export function createImageGenerateTool(options?: {
.filter((entry): entry is string => Boolean(entry));
const lines = [
`Generated ${savedImages.length} image${savedImages.length === 1 ? "" : "s"} with ${result.provider}/${result.model}.`,
// Show the actual saved paths so the model does not invent a bogus
// local path when it references the generated image in a follow-up reply.
...savedImages.map((image) => `MEDIA:${image.path}`),
];
return {