mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-02 19:50:21 +00:00
fix(agents): repair discord image generation delivery
This commit is contained in:
@@ -9,6 +9,7 @@ import type { EmbeddedPiSubscribeContext } from "./pi-embedded-subscribe.handler
|
||||
function createMockContext(overrides?: {
|
||||
shouldEmitToolOutput?: boolean;
|
||||
onToolResult?: ReturnType<typeof vi.fn>;
|
||||
toolResultFormat?: "markdown" | "plain";
|
||||
}): EmbeddedPiSubscribeContext {
|
||||
const onToolResult = overrides?.onToolResult ?? vi.fn();
|
||||
return {
|
||||
@@ -16,6 +17,7 @@ function createMockContext(overrides?: {
|
||||
runId: "test-run",
|
||||
onToolResult,
|
||||
onAgentEvent: vi.fn(),
|
||||
toolResultFormat: overrides?.toolResultFormat,
|
||||
},
|
||||
state: {
|
||||
toolMetaById: new Map(),
|
||||
@@ -217,6 +219,68 @@ describe("handleToolExecutionEnd media emission", () => {
|
||||
expect(ctx.state.pendingToolAudioAsVoice).toBe(true);
|
||||
});
|
||||
|
||||
it("does not queue structured media already emitted in plain verbose output", async () => {
|
||||
const ctx = createMockContext({
|
||||
shouldEmitToolOutput: true,
|
||||
onToolResult: vi.fn(),
|
||||
toolResultFormat: "plain",
|
||||
});
|
||||
|
||||
await handleToolExecutionEnd(ctx, {
|
||||
type: "tool_execution_end",
|
||||
toolName: "image_generate",
|
||||
toolCallId: "tc-1",
|
||||
isError: false,
|
||||
result: {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Generated 1 image with google/gemini-3.1-flash-image-preview.\nMEDIA:/tmp/generated.png",
|
||||
},
|
||||
],
|
||||
details: {
|
||||
media: {
|
||||
mediaUrls: ["/tmp/generated.png"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(ctx.emitToolOutput).toHaveBeenCalled();
|
||||
expect(ctx.state.pendingToolMediaUrls).toEqual([]);
|
||||
});
|
||||
|
||||
it("still queues structured media for markdown verbose output", async () => {
|
||||
const ctx = createMockContext({
|
||||
shouldEmitToolOutput: true,
|
||||
onToolResult: vi.fn(),
|
||||
toolResultFormat: "markdown",
|
||||
});
|
||||
|
||||
await handleToolExecutionEnd(ctx, {
|
||||
type: "tool_execution_end",
|
||||
toolName: "image_generate",
|
||||
toolCallId: "tc-1",
|
||||
isError: false,
|
||||
result: {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Generated 1 image with google/gemini-3.1-flash-image-preview.\nMEDIA:/tmp/generated.png",
|
||||
},
|
||||
],
|
||||
details: {
|
||||
media: {
|
||||
mediaUrls: ["/tmp/generated.png"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(ctx.emitToolOutput).toHaveBeenCalled();
|
||||
expect(ctx.state.pendingToolMediaUrls).toEqual(["/tmp/generated.png"]);
|
||||
});
|
||||
|
||||
it("does NOT emit media for error results", async () => {
|
||||
const onToolResult = vi.fn();
|
||||
const ctx = createMockContext({ shouldEmitToolOutput: false, onToolResult });
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
buildExecApprovalUnavailableReplyPayload,
|
||||
} from "../infra/exec-approval-reply.js";
|
||||
import type { ExecApprovalDecision } from "../infra/exec-approvals.js";
|
||||
import { splitMediaFromOutput } from "../media/parse.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import type { PluginHookAfterToolCallEvent } from "../plugins/types.js";
|
||||
import type { ApplyPatchSummary } from "./apply-patch.js";
|
||||
@@ -282,6 +283,18 @@ function queuePendingToolMedia(
|
||||
}
|
||||
}
|
||||
|
||||
function collectEmittedToolOutputMediaUrls(
|
||||
toolName: string,
|
||||
outputText: string,
|
||||
result: unknown,
|
||||
): string[] {
|
||||
const mediaUrls = splitMediaFromOutput(outputText).mediaUrls ?? [];
|
||||
if (mediaUrls.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return filterToolResultMediaUrls(toolName, mediaUrls, result);
|
||||
}
|
||||
|
||||
function readExecApprovalPendingDetails(result: unknown): {
|
||||
approvalId: string;
|
||||
approvalSlug: string;
|
||||
@@ -382,6 +395,7 @@ async function emitToolResultOutput(params: {
|
||||
"object" &&
|
||||
!Array.isArray((result as { details?: { media?: unknown } }).details?.media);
|
||||
const approvalPending = readExecApprovalPendingDetails(result);
|
||||
let emittedToolOutputMediaUrls: string[] = [];
|
||||
if (!isToolError && approvalPending) {
|
||||
if (!ctx.params.onToolResult) {
|
||||
return;
|
||||
@@ -431,6 +445,13 @@ async function emitToolResultOutput(params: {
|
||||
if (ctx.shouldEmitToolOutput()) {
|
||||
const outputText = extractToolResultText(sanitizedResult);
|
||||
if (outputText) {
|
||||
if (ctx.params.toolResultFormat === "plain") {
|
||||
emittedToolOutputMediaUrls = collectEmittedToolOutputMediaUrls(
|
||||
toolName,
|
||||
outputText,
|
||||
result,
|
||||
);
|
||||
}
|
||||
ctx.emitToolOutput(toolName, meta, outputText, result);
|
||||
}
|
||||
if (!hasStructuredMedia) {
|
||||
@@ -447,11 +468,15 @@ async function emitToolResultOutput(params: {
|
||||
return;
|
||||
}
|
||||
const mediaUrls = filterToolResultMediaUrls(toolName, mediaReply.mediaUrls, result);
|
||||
if (mediaUrls.length === 0) {
|
||||
const pendingMediaUrls =
|
||||
mediaReply.audioAsVoice || emittedToolOutputMediaUrls.length === 0
|
||||
? mediaUrls
|
||||
: mediaUrls.filter((url) => !emittedToolOutputMediaUrls.includes(url));
|
||||
if (pendingMediaUrls.length === 0) {
|
||||
return;
|
||||
}
|
||||
queuePendingToolMedia(ctx, {
|
||||
mediaUrls,
|
||||
mediaUrls: pendingMediaUrls,
|
||||
...(mediaReply.audioAsVoice ? { audioAsVoice: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -140,6 +140,7 @@ export type ToolHandlerParams = Pick<
|
||||
| "sessionKey"
|
||||
| "sessionId"
|
||||
| "agentId"
|
||||
| "toolResultFormat"
|
||||
>;
|
||||
|
||||
export type ToolHandlerState = Pick<
|
||||
|
||||
@@ -365,7 +365,57 @@ describe("createImageGenerateTool", () => {
|
||||
},
|
||||
});
|
||||
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
|
||||
expect(text).not.toContain("MEDIA:");
|
||||
expect(text).toContain("MEDIA:/tmp/generated-1.png");
|
||||
expect(text).toContain("MEDIA:/tmp/generated-2.png");
|
||||
});
|
||||
|
||||
it("includes MEDIA paths in content text so follow-up replies use the real saved file", async () => {
|
||||
vi.spyOn(imageGenerationRuntime, "generateImage").mockResolvedValue({
|
||||
provider: "google",
|
||||
model: "gemini-3.1-flash-image-preview",
|
||||
attempts: [],
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("jpg-data"),
|
||||
mimeType: "image/jpeg",
|
||||
fileName: "kodo_sawaki_zazen.jpg",
|
||||
},
|
||||
],
|
||||
});
|
||||
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
|
||||
path: "/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
|
||||
id: "kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
|
||||
size: 8,
|
||||
contentType: "image/jpeg",
|
||||
});
|
||||
|
||||
const tool = createImageGenerateTool({
|
||||
config: {
|
||||
agents: {
|
||||
defaults: {
|
||||
imageGenerationModel: { primary: "google/gemini-3.1-flash-image-preview" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(tool).not.toBeNull();
|
||||
if (!tool) {
|
||||
throw new Error("expected image_generate tool");
|
||||
}
|
||||
|
||||
const result = await tool.execute("call-regression", { prompt: "kodo sawaki zazen" });
|
||||
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
|
||||
|
||||
expect(text).toContain(
|
||||
"MEDIA:/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
|
||||
);
|
||||
expect(result.details).toMatchObject({
|
||||
media: {
|
||||
mediaUrls: [
|
||||
"/home/openclaw/.openclaw/media/tool-image-generation/kodo_sawaki_zazen---3337a0ed-898a-4572-8950-0d288719f4f8.jpg",
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects counts outside the supported range", async () => {
|
||||
|
||||
@@ -670,6 +670,9 @@ export function createImageGenerateTool(options?: {
|
||||
.filter((entry): entry is string => Boolean(entry));
|
||||
const lines = [
|
||||
`Generated ${savedImages.length} image${savedImages.length === 1 ? "" : "s"} with ${result.provider}/${result.model}.`,
|
||||
// Show the actual saved paths so the model does not invent a bogus
|
||||
// local path when it references the generated image in a follow-up reply.
|
||||
...savedImages.map((image) => `MEDIA:${image.path}`),
|
||||
];
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user