fix(codex): deliver native image outputs

Co-authored-by: Kelaw - Keshav's Agent <keshavbotagent@gmail.com>
This commit is contained in:
Peter Steinberger
2026-05-10 08:42:25 +01:00
parent 0c3c379689
commit 5821a4033c
5 changed files with 116 additions and 1 deletions

View File

@@ -21,6 +21,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- Gateway: reread config from disk after the first in-process restart loop startup, preventing SIGUSR1 restarts from reusing a stale startup snapshot and dropping config written after boot. Fixes #79947. Thanks @TheLevti.
- Codex app-server: deliver native image-generation outputs from Codex `savedPath` events as reply media, so blank-text image generation turns still attach the generated file. Thanks @keshavbotagent.
- Media/host-read: allow buffer-verified gzip, tar, and 7z archives in the shared host-local media validator alongside ZIP and document attachments.
- Plugins/doctor: invalidate persisted plugin registry snapshots when plugin diagnostics point at deleted source paths, so `openclaw doctor` stops repeating stale warnings after a local extension is replaced by a managed npm plugin. Fixes #80087. (#80134) Thanks @hclsys.
- Cron: let isolated self-cleanup runs inspect their own job run history while keeping other cron jobs and mutation actions blocked. Fixes #80019. Thanks @hclsys.

View File

@@ -200,6 +200,9 @@ settings such as `agents.defaults.imageGenerationModel`, `videoGenerationModel`,
Text, images, video, music, TTS, approvals, and messaging-tool output continue
through the normal OpenClaw delivery path. Media generation does not require PI.
When Codex emits a native image-generation item with a `savedPath`, OpenClaw
forwards that exact file through the normal reply-media path even if the Codex
turn has no assistant text.
## Related

View File

@@ -343,6 +343,55 @@ describe("CodexAppServerEventProjector", () => {
expect(result.lastAssistant?.content).toEqual([{ type: "text", text: "OK from raw" }]);
});
it("attaches native Codex image-generation saved paths as reply media", async () => {
const projector = await createProjector();
const savedPath = "/tmp/codex-home/generated_images/session-1/ig_123.png";
await projector.handleNotification(
turnCompleted([
{
type: "imageGeneration",
id: "ig_123",
status: "completed",
revisedPrompt: "A tiny blue square",
result: "Zm9v",
savedPath,
},
]),
);
const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.assistantTexts).toStrictEqual([]);
expect(result.toolMediaUrls).toEqual([savedPath]);
});
it("does not append native Codex image-generation media after explicit media delivery", async () => {
const projector = await createProjector();
const savedPath = "/tmp/codex-home/generated_images/session-1/ig_123.png";
await projector.handleNotification(
turnCompleted([
{
type: "imageGeneration",
id: "ig_123",
status: "completed",
revisedPrompt: null,
result: "Zm9v",
savedPath,
},
]),
);
const result = projector.buildResult({
...buildEmptyToolTelemetry(),
messagingToolSentMediaUrls: [savedPath],
toolMediaUrls: [],
});
expect(result.toolMediaUrls).toStrictEqual([]);
});
it("does not fail a completed reply after a retryable app-server error notification", async () => {
const projector = await createProjector();

View File

@@ -115,6 +115,7 @@ export class CodexAppServerEventProjector {
private readonly toolTranscriptMessages: AgentMessage[] = [];
private readonly toolTranscriptCallIds = new Set<string>();
private readonly toolTranscriptResultIds = new Set<string>();
private readonly nativeGeneratedMediaUrls = new Set<string>();
private assistantStarted = false;
private reasoningStarted = false;
private reasoningEnded = false;
@@ -294,7 +295,7 @@ export class CodexAppServerEventProjector {
messagingToolSentMediaUrls: toolTelemetry.messagingToolSentMediaUrls,
messagingToolSentTargets: toolTelemetry.messagingToolSentTargets,
heartbeatToolResponse: toolTelemetry.heartbeatToolResponse,
toolMediaUrls: toolTelemetry.toolMediaUrls,
toolMediaUrls: this.buildToolMediaUrls(toolTelemetry),
toolAudioAsVoice: toolTelemetry.toolAudioAsVoice,
successfulCronAdds: toolTelemetry.successfulCronAdds,
cloudCodeAssistFormatError: false,
@@ -462,6 +463,7 @@ export class CodexAppServerEventProjector {
this.rememberAssistantItem(item.id);
this.assistantTextByItem.set(item.id, item.text);
}
this.recordNativeGeneratedMedia(item);
if (item?.type === "plan" && typeof item.text === "string" && item.text) {
this.planTextByItem.set(item.id, item.text);
this.emitPlanUpdate({ explanation: undefined, steps: splitPlanText(item.text) });
@@ -597,6 +599,7 @@ export class CodexAppServerEventProjector {
this.rememberAssistantItem(item.id);
this.assistantTextByItem.set(item.id, item.text);
}
this.recordNativeGeneratedMedia(item);
if (item.type === "plan" && typeof item.text === "string" && item.text) {
this.planTextByItem.set(item.id, item.text);
this.emitPlanUpdate({ explanation: undefined, steps: splitPlanText(item.text) });
@@ -672,6 +675,28 @@ export class CodexAppServerEventProjector {
this.assistantTextByItem.set(itemId, text);
}
private recordNativeGeneratedMedia(item: CodexThreadItem | undefined): void {
if (item?.type !== "imageGeneration") {
return;
}
const savedPath = readItemString(item, "savedPath")?.trim();
if (savedPath) {
this.nativeGeneratedMediaUrls.add(savedPath);
}
}
private buildToolMediaUrls(toolTelemetry: CodexAppServerToolTelemetry): string[] | undefined {
const mediaUrls = new Set(
toolTelemetry.toolMediaUrls?.map((url) => url.trim()).filter(Boolean) ?? [],
);
if ((toolTelemetry.messagingToolSentMediaUrls?.length ?? 0) === 0) {
for (const mediaUrl of this.nativeGeneratedMediaUrls) {
mediaUrls.add(mediaUrl);
}
}
return mediaUrls.size > 0 ? [...mediaUrls] : toolTelemetry.toolMediaUrls;
}
private async maybeEndReasoning(): Promise<void> {
if (!this.reasoningStarted || this.reasoningEnded) {
return;

View File

@@ -2590,6 +2590,43 @@ describe("runCodexAppServerAttempt", () => {
expect(result.timedOut).toBe(false);
});
it("surfaces Codex-native image generation saved paths as reply media", async () => {
const harness = createStartedThreadHarness();
const params = createParams(
path.join(tempDir, "session.jsonl"),
path.join(tempDir, "workspace"),
);
const run = runCodexAppServerAttempt(params);
await harness.waitForMethod("turn/start");
await harness.notify({
method: "turn/completed",
params: {
threadId: "thread-1",
turnId: "turn-1",
turn: {
id: "turn-1",
status: "completed",
items: [
{
type: "imageGeneration",
id: "ig_123",
status: "completed",
revisedPrompt: "A tiny blue square",
result: "Zm9v",
savedPath: "/tmp/codex-home/generated_images/session-1/ig_123.png",
},
],
},
},
});
await expect(run).resolves.toMatchObject({
assistantTexts: [],
toolMediaUrls: ["/tmp/codex-home/generated_images/session-1/ig_123.png"],
});
});
it("does not complete on unscoped turn/completed notifications", async () => {
const harness = createStartedThreadHarness();
const run = runCodexAppServerAttempt(