fix(gateway): support image_url in OpenAI chat completions (#34068)

* fix(gateway): parse image_url in openai chat completions

* test(gateway): cover openai chat completions image_url flows

* docs(changelog): note openai image_url chat completions fix (#17685)

* fix(gateway): harden openai image_url parsing and limits

* test(gateway): add openai image_url regression coverage

* docs(changelog): expand #17685 openai chat completions note

* Gateway: make OpenAI image_url URL fetch opt-in and configurable

* Diagnostics: redact image base64 payload data in trace logs

* Changelog: note OpenAI image_url hardening follow-ups

* Gateway: enforce OpenAI image_url total budget incrementally

* Gateway: scope OpenAI image_url extraction to the active turn

* Update CHANGELOG.md

src/agents/anthropic-payload-log.test.ts

@@ -0,0 +1,49 @@
+import crypto from "node:crypto";
+import type { StreamFn } from "@mariozechner/pi-agent-core";
+import { describe, expect, it } from "vitest";
+import { createAnthropicPayloadLogger } from "./anthropic-payload-log.js";
+
+describe("createAnthropicPayloadLogger", () => {
+  it("redacts image base64 payload data before writing logs", async () => {
+    const lines: string[] = [];
+    const logger = createAnthropicPayloadLogger({
+      env: { OPENCLAW_ANTHROPIC_PAYLOAD_LOG: "1" },
+      writer: {
+        filePath: "memory",
+        write: (line) => lines.push(line),
+      },
+    });
+    expect(logger).not.toBeNull();
+
+    const payload = {
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image",
+              source: { type: "base64", media_type: "image/png", data: "QUJDRA==" },
+            },
+          ],
+        },
+      ],
+    };
+    const streamFn: StreamFn = ((_, __, options) => {
+      options?.onPayload?.(payload);
+      return {} as never;
+    }) as StreamFn;
+
+    const wrapped = logger?.wrapStreamFn(streamFn);
+    await wrapped?.({ api: "anthropic-messages" } as never, { messages: [] } as never, {});
+
+    const event = JSON.parse(lines[0]?.trim() ?? "{}") as Record<string, unknown>;
+    const message = ((event.payload as { messages?: unknown[] } | undefined)?.messages ??
+      []) as Array<Record<string, unknown>>;
+    const source = (((message[0]?.content as Array<Record<string, unknown>> | undefined) ?? [])[0]
+      ?.source ?? {}) as Record<string, unknown>;
+    expect(source.data).toBe("<redacted>");
+    expect(source.bytes).toBe(4);
+    expect(source.sha256).toBe(crypto.createHash("sha256").update("QUJDRA==").digest("hex"));
+    expect(event.payloadDigest).toBeDefined();
+  });
+});

src/agents/anthropic-payload-log.ts

@@ -7,6 +7,7 @@ import { createSubsystemLogger } from "../logging/subsystem.js";
 import { resolveUserPath } from "../utils.js";
 import { parseBooleanValue } from "../utils/boolean.js";
 import { safeJsonStringify } from "../utils/safe-json.js";
+import { redactImageDataForDiagnostics } from "./payload-redaction.js";
 import { getQueuedFileWriter, type QueuedFileWriter } from "./queued-file-writer.js";
 
 type PayloadLogStage = "request" | "usage";
@@ -103,6 +104,7 @@ export function createAnthropicPayloadLogger(params: {
   modelId?: string;
   modelApi?: string | null;
   workspaceDir?: string;
+  writer?: PayloadLogWriter;
 }): AnthropicPayloadLogger | null {
   const env = params.env ?? process.env;
   const cfg = resolvePayloadLogConfig(env);
@@ -110,7 +112,7 @@ export function createAnthropicPayloadLogger(params: {
     return null;
   }
 
-  const writer = getWriter(cfg.filePath);
+  const writer = params.writer ?? getWriter(cfg.filePath);
   const base: Omit<PayloadLogEvent, "ts" | "stage"> = {
     runId: params.runId,
     sessionId: params.sessionId,
@@ -135,12 +137,13 @@ export function createAnthropicPayloadLogger(params: {
         return streamFn(model, context, options);
       }
       const nextOnPayload = (payload: unknown) => {
+        const redactedPayload = redactImageDataForDiagnostics(payload);
         record({
           ...base,
           ts: new Date().toISOString(),
           stage: "request",
-          payload,
-          payloadDigest: digest(payload),
+          payload: redactedPayload,
+          payloadDigest: digest(redactedPayload),
         });
         options?.onPayload?.(payload);
       };

src/agents/cache-trace.test.ts

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import { describe, expect, it } from "vitest";
 import type { OpenClawConfig } from "../config/config.js";
 import { resolveUserPath } from "../utils.js";
@@ -89,4 +90,58 @@ describe("createCacheTrace", () => {
 
     expect(trace).toBeNull();
   });
+
+  it("redacts image data from options and messages before writing", () => {
+    const lines: string[] = [];
+    const trace = createCacheTrace({
+      cfg: {
+        diagnostics: {
+          cacheTrace: {
+            enabled: true,
+          },
+        },
+      },
+      env: {},
+      writer: {
+        filePath: "memory",
+        write: (line) => lines.push(line),
+      },
+    });
+
+    trace?.recordStage("stream:context", {
+      options: {
+        images: [{ type: "image", mimeType: "image/png", data: "QUJDRA==" }],
+      },
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image",
+              source: { type: "base64", media_type: "image/jpeg", data: "U0VDUkVU" },
+            },
+          ],
+        },
+      ] as unknown as [],
+    });
+
+    const event = JSON.parse(lines[0]?.trim() ?? "{}") as Record<string, unknown>;
+    const optionsImages = (
+      ((event.options as { images?: unknown[] } | undefined)?.images ?? []) as Array<
+        Record<string, unknown>
+      >
+    )[0];
+    expect(optionsImages?.data).toBe("<redacted>");
+    expect(optionsImages?.bytes).toBe(4);
+    expect(optionsImages?.sha256).toBe(
+      crypto.createHash("sha256").update("QUJDRA==").digest("hex"),
+    );
+
+    const firstMessage = ((event.messages as Array<Record<string, unknown>> | undefined) ?? [])[0];
+    const source = (((firstMessage?.content as Array<Record<string, unknown>> | undefined) ?? [])[0]
+      ?.source ?? {}) as Record<string, unknown>;
+    expect(source.data).toBe("<redacted>");
+    expect(source.bytes).toBe(6);
+    expect(source.sha256).toBe(crypto.createHash("sha256").update("U0VDUkVU").digest("hex"));
+  });
 });

src/agents/cache-trace.ts

@@ -6,6 +6,7 @@ import { resolveStateDir } from "../config/paths.js";
 import { resolveUserPath } from "../utils.js";
 import { parseBooleanValue } from "../utils/boolean.js";
 import { safeJsonStringify } from "../utils/safe-json.js";
+import { redactImageDataForDiagnostics } from "./payload-redaction.js";
 import { getQueuedFileWriter, type QueuedFileWriter } from "./queued-file-writer.js";
 
 export type CacheTraceStage =
@@ -198,7 +199,7 @@ export function createCacheTrace(params: CacheTraceInit): CacheTrace | null {
       event.systemDigest = digest(payload.system);
     }
     if (payload.options) {
-      event.options = payload.options;
+      event.options = redactImageDataForDiagnostics(payload.options) as Record<string, unknown>;
     }
     if (payload.model) {
       event.model = payload.model;
@@ -212,7 +213,7 @@ export function createCacheTrace(params: CacheTraceInit): CacheTrace | null {
       event.messageFingerprints = summary.messageFingerprints;
       event.messagesDigest = summary.messagesDigest;
       if (cfg.includeMessages) {
-        event.messages = messages;
+        event.messages = redactImageDataForDiagnostics(messages) as AgentMessage[];
       }
     }
 

src/agents/payload-redaction.ts

@@ -0,0 +1,64 @@
+import crypto from "node:crypto";
+import { estimateBase64DecodedBytes } from "../media/base64.js";
+
+export const REDACTED_IMAGE_DATA = "<redacted>";
+
+function toLowerTrimmed(value: unknown): string {
+  return typeof value === "string" ? value.trim().toLowerCase() : "";
+}
+
+function hasImageMime(record: Record<string, unknown>): boolean {
+  const candidates = [
+    toLowerTrimmed(record.mimeType),
+    toLowerTrimmed(record.media_type),
+    toLowerTrimmed(record.mime_type),
+  ];
+  return candidates.some((value) => value.startsWith("image/"));
+}
+
+function shouldRedactImageData(record: Record<string, unknown>): record is Record<string, string> {
+  if (typeof record.data !== "string") {
+    return false;
+  }
+  const type = toLowerTrimmed(record.type);
+  return type === "image" || hasImageMime(record);
+}
+
+function digestBase64Payload(data: string): string {
+  return crypto.createHash("sha256").update(data).digest("hex");
+}
+
+/**
+ * Redacts image/base64 payload data from diagnostic objects before persistence.
+ */
+export function redactImageDataForDiagnostics(value: unknown): unknown {
+  const seen = new WeakSet<object>();
+
+  const visit = (input: unknown): unknown => {
+    if (Array.isArray(input)) {
+      return input.map((entry) => visit(entry));
+    }
+    if (!input || typeof input !== "object") {
+      return input;
+    }
+    if (seen.has(input)) {
+      return "[Circular]";
+    }
+    seen.add(input);
+
+    const record = input as Record<string, unknown>;
+    const out: Record<string, unknown> = {};
+    for (const [key, val] of Object.entries(record)) {
+      out[key] = visit(val);
+    }
+
+    if (shouldRedactImageData(record)) {
+      out.data = REDACTED_IMAGE_DATA;
+      out.bytes = estimateBase64DecodedBytes(record.data);
+      out.sha256 = digestBase64Payload(record.data);
+    }
+    return out;
+  };
+
+  return visit(value);
+}