fix(gateway): support image_url in OpenAI chat completions (#34068)

* fix(gateway): parse image_url in openai chat completions

* test(gateway): cover openai chat completions image_url flows

* docs(changelog): note openai image_url chat completions fix (#17685)

* fix(gateway): harden openai image_url parsing and limits

* test(gateway): add openai image_url regression coverage

* docs(changelog): expand #17685 openai chat completions note

* Gateway: make OpenAI image_url URL fetch opt-in and configurable

* Diagnostics: redact image base64 payload data in trace logs

* Changelog: note OpenAI image_url hardening follow-ups

* Gateway: enforce OpenAI image_url total budget incrementally

* Gateway: scope OpenAI image_url extraction to the active turn

* Update CHANGELOG.md
This commit is contained in:
Vincent Koc
2026-03-06 00:35:50 -05:00
committed by GitHub
parent 36e2e04a32
commit 9c86a9fd23
16 changed files with 764 additions and 18 deletions

View File

@@ -1,3 +1,4 @@
import crypto from "node:crypto";
import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { resolveUserPath } from "../utils.js";
@@ -89,4 +90,58 @@ describe("createCacheTrace", () => {
expect(trace).toBeNull();
});
it("redacts image data from options and messages before writing", () => {
const lines: string[] = [];
const trace = createCacheTrace({
cfg: {
diagnostics: {
cacheTrace: {
enabled: true,
},
},
},
env: {},
writer: {
filePath: "memory",
write: (line) => lines.push(line),
},
});
trace?.recordStage("stream:context", {
options: {
images: [{ type: "image", mimeType: "image/png", data: "QUJDRA==" }],
},
messages: [
{
role: "user",
content: [
{
type: "image",
source: { type: "base64", media_type: "image/jpeg", data: "U0VDUkVU" },
},
],
},
] as unknown as [],
});
const event = JSON.parse(lines[0]?.trim() ?? "{}") as Record<string, unknown>;
const optionsImages = (
((event.options as { images?: unknown[] } | undefined)?.images ?? []) as Array<
Record<string, unknown>
>
)[0];
expect(optionsImages?.data).toBe("<redacted>");
expect(optionsImages?.bytes).toBe(4);
expect(optionsImages?.sha256).toBe(
crypto.createHash("sha256").update("QUJDRA==").digest("hex"),
);
const firstMessage = ((event.messages as Array<Record<string, unknown>> | undefined) ?? [])[0];
const source = (((firstMessage?.content as Array<Record<string, unknown>> | undefined) ?? [])[0]
?.source ?? {}) as Record<string, unknown>;
expect(source.data).toBe("<redacted>");
expect(source.bytes).toBe(6);
expect(source.sha256).toBe(crypto.createHash("sha256").update("U0VDUkVU").digest("hex"));
});
});