diff --git a/CHANGELOG.md b/CHANGELOG.md index 54ad4816e7e..808149cf885 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Docs: https://docs.openclaw.ai - Codex harness: add structured debug logging for embedded harness selection decisions so `/status` stays simple while gateway logs explain auto-selection and Pi fallback reasons. (#70760) Thanks @100yenadmin. - Providers/OpenAI: add forward-compatible `gpt-5.5` and `gpt-5.5-pro` support for OpenAI API keys, OpenAI Codex OAuth, and the Codex CLI default model. - Providers/OpenAI: add image generation and reference-image editing through Codex OAuth, so `openai/gpt-image-2` works without an `OPENAI_API_KEY`. Fixes #70703. +- Image generation: let agents request provider-supported quality and output format hints, and pass OpenAI-specific background, moderation, compression, and user hints through the `image_generate` tool. (#70503) Thanks @ottodeng. ### Fixes diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 314032b4d71..07efd0d53e0 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -748236b1167a22be8d5ae19905078df77becd4d52cec9a3bb45c78ab2abddb9f plugin-sdk-api-baseline.json -9ca0549360dd2a634e7b7e56a17d7a3978ee797d9ad80ce5a4a57c84c35a83d3 plugin-sdk-api-baseline.jsonl +64c5f94fe0234da8ae2312ab30694ebc5675091fadebac92c106210f45a66e91 plugin-sdk-api-baseline.json +fd00bb4cd8f1e32503f94e8542db95235ec641eb62ae45d6d4b653d9ff60cb09 plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 41963a195d9..25cec0d4598 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -14,7 +14,7 @@ detail unless a doc page explicitly promotes them. For the plugin authoring guide, see [Plugin SDK overview](/plugins/sdk-overview). -### Plugin entry +## Plugin entry | Subpath | Key exports | | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/tools/image-generation.md b/docs/tools/image-generation.md index 614b5673f60..8f9595cc425 100644 --- a/docs/tools/image-generation.md +++ b/docs/tools/image-generation.md @@ -92,6 +92,14 @@ Aspect ratio: `1:1`, `2:3`, `3:2`, `3:4`, `4:3`, `4:5`, `5:4`, `9:16`, `16:9`, ` Resolution hint. + +Quality hint when the provider supports it. + + + +Output format hint when the provider supports it. + + Number of images to generate (1–4). @@ -104,7 +112,11 @@ Optional provider request timeout in milliseconds. Output filename hint. -Not all providers support all parameters. When a fallback provider supports a nearby geometry option instead of the exact requested one, OpenClaw remaps to the closest supported size, aspect ratio, or resolution before submission. Truly unsupported overrides are still reported in the tool result. + +OpenAI-only hints: `background`, `moderation`, `outputCompression`, and `user`. + + +Not all providers support all parameters. When a fallback provider supports a nearby geometry option instead of the exact requested one, OpenClaw remaps to the closest supported size, aspect ratio, or resolution before submission. Unsupported output hints such as `quality` or `outputFormat` are dropped for providers that do not declare support and are reported in the tool result. Tool results report the applied settings. When OpenClaw remaps geometry during provider fallback, the returned `size`, `aspectRatio`, and `resolution` values reflect what was actually sent, and `details.normalization` captures the requested-to-applied translation. @@ -173,9 +185,29 @@ image-generation and image-editing requests should use `gpt-image-2`. `gpt-image-2` supports both text-to-image generation and reference-image editing through the same `image_generate` tool. OpenClaw forwards `prompt`, -`count`, `size`, and reference images to OpenAI. OpenAI does not receive -`aspectRatio` or `resolution` directly; when possible OpenClaw maps those into a -supported `size`, otherwise the tool reports them as ignored overrides. +`count`, `size`, `quality`, `outputFormat`, and reference images to OpenAI. +OpenAI does not receive `aspectRatio` or `resolution` directly; when possible +OpenClaw maps those into a supported `size`, otherwise the tool reports them as +ignored overrides. + +OpenAI-specific options live under the `openai` object: + +```json +{ + "quality": "low", + "outputFormat": "jpeg", + "openai": { + "background": "opaque", + "moderation": "low", + "outputCompression": 60, + "user": "end-user-42" + } +} +``` + +`openai.background` accepts `transparent`, `opaque`, or `auto`; transparent +outputs require `outputFormat` `png` or `webp`. `openai.outputCompression` +applies to JPEG/WebP outputs. Generate one 4K landscape image: diff --git a/extensions/openai/image-generation-provider.test.ts b/extensions/openai/image-generation-provider.test.ts index 4b4f35d6160..0b22155a8df 100644 --- a/extensions/openai/image-generation-provider.test.ts +++ b/extensions/openai/image-generation-provider.test.ts @@ -198,6 +198,10 @@ describe("openai image generation provider", () => { expect(provider.capabilities.geometry?.sizes).toEqual( expect.arrayContaining(["2048x2048", "3840x2160", "2160x3840"]), ); + expect(provider.capabilities.output).toEqual({ + formats: ["png", "jpeg", "webp"], + qualities: ["low", "medium", "high", "auto"], + }); }); it("reports configured when either OpenAI API key auth or Codex OAuth auth is available", () => { @@ -339,6 +343,50 @@ describe("openai image generation provider", () => { expect(result.images).toHaveLength(1); }); + it("forwards output and OpenAI-only options on direct generations", async () => { + mockGeneratedPngResponse(); + + const provider = buildOpenAIImageGenerationProvider(); + const result = await provider.generateImage({ + provider: "openai", + model: "gpt-image-2", + prompt: "Cheap JPEG preview", + cfg: {}, + quality: "low", + outputFormat: "jpeg", + providerOptions: { + openai: { + background: "opaque", + moderation: "low", + outputCompression: 60, + user: "end-user-42", + }, + }, + }); + + expect(postJsonRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://api.openai.com/v1/images/generations", + body: { + model: "gpt-image-2", + prompt: "Cheap JPEG preview", + n: 1, + size: "1024x1024", + quality: "low", + output_format: "jpeg", + background: "opaque", + moderation: "low", + output_compression: 60, + user: "end-user-42", + }, + }), + ); + expect(result.images[0]).toMatchObject({ + mimeType: "image/jpeg", + fileName: "image-1.jpg", + }); + }); + it("allows loopback image requests for the synthetic mock-openai provider", async () => { mockGeneratedPngResponse(); @@ -463,6 +511,44 @@ describe("openai image generation provider", () => { expect(result.images).toHaveLength(1); }); + it("forwards output and OpenAI-only options on multipart edits", async () => { + mockGeneratedPngResponse(); + + const provider = buildOpenAIImageGenerationProvider(); + const result = await provider.generateImage({ + provider: "openai", + model: "gpt-image-2", + prompt: "Edit as WebP", + cfg: {}, + inputImages: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }], + quality: "high", + outputFormat: "webp", + providerOptions: { + openai: { + background: "transparent", + moderation: "auto", + outputCompression: 75, + user: "end-user-99", + }, + }, + }); + + const editCallArgs = postMultipartRequestMock.mock.calls[0]?.[0] as { + body: FormData; + }; + const form = editCallArgs.body; + expect(form.get("quality")).toBe("high"); + expect(form.get("output_format")).toBe("webp"); + expect(form.get("background")).toBe("transparent"); + expect(form.get("moderation")).toBe("auto"); + expect(form.get("output_compression")).toBe("75"); + expect(form.get("user")).toBe("end-user-99"); + expect(result.images[0]).toMatchObject({ + mimeType: "image/webp", + fileName: "image-1.webp", + }); + }); + it("falls back to Codex OAuth image generation through Responses streaming", async () => { mockCodexAuthOnly(); mockCodexImageStream({ imageData: "codex-image", revisedPrompt: "revised codex prompt" }); @@ -477,6 +563,14 @@ describe("openai image generation provider", () => { authStore, count: 1, size: "1024x1536", + quality: "low", + outputFormat: "jpeg", + providerOptions: { + openai: { + background: "opaque", + outputCompression: 55, + }, + }, }); expect(resolveApiKeyForProviderMock).toHaveBeenCalledWith( @@ -517,6 +611,10 @@ describe("openai image generation provider", () => { type: "image_generation", model: "gpt-image-2", size: "1024x1536", + quality: "low", + output_format: "jpeg", + background: "opaque", + output_compression: 55, }, ], tool_choice: { type: "image_generation" }, @@ -530,8 +628,8 @@ describe("openai image generation provider", () => { expect(result.images).toEqual([ { buffer: Buffer.from("codex-image"), - mimeType: "image/png", - fileName: "image-1.png", + mimeType: "image/jpeg", + fileName: "image-1.jpg", revisedPrompt: "revised codex prompt", }, ]); diff --git a/extensions/openai/image-generation-provider.ts b/extensions/openai/image-generation-provider.ts index 4331b482c60..d6c916c6e3a 100644 --- a/extensions/openai/image-generation-provider.ts +++ b/extensions/openai/image-generation-provider.ts @@ -1,6 +1,7 @@ import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import type { + ImageGenerationOutputFormat, ImageGenerationProvider, ImageGenerationResult, ImageGenerationSourceImage, @@ -28,6 +29,7 @@ const DEFAULT_OPENAI_CODEX_IMAGE_BASE_URL = "https://chatgpt.com/backend-api/cod const OPENAI_CODEX_IMAGE_INSTRUCTIONS = "You are an image generation assistant."; const DEFAULT_OPENAI_IMAGE_TIMEOUT_MS = 180_000; const DEFAULT_OUTPUT_MIME = "image/png"; +const DEFAULT_OUTPUT_EXTENSION = "png"; const DEFAULT_SIZE = "1024x1024"; const OPENAI_SUPPORTED_SIZES = [ "1024x1024", @@ -45,6 +47,8 @@ const MAX_CODEX_IMAGE_SSE_EVENTS = 512; const MAX_CODEX_IMAGE_BASE64_CHARS = 64 * 1024 * 1024; const LOG_VALUE_MAX_CHARS = 256; const MOCK_OPENAI_PROVIDER_ID = "mock-openai"; +const OPENAI_OUTPUT_FORMATS = ["png", "jpeg", "webp"] as const; +const OPENAI_QUALITIES = ["low", "medium", "high", "auto"] as const; const log = createSubsystemLogger("image-generation/openai"); const AZURE_HOSTNAME_SUFFIXES = [ @@ -139,6 +143,46 @@ function buildAzureImageUrl( return `${cleanBase}/openai/deployments/${model}/images/${action}?api-version=${resolveAzureApiVersion()}`; } +function resolveOutputMime(outputFormat?: ImageGenerationOutputFormat): { + mimeType: string; + extension: string; +} { + switch (outputFormat) { + case "jpeg": + return { mimeType: "image/jpeg", extension: "jpg" }; + case "webp": + return { mimeType: "image/webp", extension: "webp" }; + case "png": + case undefined: + return { mimeType: DEFAULT_OUTPUT_MIME, extension: DEFAULT_OUTPUT_EXTENSION }; + } + return { mimeType: DEFAULT_OUTPUT_MIME, extension: DEFAULT_OUTPUT_EXTENSION }; +} + +function appendOpenAIImageOptions( + target: Record | FormData, + req: Parameters[0], +): void { + const openai = req.providerOptions?.openai; + const entries: Record = { + ...(req.quality !== undefined ? { quality: req.quality } : {}), + ...(req.outputFormat !== undefined ? { output_format: req.outputFormat } : {}), + ...(openai?.background !== undefined ? { background: openai.background } : {}), + ...(openai?.moderation !== undefined ? { moderation: openai.moderation } : {}), + ...(openai?.outputCompression !== undefined + ? { output_compression: openai.outputCompression } + : {}), + ...(openai?.user !== undefined ? { user: openai.user } : {}), + }; + for (const [key, value] of Object.entries(entries)) { + if (target instanceof FormData) { + target.set(key, String(value)); + } else { + target[key] = value; + } + } +} + function shouldAllowPrivateImageEndpoint(req: { provider: string; cfg: OpenClawConfig | undefined; @@ -341,15 +385,17 @@ function decodeCodexImagePayload(payload: string): Buffer { function toCodexImage( entry: { result?: string; revised_prompt?: string }, index: number, + outputFormat?: ImageGenerationOutputFormat, ): ImageGenerationResult["images"][number] | null { if (typeof entry.result !== "string" || entry.result.length === 0) { return null; } + const output = resolveOutputMime(outputFormat); return Object.assign( { buffer: decodeCodexImagePayload(entry.result), - mimeType: DEFAULT_OUTPUT_MIME, - fileName: `image-${index + 1}.png`, + mimeType: output.mimeType, + fileName: `image-${index + 1}.${output.extension}`, }, entry.revised_prompt ? { revisedPrompt: entry.revised_prompt } : {}, ); @@ -358,6 +404,7 @@ function toCodexImage( function extractCodexImageGenerationResult(params: { body: string; model: string; + outputFormat?: ImageGenerationOutputFormat; }): ImageGenerationResult { const events = parseCodexImageGenerationEvents(params.body); const failure = events.find( @@ -380,12 +427,14 @@ function extractCodexImageGenerationResult(params: { event.item.result.length > 0, ) .slice(0, OPENAI_MAX_IMAGE_RESULTS) - .map((event, index) => (event.item ? toCodexImage(event.item, index) : null)) + .map((event, index) => + event.item ? toCodexImage(event.item, index, params.outputFormat) : null, + ) .filter((image): image is NonNullable => image !== null); const completedOutputImages = (completedResponse?.response?.output ?? []) .filter((entry) => entry.type === "image_generation_call") .slice(0, OPENAI_MAX_IMAGE_RESULTS) - .map((entry, index) => toCodexImage(entry, index)) + .map((entry, index) => toCodexImage(entry, index, params.outputFormat)) .filter((image): image is NonNullable => image !== null); const images = outputItemImages.length > 0 ? outputItemImages : completedOutputImages; @@ -433,6 +482,10 @@ function createOpenAIImageGenerationProviderBase(params: { geometry: { sizes: [...OPENAI_SUPPORTED_SIZES], }, + output: { + formats: [...OPENAI_OUTPUT_FORMATS], + qualities: [...OPENAI_QUALITIES], + }, }, generateImage: params.generateImage, }; @@ -494,6 +547,7 @@ async function generateOpenAICodexImage(params: { const count = resolveOpenAIImageCount(req.count); const size = req.size ?? DEFAULT_SIZE; const timeoutMs = resolveOpenAIImageTimeoutMs(req.timeoutMs); + const openai = req.providerOptions?.openai; headers.set("Content-Type", "application/json"); const content: Array> = [ { type: "input_text", text: req.prompt }, @@ -522,6 +576,12 @@ async function generateOpenAICodexImage(params: { type: "image_generation", model, size, + ...(req.quality !== undefined ? { quality: req.quality } : {}), + ...(req.outputFormat !== undefined ? { output_format: req.outputFormat } : {}), + ...(openai?.background !== undefined ? { background: openai.background } : {}), + ...(openai?.outputCompression !== undefined + ? { output_compression: openai.outputCompression } + : {}), }, ], tool_choice: { type: "image_generation" }, @@ -540,6 +600,7 @@ async function generateOpenAICodexImage(params: { extractCodexImageGenerationResult({ body: await readResponseBodyText(response), model, + outputFormat: req.outputFormat, }), ); } finally { @@ -547,10 +608,11 @@ async function generateOpenAICodexImage(params: { } } const images = results.flatMap((result) => result.images); + const output = resolveOutputMime(req.outputFormat); return { images: images.map((image, index) => Object.assign({}, image, { - fileName: `image-${index + 1}.png`, + fileName: `image-${index + 1}.${output.extension}`, }), ), model, @@ -657,6 +719,7 @@ export function buildOpenAIImageGenerationProvider(): ImageGenerationProvider { form.set("prompt", req.prompt); form.set("n", String(count)); form.set("size", size); + appendOpenAIImageOptions(form, req); for (const [index, image] of inputImages.entries()) { const mimeType = image.mimeType?.trim() || DEFAULT_OUTPUT_MIME; form.append( @@ -685,15 +748,17 @@ export function buildOpenAIImageGenerationProvider(): ImageGenerationProvider { : await (() => { const jsonHeaders = new Headers(headers); jsonHeaders.set("Content-Type", "application/json"); + const body: Record = { + model, + prompt: req.prompt, + n: count, + size, + }; + appendOpenAIImageOptions(body, req); return postJsonRequest({ url, headers: jsonHeaders, - body: { - model, - prompt: req.prompt, - n: count, - size, - }, + body, timeoutMs, fetchFn: fetch, allowPrivateNetwork, @@ -708,6 +773,7 @@ export function buildOpenAIImageGenerationProvider(): ImageGenerationProvider { ); const data = (await response.json()) as OpenAIImageApiResponse; + const output = resolveOutputMime(req.outputFormat); const images = (data.data ?? []) .map((entry, index) => { if (!entry.b64_json) { @@ -716,8 +782,8 @@ export function buildOpenAIImageGenerationProvider(): ImageGenerationProvider { return Object.assign( { buffer: Buffer.from(entry.b64_json, `base64`), - mimeType: DEFAULT_OUTPUT_MIME, - fileName: `image-${index + 1}.png`, + mimeType: output.mimeType, + fileName: `image-${index + 1}.${output.extension}`, }, entry.revised_prompt ? { revisedPrompt: entry.revised_prompt } : {}, ); diff --git a/scripts/check-no-raw-channel-fetch.mjs b/scripts/check-no-raw-channel-fetch.mjs index bd2d9907721..c5262c5d7c4 100644 --- a/scripts/check-no-raw-channel-fetch.mjs +++ b/scripts/check-no-raw-channel-fetch.mjs @@ -59,9 +59,9 @@ const allowedRawFetchCallsites = new Set([ bundledPluginCallsite("qqbot", "src/engine/tools/channel-api.ts", 178), bundledPluginCallsite("qqbot", "src/engine/utils/stt.ts", 87), bundledPluginCallsite("signal", "src/install-signal-cli.ts", 224), - bundledPluginCallsite("slack", "src/monitor/media.ts", 99), - bundledPluginCallsite("slack", "src/monitor/media.ts", 118), - bundledPluginCallsite("slack", "src/monitor/media.ts", 123), + bundledPluginCallsite("slack", "src/monitor/media.ts", 106), + bundledPluginCallsite("slack", "src/monitor/media.ts", 125), + bundledPluginCallsite("slack", "src/monitor/media.ts", 130), bundledPluginCallsite("venice", "models.ts", 552), bundledPluginCallsite("vercel-ai-gateway", "models.ts", 181), bundledPluginCallsite("voice-call", "src/providers/twilio/api.ts", 23), diff --git a/src/agents/tools/image-generate-tool.test.ts b/src/agents/tools/image-generate-tool.test.ts index 0e03b29ab78..97a8f46dcdf 100644 --- a/src/agents/tools/image-generate-tool.test.ts +++ b/src/agents/tools/image-generate-tool.test.ts @@ -478,6 +478,62 @@ describe("createImageGenerateTool", () => { expect(text).toContain("MEDIA:/tmp/generated-2.png"); }); + it("forwards output hints and OpenAI provider options", async () => { + const generateImage = vi.spyOn(imageGenerationRuntime, "generateImage").mockResolvedValue({ + provider: "openai", + model: "gpt-image-2", + attempts: [], + ignoredOverrides: [], + images: [ + { + buffer: Buffer.from("jpg-out"), + mimeType: "image/jpeg", + fileName: "preview.jpg", + }, + ], + }); + vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValue({ + path: "/tmp/generated.jpg", + id: "generated.jpg", + size: 5, + contentType: "image/jpeg", + }); + + const tool = createToolWithPrimaryImageModel("openai/gpt-image-2"); + const result = await tool.execute("call-openai-hints", { + prompt: "Cheap preview", + quality: "low", + outputFormat: "jpeg", + openai: { + background: "opaque", + moderation: "low", + outputCompression: 60, + user: "end-user-42", + }, + }); + + expect(generateImage).toHaveBeenCalledWith( + expect.objectContaining({ + quality: "low", + outputFormat: "jpeg", + providerOptions: { + openai: { + background: "opaque", + moderation: "low", + outputCompression: 60, + user: "end-user-42", + }, + }, + }), + ); + expect(result).toMatchObject({ + details: { + quality: "low", + outputFormat: "jpeg", + }, + }); + }); + it("includes MEDIA paths in content text so follow-up replies use the real saved file", async () => { vi.spyOn(imageGenerationRuntime, "listRuntimeImageGenerationProviders").mockReturnValue([ { diff --git a/src/agents/tools/image-generate-tool.ts b/src/agents/tools/image-generate-tool.ts index 8d402d63258..e9113052b97 100644 --- a/src/agents/tools/image-generate-tool.ts +++ b/src/agents/tools/image-generate-tool.ts @@ -8,7 +8,13 @@ import { } from "../../image-generation/runtime.js"; import type { ImageGenerationIgnoredOverride, + ImageGenerationOpenAIBackground, + ImageGenerationOpenAIModeration, + ImageGenerationOpenAIOptions, + ImageGenerationOutputFormat, ImageGenerationProvider, + ImageGenerationProviderOptions, + ImageGenerationQuality, ImageGenerationResolution, ImageGenerationSourceImage, } from "../../image-generation/types.js"; @@ -18,6 +24,7 @@ import { saveMediaBuffer } from "../../media/store.js"; import { loadWebMedia } from "../../media/web-media.js"; import { getProviderEnvVars } from "../../secrets/provider-env-vars.js"; import { resolveUserPath } from "../../utils.js"; +import { optionalStringEnum } from "../schema/string-enum.js"; import { ToolInputError, readNumberParam, readStringParam } from "./common.js"; import { decodeDataUrl } from "./image-tool.helpers.js"; import { @@ -44,6 +51,10 @@ const DEFAULT_COUNT = 1; const MAX_COUNT = 4; const MAX_INPUT_IMAGES = 5; const DEFAULT_RESOLUTION: ImageGenerationResolution = "1K"; +const SUPPORTED_QUALITIES = ["low", "medium", "high", "auto"] as const; +const SUPPORTED_OUTPUT_FORMATS = ["png", "jpeg", "webp"] as const; +const SUPPORTED_OPENAI_BACKGROUNDS = ["transparent", "opaque", "auto"] as const; +const SUPPORTED_OPENAI_MODERATIONS = ["low", "auto"] as const; const SUPPORTED_ASPECT_RATIOS = new Set([ "1:1", "2:3", @@ -102,6 +113,34 @@ const ImageGenerateToolSchema = Type.Object({ "Optional resolution hint: 1K, 2K, or 4K. Useful for Google edit/generation flows.", }), ), + quality: optionalStringEnum(SUPPORTED_QUALITIES, { + description: "Optional quality hint: low, medium, high, or auto when the provider supports it.", + }), + outputFormat: optionalStringEnum(SUPPORTED_OUTPUT_FORMATS, { + description: "Optional output format hint: png, jpeg, or webp when the provider supports it.", + }), + openai: Type.Optional( + Type.Object({ + background: optionalStringEnum(SUPPORTED_OPENAI_BACKGROUNDS, { + description: "OpenAI-only background hint: transparent, opaque, or auto.", + }), + moderation: optionalStringEnum(SUPPORTED_OPENAI_MODERATIONS, { + description: "OpenAI-only moderation hint: low or auto.", + }), + outputCompression: Type.Optional( + Type.Number({ + description: "OpenAI-only compression level for jpeg/webp outputFormat, 0-100.", + minimum: 0, + maximum: 100, + }), + ), + user: Type.Optional( + Type.String({ + description: "OpenAI-only stable end-user identifier for abuse monitoring.", + }), + ), + }), + ), count: Type.Optional( Type.Number({ description: `Optional number of images to request (1-${MAX_COUNT}).`, @@ -176,6 +215,85 @@ function normalizeAspectRatio(raw: string | undefined): string | undefined { ); } +function normalizeQuality(raw: string | undefined): ImageGenerationQuality | undefined { + const normalized = raw?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + if ((SUPPORTED_QUALITIES as readonly string[]).includes(normalized)) { + return normalized as ImageGenerationQuality; + } + throw new ToolInputError("quality must be one of low, medium, high, or auto"); +} + +function normalizeOutputFormat(raw: string | undefined): ImageGenerationOutputFormat | undefined { + const normalized = raw?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + if ((SUPPORTED_OUTPUT_FORMATS as readonly string[]).includes(normalized)) { + return normalized as ImageGenerationOutputFormat; + } + throw new ToolInputError("outputFormat must be one of png, jpeg, or webp"); +} + +function normalizeOpenAIBackground( + raw: string | undefined, +): ImageGenerationOpenAIBackground | undefined { + const normalized = raw?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + if ((SUPPORTED_OPENAI_BACKGROUNDS as readonly string[]).includes(normalized)) { + return normalized as ImageGenerationOpenAIBackground; + } + throw new ToolInputError("openai.background must be one of transparent, opaque, or auto"); +} + +function normalizeOpenAIModeration( + raw: string | undefined, +): ImageGenerationOpenAIModeration | undefined { + const normalized = raw?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + if ((SUPPORTED_OPENAI_MODERATIONS as readonly string[]).includes(normalized)) { + return normalized as ImageGenerationOpenAIModeration; + } + throw new ToolInputError("openai.moderation must be one of low or auto"); +} + +function readRecordParam(params: Record, key: string): Record { + const raw = params[key]; + return raw && typeof raw === "object" && !Array.isArray(raw) + ? (raw as Record) + : {}; +} + +function normalizeOpenAIOptions(args: Record): ImageGenerationOpenAIOptions { + const raw = readRecordParam(args, "openai"); + const background = normalizeOpenAIBackground(readStringParam(raw, "background")); + const moderation = normalizeOpenAIModeration(readStringParam(raw, "moderation")); + const outputCompression = readNumberParam(raw, "outputCompression", { integer: true }); + const user = readStringParam(raw, "user"); + if (outputCompression !== undefined && (outputCompression < 0 || outputCompression > 100)) { + throw new ToolInputError("openai.outputCompression must be between 0 and 100"); + } + return { + ...(background ? { background } : {}), + ...(moderation ? { moderation } : {}), + ...(outputCompression !== undefined ? { outputCompression } : {}), + ...(user ? { user } : {}), + }; +} + +function normalizeProviderOptions( + args: Record, +): ImageGenerationProviderOptions | undefined { + const openai = normalizeOpenAIOptions(args); + return Object.keys(openai).length > 0 ? { openai } : undefined; +} + function normalizeReferenceImages(args: Record): string[] { return normalizeMediaReferenceInputs({ args, @@ -498,6 +616,9 @@ export function createImageGenerateTool(options?: { const aspectRatio = normalizeAspectRatio(readStringParam(params, "aspectRatio")); const explicitResolution = normalizeResolution(readStringParam(params, "resolution")); const timeoutMs = readGenerationTimeoutMs(params); + const quality = normalizeQuality(readStringParam(params, "quality")); + const outputFormat = normalizeOutputFormat(readStringParam(params, "outputFormat")); + const providerOptions = normalizeProviderOptions(params); const selectedProvider = resolveSelectedImageGenerationProvider({ config: effectiveCfg, imageGenerationModelConfig, @@ -541,9 +662,12 @@ export function createImageGenerateTool(options?: { size, aspectRatio, resolution, + quality, + outputFormat, count, inputImages, timeoutMs, + providerOptions, }); const ignoredOverrides = result.ignoredOverrides ?? []; const displayProvider = sanitizeInlineDirectiveText(result.provider); @@ -625,6 +749,8 @@ export function createImageGenerateTool(options?: { ...(normalizedAspectRatio || aspectRatio ? { aspectRatio: normalizedAspectRatio ?? aspectRatio } : {}), + ...(quality ? { quality } : {}), + ...(outputFormat ? { outputFormat } : {}), ...(filename ? { filename } : {}), ...(timeoutMs !== undefined ? { timeoutMs } : {}), attempts: result.attempts, diff --git a/src/image-generation/normalization.ts b/src/image-generation/normalization.ts index f20fa883c69..9d480f591a9 100644 --- a/src/image-generation/normalization.ts +++ b/src/image-generation/normalization.ts @@ -8,7 +8,9 @@ import { import type { ImageGenerationIgnoredOverride, ImageGenerationNormalization, + ImageGenerationOutputFormat, ImageGenerationProvider, + ImageGenerationQuality, ImageGenerationResolution, ImageGenerationSourceImage, } from "./types.js"; @@ -17,6 +19,8 @@ export type ResolvedImageGenerationOverrides = { size?: string; aspectRatio?: string; resolution?: ImageGenerationResolution; + quality?: ImageGenerationQuality; + outputFormat?: ImageGenerationOutputFormat; ignoredOverrides: ImageGenerationIgnoredOverride[]; normalization?: ImageGenerationNormalization; }; @@ -36,6 +40,8 @@ export function resolveImageGenerationOverrides(params: { size?: string; aspectRatio?: string; resolution?: ImageGenerationResolution; + quality?: ImageGenerationQuality; + outputFormat?: ImageGenerationOutputFormat; inputImages?: ImageGenerationSourceImage[]; }): ResolvedImageGenerationOverrides { const hasInputImages = (params.inputImages?.length ?? 0) > 0; @@ -48,6 +54,8 @@ export function resolveImageGenerationOverrides(params: { let size = params.size; let aspectRatio = params.aspectRatio; let resolution = params.resolution; + let quality = params.quality; + let outputFormat = params.outputFormat; if (size && (geometry?.sizes?.length ?? 0) > 0 && modeCaps.supportsSize) { const normalizedSize = resolveClosestSize({ @@ -155,6 +163,18 @@ export function resolveImageGenerationOverrides(params: { resolution = undefined; } + const supportedQualities = params.provider.capabilities.output?.qualities; + if (quality && !(supportedQualities ?? []).includes(quality)) { + ignoredOverrides.push({ key: "quality", value: quality }); + quality = undefined; + } + + const supportedFormats = params.provider.capabilities.output?.formats; + if (outputFormat && !(supportedFormats ?? []).includes(outputFormat)) { + ignoredOverrides.push({ key: "outputFormat", value: outputFormat }); + outputFormat = undefined; + } + if ( !normalization.aspectRatio && aspectRatio && @@ -198,6 +218,8 @@ export function resolveImageGenerationOverrides(params: { size, aspectRatio, resolution, + quality, + outputFormat, ignoredOverrides, normalization: finalizeImageNormalization(normalization), }; diff --git a/src/image-generation/runtime-types.ts b/src/image-generation/runtime-types.ts index 6eb3474e67a..cfe39bf84ce 100644 --- a/src/image-generation/runtime-types.ts +++ b/src/image-generation/runtime-types.ts @@ -5,7 +5,10 @@ import type { GeneratedImageAsset, ImageGenerationIgnoredOverride, ImageGenerationNormalization, + ImageGenerationOutputFormat, ImageGenerationProvider, + ImageGenerationProviderOptions, + ImageGenerationQuality, ImageGenerationResolution, ImageGenerationSourceImage, } from "./types.js"; @@ -20,9 +23,12 @@ export type GenerateImageParams = { size?: string; aspectRatio?: string; resolution?: ImageGenerationResolution; + quality?: ImageGenerationQuality; + outputFormat?: ImageGenerationOutputFormat; inputImages?: ImageGenerationSourceImage[]; /** Optional per-request provider timeout in milliseconds. */ timeoutMs?: number; + providerOptions?: ImageGenerationProviderOptions; }; export type GenerateImageRuntimeResult = { diff --git a/src/image-generation/runtime.test.ts b/src/image-generation/runtime.test.ts index 0d6e6698f58..e6eb1e98f83 100644 --- a/src/image-generation/runtime.test.ts +++ b/src/image-generation/runtime.test.ts @@ -221,6 +221,128 @@ describe("image-generation runtime", () => { ]); }); + it("filters image output hints by provider capabilities", async () => { + let seenRequest: + | { + quality?: string; + outputFormat?: string; + providerOptions?: unknown; + } + | undefined; + mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/gpt-image-2"); + mocks.getImageGenerationProvider.mockReturnValue({ + id: "openai", + capabilities: { + generate: { + supportsSize: true, + }, + edit: { + enabled: true, + supportsSize: true, + }, + output: { + qualities: ["low", "medium", "high", "auto"], + formats: ["png", "jpeg", "webp"], + }, + }, + async generateImage(req) { + seenRequest = { + quality: req.quality, + outputFormat: req.outputFormat, + providerOptions: req.providerOptions, + }; + return { + images: [{ buffer: Buffer.from("jpeg-bytes"), mimeType: "image/jpeg" }], + }; + }, + }); + + const result = await generateImage({ + cfg: { + agents: { + defaults: { + imageGenerationModel: { primary: "openai/gpt-image-2" }, + }, + }, + } as OpenClawConfig, + prompt: "draw a cheap preview", + quality: "low", + outputFormat: "jpeg", + providerOptions: { + openai: { + background: "opaque", + moderation: "low", + outputCompression: 60, + user: "end-user-42", + }, + }, + }); + + expect(seenRequest).toEqual({ + quality: "low", + outputFormat: "jpeg", + providerOptions: { + openai: { + background: "opaque", + moderation: "low", + outputCompression: 60, + user: "end-user-42", + }, + }, + }); + expect(result.ignoredOverrides).toEqual([]); + }); + + it("drops unsupported image output hints and reports them", async () => { + let seenRequest: + | { + quality?: string; + outputFormat?: string; + } + | undefined; + mocks.resolveAgentModelPrimaryValue.mockReturnValue("vydra/grok-imagine"); + mocks.getImageGenerationProvider.mockReturnValue({ + id: "vydra", + capabilities: { + generate: {}, + edit: { + enabled: false, + }, + }, + async generateImage(req) { + seenRequest = { + quality: req.quality, + outputFormat: req.outputFormat, + }; + return { + images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }], + }; + }, + }); + + const result = await generateImage({ + cfg: { + agents: { + defaults: { + imageGenerationModel: { primary: "vydra/grok-imagine" }, + }, + }, + } as OpenClawConfig, + prompt: "draw a cat", + quality: "low", + outputFormat: "jpeg", + }); + + expect(seenRequest).toEqual({ + quality: undefined, + outputFormat: undefined, + }); + expect(result.ignoredOverrides).toEqual([ + { key: "quality", value: "low" }, + { key: "outputFormat", value: "jpeg" }, + ]); + }); + it("maps requested size to the closest supported fallback geometry", async () => { let seenRequest: | { diff --git a/src/image-generation/runtime.ts b/src/image-generation/runtime.ts index 365d77c0c0c..1ac50d4f008 100644 --- a/src/image-generation/runtime.ts +++ b/src/image-generation/runtime.ts @@ -71,6 +71,8 @@ export async function generateImage( size: params.size, aspectRatio: params.aspectRatio, resolution: params.resolution, + quality: params.quality, + outputFormat: params.outputFormat, inputImages: params.inputImages, }); const result: ImageGenerationResult = await provider.generateImage({ @@ -84,8 +86,11 @@ export async function generateImage( size: sanitized.size, aspectRatio: sanitized.aspectRatio, resolution: sanitized.resolution, + quality: sanitized.quality, + outputFormat: sanitized.outputFormat, inputImages: params.inputImages, ...(params.timeoutMs !== undefined ? { timeoutMs: params.timeoutMs } : {}), + providerOptions: params.providerOptions, }); if (!Array.isArray(result.images) || result.images.length === 0) { throw new Error("Image generation provider returned no images."); diff --git a/src/image-generation/types.ts b/src/image-generation/types.ts index 653c04c7d3a..5296a771f75 100644 --- a/src/image-generation/types.ts +++ b/src/image-generation/types.ts @@ -12,7 +12,31 @@ export type GeneratedImageAsset = { export type ImageGenerationResolution = "1K" | "2K" | "4K"; -export type ImageGenerationIgnoredOverrideKey = "size" | "aspectRatio" | "resolution"; +export type ImageGenerationQuality = "low" | "medium" | "high" | "auto"; + +export type ImageGenerationOutputFormat = "png" | "jpeg" | "webp"; + +export type ImageGenerationOpenAIBackground = "transparent" | "opaque" | "auto"; + +export type ImageGenerationOpenAIModeration = "low" | "auto"; + +export type ImageGenerationOpenAIOptions = { + background?: ImageGenerationOpenAIBackground; + moderation?: ImageGenerationOpenAIModeration; + outputCompression?: number; + user?: string; +}; + +export type ImageGenerationProviderOptions = { + openai?: ImageGenerationOpenAIOptions; +}; + +export type ImageGenerationIgnoredOverrideKey = + | "size" + | "aspectRatio" + | "resolution" + | "quality" + | "outputFormat"; export type ImageGenerationIgnoredOverride = { key: ImageGenerationIgnoredOverrideKey; @@ -43,7 +67,10 @@ export type ImageGenerationRequest = { size?: string; aspectRatio?: string; resolution?: ImageGenerationResolution; + quality?: ImageGenerationQuality; + outputFormat?: ImageGenerationOutputFormat; inputImages?: ImageGenerationSourceImage[]; + providerOptions?: ImageGenerationProviderOptions; }; export type ImageGenerationResult = { @@ -70,6 +97,11 @@ export type ImageGenerationGeometryCapabilities = { resolutions?: ImageGenerationResolution[]; }; +export type ImageGenerationOutputCapabilities = { + qualities?: ImageGenerationQuality[]; + formats?: ImageGenerationOutputFormat[]; +}; + export type ImageGenerationNormalization = { size?: MediaNormalizationEntry; aspectRatio?: MediaNormalizationEntry; @@ -80,6 +112,7 @@ export type ImageGenerationProviderCapabilities = { generate: ImageGenerationModeCapabilities; edit: ImageGenerationEditCapabilities; geometry?: ImageGenerationGeometryCapabilities; + output?: ImageGenerationOutputCapabilities; }; export type ImageGenerationProvider = { diff --git a/src/infra/exec-safe-bin-policy.test.ts b/src/infra/exec-safe-bin-policy.test.ts index 110acff27d1..44d7f2def15 100644 --- a/src/infra/exec-safe-bin-policy.test.ts +++ b/src/infra/exec-safe-bin-policy.test.ts @@ -16,6 +16,7 @@ const SAFE_BIN_DOC_DEFAULTS_START = '[//]: # "SAFE_BIN_DEFAULTS:START"'; const SAFE_BIN_DOC_DEFAULTS_END = '[//]: # "SAFE_BIN_DEFAULTS:END"'; const SAFE_BIN_DOC_DENIED_FLAGS_START = '[//]: # "SAFE_BIN_DENIED_FLAGS:START"'; const SAFE_BIN_DOC_DENIED_FLAGS_END = '[//]: # "SAFE_BIN_DENIED_FLAGS:END"'; +const SAFE_BIN_DOC_PATH = "docs/tools/exec-approvals-advanced.md"; function normalizeGeneratedDocBlock(block: string): string { const lines = block.split("\n"); @@ -187,7 +188,7 @@ describe("exec safe bin policy denied-flag matrix", () => { describe("exec safe bin policy docs parity", () => { it("keeps default safe-bin docs in sync with policy defaults", () => { - const docsPath = path.resolve(process.cwd(), "docs/tools/exec-approvals.md"); + const docsPath = path.resolve(process.cwd(), SAFE_BIN_DOC_PATH); const docs = fs.readFileSync(docsPath, "utf8").replaceAll("\r\n", "\n"); const start = docs.indexOf(SAFE_BIN_DOC_DEFAULTS_START); const end = docs.indexOf(SAFE_BIN_DOC_DEFAULTS_END); @@ -199,7 +200,7 @@ describe("exec safe bin policy docs parity", () => { }); it("keeps denied-flag docs in sync with policy fixtures", () => { - const docsPath = path.resolve(process.cwd(), "docs/tools/exec-approvals.md"); + const docsPath = path.resolve(process.cwd(), SAFE_BIN_DOC_PATH); const docs = fs.readFileSync(docsPath, "utf8").replaceAll("\r\n", "\n"); const start = docs.indexOf(SAFE_BIN_DOC_DENIED_FLAGS_START); const end = docs.indexOf(SAFE_BIN_DOC_DENIED_FLAGS_END); diff --git a/src/plugin-sdk/image-generation.ts b/src/plugin-sdk/image-generation.ts index 4c0f01c30c5..ae10d39bf37 100644 --- a/src/plugin-sdk/image-generation.ts +++ b/src/plugin-sdk/image-generation.ts @@ -2,8 +2,14 @@ export type { GeneratedImageAsset, + ImageGenerationOpenAIBackground, + ImageGenerationOpenAIModeration, + ImageGenerationOpenAIOptions, + ImageGenerationOutputFormat, ImageGenerationProvider, ImageGenerationProviderConfiguredContext, + ImageGenerationProviderOptions, + ImageGenerationQuality, ImageGenerationResolution, ImageGenerationRequest, ImageGenerationResult,