From 0ff82497e9a27db4a7ac20fffa735d84831452a4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 17 Mar 2026 01:09:36 -0700 Subject: [PATCH] test(image-generation): add live variant coverage --- .../live-test-helpers.test.ts | 90 +++++++ src/image-generation/live-test-helpers.ts | 96 +++++++ .../providers/google.live.test.ts | 51 ---- src/image-generation/runtime.live.test.ts | 237 ++++++++++++++++++ 4 files changed, 423 insertions(+), 51 deletions(-) create mode 100644 src/image-generation/live-test-helpers.test.ts create mode 100644 src/image-generation/live-test-helpers.ts delete mode 100644 src/image-generation/providers/google.live.test.ts create mode 100644 src/image-generation/runtime.live.test.ts diff --git a/src/image-generation/live-test-helpers.test.ts b/src/image-generation/live-test-helpers.test.ts new file mode 100644 index 00000000000..3a7058569cf --- /dev/null +++ b/src/image-generation/live-test-helpers.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import { + parseCaseFilter, + parseCsvFilter, + parseProviderModelMap, + redactLiveApiKey, + resolveConfiguredLiveImageModels, + resolveLiveImageAuthStore, +} from "./live-test-helpers.js"; + +describe("image-generation live-test helpers", () => { + it("parses provider filters and treats empty/all as unfiltered", () => { + expect(parseCsvFilter()).toBeNull(); + expect(parseCsvFilter("all")).toBeNull(); + expect(parseCsvFilter(" openai , google ")).toEqual(new Set(["openai", "google"])); + }); + + it("parses live case filters and treats empty/all as unfiltered", () => { + expect(parseCaseFilter()).toBeNull(); + expect(parseCaseFilter("all")).toBeNull(); + expect(parseCaseFilter(" google:flash , openai:default ")).toEqual( + new Set(["google:flash", "openai:default"]), + ); + }); + + it("parses provider model overrides by provider id", () => { + expect( + parseProviderModelMap("openai/gpt-image-1, google/gemini-3.1-flash-image-preview, invalid"), + ).toEqual( + new Map([ + ["openai", "openai/gpt-image-1"], + ["google", "google/gemini-3.1-flash-image-preview"], + ]), + ); + }); + + it("collects configured models from primary and fallbacks", () => { + const cfg = { + agents: { + defaults: { + imageGenerationModel: { + primary: "openai/gpt-image-1", + fallbacks: ["google/gemini-3.1-flash-image-preview", "invalid"], + }, + }, + }, + } as OpenClawConfig; + + expect(resolveConfiguredLiveImageModels(cfg)).toEqual( + new Map([ + ["openai", "openai/gpt-image-1"], + ["google", "google/gemini-3.1-flash-image-preview"], + ]), + ); + }); + + it("uses an empty auth store when live env keys should override stale profiles", () => { + expect( + resolveLiveImageAuthStore({ + requireProfileKeys: false, + hasLiveKeys: true, + }), + ).toEqual({ + version: 1, + profiles: {}, + }); + }); + + it("keeps profile-store mode when requested or when no live keys exist", () => { + expect( + resolveLiveImageAuthStore({ + requireProfileKeys: true, + hasLiveKeys: true, + }), + ).toBeUndefined(); + expect( + resolveLiveImageAuthStore({ + requireProfileKeys: false, + hasLiveKeys: false, + }), + ).toBeUndefined(); + }); + + it("redacts live API keys for diagnostics", () => { + expect(redactLiveApiKey(undefined)).toBe("none"); + expect(redactLiveApiKey("short-key")).toBe("short-key"); + expect(redactLiveApiKey("sk-proj-1234567890")).toBe("sk-proj-...7890"); + }); +}); diff --git a/src/image-generation/live-test-helpers.ts b/src/image-generation/live-test-helpers.ts new file mode 100644 index 00000000000..0063bab89fa --- /dev/null +++ b/src/image-generation/live-test-helpers.ts @@ -0,0 +1,96 @@ +import type { AuthProfileStore } from "../agents/auth-profiles.js"; +import type { OpenClawConfig } from "../config/config.js"; + +export const DEFAULT_LIVE_IMAGE_MODELS: Record = { + google: "google/gemini-3.1-flash-image-preview", + openai: "openai/gpt-image-1", +}; + +export function parseCaseFilter(raw?: string): Set | null { + const trimmed = raw?.trim(); + if (!trimmed || trimmed === "all") { + return null; + } + const values = trimmed + .split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean); + return values.length > 0 ? new Set(values) : null; +} + +export function redactLiveApiKey(value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + return "none"; + } + if (trimmed.length <= 12) { + return trimmed; + } + return `${trimmed.slice(0, 8)}...${trimmed.slice(-4)}`; +} + +export function parseCsvFilter(raw?: string): Set | null { + const trimmed = raw?.trim(); + if (!trimmed || trimmed === "all") { + return null; + } + const values = trimmed + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); + return values.length > 0 ? new Set(values) : null; +} + +export function parseProviderModelMap(raw?: string): Map { + const entries = new Map(); + for (const token of raw?.split(",") ?? []) { + const trimmed = token.trim(); + if (!trimmed) { + continue; + } + const slash = trimmed.indexOf("/"); + if (slash <= 0 || slash === trimmed.length - 1) { + continue; + } + entries.set(trimmed.slice(0, slash).trim().toLowerCase(), trimmed); + } + return entries; +} + +export function resolveConfiguredLiveImageModels(cfg: OpenClawConfig): Map { + const resolved = new Map(); + const configured = cfg.agents?.defaults?.imageGenerationModel; + const add = (value: string | undefined) => { + const trimmed = value?.trim(); + if (!trimmed) { + return; + } + const slash = trimmed.indexOf("/"); + if (slash <= 0 || slash === trimmed.length - 1) { + return; + } + resolved.set(trimmed.slice(0, slash).trim().toLowerCase(), trimmed); + }; + if (typeof configured === "string") { + add(configured); + return resolved; + } + add(configured?.primary); + for (const fallback of configured?.fallbacks ?? []) { + add(fallback); + } + return resolved; +} + +export function resolveLiveImageAuthStore(params: { + requireProfileKeys: boolean; + hasLiveKeys: boolean; +}): AuthProfileStore | undefined { + if (params.requireProfileKeys || !params.hasLiveKeys) { + return undefined; + } + return { + version: 1, + profiles: {}, + }; +} diff --git a/src/image-generation/providers/google.live.test.ts b/src/image-generation/providers/google.live.test.ts deleted file mode 100644 index dcf2ddd1108..00000000000 --- a/src/image-generation/providers/google.live.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { OpenClawConfig } from "../../config/config.js"; -import { isTruthyEnvValue } from "../../infra/env.js"; -import { buildGoogleImageGenerationProvider } from "./google.js"; - -const LIVE = - isTruthyEnvValue(process.env.GOOGLE_LIVE_TEST) || - isTruthyEnvValue(process.env.LIVE) || - isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST); -const HAS_KEY = Boolean(process.env.GEMINI_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim()); -const MODEL = - process.env.GOOGLE_IMAGE_GENERATION_MODEL?.trim() || - process.env.GEMINI_IMAGE_GENERATION_MODEL?.trim() || - "gemini-3.1-flash-image-preview"; -const BASE_URL = process.env.GOOGLE_IMAGE_BASE_URL?.trim(); - -const describeLive = LIVE && HAS_KEY ? describe : describe.skip; - -function buildLiveConfig(): OpenClawConfig { - if (!BASE_URL) { - return {}; - } - return { - models: { - providers: { - google: { - baseUrl: BASE_URL, - }, - }, - }, - } as unknown as OpenClawConfig; -} - -describeLive("google image-generation live", () => { - it("generates a real image", async () => { - const provider = buildGoogleImageGenerationProvider(); - const result = await provider.generateImage({ - provider: "google", - model: MODEL, - prompt: - "Create a minimal flat illustration of an orange cat face sticker on a white background.", - cfg: buildLiveConfig(), - size: "1024x1024", - }); - - expect(result.model).toBeTruthy(); - expect(result.images.length).toBeGreaterThan(0); - expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true); - expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(512); - }, 120_000); -}); diff --git a/src/image-generation/runtime.live.test.ts b/src/image-generation/runtime.live.test.ts new file mode 100644 index 00000000000..f0132414a6c --- /dev/null +++ b/src/image-generation/runtime.live.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from "vitest"; +import { resolveOpenClawAgentDir } from "../agents/agent-paths.js"; +import { collectProviderApiKeys } from "../agents/live-auth-keys.js"; +import { resolveApiKeyForProvider } from "../agents/model-auth.js"; +import type { OpenClawConfig } from "../config/config.js"; +import { loadConfig } from "../config/config.js"; +import { isTruthyEnvValue } from "../infra/env.js"; +import { getShellEnvAppliedKeys, loadShellEnvFallback } from "../infra/shell-env.js"; +import { encodePngRgba, fillPixel } from "../media/png-encode.js"; +import { + imageGenerationProviderContractRegistry, + providerContractRegistry, +} from "../plugins/contracts/registry.js"; +import { + DEFAULT_LIVE_IMAGE_MODELS, + parseCaseFilter, + parseCsvFilter, + parseProviderModelMap, + redactLiveApiKey, + resolveConfiguredLiveImageModels, + resolveLiveImageAuthStore, +} from "./live-test-helpers.js"; +import { generateImage } from "./runtime.js"; + +const LIVE = isTruthyEnvValue(process.env.LIVE) || isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST); +const REQUIRE_PROFILE_KEYS = isTruthyEnvValue(process.env.OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS); +const describeLive = LIVE ? describe : describe.skip; + +type LiveImageCase = { + id: string; + providerId: string; + modelRef: string; + prompt: string; + size?: string; + resolution?: "1K" | "2K" | "4K"; + inputImages?: Array<{ buffer: Buffer; mimeType: string; fileName?: string }>; +}; + +function createEditReferencePng(): Buffer { + const width = 192; + const height = 192; + const buf = Buffer.alloc(width * height * 4, 255); + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + fillPixel(buf, x, y, width, 245, 248, 255, 255); + } + } + + for (let y = 24; y < 168; y += 1) { + for (let x = 24; x < 168; x += 1) { + fillPixel(buf, x, y, width, 255, 189, 89, 255); + } + } + + for (let y = 48; y < 144; y += 1) { + for (let x = 48; x < 144; x += 1) { + fillPixel(buf, x, y, width, 41, 47, 54, 255); + } + } + + return encodePngRgba(buf, width, height); +} + +function withPluginsEnabled(cfg: OpenClawConfig): OpenClawConfig { + return { + ...cfg, + plugins: { + ...cfg.plugins, + enabled: true, + }, + }; +} + +function resolveProviderEnvVars(providerId: string): string[] { + const entry = providerContractRegistry.find((candidate) => candidate.provider.id === providerId); + return entry?.provider.envVars ?? []; +} + +function maybeLoadShellEnvForImageProviders(providerIds: string[]): void { + const expectedKeys = [ + ...new Set(providerIds.flatMap((providerId) => resolveProviderEnvVars(providerId))), + ]; + if (expectedKeys.length === 0) { + return; + } + loadShellEnvFallback({ + enabled: true, + env: process.env, + expectedKeys, + logger: { warn: (message: string) => console.warn(message) }, + }); +} + +async function resolveLiveAuthForProvider( + provider: string, + cfg: ReturnType, + agentDir: string, +) { + const authStore = resolveLiveImageAuthStore({ + requireProfileKeys: REQUIRE_PROFILE_KEYS, + hasLiveKeys: collectProviderApiKeys(provider).length > 0, + }); + try { + const auth = await resolveApiKeyForProvider({ provider, cfg, agentDir, store: authStore }); + return { auth, authStore }; + } catch { + return null; + } +} + +describeLive("image generation live (provider sweep)", () => { + it("generates images for every configured image-generation variant with available auth", async () => { + const cfg = withPluginsEnabled(loadConfig()); + const agentDir = resolveOpenClawAgentDir(); + const providerFilter = parseCsvFilter(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS); + const caseFilter = parseCaseFilter(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_CASES); + const envModelMap = parseProviderModelMap(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_MODELS); + const configuredModels = resolveConfiguredLiveImageModels(cfg); + const availableProviders = imageGenerationProviderContractRegistry + .map((entry) => entry.provider.id) + .toSorted((left, right) => left.localeCompare(right)) + .filter((providerId) => (providerFilter ? providerFilter.has(providerId) : true)); + const liveCases: LiveImageCase[] = []; + + if (availableProviders.includes("google")) { + liveCases.push( + { + id: "google:flash-generate", + providerId: "google", + modelRef: + envModelMap.get("google") ?? + configuredModels.get("google") ?? + DEFAULT_LIVE_IMAGE_MODELS.google, + prompt: + "Create a minimal flat illustration of an orange cat face sticker on a white background.", + size: "1024x1024", + }, + { + id: "google:pro-generate", + providerId: "google", + modelRef: "google/gemini-3-pro-image-preview", + prompt: + "Create a minimal flat illustration of an orange cat face sticker on a white background.", + size: "1024x1024", + }, + { + id: "google:pro-edit", + providerId: "google", + modelRef: "google/gemini-3-pro-image-preview", + prompt: + "Change ONLY the background to a pale blue gradient. Keep the subject, framing, and style identical.", + resolution: "2K", + inputImages: [ + { + buffer: createEditReferencePng(), + mimeType: "image/png", + fileName: "reference.png", + }, + ], + }, + ); + } + if (availableProviders.includes("openai")) { + liveCases.push({ + id: "openai:default-generate", + providerId: "openai", + modelRef: + envModelMap.get("openai") ?? + configuredModels.get("openai") ?? + DEFAULT_LIVE_IMAGE_MODELS.openai, + prompt: + "Create a minimal flat illustration of an orange cat face sticker on a white background.", + size: "1024x1024", + }); + } + + const selectedCases = liveCases.filter((entry) => + caseFilter ? caseFilter.has(entry.id.toLowerCase()) : true, + ); + + maybeLoadShellEnvForImageProviders(availableProviders); + + const attempted: string[] = []; + const skipped: string[] = []; + const failures: string[] = []; + + for (const testCase of selectedCases) { + if (!testCase.modelRef) { + skipped.push(`${testCase.id}: no model configured`); + continue; + } + const resolvedAuth = await resolveLiveAuthForProvider(testCase.providerId, cfg, agentDir); + if (!resolvedAuth) { + skipped.push(`${testCase.id}: no auth`); + continue; + } + + try { + const result = await generateImage({ + cfg, + agentDir, + authStore: resolvedAuth.authStore, + modelOverride: testCase.modelRef, + prompt: testCase.prompt, + size: testCase.size, + resolution: testCase.resolution, + inputImages: testCase.inputImages, + }); + + attempted.push( + `${testCase.id}:${result.model} (${resolvedAuth.auth.source} ${redactLiveApiKey(resolvedAuth.auth.apiKey)})`, + ); + expect(result.provider).toBe(testCase.providerId); + expect(result.images.length).toBeGreaterThan(0); + expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true); + expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(512); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push( + `${testCase.id} (${resolvedAuth.auth.source} ${redactLiveApiKey(resolvedAuth.auth.apiKey)}): ${message}`, + ); + } + } + + console.log( + `[live:image-generation] attempted=${attempted.join(", ") || "none"} skipped=${skipped.join(", ") || "none"} failures=${failures.join(" | ") || "none"} shellEnv=${getShellEnvAppliedKeys().join(", ") || "none"}`, + ); + + if (attempted.length === 0) { + console.warn("[live:image-generation] no provider had usable auth; skipping assertions"); + return; + } + expect(failures).toEqual([]); + expect(attempted.length).toBeGreaterThan(0); + }, 180_000); +});