test(image-generation): add live variant coverage

src/image-generation/live-test-helpers.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, it } from "vitest";
+import type { OpenClawConfig } from "../config/config.js";
+import {
+  parseCaseFilter,
+  parseCsvFilter,
+  parseProviderModelMap,
+  redactLiveApiKey,
+  resolveConfiguredLiveImageModels,
+  resolveLiveImageAuthStore,
+} from "./live-test-helpers.js";
+
+describe("image-generation live-test helpers", () => {
+  it("parses provider filters and treats empty/all as unfiltered", () => {
+    expect(parseCsvFilter()).toBeNull();
+    expect(parseCsvFilter("all")).toBeNull();
+    expect(parseCsvFilter(" openai , google ")).toEqual(new Set(["openai", "google"]));
+  });
+
+  it("parses live case filters and treats empty/all as unfiltered", () => {
+    expect(parseCaseFilter()).toBeNull();
+    expect(parseCaseFilter("all")).toBeNull();
+    expect(parseCaseFilter(" google:flash , openai:default ")).toEqual(
+      new Set(["google:flash", "openai:default"]),
+    );
+  });
+
+  it("parses provider model overrides by provider id", () => {
+    expect(
+      parseProviderModelMap("openai/gpt-image-1, google/gemini-3.1-flash-image-preview, invalid"),
+    ).toEqual(
+      new Map([
+        ["openai", "openai/gpt-image-1"],
+        ["google", "google/gemini-3.1-flash-image-preview"],
+      ]),
+    );
+  });
+
+  it("collects configured models from primary and fallbacks", () => {
+    const cfg = {
+      agents: {
+        defaults: {
+          imageGenerationModel: {
+            primary: "openai/gpt-image-1",
+            fallbacks: ["google/gemini-3.1-flash-image-preview", "invalid"],
+          },
+        },
+      },
+    } as OpenClawConfig;
+
+    expect(resolveConfiguredLiveImageModels(cfg)).toEqual(
+      new Map([
+        ["openai", "openai/gpt-image-1"],
+        ["google", "google/gemini-3.1-flash-image-preview"],
+      ]),
+    );
+  });
+
+  it("uses an empty auth store when live env keys should override stale profiles", () => {
+    expect(
+      resolveLiveImageAuthStore({
+        requireProfileKeys: false,
+        hasLiveKeys: true,
+      }),
+    ).toEqual({
+      version: 1,
+      profiles: {},
+    });
+  });
+
+  it("keeps profile-store mode when requested or when no live keys exist", () => {
+    expect(
+      resolveLiveImageAuthStore({
+        requireProfileKeys: true,
+        hasLiveKeys: true,
+      }),
+    ).toBeUndefined();
+    expect(
+      resolveLiveImageAuthStore({
+        requireProfileKeys: false,
+        hasLiveKeys: false,
+      }),
+    ).toBeUndefined();
+  });
+
+  it("redacts live API keys for diagnostics", () => {
+    expect(redactLiveApiKey(undefined)).toBe("none");
+    expect(redactLiveApiKey("short-key")).toBe("short-key");
+    expect(redactLiveApiKey("sk-proj-1234567890")).toBe("sk-proj-...7890");
+  });
+});

src/image-generation/live-test-helpers.ts

@@ -0,0 +1,96 @@
+import type { AuthProfileStore } from "../agents/auth-profiles.js";
+import type { OpenClawConfig } from "../config/config.js";
+
+export const DEFAULT_LIVE_IMAGE_MODELS: Record<string, string> = {
+  google: "google/gemini-3.1-flash-image-preview",
+  openai: "openai/gpt-image-1",
+};
+
+export function parseCaseFilter(raw?: string): Set<string> | null {
+  const trimmed = raw?.trim();
+  if (!trimmed || trimmed === "all") {
+    return null;
+  }
+  const values = trimmed
+    .split(",")
+    .map((entry) => entry.trim().toLowerCase())
+    .filter(Boolean);
+  return values.length > 0 ? new Set(values) : null;
+}
+
+export function redactLiveApiKey(value: string | undefined): string {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    return "none";
+  }
+  if (trimmed.length <= 12) {
+    return trimmed;
+  }
+  return `${trimmed.slice(0, 8)}...${trimmed.slice(-4)}`;
+}
+
+export function parseCsvFilter(raw?: string): Set<string> | null {
+  const trimmed = raw?.trim();
+  if (!trimmed || trimmed === "all") {
+    return null;
+  }
+  const values = trimmed
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+  return values.length > 0 ? new Set(values) : null;
+}
+
+export function parseProviderModelMap(raw?: string): Map<string, string> {
+  const entries = new Map<string, string>();
+  for (const token of raw?.split(",") ?? []) {
+    const trimmed = token.trim();
+    if (!trimmed) {
+      continue;
+    }
+    const slash = trimmed.indexOf("/");
+    if (slash <= 0 || slash === trimmed.length - 1) {
+      continue;
+    }
+    entries.set(trimmed.slice(0, slash).trim().toLowerCase(), trimmed);
+  }
+  return entries;
+}
+
+export function resolveConfiguredLiveImageModels(cfg: OpenClawConfig): Map<string, string> {
+  const resolved = new Map<string, string>();
+  const configured = cfg.agents?.defaults?.imageGenerationModel;
+  const add = (value: string | undefined) => {
+    const trimmed = value?.trim();
+    if (!trimmed) {
+      return;
+    }
+    const slash = trimmed.indexOf("/");
+    if (slash <= 0 || slash === trimmed.length - 1) {
+      return;
+    }
+    resolved.set(trimmed.slice(0, slash).trim().toLowerCase(), trimmed);
+  };
+  if (typeof configured === "string") {
+    add(configured);
+    return resolved;
+  }
+  add(configured?.primary);
+  for (const fallback of configured?.fallbacks ?? []) {
+    add(fallback);
+  }
+  return resolved;
+}
+
+export function resolveLiveImageAuthStore(params: {
+  requireProfileKeys: boolean;
+  hasLiveKeys: boolean;
+}): AuthProfileStore | undefined {
+  if (params.requireProfileKeys || !params.hasLiveKeys) {
+    return undefined;
+  }
+  return {
+    version: 1,
+    profiles: {},
+  };
+}

src/image-generation/providers/google.live.test.ts

@@ -1,51 +0,0 @@
-import { describe, expect, it } from "vitest";
-import type { OpenClawConfig } from "../../config/config.js";
-import { isTruthyEnvValue } from "../../infra/env.js";
-import { buildGoogleImageGenerationProvider } from "./google.js";
-
-const LIVE =
-  isTruthyEnvValue(process.env.GOOGLE_LIVE_TEST) ||
-  isTruthyEnvValue(process.env.LIVE) ||
-  isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST);
-const HAS_KEY = Boolean(process.env.GEMINI_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim());
-const MODEL =
-  process.env.GOOGLE_IMAGE_GENERATION_MODEL?.trim() ||
-  process.env.GEMINI_IMAGE_GENERATION_MODEL?.trim() ||
-  "gemini-3.1-flash-image-preview";
-const BASE_URL = process.env.GOOGLE_IMAGE_BASE_URL?.trim();
-
-const describeLive = LIVE && HAS_KEY ? describe : describe.skip;
-
-function buildLiveConfig(): OpenClawConfig {
-  if (!BASE_URL) {
-    return {};
-  }
-  return {
-    models: {
-      providers: {
-        google: {
-          baseUrl: BASE_URL,
-        },
-      },
-    },
-  } as unknown as OpenClawConfig;
-}
-
-describeLive("google image-generation live", () => {
-  it("generates a real image", async () => {
-    const provider = buildGoogleImageGenerationProvider();
-    const result = await provider.generateImage({
-      provider: "google",
-      model: MODEL,
-      prompt:
-        "Create a minimal flat illustration of an orange cat face sticker on a white background.",
-      cfg: buildLiveConfig(),
-      size: "1024x1024",
-    });
-
-    expect(result.model).toBeTruthy();
-    expect(result.images.length).toBeGreaterThan(0);
-    expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true);
-    expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(512);
-  }, 120_000);
-});

src/image-generation/runtime.live.test.ts

@@ -0,0 +1,237 @@
+import { describe, expect, it } from "vitest";
+import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
+import { collectProviderApiKeys } from "../agents/live-auth-keys.js";
+import { resolveApiKeyForProvider } from "../agents/model-auth.js";
+import type { OpenClawConfig } from "../config/config.js";
+import { loadConfig } from "../config/config.js";
+import { isTruthyEnvValue } from "../infra/env.js";
+import { getShellEnvAppliedKeys, loadShellEnvFallback } from "../infra/shell-env.js";
+import { encodePngRgba, fillPixel } from "../media/png-encode.js";
+import {
+  imageGenerationProviderContractRegistry,
+  providerContractRegistry,
+} from "../plugins/contracts/registry.js";
+import {
+  DEFAULT_LIVE_IMAGE_MODELS,
+  parseCaseFilter,
+  parseCsvFilter,
+  parseProviderModelMap,
+  redactLiveApiKey,
+  resolveConfiguredLiveImageModels,
+  resolveLiveImageAuthStore,
+} from "./live-test-helpers.js";
+import { generateImage } from "./runtime.js";
+
+const LIVE = isTruthyEnvValue(process.env.LIVE) || isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST);
+const REQUIRE_PROFILE_KEYS = isTruthyEnvValue(process.env.OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS);
+const describeLive = LIVE ? describe : describe.skip;
+
+type LiveImageCase = {
+  id: string;
+  providerId: string;
+  modelRef: string;
+  prompt: string;
+  size?: string;
+  resolution?: "1K" | "2K" | "4K";
+  inputImages?: Array<{ buffer: Buffer; mimeType: string; fileName?: string }>;
+};
+
+function createEditReferencePng(): Buffer {
+  const width = 192;
+  const height = 192;
+  const buf = Buffer.alloc(width * height * 4, 255);
+
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      fillPixel(buf, x, y, width, 245, 248, 255, 255);
+    }
+  }
+
+  for (let y = 24; y < 168; y += 1) {
+    for (let x = 24; x < 168; x += 1) {
+      fillPixel(buf, x, y, width, 255, 189, 89, 255);
+    }
+  }
+
+  for (let y = 48; y < 144; y += 1) {
+    for (let x = 48; x < 144; x += 1) {
+      fillPixel(buf, x, y, width, 41, 47, 54, 255);
+    }
+  }
+
+  return encodePngRgba(buf, width, height);
+}
+
+function withPluginsEnabled(cfg: OpenClawConfig): OpenClawConfig {
+  return {
+    ...cfg,
+    plugins: {
+      ...cfg.plugins,
+      enabled: true,
+    },
+  };
+}
+
+function resolveProviderEnvVars(providerId: string): string[] {
+  const entry = providerContractRegistry.find((candidate) => candidate.provider.id === providerId);
+  return entry?.provider.envVars ?? [];
+}
+
+function maybeLoadShellEnvForImageProviders(providerIds: string[]): void {
+  const expectedKeys = [
+    ...new Set(providerIds.flatMap((providerId) => resolveProviderEnvVars(providerId))),
+  ];
+  if (expectedKeys.length === 0) {
+    return;
+  }
+  loadShellEnvFallback({
+    enabled: true,
+    env: process.env,
+    expectedKeys,
+    logger: { warn: (message: string) => console.warn(message) },
+  });
+}
+
+async function resolveLiveAuthForProvider(
+  provider: string,
+  cfg: ReturnType<typeof loadConfig>,
+  agentDir: string,
+) {
+  const authStore = resolveLiveImageAuthStore({
+    requireProfileKeys: REQUIRE_PROFILE_KEYS,
+    hasLiveKeys: collectProviderApiKeys(provider).length > 0,
+  });
+  try {
+    const auth = await resolveApiKeyForProvider({ provider, cfg, agentDir, store: authStore });
+    return { auth, authStore };
+  } catch {
+    return null;
+  }
+}
+
+describeLive("image generation live (provider sweep)", () => {
+  it("generates images for every configured image-generation variant with available auth", async () => {
+    const cfg = withPluginsEnabled(loadConfig());
+    const agentDir = resolveOpenClawAgentDir();
+    const providerFilter = parseCsvFilter(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS);
+    const caseFilter = parseCaseFilter(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_CASES);
+    const envModelMap = parseProviderModelMap(process.env.OPENCLAW_LIVE_IMAGE_GENERATION_MODELS);
+    const configuredModels = resolveConfiguredLiveImageModels(cfg);
+    const availableProviders = imageGenerationProviderContractRegistry
+      .map((entry) => entry.provider.id)
+      .toSorted((left, right) => left.localeCompare(right))
+      .filter((providerId) => (providerFilter ? providerFilter.has(providerId) : true));
+    const liveCases: LiveImageCase[] = [];
+
+    if (availableProviders.includes("google")) {
+      liveCases.push(
+        {
+          id: "google:flash-generate",
+          providerId: "google",
+          modelRef:
+            envModelMap.get("google") ??
+            configuredModels.get("google") ??
+            DEFAULT_LIVE_IMAGE_MODELS.google,
+          prompt:
+            "Create a minimal flat illustration of an orange cat face sticker on a white background.",
+          size: "1024x1024",
+        },
+        {
+          id: "google:pro-generate",
+          providerId: "google",
+          modelRef: "google/gemini-3-pro-image-preview",
+          prompt:
+            "Create a minimal flat illustration of an orange cat face sticker on a white background.",
+          size: "1024x1024",
+        },
+        {
+          id: "google:pro-edit",
+          providerId: "google",
+          modelRef: "google/gemini-3-pro-image-preview",
+          prompt:
+            "Change ONLY the background to a pale blue gradient. Keep the subject, framing, and style identical.",
+          resolution: "2K",
+          inputImages: [
+            {
+              buffer: createEditReferencePng(),
+              mimeType: "image/png",
+              fileName: "reference.png",
+            },
+          ],
+        },
+      );
+    }
+    if (availableProviders.includes("openai")) {
+      liveCases.push({
+        id: "openai:default-generate",
+        providerId: "openai",
+        modelRef:
+          envModelMap.get("openai") ??
+          configuredModels.get("openai") ??
+          DEFAULT_LIVE_IMAGE_MODELS.openai,
+        prompt:
+          "Create a minimal flat illustration of an orange cat face sticker on a white background.",
+        size: "1024x1024",
+      });
+    }
+
+    const selectedCases = liveCases.filter((entry) =>
+      caseFilter ? caseFilter.has(entry.id.toLowerCase()) : true,
+    );
+
+    maybeLoadShellEnvForImageProviders(availableProviders);
+
+    const attempted: string[] = [];
+    const skipped: string[] = [];
+    const failures: string[] = [];
+
+    for (const testCase of selectedCases) {
+      if (!testCase.modelRef) {
+        skipped.push(`${testCase.id}: no model configured`);
+        continue;
+      }
+      const resolvedAuth = await resolveLiveAuthForProvider(testCase.providerId, cfg, agentDir);
+      if (!resolvedAuth) {
+        skipped.push(`${testCase.id}: no auth`);
+        continue;
+      }
+
+      try {
+        const result = await generateImage({
+          cfg,
+          agentDir,
+          authStore: resolvedAuth.authStore,
+          modelOverride: testCase.modelRef,
+          prompt: testCase.prompt,
+          size: testCase.size,
+          resolution: testCase.resolution,
+          inputImages: testCase.inputImages,
+        });
+
+        attempted.push(
+          `${testCase.id}:${result.model} (${resolvedAuth.auth.source} ${redactLiveApiKey(resolvedAuth.auth.apiKey)})`,
+        );
+        expect(result.provider).toBe(testCase.providerId);
+        expect(result.images.length).toBeGreaterThan(0);
+        expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true);
+        expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(512);
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        failures.push(
+          `${testCase.id} (${resolvedAuth.auth.source} ${redactLiveApiKey(resolvedAuth.auth.apiKey)}): ${message}`,
+        );
+      }
+    }
+
+    console.log(
+      `[live:image-generation] attempted=${attempted.join(", ") || "none"} skipped=${skipped.join(", ") || "none"} failures=${failures.join(" | ") || "none"} shellEnv=${getShellEnvAppliedKeys().join(", ") || "none"}`,
+    );
+
+    if (attempted.length === 0) {
+      console.warn("[live:image-generation] no provider had usable auth; skipping assertions");
+      return;
+    }
+    expect(failures).toEqual([]);
+    expect(attempted.length).toBeGreaterThan(0);
+  }, 180_000);
+});