From f0bfb3fc338abe638a64c970e799062b529fd6fd Mon Sep 17 00:00:00 2001 From: Mason Huang Date: Mon, 25 May 2026 22:19:04 +0800 Subject: [PATCH] test(tools): add unmocked image custom-provider auth regression (#85733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: - The branch adds an unmocked image-tool custom-provider auth regression test, fixes split agents Vitest config routing, adds routing coverage, and records a changelog entry. - PR surface: Tests +203, Docs +1, Other +8. Total +212 across 4 files. - Reproducibility: not applicable. as a current-main failing issue: the production runtime bug was addressed by the linked predecessor, and this PR adds regression coverage plus test-routing verification for that path. Automerge notes: - PR branch already contained follow-up commit before automerge: test(tools): polish image auth regression and fix agents vitest routing - PR branch already contained follow-up commit before automerge: test(tools): remove proof test filename after regression rename - PR branch already contained follow-up commit before automerge: fix(test): remove duplicate agent shard constants - PR branch already contained follow-up commit before automerge: test(tools): add unmocked image custom-provider auth regression - PR branch already contained follow-up commit before automerge: fix(clawsweeper): address review for automerge-openclaw-openclaw-8573… Validation: - ClawSweeper review passed for head cff5476aeb8f228b33d8d9356a4da400e91f6498. - Required merge gates passed before the squash merge. Prepared head SHA: cff5476aeb8f228b33d8d9356a4da400e91f6498 Review: https://github.com/openclaw/openclaw/pull/85733#issuecomment-4525628364 Co-authored-by: Mason Huang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: hxy91819 Co-authored-by: hxy91819 <8814856+hxy91819@users.noreply.github.com> --- CHANGELOG.md | 1 + scripts/test-projects.test-support.mjs | 8 + ...ol.custom-provider-auth.regression.test.ts | 185 ++++++++++++++++++ test/scripts/test-projects.test.ts | 18 ++ 4 files changed, 212 insertions(+) create mode 100644 src/agents/tools/image-tool.custom-provider-auth.regression.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4c2f37618..9eea3536e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Docs: https://docs.openclaw.ai - Agents: release embedded-attempt session locks from outer teardown so post-prompt exceptions cannot wedge later requests behind `SessionWriteLockTimeoutError`. Fixes #86014. Thanks @openperf. - Discord/OpenAI voice: rotate Realtime sessions at provider max duration without logging the expected session-expiry event as an error. - Agents/media: derive bundled plugin local-media trust from plugin tool metadata instead of importing the full plugin registry on subscription paths. (#84409) Thanks @samzong. +- Image tool: keep config-backed custom-provider API keys usable for auto-discovered vision models, including deferred image-tool execution without env keys or auth profiles. (#85733) - Memory/local embeddings: run local GGUF embeddings in an isolated worker sidecar and degrade to configured fallback or keyword search on worker failure so native embedding crashes do not take down the Gateway. (#85348) Thanks @osolmaz. - Gateway: clear the runtime config snapshot before `SIGUSR1` in-process restarts so config changes survive the next gateway loop. (#86388) Thanks @XuZehan-iCenter. - Models: show OAuth delegation markers as configured `models.json` auth while keeping runtime route usability checks strict. (#86378) Thanks @rohitjavvadi. diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 5f4b4ecf862..d8800ccfdcc 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -240,6 +240,10 @@ const VITEST_CONFIG_BY_KIND = { agentSupport: AGENTS_SUPPORT_VITEST_CONFIG, agentTools: AGENTS_TOOLS_VITEST_CONFIG, agent: AGENTS_VITEST_CONFIG, + agentsCore: AGENTS_CORE_VITEST_CONFIG, + agentsPiEmbedded: AGENTS_PI_EMBEDDED_VITEST_CONFIG, + agentsSupport: AGENTS_SUPPORT_VITEST_CONFIG, + agentsTools: AGENTS_TOOLS_VITEST_CONFIG, autoReplyCore: AUTO_REPLY_CORE_VITEST_CONFIG, autoReplyReply: AUTO_REPLY_REPLY_VITEST_CONFIG, autoReplyTopLevel: AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG, @@ -1713,6 +1717,10 @@ export function buildVitestRunPlans( "agentSupport", "agentTools", "agent", + "agentsCore", + "agentsPiEmbedded", + "agentsSupport", + "agentsTools", "plugin", "ui", "uiE2e", diff --git a/src/agents/tools/image-tool.custom-provider-auth.regression.test.ts b/src/agents/tools/image-tool.custom-provider-auth.regression.test.ts new file mode 100644 index 00000000000..fae049563c3 --- /dev/null +++ b/src/agents/tools/image-tool.custom-provider-auth.regression.test.ts @@ -0,0 +1,185 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/config.js"; +import type { ModelDefinitionConfig } from "../../config/types.models.js"; +import type { ImageDescriptionRequest } from "../../plugin-sdk/media-understanding.js"; +import { getApiKeyForModel, hasUsableCustomProviderApiKey } from "../model-auth.js"; +import { resolveImageToolFactoryAvailable } from "../openclaw-tools.media-factory-plan.js"; +import { createImageTool, resolveImageModelConfigForTool, testing } from "./image-tool.js"; +import { hasProviderAuthForTool } from "./model-config.helpers.js"; + +const USER_PROVIDER = "hatchery-qwen3.6-plus"; +const USER_MODEL = "qwen3.6-plus"; +const USER_PRIMARY = `${USER_PROVIDER}/${USER_MODEL}`; +const CONFIG_API_KEY = "sk-user-configured-key"; // pragma: allowlist secret + +const ONE_PIXEL_PNG_B64 = + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRAD/AP8A/6C9p5MAAAAHdElNRQfqBBsGAQr00ED3AAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDI2LTA0LTI3VDA2OjAxOjEwKzAwOjAwPU3tXwAAACV0RVh0ZGF0ZTptb2RpZnkAMjAyNi0wNC0yN1QwNjowMToxMCswMDowMEwQVeMAAAAodEVYdGRhdGU6dGltZXN0YW1wADIwMjYtMDQtMjdUMDY6MDE6MTArMDA6MDAbBXQ8AAAAeElEQVRo3u3awQnDQBAEwT2Q8w/YAikIP5rF1RFMca+FO8/s7rrnqjcA1BsA6g0A9QaAesOfA77zqTf8Blj/AgAAAAAAAJsDqAOoA6gDqAOoc9TXAdQB1AHUAdQB1AHUAdQB1AHU7Qc46gEAAAAANrcecGZ2f8B/ASYSQPlKoEJ/AAAAAElFTkSuQmCC"; + +function makeVisionModel(id: string): ModelDefinitionConfig { + return { + id, + name: id, + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }; +} + +function createUserReportedConfig(params?: { includeApiKey?: boolean }): OpenClawConfig { + const includeApiKey = params?.includeApiKey ?? true; + return { + agents: { + defaults: { + model: { primary: USER_PRIMARY }, + }, + }, + models: { + providers: { + [USER_PROVIDER]: { + baseUrl: "https://example.com/v1", + api: "openai-completions", + ...(includeApiKey ? { apiKey: CONFIG_API_KEY } : {}), + models: [makeVisionModel(USER_MODEL)], + }, + }, + }, + }; +} + +async function withEmptyAgentDir(run: (agentDir: string) => Promise): Promise { + const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-auth-regression-")); + try { + return await run(agentDir); + } finally { + await fs.rm(agentDir, { recursive: true, force: true }); + } +} + +describe("image custom provider auth regression", () => { + const priorFetch = global.fetch; + + beforeEach(() => { + for (const key of Object.keys(process.env)) { + if (key.endsWith("_API_KEY") || key.endsWith("_OAUTH_TOKEN")) { + vi.stubEnv(key, ""); + } + } + testing.setProviderDepsForTest({ + buildProviderRegistry: () => new Map(), + getMediaUnderstandingProvider: () => undefined, + describeImageWithModel: async (params: ImageDescriptionRequest) => ({ + text: `seen:${params.provider}/${params.model}`, + model: params.model, + }), + describeImagesWithModel: async (params) => ({ + text: `seen:${params.provider}/${params.model}`, + model: params.model, + }), + resolveAutoMediaKeyProviders: () => [], + resolveDefaultMediaModel: () => undefined, + }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + global.fetch = priorFetch; + testing.setProviderDepsForTest(undefined); + }); + + it("uses real model-auth to accept config-only custom provider credentials", async () => { + const cfg = createUserReportedConfig(); + expect(hasUsableCustomProviderApiKey(cfg, USER_PROVIDER)).toBe(true); + expect(hasProviderAuthForTool({ provider: USER_PROVIDER, cfg })).toBe(true); + }); + + it("auto-discovers the user-reported vision model without env key or auth profile", async () => { + await withEmptyAgentDir(async (agentDir) => { + const cfg = createUserReportedConfig(); + expect(resolveImageModelConfigForTool({ cfg, agentDir })).toEqual({ + primary: USER_PRIMARY, + }); + }); + }); + + it("registers the image tool on the production factory path when the primary model has vision", async () => { + await withEmptyAgentDir(async (agentDir) => { + const cfg = createUserReportedConfig(); + expect( + resolveImageToolFactoryAvailable({ + config: cfg, + agentDir, + modelHasVision: true, + }), + ).toBe(true); + }); + }); + + it("executes deferred image tool discovery with config-backed auth and runtime key resolution", async () => { + await withEmptyAgentDir(async (agentDir) => { + const cfg = createUserReportedConfig(); + const auth = await getApiKeyForModel({ + model: { + id: USER_MODEL, + name: USER_MODEL, + provider: USER_PROVIDER, + api: "openai-completions", + baseUrl: "https://example.com/v1", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8_192, + }, + cfg, + agentDir, + }); + expect(auth.apiKey).toBe(CONFIG_API_KEY); + expect(auth.source).toContain("models.json"); + + const tool = createImageTool({ + config: cfg, + agentDir, + deferAutoModelResolution: true, + modelHasVision: true, + }); + expect(typeof tool?.execute).toBe("function"); + + const result = await tool!.execute("regression-1", { + prompt: "Read this screenshot.", + image: `data:image/png;base64,${ONE_PIXEL_PNG_B64}`, + }); + + const payload = result as { content?: Array<{ type?: string; text?: string }> }; + const text = payload.content?.find((entry) => entry.type === "text")?.text ?? ""; + expect(text).toContain(`seen:${USER_PRIMARY}`); + expect(text).not.toMatch(/No image model is configured/i); + }); + }); + + it("still rejects the same config when apiKey is missing", async () => { + await withEmptyAgentDir(async (agentDir) => { + const cfg = createUserReportedConfig({ includeApiKey: false }); + expect(hasUsableCustomProviderApiKey(cfg, USER_PROVIDER)).toBe(false); + expect(hasProviderAuthForTool({ provider: USER_PROVIDER, cfg })).toBe(false); + expect(resolveImageModelConfigForTool({ cfg, agentDir })).toBeNull(); + + const tool = createImageTool({ + config: cfg, + agentDir, + deferAutoModelResolution: true, + modelHasVision: true, + }); + await expect( + tool!.execute("regression-2", { + prompt: "Read this screenshot.", + image: `data:image/png;base64,${ONE_PIXEL_PNG_B64}`, + }), + ).rejects.toThrow(/No image model is configured/); + }); + }); +}); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 03ad1a5c77c..4b01fe945f8 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -936,6 +936,24 @@ describe("scripts/test-projects changed-target routing", () => { expect(repoSourceReads.length).toBeLessThan(100); }); + it.each([ + "test/vitest/vitest.agents-core.config.ts", + "test/vitest/vitest.agents-pi-embedded.config.ts", + "test/vitest/vitest.agents-support.config.ts", + "test/vitest/vitest.agents-tools.config.ts", + ])("routes split agents vitest config %s to itself", (target) => { + const plans = buildVitestRunPlans([target], process.cwd()); + + expect(plans).toEqual([ + { + config: target, + forwardedArgs: [], + includePatterns: null, + watchMode: false, + }, + ]); + }); + it.each([ "src/gateway/gateway.test.ts", "src/gateway/server.startup-matrix-migration.integration.test.ts",