vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 06:20:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(test): dedupe model config fixtures

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-20 12:26:47 +01:00
parent 2c814d33e6
commit 69c78fbef0
12 changed files with 577 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/agents/agent-command.live-model-switch.test.ts
View File

@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { LiveSessionModelSwitchError } from "./live-model-switch.js";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { LiveSessionModelSwitchError } from "./live-model-switch-error.js";
const state = vi.hoisted(() => ({
runWithModelFallbackMock: vi.fn(),
@@ -57,10 +57,14 @@ vi.mock("./command/session.js", () => ({
resolveSession: () => ({
sessionId: "session-1",
sessionKey: "agent:main",
sessionEntry: { sessionId: "session-1", updatedAt: Date.now() },
sessionStore: {},
storePath: "/tmp/store.json",
isNewSession: true,
sessionEntry: {
sessionId: "session-1",
updatedAt: Date.now(),
skillsSnapshot: { prompt: "", skills: [], version: 0 },
},
sessionStore: undefined,
storePath: undefined,
isNewSession: false,
persistedThinking: undefined,
persistedVerbose: undefined,
}),
@@ -125,6 +129,27 @@ vi.mock("../config/io.js", () => ({
}),
}));
vi.mock("./agent-runtime-config.js", () => {
const cfg = {
agents: {
defaults: {
models: {
"anthropic/claude": {},
"openai/claude": {},
"openai/gpt-5.4": {},
},
},
},
};
return {
resolveAgentRuntimeConfig: async () => ({
loadedRaw: cfg,
sourceConfig: cfg,
cfg,
}),
};
});
vi.mock("../config/runtime-snapshot.js", () => ({
setRuntimeConfigSnapshot: vi.fn(),
}));
@@ -258,8 +283,13 @@ vi.mock("./skills.js", () => ({
buildWorkspaceSkillSnapshot: () => ({}),
}));
vi.mock("./skills/refresh.js", () => ({
vi.mock("./skills/filter.js", () => ({
matchesSkillFilter: () => true,
}));
vi.mock("./skills/refresh-state.js", () => ({
getSkillsSnapshotVersion: () => 0,
shouldRefreshSnapshotForVersion: () => false,
}));
vi.mock("./spawned-context.js", () => ({
@@ -280,9 +310,11 @@ vi.mock("../acp/control-plane/manager.js", () => ({
}),
}));
async function getAgentCommand() {
return (await import("./agent-command.js")).agentCommand;
}
let agentCommand: typeof import("./agent-command.js").agentCommand;
beforeAll(async () => {
agentCommand ??= (await import("./agent-command.js")).agentCommand;
});
type FallbackRunnerParams = {
provider: string;
@@ -322,7 +354,6 @@ function setupModelSwitchRetry(switchOptions: ModelSwitchOptions) {
}
async function runBasicAgentCommand() {
const agentCommand = await getAgentCommand();
await agentCommand({
message: "hello",
to: "+1234567890",
@@ -379,7 +410,6 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => {
it("propagates non-switch errors without retrying and emits lifecycle error", async () => {
state.runWithModelFallbackMock.mockRejectedValueOnce(new Error("provider down"));
const agentCommand = await getAgentCommand();
await expect(
agentCommand({
message: "hello",

10
src/agents/failover-error.test.ts
View File

@@ -510,8 +510,13 @@ describe("failover-error", () => {
it("infers timeout from common node error codes", () => {
expect(resolveFailoverReasonFromError({ code: "ETIMEDOUT" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "ECONNREFUSED" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "ECONNRESET" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "EAI_AGAIN" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "EHOSTUNREACH" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "EHOSTDOWN" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "ENETRESET" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "ENETUNREACH" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ code: "EPIPE" })).toBe("timeout");
});
@@ -538,6 +543,11 @@ describe("failover-error", () => {
});
it("infers timeout from connection/network error messages", () => {
expect(
resolveFailoverReasonFromError({
message: "model_cooldown: All credentials for model gpt-5 are cooling down",
}),
).toBe("rate_limit");
expect(resolveFailoverReasonFromError({ message: "Connection error." })).toBe("timeout");
expect(resolveFailoverReasonFromError({ message: "fetch failed" })).toBe("timeout");
expect(resolveFailoverReasonFromError({ message: "Network error: ECONNREFUSED" })).toBe(

44
src/agents/live-auth-keys.test.ts
View File

@@ -3,18 +3,19 @@ import { beforeAll, describe, expect, it, vi } from "vitest";
vi.unmock("../secrets/provider-env-vars.js");
let collectProviderApiKeys: typeof import("./live-auth-keys.js").collectProviderApiKeys;
let isAnthropicBillingError: typeof import("./live-auth-keys.js").isAnthropicBillingError;
async function loadModulesForTest(): Promise<void> {
vi.resetModules();
vi.doUnmock("../secrets/provider-env-vars.js");
({ collectProviderApiKeys } = await import("./live-auth-keys.js"));
({ collectProviderApiKeys, isAnthropicBillingError } = await import("./live-auth-keys.js"));
}
describe("collectProviderApiKeys", () => {
beforeAll(async () => {
await loadModulesForTest();
});
beforeAll(async () => {
await loadModulesForTest();
});
describe("collectProviderApiKeys", () => {
it("honors provider auth env vars with nonstandard names", async () => {
const env = { MODELSTUDIO_API_KEY: "modelstudio-live-key" };
@@ -37,3 +38,36 @@ describe("collectProviderApiKeys", () => {
).toEqual(["xai-live-key"]);
});
});
describe("isAnthropicBillingError", () => {
it("does not false-positive on plain 'a 402' prose", () => {
const samples = [
"Use a 402 stainless bolt",
"Book a 402 room",
"There is a 402 near me",
"The building at 402 Main Street",
];
for (const sample of samples) {
expect(isAnthropicBillingError(sample)).toBe(false);
}
});
it("matches real 402 billing payload contexts including JSON keys", () => {
const samples = [
"HTTP 402 Payment Required",
"status: 402",
"error code 402",
'{"status":402,"type":"error"}',
'{"code":402,"message":"payment required"}',
'{"error":{"code":402,"message":"billing hard limit reached"}}',
"got a 402 from the API",
"returned 402",
"received a 402 response",
];
for (const sample of samples) {
expect(isAnthropicBillingError(sample)).toBe(true);
}
});
});

18
src/agents/live-target-matcher.test.ts
View File

@@ -1,13 +1,13 @@
import { beforeAll, describe, expect, it, vi } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { createLiveTargetMatcher } from "./live-target-matcher.js";
type CreateLiveTargetMatcher = typeof import("./live-target-matcher.js").createLiveTargetMatcher;
let createLiveTargetMatcher: CreateLiveTargetMatcher;
beforeAll(async () => {
vi.doUnmock("../plugins/providers.js");
vi.doUnmock("../plugins/manifest-registry.js");
vi.resetModules();
({ createLiveTargetMatcher } = await import("./live-target-matcher.js"));
vi.mock("./live-provider-owner.js", () => {
const anthropicOwned = new Set(["anthropic", "claude-cli"]);
return {
liveProvidersShareOwningPlugin(left: string, right: string): boolean {
return anthropicOwned.has(left) && anthropicOwned.has(right);
},
};
});
describe("createLiveTargetMatcher", () => {

6
src/agents/model-compat.test.ts
View File

@@ -5,12 +5,8 @@ const providerRuntimeMocks = vi.hoisted(() => ({
resolveProviderModernModelRef: vi.fn(),
}));
vi.mock("../plugins/provider-runtime.js", async () => {
const actual = await vi.importActual<typeof import("../plugins/provider-runtime.js")>(
"../plugins/provider-runtime.js",
);
vi.mock("../plugins/provider-runtime.js", () => {
return {
...actual,
resolveProviderModernModelRef: providerRuntimeMocks.resolveProviderModernModelRef,
};
});

308
src/agents/model-fallback.test.ts
View File

@@ -1,20 +1,11 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { resetLogger, setLoggerOverride } from "../logging/logger.js";
import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.js";
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
import * as authProfileSourceCheckModule from "./auth-profiles/source-check.js";
import * as authProfileStoreModule from "./auth-profiles/store.js";
import {
clearRuntimeAuthProfileStoreSnapshots,
replaceRuntimeAuthProfileStoreSnapshots,
} from "./auth-profiles/store.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
import { isAnthropicBillingError } from "./live-auth-keys.js";
import { LiveSessionModelSwitchError } from "./live-model-switch-error.js";
import { runWithImageModelFallback, runWithModelFallback } from "./model-fallback.js";
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
@@ -28,15 +19,20 @@ vi.mock("../plugins/provider-runtime.js", () => ({
resolveExternalAuthProfilesWithPlugins: () => [],
}));
const authSourceCheckMock = vi.hoisted(() => ({
hasAnyAuthProfileStoreSource: vi.fn(() => true),
}));
vi.mock("./auth-profiles/source-check.js", () => authSourceCheckMock);
const authRuntimeMock = vi.hoisted(() => {
const stores = new Map<string, AuthProfileStore>();
const keyFor = (agentDir?: string) => agentDir ?? "__main__";
const now = () => Date.now();
const isActive = (value: unknown, ts = now()) =>
typeof value === "number" && Number.isFinite(value) && value > ts;
const cloneStore = (store: AuthProfileStore): AuthProfileStore => structuredClone(store);
const getStore = (agentDir?: string): AuthProfileStore =>
cloneStore(stores.get(keyFor(agentDir)) ?? { version: 1, profiles: {} });
stores.get(keyFor(agentDir)) ?? { version: 1, profiles: {} };
const getProfileIds = (store: AuthProfileStore, provider: string) =>
Object.entries(store.profiles)
.filter(([, profile]) => profile.provider === provider)
@@ -93,11 +89,11 @@ const authRuntimeMock = vi.hoisted(() => {
return {
clear: () => stores.clear(),
setStore: (agentDir: string | undefined, store: AuthProfileStore) => {
stores.set(keyFor(agentDir), cloneStore(store));
stores.set(keyFor(agentDir), store);
},
runtime: {
ensureAuthProfileStore: (agentDir?: string) => getStore(agentDir),
loadAuthProfileStoreForRuntime: (agentDir?: string) => getStore(agentDir),
ensureAuthProfileStore: vi.fn((agentDir?: string) => getStore(agentDir)),
loadAuthProfileStoreForRuntime: vi.fn((agentDir?: string) => getStore(agentDir)),
resolveAuthProfileOrder: (params: { store: AuthProfileStore; provider: string }) =>
getProfileIds(params.store, params.provider),
isProfileInCooldown,
@@ -139,19 +135,11 @@ const OPENROUTER_MODEL_NOT_FOUND_PAYLOAD =
let authTempRoot = "";
let authTempCounter = 0;
beforeAll(async () => {
authTempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-suite-"));
});
afterAll(async () => {
if (authTempRoot) {
await fs.rm(authTempRoot, { recursive: true, force: true });
}
});
afterEach(() => {
clearRuntimeAuthProfileStoreSnapshots();
authRuntimeMock.clear();
authRuntimeMock.runtime.ensureAuthProfileStore.mockClear();
authRuntimeMock.runtime.loadAuthProfileStoreForRuntime.mockClear();
authSourceCheckMock.hasAnyAuthProfileStoreSource.mockReset().mockReturnValue(true);
});
function makeFallbacksOnlyCfg(): OpenClawConfig {
@@ -189,9 +177,8 @@ async function withTempAuthStore<T>(
}
async function makeAuthTempDir(): Promise<string> {
const tempDir = path.join(authTempRoot, `case-${++authTempCounter}`);
await fs.mkdir(tempDir, { recursive: true });
return tempDir;
authTempRoot ||= path.join("/tmp", "openclaw-auth-suite-mock");
return path.join(authTempRoot, `case-${++authTempCounter}`);
}
async function runWithStoredAuth(params: {
@@ -212,7 +199,6 @@ async function runWithStoredAuth(params: {
}
function setAuthRuntimeStore(agentDir: string | undefined, store: AuthProfileStore): void {
replaceRuntimeAuthProfileStoreSnapshots([{ agentDir, store }]);
authRuntimeMock.setStore(agentDir, store);
}
@@ -331,36 +317,26 @@ const ANTHROPIC_OVERLOADED_PAYLOAD =
// https://github.com/openclaw/openclaw/issues/23440
const INSUFFICIENT_QUOTA_PAYLOAD =
'{"type":"error","error":{"type":"insufficient_quota","message":"Your account has insufficient quota balance to run this request."}}';
// Internal OpenClaw compatibility marker, not a provider API contract.
const MODEL_COOLDOWN_MESSAGE = "model_cooldown: All credentials for model gpt-5 are cooling down";
// SDK/transport compatibility marker, not a provider API contract.
const CONNECTION_ERROR_MESSAGE = "Connection error.";
describe("runWithModelFallback", () => {
it("skips auth store bootstrap when no auth profile sources exist", async () => {
const hasSourcesSpy = vi
.spyOn(authProfileSourceCheckModule, "hasAnyAuthProfileStoreSource")
.mockReturnValue(false);
const ensureStoreSpy = vi.spyOn(authProfileStoreModule, "ensureAuthProfileStore");
authSourceCheckMock.hasAnyAuthProfileStoreSource.mockReturnValue(false);
const run = vi.fn().mockResolvedValueOnce("ok");
try {
const result = await runWithModelFallback({
cfg: makeCfg(),
provider: "openai",
model: "gpt-4.1-mini",
agentDir: "/tmp/openclaw-no-auth-profiles",
run,
});
const result = await runWithModelFallback({
cfg: makeCfg(),
provider: "openai",
model: "gpt-4.1-mini",
agentDir: "/tmp/openclaw-no-auth-profiles",
run,
});
expect(result.result).toBe("ok");
expect(hasSourcesSpy).toHaveBeenCalledWith("/tmp/openclaw-no-auth-profiles");
expect(ensureStoreSpy).not.toHaveBeenCalled();
expect(run).toHaveBeenCalledWith("openai", "gpt-4.1-mini");
} finally {
hasSourcesSpy.mockRestore();
ensureStoreSpy.mockRestore();
}
expect(result.result).toBe("ok");
expect(authSourceCheckMock.hasAnyAuthProfileStoreSource).toHaveBeenCalledWith(
"/tmp/openclaw-no-auth-profiles",
);
expect(authRuntimeMock.runtime.ensureAuthProfileStore).not.toHaveBeenCalled();
expect(run).toHaveBeenCalledWith("openai", "gpt-4.1-mini");
});
it("keeps openai gpt-5.3 codex on the openai provider before running", async () => {
@@ -1169,162 +1145,43 @@ describe("runWithModelFallback", () => {
expect(calls).toEqual([{ provider: "openai", model: "gpt-4.1-mini" }]);
});
it("falls back on missing API key errors", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error("No API key found for profile openai."),
});
});
it("falls back on representative transient/provider error shapes", async () => {
// The full classification matrix lives in failover-error tests; keep this as integration
// coverage that classified errors actually advance the model fallback chain.
const cases: Array<{ name: string; firstError: Error }> = [
{
name: "missing API key",
firstError: new Error("No API key found for profile openai."),
},
{
name: "lowercase credential",
firstError: new Error("no api key found for profile openai"),
},
{
name: "documented OpenAI 429 rate limit",
firstError: Object.assign(new Error(OPENAI_RATE_LIMIT_MESSAGE), { status: 429 }),
},
{
name: "documented overloaded_error payload",
firstError: new Error(ANTHROPIC_OVERLOADED_PAYLOAD),
},
{
name: "provider request aborted",
firstError: Object.assign(new Error("Request was aborted"), { name: "AbortError" }),
},
];
it("falls back on lowercase credential errors", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error("no api key found for profile openai"),
});
});
it("falls back on documented OpenAI 429 rate limit responses", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error(OPENAI_RATE_LIMIT_MESSAGE), { status: 429 }),
});
});
it("falls back on documented overloaded_error payloads", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error(ANTHROPIC_OVERLOADED_PAYLOAD),
});
});
it("falls back on internal model cooldown markers", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error(MODEL_COOLDOWN_MESSAGE),
});
});
it("falls back on compatibility connection error messages", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error(CONNECTION_ERROR_MESSAGE),
});
});
it("falls back on timeout abort errors", async () => {
const timeoutCause = Object.assign(new Error("request timed out"), { name: "TimeoutError" });
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("aborted"), { name: "AbortError", cause: timeoutCause }),
});
});
it("falls back on abort errors with timeout reasons", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("aborted"), {
name: "AbortError",
reason: "deadline exceeded",
}),
});
});
it("falls back on abort errors with reason: abort", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("aborted"), {
name: "AbortError",
reason: "reason: abort",
}),
});
});
it("falls back on unhandled stop reason error responses", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: new Error("Unhandled stop reason: error"),
});
});
it("falls back on abort errors with reason: error", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("aborted"), {
name: "AbortError",
reason: "reason: error",
}),
});
});
it("falls back when message says aborted but error is a timeout", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("request aborted"), { code: "ETIMEDOUT" }),
});
});
it("falls back on ECONNREFUSED (local server down or remote unreachable)", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:11434"), {
code: "ECONNREFUSED",
}),
});
});
it("falls back on ENETUNREACH (network disconnected)", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("connect ENETUNREACH"), { code: "ENETUNREACH" }),
});
});
it("falls back on EHOSTUNREACH (host unreachable)", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("connect EHOSTUNREACH"), { code: "EHOSTUNREACH" }),
});
});
it("falls back on EAI_AGAIN (DNS resolution failure)", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("getaddrinfo EAI_AGAIN api.openai.com"), {
code: "EAI_AGAIN",
}),
});
});
it("falls back on ENETRESET (connection reset by network)", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("connect ENETRESET"), { code: "ENETRESET" }),
});
});
it("falls back on provider abort errors with request-aborted messages", async () => {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError: Object.assign(new Error("Request was aborted"), { name: "AbortError" }),
});
for (const { name, firstError } of cases) {
try {
await expectFallsBackToHaiku({
provider: "openai",
model: "gpt-4.1-mini",
firstError,
});
} catch (error) {
throw new Error(`fallback case failed: ${name}`, { cause: error });
}
}
});
it("does not fall back on user aborts", async () => {
@@ -1833,36 +1690,3 @@ describe("runWithImageModelFallback", () => {
]);
});
});
describe("isAnthropicBillingError", () => {
it("does not false-positive on plain 'a 402' prose", () => {
const samples = [
"Use a 402 stainless bolt",
"Book a 402 room",
"There is a 402 near me",
"The building at 402 Main Street",
];
for (const sample of samples) {
expect(isAnthropicBillingError(sample)).toBe(false);
}
});
it("matches real 402 billing payload contexts including JSON keys", () => {
const samples = [
"HTTP 402 Payment Required",
"status: 402",
"error code 402",
'{"status":402,"type":"error"}',
'{"code":402,"message":"payment required"}',
'{"error":{"code":402,"message":"billing hard limit reached"}}',
"got a 402 from the API",
"returned 402",
"received a 402 response",
];
for (const sample of samples) {
expect(isAnthropicBillingError(sample)).toBe(true);
}
});
});

364
src/agents/models-config.runtime-source-snapshot.test.ts
View File

@@ -1,13 +1,25 @@
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createFixtureSuite } from "../test-utils/fixture-suite.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import {
installModelsConfigTestHooks,
MODELS_CONFIG_IMPLICIT_ENV_VARS,
unsetEnv,
withModelsTempHome as withTempHome,
withTempEnv,
} from "./models-config.e2e-harness.js";
import { enforceSourceManagedProviderSecrets } from "./models-config.providers.source-managed.js";
vi.mock("../plugins/manifest-registry.js", () => ({
clearPluginManifestRegistryCache: () => undefined,
loadPluginManifestRegistry: () => ({ plugins: [] }),
}));
vi.mock("./model-auth-env-vars.js", () => ({
listKnownProviderEnvApiKeyNames: () => ["OPENAI_API_KEY"],
PROVIDER_ENV_API_KEY_CANDIDATES: { openai: ["OPENAI_API_KEY"] },
resolveProviderEnvApiKeyCandidates: () => ({ openai: ["OPENAI_API_KEY"] }),
}));
vi.mock("../plugins/provider-runtime.js", () => ({
applyProviderConfigDefaultsWithPlugin: (config: OpenClawConfig) => config,
@@ -32,17 +44,20 @@ installModelsConfigTestHooks();
let clearConfigCache: typeof import("../config/io.js").clearConfigCache;
let clearRuntimeConfigSnapshot: typeof import("../config/io.js").clearRuntimeConfigSnapshot;
let loadConfig: typeof import("../config/io.js").loadConfig;
let setRuntimeConfigSnapshot: typeof import("../config/io.js").setRuntimeConfigSnapshot;
let ensureOpenClawModelsJson: typeof import("./models-config.js").ensureOpenClawModelsJson;
let resetModelsJsonReadyCacheForTest: typeof import("./models-config.js").resetModelsJsonReadyCacheForTest;
let planOpenClawModelsJsonWithDeps: typeof import("./models-config.plan.js").planOpenClawModelsJsonWithDeps;
let readGeneratedModelsJson: typeof import("./models-config.test-utils.js").readGeneratedModelsJson;
const fixtureSuite = createFixtureSuite("openclaw-models-runtime-source-");
beforeAll(async () => {
({ clearConfigCache, clearRuntimeConfigSnapshot, loadConfig, setRuntimeConfigSnapshot } =
await fixtureSuite.setup();
({ clearConfigCache, clearRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
await import("../config/io.js"));
({ ensureOpenClawModelsJson, resetModelsJsonReadyCacheForTest } =
await import("./models-config.js"));
({ planOpenClawModelsJsonWithDeps } = await import("./models-config.plan.js"));
({ readGeneratedModelsJson } = await import("./models-config.test-utils.js"));
});
@@ -52,6 +67,10 @@ afterEach(() => {
resetModelsJsonReadyCacheForTest();
});
afterAll(async () => {
await fixtureSuite.cleanup();
});
function createOpenAiApiKeySourceConfig(): OpenClawConfig {
return {
models: {
@@ -153,208 +172,195 @@ function withGatewayTokenMode(config: OpenClawConfig): OpenClawConfig {
};
}
async function withGeneratedModelsFromRuntimeSource(
params: {
sourceConfig: OpenClawConfig;
runtimeConfig: OpenClawConfig;
candidateConfig?: OpenClawConfig;
},
runAssertions: () => Promise<void>,
async function expectGeneratedProviderApiKey(
agentDir: string,
providerId: string,
expected: string,
) {
await withTempHome(async () => {
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>(agentDir);
expect(parsed.providers[providerId]?.apiKey).toBe(expected);
}
async function planGeneratedProviders(params: {
config: OpenClawConfig;
sourceConfigForSecrets: OpenClawConfig;
}) {
const plan = await planOpenClawModelsJsonWithDeps(
{
cfg: params.config,
sourceConfigForSecrets: params.sourceConfigForSecrets,
agentDir: "/tmp/openclaw-models-plan",
env: {},
existingRaw: "",
existingParsed: null,
},
{
resolveImplicitProviders: async () => ({}),
},
);
expect(plan.action).toBe("write");
if (plan.action !== "write") {
throw new Error(`expected models.json write plan, got ${plan.action}`);
}
return JSON.parse(plan.contents).providers as Record<
string,
{ apiKey?: string; headers?: Record<string, string> }
>;
}
function expectOpenAiHeaderMarkers(
providers: Record<string, { headers?: Record<string, string> }>,
) {
expect(providers.openai?.headers?.Authorization).toBe(
"secretref-env:OPENAI_HEADER_TOKEN", // pragma: allowlist secret
);
expect(providers.openai?.headers?.["X-Tenant-Token"]).toBe(NON_ENV_SECRETREF_MARKER);
}
describe("models-config runtime source snapshot", () => {
it("uses runtime source snapshot markers when passed the active runtime config", async () => {
const sourceConfig: OpenClawConfig = {
models: {
providers: {
openai: createOpenAiApiKeySourceConfig().models!.providers!.openai,
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: { source: "file", provider: "vault", id: "/moonshot/apiKey" },
api: "openai-completions" as const,
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
openai: createOpenAiApiKeyRuntimeConfig().models!.providers!.openai,
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: "sk-runtime-moonshot", // pragma: allowlist secret
api: "openai-completions" as const,
models: [],
},
},
},
};
const providers = enforceSourceManagedProviderSecrets({
providers: runtimeConfig.models!.providers!,
sourceProviders: sourceConfig.models!.providers,
})!;
expect(providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expect(providers.moonshot?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
});
it("projects cloned runtime configs onto source snapshot when preserving provider auth", async () => {
const agentDir = await fixtureSuite.createCaseDir("agent");
await withTempEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS, async () => {
unsetEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS);
const sourceConfig = createOpenAiApiKeySourceConfig();
const runtimeConfig = createOpenAiApiKeyRuntimeConfig();
const clonedRuntimeConfig: OpenClawConfig = {
...runtimeConfig,
agents: {
defaults: {
imageModel: "openai/gpt-image-1",
},
},
};
try {
setRuntimeConfigSnapshot(params.runtimeConfig, params.sourceConfig);
await ensureOpenClawModelsJson(params.candidateConfig ?? loadConfig());
await runAssertions();
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(clonedRuntimeConfig, agentDir);
await expectGeneratedProviderApiKey(agentDir, "openai", "OPENAI_API_KEY"); // pragma: allowlist secret
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
}
async function expectGeneratedProviderApiKey(providerId: string, expected: string) {
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>();
expect(parsed.providers[providerId]?.apiKey).toBe(expected);
}
async function expectGeneratedOpenAiHeaderMarkers() {
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { headers?: Record<string, string> }>;
}>();
expect(parsed.providers.openai?.headers?.Authorization).toBe(
"secretref-env:OPENAI_HEADER_TOKEN", // pragma: allowlist secret
);
expect(parsed.providers.openai?.headers?.["X-Tenant-Token"]).toBe(NON_ENV_SECRETREF_MARKER);
}
describe("models-config runtime source snapshot", () => {
it("uses runtime source snapshot markers when passed the active runtime config", async () => {
await withGeneratedModelsFromRuntimeSource(
{
sourceConfig: {
models: {
providers: {
openai: createOpenAiApiKeySourceConfig().models!.providers!.openai,
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: { source: "file", provider: "vault", id: "/moonshot/apiKey" },
api: "openai-completions" as const,
models: [],
},
},
},
},
runtimeConfig: {
models: {
providers: {
openai: createOpenAiApiKeyRuntimeConfig().models!.providers!.openai,
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: "sk-runtime-moonshot", // pragma: allowlist secret
api: "openai-completions" as const,
models: [],
},
},
},
},
},
async () => {
await expectGeneratedProviderApiKey("openai", "OPENAI_API_KEY"); // pragma: allowlist secret
await expectGeneratedProviderApiKey("moonshot", NON_ENV_SECRETREF_MARKER);
},
);
});
it("projects cloned runtime configs onto source snapshot when preserving provider auth", async () => {
await withTempHome(async () => {
await withTempEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS, async () => {
unsetEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS);
const sourceConfig = createOpenAiApiKeySourceConfig();
const runtimeConfig = createOpenAiApiKeyRuntimeConfig();
const clonedRuntimeConfig: OpenClawConfig = {
...runtimeConfig,
agents: {
defaults: {
imageModel: "openai/gpt-image-1",
},
},
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(clonedRuntimeConfig);
await expectGeneratedProviderApiKey("openai", "OPENAI_API_KEY"); // pragma: allowlist secret
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
});
it("invalidates cached readiness when projected config changes under the same runtime snapshot", async () => {
await withTempHome(async () => {
await withTempEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS, async () => {
unsetEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS);
const sourceConfig = createOpenAiApiKeySourceConfig();
const runtimeConfig = createOpenAiApiKeyRuntimeConfig();
const firstCandidate: OpenClawConfig = {
...runtimeConfig,
models: {
providers: {
openai: {
...runtimeConfig.models!.providers!.openai,
baseUrl: "https://api.openai.com/v1",
headers: {
"X-OpenClaw-Test": "one",
},
const agentDir = await fixtureSuite.createCaseDir("agent");
await withTempEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS, async () => {
unsetEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS);
const sourceConfig = createOpenAiApiKeySourceConfig();
const runtimeConfig = createOpenAiApiKeyRuntimeConfig();
const firstCandidate: OpenClawConfig = {
...runtimeConfig,
models: {
providers: {
openai: {
...runtimeConfig.models!.providers!.openai,
baseUrl: "https://api.openai.com/v1",
headers: {
"X-OpenClaw-Test": "one",
},
},
},
};
const secondCandidate: OpenClawConfig = {
...runtimeConfig,
models: {
providers: {
openai: {
...runtimeConfig.models!.providers!.openai,
baseUrl: "https://mirror.example/v1",
headers: {
"X-OpenClaw-Test": "two",
},
},
};
const secondCandidate: OpenClawConfig = {
...runtimeConfig,
models: {
providers: {
openai: {
...runtimeConfig.models!.providers!.openai,
baseUrl: "https://mirror.example/v1",
headers: {
"X-OpenClaw-Test": "two",
},
},
},
};
},
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(firstCandidate);
let parsed = await readGeneratedModelsJson<{
providers: Record<
string,
{ baseUrl?: string; apiKey?: string; headers?: Record<string, string> }
>;
}>();
expect(parsed.providers.openai?.baseUrl).toBe("https://api.openai.com/v1");
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expect(parsed.providers.openai?.headers?.["X-OpenClaw-Test"]).toBe("one");
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(firstCandidate, agentDir);
let parsed = await readGeneratedModelsJson<{
providers: Record<
string,
{ baseUrl?: string; apiKey?: string; headers?: Record<string, string> }
>;
}>(agentDir);
expect(parsed.providers.openai?.baseUrl).toBe("https://api.openai.com/v1");
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expect(parsed.providers.openai?.headers?.["X-OpenClaw-Test"]).toBe("one");
// Header changes still rewrite models.json, but merge mode preserves the existing baseUrl.
await ensureOpenClawModelsJson(secondCandidate);
parsed = await readGeneratedModelsJson<{
providers: Record<
string,
{ baseUrl?: string; apiKey?: string; headers?: Record<string, string> }
>;
}>();
expect(parsed.providers.openai?.baseUrl).toBe("https://api.openai.com/v1");
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expect(parsed.providers.openai?.headers?.["X-OpenClaw-Test"]).toBe("two");
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
// Header changes still rewrite models.json, but merge mode preserves the existing baseUrl.
await ensureOpenClawModelsJson(secondCandidate, agentDir);
parsed = await readGeneratedModelsJson<{
providers: Record<
string,
{ baseUrl?: string; apiKey?: string; headers?: Record<string, string> }
>;
}>(agentDir);
expect(parsed.providers.openai?.baseUrl).toBe("https://api.openai.com/v1");
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expect(parsed.providers.openai?.headers?.["X-OpenClaw-Test"]).toBe("two");
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
it("uses header markers from runtime source snapshot instead of resolved runtime values", async () => {
await withGeneratedModelsFromRuntimeSource(
{
sourceConfig: createOpenAiHeaderSourceConfig(),
runtimeConfig: createOpenAiHeaderRuntimeConfig(),
},
expectGeneratedOpenAiHeaderMarkers,
);
const providers = await planGeneratedProviders({
config: createOpenAiHeaderRuntimeConfig(),
sourceConfigForSecrets: createOpenAiHeaderSourceConfig(),
});
expectOpenAiHeaderMarkers(providers);
});
it("keeps source markers when runtime projection is skipped for incompatible top-level shape", async () => {
await withTempHome(async () => {
await withTempEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS, async () => {
unsetEnv(MODELS_CONFIG_IMPLICIT_ENV_VARS);
const sourceConfig = withGatewayTokenMode(createOpenAiSourceConfigWithHeadersAndApiKey());
const runtimeConfig = withGatewayTokenMode(createOpenAiRuntimeConfigWithHeadersAndApiKey());
const incompatibleCandidate: OpenClawConfig = {
...createOpenAiRuntimeConfigWithHeadersAndApiKey(),
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(incompatibleCandidate);
await expectGeneratedProviderApiKey("openai", "OPENAI_API_KEY"); // pragma: allowlist secret
await expectGeneratedOpenAiHeaderMarkers();
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
const providers = await planGeneratedProviders({
config: createOpenAiRuntimeConfigWithHeadersAndApiKey(),
sourceConfigForSecrets: withGatewayTokenMode(createOpenAiSourceConfigWithHeadersAndApiKey()),
});
expect(providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
expectOpenAiHeaderMarkers(providers);
});
});

4
src/agents/models-config.test-utils.ts
View File

@@ -2,8 +2,8 @@ import fs from "node:fs/promises";
import path from "node:path";
import { resolveOpenClawAgentDir } from "./agent-paths.js";
export async function readGeneratedModelsJson<T>(): Promise<T> {
const modelPath = path.join(resolveOpenClawAgentDir(), "models.json");
export async function readGeneratedModelsJson<T>(agentDir = resolveOpenClawAgentDir()): Promise<T> {
const modelPath = path.join(agentDir, "models.json");
const raw = await fs.readFile(modelPath, "utf8");
return JSON.parse(raw) as T;
}

18
src/agents/models-config.uses-first-github-copilot-profile-env-tokens.test.ts
View File

@@ -7,6 +7,24 @@ import {
import type { ProviderConfig } from "./models-config.providers.secrets.js";
import { createProviderAuthResolver } from "./models-config.providers.secrets.js";
vi.mock("./model-auth-env.js", () => ({
resolveEnvApiKey: () => null,
}));
vi.mock("./provider-auth-aliases.js", () => ({
resolveProviderIdForAuth: (provider: string) => provider.trim().toLowerCase(),
}));
vi.mock("./model-auth-env-vars.js", () => ({
PROVIDER_ENV_API_KEY_CANDIDATES: {},
listKnownProviderEnvApiKeyNames: () => [],
resolveProviderEnvApiKeyCandidates: () => ({}),
}));
vi.mock("../plugins/provider-runtime.js", () => ({
resolveProviderSyntheticAuthWithPlugin: () => undefined,
}));
vi.mock("./models-config.providers.js", () => ({
applyNativeStreamingUsageCompat: (providers: unknown) => providers,
enforceSourceManagedProviderSecrets: ({ providers }: { providers: unknown }) => providers,

147
src/agents/pi-bundle-mcp-runtime.test.ts
View File

@@ -1,21 +1,15 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
cleanupBundleMcpHarness,
makeTempDir,
waitForFileText,
writeBundleProbeMcpServer,
writeClaudeBundle,
} from "./pi-bundle-mcp-test-harness.js";
import {
__testing,
disposeSessionMcpRuntime,
getOrCreateSessionMcpRuntime,
materializeBundleMcpToolsForRun,
} from "./pi-bundle-mcp-tools.js";
import { cleanupBundleMcpHarness } from "./pi-bundle-mcp-test-harness.js";
import { __testing, materializeBundleMcpToolsForRun } from "./pi-bundle-mcp-tools.js";
import type { SessionMcpRuntime } from "./pi-bundle-mcp-types.js";
vi.mock("./embedded-pi-mcp.js", () => ({
loadEmbeddedPiMcpConfig: (params: { cfg?: { mcp?: { servers?: Record<string, unknown> } } }) => ({
diagnostics: [],
mcpServers: params.cfg?.mcp?.servers ?? {},
}),
}));
type RuntimeFactoryOptions = NonNullable<
Parameters<typeof __testing.createSessionMcpRuntimeManager>[0]
>;
@@ -117,6 +111,7 @@ describe("session MCP runtime", () => {
it("reuses repeated materialization and recreates after explicit disposal", async () => {
const created: SessionMcpRuntime[] = [];
const disposed: string[] = [];
const createRuntime: RuntimeFactory = (params) => {
const runtime = makeRuntime([{ toolName: "bundle_probe", description: "Bundle MCP probe" }]);
created.push(runtime);
@@ -126,6 +121,9 @@ describe("session MCP runtime", () => {
sessionKey: params.sessionKey,
workspaceDir: params.workspaceDir,
configFingerprint: params.configFingerprint ?? "fingerprint",
dispose: async () => {
disposed.push(params.sessionId);
},
};
};
const manager = __testing.createSessionMcpRuntimeManager({ createRuntime });
@@ -154,6 +152,7 @@ describe("session MCP runtime", () => {
expect(manager.listSessionIds()).toEqual(["session-a"]);
await manager.disposeSession("session-a");
expect(disposed).toEqual(["session-a"]);
const runtimeC = await manager.getOrCreate({
sessionId: "session-a",
@@ -164,6 +163,19 @@ describe("session MCP runtime", () => {
expect(runtimeC).not.toBe(runtimeA);
expect(created).toHaveLength(2);
const materializedC = await materializeBundleMcpToolsForRun({
runtime: runtimeC,
disposeRuntime: async () => {
await manager.disposeSession("session-a");
},
});
expect(materializedC.tools.map((tool) => tool.name)).toEqual(["bundleProbe__bundle_probe"]);
await materializedC.dispose();
expect(disposed).toEqual(["session-a", "session-a"]);
expect(manager.listSessionIds()).not.toContain("session-a");
});
it("recreates the session runtime when MCP config changes", async () => {
@@ -242,41 +254,41 @@ describe("session MCP runtime", () => {
expect(resultB.content[0]).toMatchObject({ type: "text", text: "FROM-CONFIG-B" });
});
it("disposes startup-in-flight runtimes without leaking MCP processes", async () => {
vi.useRealTimers();
const workspaceDir = await makeTempDir("openclaw-bundle-mcp-tools-");
const startupCounterPath = path.join(workspaceDir, "bundle-starts.txt");
const pidPath = path.join(workspaceDir, "bundle.pid");
const exitMarkerPath = path.join(workspaceDir, "bundle.exit");
const pluginRoot = path.join(workspaceDir, ".openclaw", "extensions", "bundle-probe");
const serverScriptPath = path.join(pluginRoot, "servers", "bundle-probe.mjs");
await writeBundleProbeMcpServer(serverScriptPath, {
startupCounterPath,
startupDelayMs: 10,
pidPath,
exitMarkerPath,
it("disposes catalog startup in-flight without leaving cached runtimes", async () => {
let notifyCatalogStarted!: () => void;
const catalogStarted = new Promise<void>((resolve) => {
notifyCatalogStarted = resolve;
});
await writeClaudeBundle({ pluginRoot, serverScriptPath });
const runtime = await getOrCreateSessionMcpRuntime({
let rejectCatalog: ((error: Error) => void) | undefined;
const createRuntime: RuntimeFactory = (params) => ({
...makeRuntime([{ toolName: "bundle_probe", description: "Bundle MCP probe" }]),
sessionId: params.sessionId,
sessionKey: params.sessionKey,
workspaceDir: params.workspaceDir,
configFingerprint: params.configFingerprint ?? "fingerprint",
getCatalog: async () => {
notifyCatalogStarted();
return await new Promise((_, reject) => {
rejectCatalog = reject;
});
},
dispose: async () => {
rejectCatalog?.(new Error(`bundle-mcp runtime disposed for session ${params.sessionId}`));
},
});
const manager = __testing.createSessionMcpRuntimeManager({ createRuntime });
const runtime = await manager.getOrCreate({
sessionId: "session-d",
sessionKey: "agent:test:session-d",
workspaceDir,
cfg: {
plugins: {
entries: {
"bundle-probe": { enabled: true },
},
},
},
workspaceDir: "/workspace",
});
const materializeResult = materializeBundleMcpToolsForRun({ runtime }).then(
() => ({ status: "resolved" as const }),
(error: unknown) => ({ status: "rejected" as const, error }),
);
await waitForFileText(pidPath);
await disposeSessionMcpRuntime("session-d");
await catalogStarted;
await manager.disposeSession("session-d");
const result = await materializeResult;
if (result.status !== "rejected") {
@@ -284,57 +296,6 @@ describe("session MCP runtime", () => {
}
expect(result.error).toBeInstanceOf(Error);
expect((result.error as Error).message).toMatch(/disposed/);
expect(await waitForFileText(exitMarkerPath)).toBe("exited");
expect(await fs.readFile(startupCounterPath, "utf8")).toBe("1");
expect(__testing.getCachedSessionIds()).not.toContain("session-d");
});
it("materialized disposal can retire a manager-owned runtime", async () => {
const disposed: string[] = [];
const created: SessionMcpRuntime[] = [];
const createRuntime: RuntimeFactory = (params) => {
const runtime = {
...makeRuntime([{ toolName: "bundle_probe", description: "Bundle MCP probe" }]),
sessionId: params.sessionId,
sessionKey: params.sessionKey,
workspaceDir: params.workspaceDir,
configFingerprint: params.configFingerprint ?? "fingerprint",
dispose: async () => {
disposed.push(params.sessionId);
},
};
created.push(runtime);
return runtime;
};
const manager = __testing.createSessionMcpRuntimeManager({ createRuntime });
const runtimeA = await manager.getOrCreate({
sessionId: "session-e",
sessionKey: "agent:test:session-e",
workspaceDir: "/workspace",
});
const materialized = await materializeBundleMcpToolsForRun({
runtime: runtimeA,
disposeRuntime: async () => {
await manager.disposeSession("session-e");
},
});
expect(materialized.tools.map((tool) => tool.name)).toEqual(["bundleProbe__bundle_probe"]);
await materialized.dispose();
expect(disposed).toEqual(["session-e"]);
expect(manager.listSessionIds()).not.toContain("session-e");
const runtimeB = await manager.getOrCreate({
sessionId: "session-e",
sessionKey: "agent:test:session-e",
workspaceDir: "/workspace",
});
expect(runtimeB).not.toBe(runtimeA);
await materializeBundleMcpToolsForRun({ runtime: runtimeB });
expect(created).toHaveLength(2);
expect(manager.listSessionIds()).not.toContain("session-d");
});
});

84
src/agents/pi-project-settings.bundle.test.ts
View File

@@ -1,8 +1,90 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
vi.mock("../infra/boundary-file-read.js", async () => {
const fs = await import("node:fs");
return {
openBoundaryFileSync: ({ absolutePath }: { absolutePath: string }) => ({
ok: true,
fd: fs.openSync(absolutePath, "r"),
}),
};
});
vi.mock("../plugins/manifest-registry.js", async () => {
const fs = await import("node:fs");
const path = await import("node:path");
return {
loadPluginManifestRegistry: (params: { workspaceDir?: string }) => {
const rootDir = path.join(
params.workspaceDir ?? "",
".openclaw",
"extensions",
"claude-bundle",
);
if (!fs.existsSync(path.join(rootDir, ".claude-plugin", "plugin.json"))) {
return { plugins: [], diagnostics: [] };
}
const resolvedRootDir = fs.realpathSync(rootDir);
return {
diagnostics: [],
plugins: [
{
id: "claude-bundle",
origin: "workspace",
format: "bundle",
bundleFormat: "claude",
settingsFiles: ["settings.json"],
rootDir: resolvedRootDir,
},
],
};
},
};
});
vi.mock("./embedded-pi-mcp.js", async () => {
const fs = await import("node:fs");
const path = await import("node:path");
return {
loadEmbeddedPiMcpConfig: (params: {
workspaceDir: string;
cfg?: { mcp?: { servers?: Record<string, unknown> } };
}) => {
const pluginRoot = path.join(params.workspaceDir, ".openclaw", "extensions", "claude-bundle");
const mcpPath = path.join(pluginRoot, ".mcp.json");
let bundleServers: Record<string, unknown> = {};
if (fs.existsSync(mcpPath)) {
const raw = JSON.parse(fs.readFileSync(mcpPath, "utf-8")) as {
mcpServers?: Record<string, { args?: string[]; command?: string }>;
};
const resolvedRoot = fs.realpathSync(pluginRoot);
bundleServers = Object.fromEntries(
Object.entries(raw.mcpServers ?? {}).map(([id, server]) => [
id,
{
...server,
args: server.args?.map((arg) =>
arg.startsWith("./") ? path.join(resolvedRoot, arg) : arg,
),
cwd: resolvedRoot,
},
]),
);
}
return {
diagnostics: [],
mcpServers: {
...bundleServers,
...params.cfg?.mcp?.servers,
},
};
},
};
});
const { loadEnabledBundlePiSettingsSnapshot } = await import("./pi-project-settings-snapshot.js");
const tempDirs = createTrackedTempDirs();

106
src/agents/tools/pdf-tool.model-config.test.ts
View File

@@ -1,9 +1,45 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import { resolvePdfModelConfigForTool } from "./pdf-tool.model-config.js";
import { resetPdfToolAuthEnv, withTempPdfAgentDir } from "./pdf-tool.test-support.js";
import { resetPdfToolAuthEnv } from "./pdf-tool.test-support.js";
const ANTHROPIC_PDF_MODEL = "anthropic/claude-opus-4-7";
const TEST_AGENT_DIR = "/tmp/openclaw-pdf-model-config";
vi.mock("./model-config.helpers.js", () => ({
coerceToolModelConfig: (model?: unknown) => {
if (typeof model === "string") {
const primary = model.trim();
return primary ? { primary } : {};
}
const objectModel = model as { primary?: string; fallbacks?: string[] } | undefined;
return {
...(objectModel?.primary?.trim() ? { primary: objectModel.primary.trim() } : {}),
...(objectModel?.fallbacks?.length ? { fallbacks: objectModel.fallbacks } : {}),
};
},
hasAuthForProvider: ({ provider }: { provider: string }) => {
if (provider === "anthropic") {
return Boolean(process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_OAUTH_TOKEN);
}
if (provider === "openai") {
return Boolean(process.env.OPENAI_API_KEY);
}
if (provider === "google") {
return Boolean(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
}
return false;
},
resolveDefaultModelRef: (cfg?: OpenClawConfig) => {
const modelCfg = cfg?.agents?.defaults?.model;
const primary =
(typeof modelCfg === "string"
? modelCfg
: (modelCfg as { primary?: string } | undefined)?.primary) ?? "anthropic/claude-sonnet-4-5";
const [provider = "anthropic", model = "claude-sonnet-4-5"] = primary.split("/", 2);
return { provider, model };
},
}));
function withDefaultModel(primary: string): OpenClawConfig {
return {
@@ -21,58 +57,52 @@ describe("resolvePdfModelConfigForTool", () => {
});
it("returns null without any auth", async () => {
await withTempPdfAgentDir(async (agentDir) => {
const cfg = withDefaultModel("openai/gpt-5.4");
expect(resolvePdfModelConfigForTool({ cfg, agentDir })).toBeNull();
});
const cfg = withDefaultModel("openai/gpt-5.4");
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR })).toBeNull();
});
it("prefers explicit pdfModel config", async () => {
await withTempPdfAgentDir(async (agentDir) => {
const cfg = {
agents: {
defaults: {
model: { primary: "openai/gpt-5.4" },
pdfModel: { primary: ANTHROPIC_PDF_MODEL },
},
const cfg = {
agents: {
defaults: {
model: { primary: "openai/gpt-5.4" },
pdfModel: { primary: ANTHROPIC_PDF_MODEL },
},
} as OpenClawConfig;
expect(resolvePdfModelConfigForTool({ cfg, agentDir })).toEqual({
primary: ANTHROPIC_PDF_MODEL,
});
},
} as OpenClawConfig;
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR })).toEqual({
primary: ANTHROPIC_PDF_MODEL,
});
});
it("falls back to imageModel config when no pdfModel set", async () => {
await withTempPdfAgentDir(async (agentDir) => {
const cfg = {
agents: {
defaults: {
model: { primary: "openai/gpt-5.4" },
imageModel: { primary: "openai/gpt-5.4-mini" },
},
const cfg = {
agents: {
defaults: {
model: { primary: "openai/gpt-5.4" },
imageModel: { primary: "openai/gpt-5.4-mini" },
},
} as OpenClawConfig;
expect(resolvePdfModelConfigForTool({ cfg, agentDir })).toEqual({
primary: "openai/gpt-5.4-mini",
});
},
} as OpenClawConfig;
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR })).toEqual({
primary: "openai/gpt-5.4-mini",
});
});
it("prefers anthropic when available for native PDF support", async () => {
await withTempPdfAgentDir(async (agentDir) => {
vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-test");
vi.stubEnv("OPENAI_API_KEY", "openai-test");
const cfg = withDefaultModel("openai/gpt-5.4");
expect(resolvePdfModelConfigForTool({ cfg, agentDir })?.primary).toBe(ANTHROPIC_PDF_MODEL);
});
vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-test");
vi.stubEnv("OPENAI_API_KEY", "openai-test");
const cfg = withDefaultModel("openai/gpt-5.4");
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR })?.primary).toBe(
ANTHROPIC_PDF_MODEL,
);
});
it("uses anthropic primary when provider is anthropic", async () => {
await withTempPdfAgentDir(async (agentDir) => {
vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-test");
const cfg = withDefaultModel(ANTHROPIC_PDF_MODEL);
expect(resolvePdfModelConfigForTool({ cfg, agentDir })?.primary).toBe(ANTHROPIC_PDF_MODEL);
});
vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-test");
const cfg = withDefaultModel(ANTHROPIC_PDF_MODEL);
expect(resolvePdfModelConfigForTool({ cfg, agentDir: TEST_AGENT_DIR })?.primary).toBe(
ANTHROPIC_PDF_MODEL,
);
});
});