perf(agents): trim pi model discovery auth tests

This commit is contained in:
Peter Steinberger
2026-04-07 04:13:36 +01:00
parent 53f6d962c2
commit d34b3ec701
5 changed files with 128 additions and 433 deletions

View File

@@ -1,6 +1,10 @@
export {
AuthStorage,
addEnvBackedPiCredentials,
discoverAuthStorage,
discoverModels,
ModelRegistry,
normalizeDiscoveredPiModel,
resolvePiCredentialsForDiscovery,
scrubLegacyStaticAuthJsonEntriesForDiscovery,
} from "./pi-model-discovery.js";

View File

@@ -2,8 +2,11 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { saveAuthProfileStore } from "./auth-profiles.js";
import { discoverAuthStorage, discoverModels } from "./pi-model-discovery.js";
import {
addEnvBackedPiCredentials,
scrubLegacyStaticAuthJsonEntriesForDiscovery,
} from "./pi-model-discovery.js";
import { resolvePiCredentialMapFromStore } from "./pi-auth-credentials.js";
async function createAgentDir(): Promise<string> {
return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pi-auth-storage-"));
@@ -18,31 +21,6 @@ async function withAgentDir(run: (agentDir: string) => Promise<void>): Promise<v
}
}
async function pathExists(pathname: string): Promise<boolean> {
try {
await fs.stat(pathname);
return true;
} catch {
return false;
}
}
function writeRuntimeOpenRouterProfile(agentDir: string): void {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
},
},
agentDir,
);
}
async function writeLegacyAuthJson(
agentDir: string,
authEntries: Record<string, unknown>,
@@ -57,58 +35,48 @@ async function readLegacyAuthJson(agentDir: string): Promise<Record<string, unkn
>;
}
async function writeModelsJson(agentDir: string, payload: unknown): Promise<void> {
await fs.writeFile(path.join(agentDir, "models.json"), `${JSON.stringify(payload, null, 2)}\n`);
}
describe("discoverAuthStorage", () => {
it("loads runtime credentials from auth-profiles without writing auth.json", async () => {
await withAgentDir(async (agentDir) => {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
"anthropic:default": {
type: "token",
provider: "anthropic",
token: "sk-ant-runtime",
},
"openai-codex:default": {
type: "oauth",
provider: "openai-codex",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
it("converts runtime auth profiles into pi discovery credentials", () => {
const credentials = resolvePiCredentialMapFromStore({
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
agentDir,
);
"anthropic:default": {
type: "token",
provider: "anthropic",
token: "sk-ant-runtime",
},
"openai-codex:default": {
type: "oauth",
provider: "openai-codex",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
});
const authStorage = discoverAuthStorage(agentDir);
expect(authStorage.hasAuth("openrouter")).toBe(true);
expect(authStorage.hasAuth("anthropic")).toBe(true);
expect(authStorage.hasAuth("openai-codex")).toBe(true);
await expect(authStorage.getApiKey("openrouter")).resolves.toBe("sk-or-v1-runtime");
await expect(authStorage.getApiKey("anthropic")).resolves.toBe("sk-ant-runtime");
expect(authStorage.get("openai-codex")).toMatchObject({
type: "oauth",
access: "oauth-access",
});
expect(await pathExists(path.join(agentDir, "auth.json"))).toBe(false);
expect(credentials.openrouter).toEqual({
type: "api_key",
key: "sk-or-v1-runtime",
});
expect(credentials.anthropic).toEqual({
type: "api_key",
key: "sk-ant-runtime",
});
expect(credentials["openai-codex"]).toMatchObject({
type: "oauth",
access: "oauth-access",
refresh: "oauth-refresh",
});
});
it("scrubs static api_key entries from legacy auth.json and keeps oauth entries", async () => {
await withAgentDir(async (agentDir) => {
writeRuntimeOpenRouterProfile(agentDir);
await writeLegacyAuthJson(agentDir, {
openrouter: { type: "api_key", key: "legacy-static-key" },
"openai-codex": {
@@ -119,7 +87,7 @@ describe("discoverAuthStorage", () => {
},
});
discoverAuthStorage(agentDir);
scrubLegacyStaticAuthJsonEntriesForDiscovery(path.join(agentDir, "auth.json"));
const parsed = await readLegacyAuthJson(agentDir);
expect(parsed.openrouter).toBeUndefined();
@@ -135,12 +103,11 @@ describe("discoverAuthStorage", () => {
const previous = process.env.OPENCLAW_AUTH_STORE_READONLY;
process.env.OPENCLAW_AUTH_STORE_READONLY = "1";
try {
writeRuntimeOpenRouterProfile(agentDir);
await writeLegacyAuthJson(agentDir, {
openrouter: { type: "api_key", key: "legacy-static-key" },
});
discoverAuthStorage(agentDir);
scrubLegacyStaticAuthJsonEntriesForDiscovery(path.join(agentDir, "auth.json"));
const parsed = await readLegacyAuthJson(agentDir);
expect(parsed.openrouter).toMatchObject({ type: "api_key", key: "legacy-static-key" });
@@ -155,326 +122,22 @@ describe("discoverAuthStorage", () => {
});
it("includes env-backed provider auth when no auth profile exists", async () => {
await withAgentDir(async (agentDir) => {
const previous = process.env.MISTRAL_API_KEY;
process.env.MISTRAL_API_KEY = "mistral-env-test-key";
try {
saveAuthProfileStore(
{
version: 1,
profiles: {},
},
agentDir,
);
const previous = process.env.MISTRAL_API_KEY;
process.env.MISTRAL_API_KEY = "mistral-env-test-key";
try {
const credentials = addEnvBackedPiCredentials({}, process.env);
const authStorage = discoverAuthStorage(agentDir);
expect(authStorage.hasAuth("mistral")).toBe(true);
await expect(authStorage.getApiKey("mistral")).resolves.toBe("mistral-env-test-key");
} finally {
if (previous === undefined) {
delete process.env.MISTRAL_API_KEY;
} else {
process.env.MISTRAL_API_KEY = previous;
}
expect(credentials.mistral).toEqual({
type: "api_key",
key: "mistral-env-test-key",
});
} finally {
if (previous === undefined) {
delete process.env.MISTRAL_API_KEY;
} else {
process.env.MISTRAL_API_KEY = previous;
}
});
}
});
it("normalizes discovered Mistral compat flags for direct callers", async () => {
await withAgentDir(async (agentDir) => {
const previous = process.env.MISTRAL_API_KEY;
process.env.MISTRAL_API_KEY = "mistral-env-test-key";
try {
saveAuthProfileStore(
{
version: 1,
profiles: {},
},
agentDir,
);
await writeModelsJson(agentDir, {
providers: {
mistral: {
api: "openai-completions",
baseUrl: "https://api.mistral.ai/v1",
apiKey: "MISTRAL_API_KEY",
models: [
{
id: "mistral-large-latest",
name: "Mistral Large",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 16384,
},
],
},
},
});
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
expect(modelRegistry.getError?.()).toBeUndefined();
const model = modelRegistry.find("mistral", "mistral-large-latest") as {
api?: string;
compat?: {
supportsStore?: boolean;
supportsReasoningEffort?: boolean;
maxTokensField?: string;
};
} | null;
const all = modelRegistry.getAll() as Array<{
provider?: string;
id?: string;
api?: string;
compat?: {
supportsStore?: boolean;
supportsReasoningEffort?: boolean;
maxTokensField?: string;
};
}>;
const available = modelRegistry.getAvailable() as Array<{
provider?: string;
id?: string;
api?: string;
compat?: {
supportsStore?: boolean;
supportsReasoningEffort?: boolean;
maxTokensField?: string;
};
}>;
const fromAll = all.find(
(entry) => entry.provider === "mistral" && entry.id === "mistral-large-latest",
);
const fromAvailable = available.find(
(entry) => entry.provider === "mistral" && entry.id === "mistral-large-latest",
);
expect(model?.api).toBe("openai-completions");
expect(fromAll?.api).toBe("openai-completions");
expect(fromAvailable?.api).toBe("openai-completions");
expect(model?.compat?.supportsStore).toBe(false);
expect(model?.compat?.supportsReasoningEffort).toBe(false);
expect(model?.compat?.maxTokensField).toBe("max_tokens");
expect(fromAll?.compat?.supportsStore).toBe(false);
expect(fromAll?.compat?.supportsReasoningEffort).toBe(false);
expect(fromAll?.compat?.maxTokensField).toBe("max_tokens");
expect(fromAvailable?.compat?.supportsStore).toBe(false);
expect(fromAvailable?.compat?.supportsReasoningEffort).toBe(false);
expect(fromAvailable?.compat?.maxTokensField).toBe("max_tokens");
} finally {
if (previous === undefined) {
delete process.env.MISTRAL_API_KEY;
} else {
process.env.MISTRAL_API_KEY = previous;
}
}
});
});
it("normalizes discovered Mistral compat flags for custom Mistral-hosted providers", async () => {
await withAgentDir(async (agentDir) => {
saveAuthProfileStore(
{
version: 1,
profiles: {
"custom-api-mistral-ai:default": {
type: "api_key",
provider: "custom-api-mistral-ai",
key: "mistral-custom-key",
},
},
},
agentDir,
);
await writeModelsJson(agentDir, {
providers: {
"custom-api-mistral-ai": {
api: "openai-completions",
baseUrl: "https://api.mistral.ai/v1",
apiKey: "custom-api-mistral-ai",
models: [
{
id: "mistral-small-latest",
name: "Mistral Small",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 16384,
},
],
},
},
});
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
const model = modelRegistry.find("custom-api-mistral-ai", "mistral-small-latest") as {
compat?: {
supportsStore?: boolean;
supportsReasoningEffort?: boolean;
maxTokensField?: string;
};
} | null;
expect(model?.compat?.supportsStore).toBe(false);
expect(model?.compat?.supportsReasoningEffort).toBe(false);
expect(model?.compat?.maxTokensField).toBe("max_tokens");
});
});
it("normalizes discovered Mistral compat flags for OpenRouter Mistral model ids", async () => {
await withAgentDir(async (agentDir) => {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
},
},
agentDir,
);
await writeModelsJson(agentDir, {
providers: {
openrouter: {
api: "openai-completions",
baseUrl: "https://openrouter.ai/api/v1",
apiKey: "OPENROUTER_API_KEY",
models: [
{
id: "mistralai/mistral-small-3.2-24b-instruct",
name: "Mistral Small via OpenRouter",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 16384,
},
],
},
},
});
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
const model = modelRegistry.find(
"openrouter",
"mistralai/mistral-small-3.2-24b-instruct",
) as {
compat?: {
supportsStore?: boolean;
supportsReasoningEffort?: boolean;
maxTokensField?: string;
};
} | null;
expect(model?.compat?.supportsStore).toBe(false);
expect(model?.compat?.supportsReasoningEffort).toBe(false);
expect(model?.compat?.maxTokensField).toBe("max_tokens");
});
});
it("normalizes discovered xAI compat flags for OpenRouter x-ai model ids", async () => {
await withAgentDir(async (agentDir) => {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
},
},
agentDir,
);
await writeModelsJson(agentDir, {
providers: {
openrouter: {
api: "openai-completions",
baseUrl: "https://openrouter.ai/api/v1",
apiKey: "OPENROUTER_API_KEY",
models: [
{
id: "x-ai/grok-4.1-fast",
name: "Grok via OpenRouter",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 256000,
maxTokens: 8192,
},
],
},
},
});
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
const model = modelRegistry.find("openrouter", "x-ai/grok-4.1-fast") as {
compat?: {
toolSchemaProfile?: string;
nativeWebSearchTool?: boolean;
toolCallArgumentsEncoding?: string;
};
} | null;
expect(model?.compat?.toolSchemaProfile).toBe("xai");
expect(model?.compat?.nativeWebSearchTool).toBe(true);
expect(model?.compat?.toolCallArgumentsEncoding).toBe("html-entities");
});
});
it("normalizes discovered custom xAI-compatible providers by host", async () => {
await withAgentDir(async (agentDir) => {
await writeModelsJson(agentDir, {
providers: {
"custom-xai": {
api: "openai-completions",
baseUrl: "https://api.x.ai/v1",
apiKey: "XAI_API_KEY",
models: [
{
id: "grok-4.1-fast",
name: "Custom Grok",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 256000,
maxTokens: 8192,
},
],
},
},
});
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
const model = modelRegistry
.getAll()
.find((entry) => entry.provider === "custom-xai" && entry.id === "grok-4.1-fast") as
| {
api?: string;
compat?: {
toolSchemaProfile?: string;
nativeWebSearchTool?: boolean;
toolCallArgumentsEncoding?: string;
};
}
| undefined;
expect(model?.api).toBe("openai-responses");
expect(model?.compat?.toolSchemaProfile).toBe("xai");
expect(model?.compat?.nativeWebSearchTool).toBe(true);
expect(model?.compat?.toolCallArgumentsEncoding).toBe("html-entities");
});
});
});

View File

@@ -4,19 +4,7 @@ import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { saveAuthProfileStore } from "./auth-profiles.js";
const loadPluginManifestRegistry = vi.hoisted(() =>
vi.fn(() => ({
plugins: [
{
id: "anthropic",
origin: "bundled",
providers: ["anthropic"],
cliBackends: ["claude-cli"],
},
],
diagnostics: [],
})),
);
const resolveRuntimeSyntheticAuthProviderRefs = vi.hoisted(() => vi.fn(() => ["claude-cli"]));
const resolveProviderSyntheticAuthWithPlugin = vi.hoisted(() =>
vi.fn((params: { provider: string }) =>
@@ -30,8 +18,8 @@ const resolveProviderSyntheticAuthWithPlugin = vi.hoisted(() =>
),
);
vi.mock("../plugins/manifest-registry.js", () => ({
loadPluginManifestRegistry,
vi.mock("../plugins/synthetic-auth.runtime.js", () => ({
resolveRuntimeSyntheticAuthProviderRefs,
}));
vi.mock("../plugins/provider-runtime.js", () => ({
@@ -54,7 +42,7 @@ async function withAgentDir(run: (agentDir: string) => Promise<void>): Promise<v
describe("pi model discovery synthetic auth", () => {
beforeEach(() => {
vi.resetModules();
loadPluginManifestRegistry.mockClear();
resolveRuntimeSyntheticAuthProviderRefs.mockClear();
resolveProviderSyntheticAuthWithPlugin.mockClear();
});
@@ -75,7 +63,7 @@ describe("pi model discovery synthetic auth", () => {
const { discoverAuthStorage } = await import("./pi-model-discovery.js");
const authStorage = discoverAuthStorage(agentDir);
expect(loadPluginManifestRegistry).toHaveBeenCalled();
expect(resolveRuntimeSyntheticAuthProviderRefs).toHaveBeenCalled();
expect(resolveProviderSyntheticAuthWithPlugin).toHaveBeenCalledWith({
provider: "claude-cli",
context: {

View File

@@ -6,7 +6,6 @@ import type {
AuthStorage as PiAuthStorage,
ModelRegistry as PiModelRegistry,
} from "@mariozechner/pi-coding-agent";
import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js";
import { normalizeModelCompat } from "../plugins/provider-model-compat.js";
import {
applyProviderResolvedModelCompatWithPlugins,
@@ -14,6 +13,7 @@ import {
normalizeProviderResolvedModelWithPlugin,
resolveProviderSyntheticAuthWithPlugin,
} from "../plugins/provider-runtime.js";
import { resolveRuntimeSyntheticAuthProviderRefs } from "../plugins/synthetic-auth.runtime.js";
import type { ProviderRuntimeModel } from "../plugins/types.js";
import { isRecord } from "../utils.js";
import { ensureAuthProfileStore } from "./auth-profiles.js";
@@ -55,7 +55,7 @@ function createInMemoryAuthStorageBackend(
};
}
function normalizeRegistryModel<T>(value: T, agentDir: string): T {
export function normalizeDiscoveredPiModel<T>(value: T, agentDir: string): T {
if (!isRecord(value)) {
return value;
}
@@ -128,16 +128,16 @@ function createOpenClawModelRegistry(
const find = registry.find.bind(registry);
registry.getAll = () =>
getAll().map((entry: Model<Api>) => normalizeRegistryModel(entry, agentDir));
getAll().map((entry: Model<Api>) => normalizeDiscoveredPiModel(entry, agentDir));
registry.getAvailable = () =>
getAvailable().map((entry: Model<Api>) => normalizeRegistryModel(entry, agentDir));
getAvailable().map((entry: Model<Api>) => normalizeDiscoveredPiModel(entry, agentDir));
registry.find = (provider: string, modelId: string) =>
normalizeRegistryModel(find(provider, modelId), agentDir);
normalizeDiscoveredPiModel(find(provider, modelId), agentDir);
return registry;
}
function scrubLegacyStaticAuthJsonEntries(pathname: string): void {
export function scrubLegacyStaticAuthJsonEntriesForDiscovery(pathname: string): void {
if (process.env.OPENCLAW_AUTH_STORE_READONLY === "1") {
return;
}
@@ -225,35 +225,34 @@ function createAuthStorage(AuthStorageLike: unknown, path: string, creds: PiCred
return withRuntimeOverride;
}
function resolvePiCredentials(agentDir: string): PiCredentialMap {
const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
const credentials = resolvePiCredentialMapFromStore(store);
export function addEnvBackedPiCredentials(
credentials: PiCredentialMap,
env: NodeJS.ProcessEnv = process.env,
): PiCredentialMap {
const next = { ...credentials };
// pi-coding-agent hides providers from its registry when auth storage lacks
// a matching credential entry. Mirror env-backed provider auth here so
// live/model discovery sees the same providers runtime auth can use.
for (const provider of Object.keys(resolveProviderEnvApiKeyCandidates())) {
if (credentials[provider]) {
if (next[provider]) {
continue;
}
const resolved = resolveEnvApiKey(provider);
const resolved = resolveEnvApiKey(provider, env);
if (!resolved?.apiKey) {
continue;
}
credentials[provider] = {
next[provider] = {
type: "api_key",
key: resolved.apiKey,
};
}
const syntheticProviders = new Set<string>();
for (const plugin of loadPluginManifestRegistry().plugins) {
for (const provider of plugin.providers) {
syntheticProviders.add(provider);
}
for (const backend of plugin.cliBackends) {
syntheticProviders.add(backend);
}
}
for (const provider of syntheticProviders) {
return next;
}
export function resolvePiCredentialsForDiscovery(agentDir: string): PiCredentialMap {
const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
const credentials = addEnvBackedPiCredentials(resolvePiCredentialMapFromStore(store));
for (const provider of resolveRuntimeSyntheticAuthProviderRefs()) {
if (credentials[provider]) {
continue;
}
@@ -279,9 +278,9 @@ function resolvePiCredentials(agentDir: string): PiCredentialMap {
// Compatibility helpers for pi-coding-agent 0.50+ (discover* helpers removed).
export function discoverAuthStorage(agentDir: string): PiAuthStorage {
const credentials = resolvePiCredentials(agentDir);
const credentials = resolvePiCredentialsForDiscovery(agentDir);
const authPath = path.join(agentDir, "auth.json");
scrubLegacyStaticAuthJsonEntries(authPath);
scrubLegacyStaticAuthJsonEntriesForDiscovery(authPath);
return createAuthStorage(PiAuthStorageClass, authPath, credentials);
}

View File

@@ -0,0 +1,41 @@
import { normalizeProviderId } from "../agents/provider-id.js";
import { getPluginRegistryState } from "./runtime-state.js";
const BUNDLED_SYNTHETIC_AUTH_PROVIDER_REFS = ["claude-cli", "ollama", "xai"] as const;
function uniqueProviderRefs(values: readonly string[]): string[] {
const seen = new Set<string>();
const next: string[] = [];
for (const raw of values) {
const trimmed = raw.trim();
const normalized = normalizeProviderId(trimmed);
if (!trimmed || seen.has(normalized)) {
continue;
}
seen.add(normalized);
next.push(trimmed);
}
return next;
}
export function resolveRuntimeSyntheticAuthProviderRefs(): string[] {
const registry = getPluginRegistryState()?.activeRegistry;
if (registry) {
return uniqueProviderRefs([
...(registry.providers ?? [])
.filter(
(entry) =>
"resolveSyntheticAuth" in entry.provider &&
typeof entry.provider.resolveSyntheticAuth === "function",
)
.map((entry) => entry.provider.id),
...(registry.cliBackends ?? [])
.filter(
(entry) =>
"resolveSyntheticAuth" in entry.backend &&
typeof entry.backend.resolveSyntheticAuth === "function",
)
.map((entry) => entry.backend.id),
]);
}
return uniqueProviderRefs(BUNDLED_SYNTHETIC_AUTH_PROVIDER_REFS);
}