From dc7fab4dc5f604d8e84be4df07ee38b4c73b1a4a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 14 May 2026 00:12:53 +0100 Subject: [PATCH] perf: cache pi model discovery --- CHANGELOG.md | 1 + package.json | 1 + scripts/perf/issue-78851-model-resolution.ts | 499 ++++++++++++++++++ .../model-discovery-cache.ts | 124 +++++ src/agents/pi-embedded-runner/model.test.ts | 181 ++++++- src/agents/pi-embedded-runner/model.ts | 32 +- src/agents/pi-model-discovery.test.ts | 38 ++ src/agents/pi-model-discovery.ts | 23 +- src/plugins/synthetic-auth.runtime.test.ts | 104 +++- src/plugins/synthetic-auth.runtime.ts | 42 ++ 10 files changed, 1037 insertions(+), 8 deletions(-) create mode 100644 scripts/perf/issue-78851-model-resolution.ts create mode 100644 src/agents/pi-embedded-runner/model-discovery-cache.ts create mode 100644 src/agents/pi-model-discovery.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b18dd882f8..e4c1dc70668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Docs: https://docs.openclaw.ai ### Fixes - ACP: preserve redacted numeric JSON-RPC `RequestError` details in runtime failure text, so backend diagnostics are visible instead of only `Internal error`. Fixes #81126. (#81188) Thanks @vyctorbrzezowski. +- Agents: cache unchanged PI model discovery stores and model lookups, reducing repeated model-resolution startup latency under large model configs. Fixes #78851. - Security/Windows ACL audit: classify Anonymous Logon, Guests, Interactive, Local, and Network SIDs as world-equivalent principals so broadly writable paths stay critical instead of being downgraded to group-writable. Fixes #74350. (#74383) Thanks @dwc1997. - Media-understanding: retry transient remote attachment fetch failures before audio or vision processing, so Discord voice notes are not lost after one network/CDN blip. Fixes #74316. Thanks @vyctorbrzezowski and @gabrielexito-stack. - Control UI: order timestamped live stream and tool items before untimestamped history fallbacks, keeping chat history in visible time order. Fixes #80759. (#81016) Thanks @akrimm702. diff --git a/package.json b/package.json index 36a1d9eaa31..14df1396fda 100644 --- a/package.json +++ b/package.json @@ -1487,6 +1487,7 @@ "moltbot:rpc": "node scripts/run-node.mjs agent --mode rpc --json", "openclaw": "node scripts/run-node.mjs", "openclaw:rpc": "node scripts/run-node.mjs agent --mode rpc --json", + "perf:issue-78851": "node --import tsx scripts/perf/issue-78851-model-resolution.ts", "perf:kova:summary": "node scripts/kova-ci-summary.mjs", "perf:source:summary": "node scripts/openclaw-performance-source-summary.mjs", "plugin-sdk:api:check": "node --max-old-space-size=8192 --import tsx scripts/generate-plugin-sdk-api-baseline.ts --check", diff --git a/scripts/perf/issue-78851-model-resolution.ts b/scripts/perf/issue-78851-model-resolution.ts new file mode 100644 index 00000000000..8ce3ba19937 --- /dev/null +++ b/scripts/perf/issue-78851-model-resolution.ts @@ -0,0 +1,499 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import * as inspector from "node:inspector"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { + ensureOpenClawModelsJson, + resetModelsJsonReadyCacheForTest, +} from "../../src/agents/models-config.js"; +import { resolveModelAsync } from "../../src/agents/pi-embedded-runner/model.js"; +import type { OpenClawConfig } from "../../src/config/types.openclaw.js"; + +type Options = { + agentCount: number; + cpuProfDir?: string; + cpuProfOutput?: string; + json: boolean; + keepTemp: boolean; + lookupsPerRun: number; + modelsPerProvider: number; + output?: string; + providers: number; + runs: number; + runtimeHooks: boolean; + warmup: number; +}; + +type PhaseSample = { + ensureMs: number; + resolveMs: number; + totalMs: number; + wrote: boolean; +}; + +type RunSample = { + cold: PhaseSample; + eventLoopDelayMaxMs: number; + eventLoopDelayMeanMs: number; + index: number; + rssMb: number; + warm: PhaseSample; +}; + +type SummaryStats = { + avg: number; + max: number; + min: number; + p50: number; + p95: number; +}; + +type Report = { + scenario: string; + options: Omit; + samples: RunSample[]; + summary: { + coldEnsureMs: SummaryStats; + coldResolveMs: SummaryStats; + coldTotalMs: SummaryStats; + warmEnsureMs: SummaryStats; + warmResolveMs: SummaryStats; + warmTotalMs: SummaryStats; + eventLoopDelayMaxMs: SummaryStats; + rssMb: SummaryStats; + }; + tempRoot: string; + cpuProfilePath?: string; +}; + +function parseFlagValue(flag: string): string | undefined { + const index = process.argv.indexOf(flag); + if (index === -1) { + return undefined; + } + return process.argv[index + 1]; +} + +function hasFlag(flag: string): boolean { + return process.argv.includes(flag); +} + +function parsePositiveInt(flag: string, fallback: number): number { + const raw = parseFlagValue(flag); + if (!raw) { + return fallback; + } + const value = Number.parseInt(raw, 10); + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${flag} must be a positive integer`); + } + return value; +} + +function parseNonNegativeInt(flag: string, fallback: number): number { + const raw = parseFlagValue(flag); + if (!raw) { + return fallback; + } + const value = Number.parseInt(raw, 10); + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${flag} must be a non-negative integer`); + } + return value; +} + +function parseOptions(): Options { + return { + agentCount: parsePositiveInt("--agents", 8), + cpuProfDir: parseFlagValue("--cpu-prof-dir"), + cpuProfOutput: parseFlagValue("--cpu-prof-output"), + json: hasFlag("--json"), + keepTemp: hasFlag("--keep-temp"), + lookupsPerRun: parsePositiveInt("--lookups", 32), + modelsPerProvider: parsePositiveInt("--models-per-provider", 16), + output: parseFlagValue("--output"), + providers: parsePositiveInt("--providers", 48), + runs: parsePositiveInt("--runs", 8), + runtimeHooks: hasFlag("--runtime-hooks"), + warmup: parseNonNegativeInt("--warmup", 1), + }; +} + +function printUsage(): void { + process.stdout.write(`OpenClaw issue #78851 model-resolution profiler + +Usage: + pnpm perf:issue-78851 -- [options] + node --import tsx scripts/perf/issue-78851-model-resolution.ts [options] + +Options: + --providers Synthetic configured providers (default: 48) + --models-per-provider Models per provider (default: 16) + --agents Agent configs/fallback chains (default: 8) + --lookups resolveModelAsync calls per phase (default: 32) + --runs Measured runs (default: 8) + --warmup Warmup runs before measurement (default: 1) + --cpu-prof-dir Write a V8 .cpuprofile for the measured loop + --cpu-prof-output Write the V8 .cpuprofile to this exact path + --runtime-hooks Include provider runtime hook resolution + --output Write JSON report + --json Print JSON report + --keep-temp Keep generated temp state + --help, -h Show this text +`); +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} + +function percentile(values: number[], p: number): number { + if (values.length === 0) { + return 0; + } + const sorted = values.toSorted((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.floor((sorted.length - 1) * p)); + return round(sorted[index] ?? 0); +} + +function stats(values: number[]): SummaryStats { + if (values.length === 0) { + return { avg: 0, max: 0, min: 0, p50: 0, p95: 0 }; + } + const total = values.reduce((sum, value) => sum + value, 0); + return { + avg: round(total / values.length), + max: round(Math.max(...values)), + min: round(Math.min(...values)), + p50: percentile(values, 0.5), + p95: percentile(values, 0.95), + }; +} + +function modelRef(providerIndex: number, modelIndex: number): string { + return `perf-${providerIndex}/perf-model-${modelIndex}`; +} + +function buildConfig(options: Options, workspaceDir: string): OpenClawConfig { + const providers: NonNullable["providers"]> = {}; + for (let providerIndex = 0; providerIndex < options.providers; providerIndex += 1) { + providers[`perf-${providerIndex}`] = { + api: providerIndex % 2 === 0 ? "openai-responses" : "openai-completions", + apiKey: "perf-key", + baseUrl: `http://127.0.0.1:${20_000 + providerIndex}/v1`, + models: Array.from({ length: options.modelsPerProvider }, (_, modelIndex) => ({ + api: modelIndex % 2 === 0 ? "openai-responses" : "openai-completions", + baseUrl: `http://127.0.0.1:${20_000 + providerIndex}/v1`, + contextWindow: 128_000 + modelIndex, + cost: { cacheRead: 0, cacheWrite: 0, input: 0, output: 0 }, + id: `perf-model-${modelIndex}`, + input: modelIndex % 5 === 0 ? ["text", "image"] : ["text"], + maxTokens: 8192, + name: `Perf Model ${providerIndex}.${modelIndex}`, + params: { + cacheRetention: modelIndex % 3 === 0 ? "ephemeral" : undefined, + syntheticRank: providerIndex * options.modelsPerProvider + modelIndex, + }, + reasoning: modelIndex % 3 === 0, + })), + params: { + syntheticProviderRank: providerIndex, + }, + }; + } + + const fallbacks = Array.from({ length: Math.min(12, options.providers) }, (_, index) => + modelRef(index, index % options.modelsPerProvider), + ); + + return { + browser: { enabled: false }, + agents: { + defaults: { + contextInjection: "never", + model: { + primary: modelRef(0, 0), + fallbacks, + }, + skipBootstrap: true, + workspace: workspaceDir, + }, + list: Array.from({ length: options.agentCount }, (_, index) => ({ + default: index === 0, + id: `agent-${index}`, + model: { + primary: modelRef(index % options.providers, index % options.modelsPerProvider), + fallbacks: fallbacks.toReversed(), + }, + workspace: path.join(workspaceDir, `agent-${index}`), + })), + }, + gateway: { + auth: { mode: "none" }, + bind: "loopback", + controlUi: { enabled: false }, + mode: "local", + }, + memory: { + active: { + allowedChatTypes: ["direct"], + agents: ["main"], + logging: false, + maxSummaryChars: 220, + persistTranscripts: false, + promptStyle: "balanced", + queryMode: "recent", + timeoutMs: 15_000, + }, + }, + models: { + mode: "replace", + providers, + }, + plugins: { + enabled: true, + entries: { + browser: { enabled: false }, + }, + }, + }; +} + +async function startCpuProfile(params: { dir?: string; output?: string }): Promise<{ + stop: () => Promise; +}> { + const fallbackDir = ".artifacts/perf/issue-78851/cpu"; + const cpuProfDir = params.dir ?? path.dirname(params.output ?? fallbackDir); + await mkdir(cpuProfDir, { recursive: true }); + const session = new inspector.Session(); + session.connect(); + const post = (method: string, params?: Record) => + new Promise((resolve, reject) => { + session.post(method, params ?? {}, (error, result) => { + if (error) { + reject(error); + } else { + resolve(result as T); + } + }); + }); + await post("Profiler.enable"); + await post("Profiler.start"); + return { + async stop() { + const result = await post<{ profile: unknown }>("Profiler.stop"); + session.disconnect(); + const profilePath = + params.output ?? + path.join(cpuProfDir, `issue-78851-${process.pid}-${Date.now()}.cpuprofile`); + await mkdir(path.dirname(profilePath), { recursive: true }); + await writeFile(profilePath, JSON.stringify(result.profile)); + return profilePath; + }, + }; +} + +async function measurePhase(params: { + agentDir: string; + config: OpenClawConfig; + lookups: number; + modelIndexOffset: number; + providerCount: number; + modelsPerProvider: number; + workspaceDir: string; + runtimeHooks: boolean; +}): Promise { + const started = performance.now(); + const ensureStarted = performance.now(); + const ensureResult = await ensureOpenClawModelsJson(params.config, params.agentDir, { + // Keep this harness deterministic by measuring configured-model scale. + // Live provider catalog timing belongs in a separate Crabbox lane with secrets. + providerDiscoveryProviderIds: [], + providerDiscoveryTimeoutMs: 5_000, + workspaceDir: params.workspaceDir, + }); + const ensureMs = performance.now() - ensureStarted; + const resolveStarted = performance.now(); + for (let lookupIndex = 0; lookupIndex < params.lookups; lookupIndex += 1) { + const providerIndex = lookupIndex % params.providerCount; + const modelIndex = (lookupIndex + params.modelIndexOffset) % params.modelsPerProvider; + const resolved = await resolveModelAsync( + `perf-${providerIndex}`, + `perf-model-${modelIndex}`, + params.agentDir, + params.config, + { + skipProviderRuntimeHooks: !params.runtimeHooks, + workspaceDir: params.workspaceDir, + }, + ); + if (!resolved.model) { + throw new Error(resolved.error ?? `failed to resolve ${modelRef(providerIndex, modelIndex)}`); + } + } + const resolveMs = performance.now() - resolveStarted; + return { + ensureMs: round(ensureMs), + resolveMs: round(resolveMs), + totalMs: round(performance.now() - started), + wrote: ensureResult.wrote, + }; +} + +async function runOne(params: { + config: OpenClawConfig; + index: number; + options: Options; + tempRoot: string; + workspaceDir: string; +}): Promise { + const agentDir = path.join(params.tempRoot, `agent-state-${params.index}`); + await mkdir(agentDir, { recursive: true }); + resetModelsJsonReadyCacheForTest(); + const histogram = monitorEventLoopDelay({ resolution: 10 }); + histogram.enable(); + const cold = await measurePhase({ + agentDir, + config: params.config, + lookups: params.options.lookupsPerRun, + modelIndexOffset: params.index, + modelsPerProvider: params.options.modelsPerProvider, + providerCount: params.options.providers, + workspaceDir: params.workspaceDir, + runtimeHooks: params.options.runtimeHooks, + }); + const warm = await measurePhase({ + agentDir, + config: params.config, + lookups: params.options.lookupsPerRun, + modelIndexOffset: params.index + 1, + modelsPerProvider: params.options.modelsPerProvider, + providerCount: params.options.providers, + workspaceDir: params.workspaceDir, + runtimeHooks: params.options.runtimeHooks, + }); + histogram.disable(); + return { + cold, + eventLoopDelayMaxMs: round(histogram.max / 1_000_000), + eventLoopDelayMeanMs: round(histogram.mean / 1_000_000), + index: params.index, + rssMb: round(process.memoryUsage().rss / 1024 / 1024), + warm, + }; +} + +function summarize(samples: RunSample[]): Report["summary"] { + return { + coldEnsureMs: stats(samples.map((sample) => sample.cold.ensureMs)), + coldResolveMs: stats(samples.map((sample) => sample.cold.resolveMs)), + coldTotalMs: stats(samples.map((sample) => sample.cold.totalMs)), + eventLoopDelayMaxMs: stats(samples.map((sample) => sample.eventLoopDelayMaxMs)), + rssMb: stats(samples.map((sample) => sample.rssMb)), + warmEnsureMs: stats(samples.map((sample) => sample.warm.ensureMs)), + warmResolveMs: stats(samples.map((sample) => sample.warm.resolveMs)), + warmTotalMs: stats(samples.map((sample) => sample.warm.totalMs)), + }; +} + +function printHuman(report: Report, cpuProfilePath?: string): void { + const lines = [ + `scenario: ${report.scenario}`, + `providers: ${report.options.providers}`, + `modelsPerProvider: ${report.options.modelsPerProvider}`, + `agents: ${report.options.agentCount}`, + `lookups: ${report.options.lookupsPerRun}`, + `runs: ${report.options.runs}`, + `runtimeHooks: ${report.options.runtimeHooks}`, + `coldTotalMs: avg=${report.summary.coldTotalMs.avg} p50=${report.summary.coldTotalMs.p50} p95=${report.summary.coldTotalMs.p95} max=${report.summary.coldTotalMs.max}`, + `coldEnsureMs: avg=${report.summary.coldEnsureMs.avg} p50=${report.summary.coldEnsureMs.p50} p95=${report.summary.coldEnsureMs.p95} max=${report.summary.coldEnsureMs.max}`, + `coldResolveMs: avg=${report.summary.coldResolveMs.avg} p50=${report.summary.coldResolveMs.p50} p95=${report.summary.coldResolveMs.p95} max=${report.summary.coldResolveMs.max}`, + `warmTotalMs: avg=${report.summary.warmTotalMs.avg} p50=${report.summary.warmTotalMs.p50} p95=${report.summary.warmTotalMs.p95} max=${report.summary.warmTotalMs.max}`, + `warmEnsureMs: avg=${report.summary.warmEnsureMs.avg} p50=${report.summary.warmEnsureMs.p50} p95=${report.summary.warmEnsureMs.p95} max=${report.summary.warmEnsureMs.max}`, + `warmResolveMs: avg=${report.summary.warmResolveMs.avg} p50=${report.summary.warmResolveMs.p50} p95=${report.summary.warmResolveMs.p95} max=${report.summary.warmResolveMs.max}`, + `eventLoopDelayMaxMs: avg=${report.summary.eventLoopDelayMaxMs.avg} max=${report.summary.eventLoopDelayMaxMs.max}`, + `rssMb: avg=${report.summary.rssMb.avg} max=${report.summary.rssMb.max}`, + ]; + if (report.options.output) { + lines.push(`output: ${report.options.output}`); + } + if (report.cpuProfilePath ?? cpuProfilePath) { + lines.push(`cpuProfile: ${report.cpuProfilePath ?? cpuProfilePath}`); + } + process.stdout.write(`${lines.join("\n")}\n`); +} + +async function main(): Promise { + if (hasFlag("--help") || hasFlag("-h")) { + printUsage(); + return; + } + const options = parseOptions(); + const tempRoot = await mkdtemp(path.join(tmpdir(), "openclaw-issue-78851-")); + const workspaceDir = path.join(tempRoot, "workspace"); + await mkdir(workspaceDir, { recursive: true }); + const config = buildConfig(options, workspaceDir); + let profiler: Awaited> | undefined; + let cpuProfilePath: string | undefined; + try { + if (options.cpuProfDir ?? options.cpuProfOutput) { + profiler = await startCpuProfile({ + dir: options.cpuProfDir, + output: options.cpuProfOutput, + }); + } + for (let index = 0; index < options.warmup; index += 1) { + await runOne({ config, index: -index - 1, options, tempRoot, workspaceDir }); + } + const samples: RunSample[] = []; + for (let index = 0; index < options.runs; index += 1) { + samples.push(await runOne({ config, index, options, tempRoot, workspaceDir })); + } + if (profiler) { + cpuProfilePath = await profiler.stop(); + profiler = undefined; + } + const report: Report = { + options: { + agentCount: options.agentCount, + cpuProfDir: options.cpuProfDir, + cpuProfOutput: options.cpuProfOutput, + lookupsPerRun: options.lookupsPerRun, + modelsPerProvider: options.modelsPerProvider, + output: options.output, + providers: options.providers, + runs: options.runs, + runtimeHooks: options.runtimeHooks, + warmup: options.warmup, + }, + samples, + scenario: "issue-78851-model-resolution", + summary: summarize(samples), + tempRoot, + ...(cpuProfilePath ? { cpuProfilePath } : {}), + }; + if (options.output) { + await mkdir(path.dirname(options.output), { recursive: true }); + await writeFile(options.output, `${JSON.stringify(report, null, 2)}\n`); + } + if (options.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + printHuman(report, cpuProfilePath); + } + } finally { + if (profiler) { + await profiler.stop().catch(() => undefined); + } + if (!options.keepTemp) { + await rm(tempRoot, { recursive: true, force: true }); + } + } +} + +main().catch((error: unknown) => { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + process.stderr.write(`${message}\n`); + process.exit(1); +}); diff --git a/src/agents/pi-embedded-runner/model-discovery-cache.ts b/src/agents/pi-embedded-runner/model-discovery-cache.ts new file mode 100644 index 00000000000..bfb3ef5e432 --- /dev/null +++ b/src/agents/pi-embedded-runner/model-discovery-cache.ts @@ -0,0 +1,124 @@ +import { statSync } from "node:fs"; +import path from "node:path"; +import type { AuthStorage, ModelRegistry } from "@earendil-works/pi-coding-agent"; +import { + resolveRuntimeExternalAuthProviderRefs, + resolveRuntimeSyntheticAuthProviderRefs, +} from "../../plugins/synthetic-auth.runtime.js"; +import { resolveDefaultAgentDir } from "../agent-scope.js"; +import { hasAnyRuntimeAuthProfileStoreSource } from "../auth-profiles/runtime-snapshots.js"; +import { discoverAuthStorage, discoverModels } from "../pi-model-discovery.js"; + +type DiscoveryStores = { + authStorage: AuthStorage; + modelRegistry: ModelRegistry; +}; + +type DiscoverCachedPiStoresOptions = { + agentDir: string; + inheritedAuthDir?: string; +}; + +type CacheEntry = DiscoveryStores & { + fingerprint: string; + lastUsedAt: number; +}; + +const MAX_DISCOVERY_STORE_CACHE_ENTRIES = 64; +const DISCOVERY_STORE_CACHE = new Map(); + +function fileFingerprint(pathname: string): { mtimeMs: number; size: number } | null { + try { + const stat = statSync(pathname); + return Number.isFinite(stat.mtimeMs) ? { mtimeMs: stat.mtimeMs, size: stat.size } : null; + } catch { + return null; + } +} + +function normalizeCacheDir(dirname: string | undefined): string | undefined { + return dirname ? path.resolve(dirname) : undefined; +} + +function authFingerprint(agentDir: string): object { + return { + authJson: fileFingerprint(path.join(agentDir, "auth.json")), + authProfilesJson: fileFingerprint(path.join(agentDir, "auth-profiles.json")), + }; +} + +function discoveryFingerprint(params: DiscoverCachedPiStoresOptions): string { + const inheritedAuthDir = + params.inheritedAuthDir && params.inheritedAuthDir !== params.agentDir + ? params.inheritedAuthDir + : undefined; + return JSON.stringify({ + agentDir: params.agentDir, + inheritedAuthDir, + localAuth: authFingerprint(params.agentDir), + inheritedAuth: inheritedAuthDir ? authFingerprint(inheritedAuthDir) : undefined, + modelsJson: fileFingerprint(path.join(params.agentDir, "models.json")), + }); +} + +function hasRuntimePluginAuthSources(): boolean { + return ( + resolveRuntimeSyntheticAuthProviderRefs().length > 0 || + resolveRuntimeExternalAuthProviderRefs().length > 0 + ); +} + +function pruneDiscoveryStoreCache(): void { + if (DISCOVERY_STORE_CACHE.size <= MAX_DISCOVERY_STORE_CACHE_ENTRIES) { + return; + } + const overflow = DISCOVERY_STORE_CACHE.size - MAX_DISCOVERY_STORE_CACHE_ENTRIES; + const oldestKeys = [...DISCOVERY_STORE_CACHE.entries()] + .toSorted((left, right) => left[1].lastUsedAt - right[1].lastUsedAt) + .slice(0, overflow) + .map(([key]) => key); + for (const key of oldestKeys) { + DISCOVERY_STORE_CACHE.delete(key); + } +} + +function discoverFreshPiStores(agentDir: string): DiscoveryStores { + const authStorage = discoverAuthStorage(agentDir); + const modelRegistry = discoverModels(authStorage, agentDir); + return { authStorage, modelRegistry }; +} + +export function discoverCachedPiStores(options: DiscoverCachedPiStoresOptions): DiscoveryStores { + const agentDir = normalizeCacheDir(options.agentDir) ?? options.agentDir; + const inheritedAuthDir = normalizeCacheDir( + options.inheritedAuthDir ?? resolveDefaultAgentDir({}), + ); + if (hasAnyRuntimeAuthProfileStoreSource(agentDir) || hasRuntimePluginAuthSources()) { + return discoverFreshPiStores(agentDir); + } + + const cacheKey = JSON.stringify({ agentDir, inheritedAuthDir }); + const fingerprint = discoveryFingerprint({ agentDir, inheritedAuthDir }); + const cached = DISCOVERY_STORE_CACHE.get(cacheKey); + if (cached?.fingerprint === fingerprint) { + cached.lastUsedAt = Date.now(); + return { + authStorage: cached.authStorage, + modelRegistry: cached.modelRegistry, + }; + } + + const stores = discoverFreshPiStores(agentDir); + DISCOVERY_STORE_CACHE.set(cacheKey, { + authStorage: stores.authStorage, + fingerprint, + lastUsedAt: Date.now(), + modelRegistry: stores.modelRegistry, + }); + pruneDiscoveryStoreCache(); + return stores; +} + +export function resetModelDiscoveryCacheForTest(): void { + DISCOVERY_STORE_CACHE.clear(); +} diff --git a/src/agents/pi-embedded-runner/model.test.ts b/src/agents/pi-embedded-runner/model.test.ts index c369bf988cc..84e7b8e48a7 100644 --- a/src/agents/pi-embedded-runner/model.test.ts +++ b/src/agents/pi-embedded-runner/model.test.ts @@ -1,8 +1,18 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + clearRuntimeAuthProfileStoreSnapshots, + replaceRuntimeAuthProfileStoreSnapshots, +} from "../auth-profiles.js"; import { discoverAuthStorage, discoverModels } from "../pi-model-discovery.js"; +import { resetModelDiscoveryCacheForTest } from "./model-discovery-cache.js"; import { createProviderRuntimeTestMock } from "./model.provider-runtime.test-support.js"; const resolveBundledStaticCatalogModelMock = vi.hoisted(() => vi.fn()); +const resolveRuntimeSyntheticAuthProviderRefsMock = vi.hoisted(() => vi.fn((): string[] => [])); +const resolveRuntimeExternalAuthProviderRefsMock = vi.hoisted(() => vi.fn((): string[] => [])); vi.mock("../model-suppression.js", () => { // Mirrors the canonical manifest-driven suppression in @@ -142,6 +152,11 @@ vi.mock("../pi-model-discovery.js", () => ({ discoverModels: vi.fn(() => ({ find: vi.fn(() => null) })), })); +vi.mock("../../plugins/synthetic-auth.runtime.js", () => ({ + resolveRuntimeSyntheticAuthProviderRefs: resolveRuntimeSyntheticAuthProviderRefsMock, + resolveRuntimeExternalAuthProviderRefs: resolveRuntimeExternalAuthProviderRefsMock, +})); + vi.mock("./model.static-catalog.js", () => ({ resolveBundledStaticCatalogModel: resolveBundledStaticCatalogModelMock, })); @@ -180,9 +195,15 @@ import { } from "./model.test-harness.js"; beforeEach(() => { + clearRuntimeAuthProfileStoreSnapshots(); + resetModelDiscoveryCacheForTest(); resetMockDiscoverModels(discoverModels); vi.mocked(discoverModels).mockClear(); vi.mocked(discoverAuthStorage).mockClear(); + resolveRuntimeSyntheticAuthProviderRefsMock.mockReset(); + resolveRuntimeSyntheticAuthProviderRefsMock.mockReturnValue([]); + resolveRuntimeExternalAuthProviderRefsMock.mockReset(); + resolveRuntimeExternalAuthProviderRefsMock.mockReturnValue([]); mockGetOpenRouterModelCapabilities.mockReset(); mockGetOpenRouterModelCapabilities.mockReturnValue(undefined); mockLoadOpenRouterModelCapabilities.mockReset(); @@ -190,6 +211,10 @@ beforeEach(() => { resolveBundledStaticCatalogModelMock.mockReset(); }); +afterEach(() => { + vi.unstubAllEnvs(); +}); + function createRuntimeHooks() { return createProviderRuntimeTestMock({ handledDynamicProviders: [ @@ -272,6 +297,160 @@ function mockCallArg(mock: ReturnType, callIndex = 0): Record { + it("reuses PI discovery stores while the agent model files are unchanged", async () => { + mockDiscoveredModel(discoverModels, { + provider: "openai", + modelId: "gpt-5.5", + templateModel: { + provider: "openai", + ...makeModel("gpt-5.5"), + }, + }); + + const first = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + const second = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + + expectResolvedModel(first); + expectResolvedModel(second); + expect(discoverAuthStorage).toHaveBeenCalledTimes(1); + expect(discoverModels).toHaveBeenCalledTimes(1); + }); + + it("invalidates PI discovery stores when inherited default auth changes", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-cache-")); + const agentDir = path.join(rootDir, "agent"); + const defaultAgentDir = path.join(rootDir, "default-agent"); + fs.mkdirSync(agentDir, { recursive: true }); + fs.mkdirSync(defaultAgentDir, { recursive: true }); + const cfg = { + agents: { + list: [ + { id: "main", default: true, agentDir: defaultAgentDir }, + { id: "worker", agentDir }, + ], + }, + } as unknown as OpenClawConfig; + mockDiscoveredModel(discoverModels, { + provider: "openai", + modelId: "gpt-5.5", + templateModel: { + provider: "openai", + ...makeModel("gpt-5.5"), + }, + }); + + const first = await resolveModelAsync("openai", "gpt-5.5", agentDir, cfg, { + runtimeHooks: createRuntimeHooks(), + }); + fs.writeFileSync( + path.join(defaultAgentDir, "auth-profiles.json"), + JSON.stringify({ version: 1, profiles: { openai: { type: "api_key", key: "one" } } }), + ); + const second = await resolveModelAsync("openai", "gpt-5.5", agentDir, cfg, { + runtimeHooks: createRuntimeHooks(), + }); + + expectResolvedModel(first); + expectResolvedModel(second); + expect(discoverAuthStorage).toHaveBeenCalledTimes(2); + expect(discoverModels).toHaveBeenCalledTimes(2); + }); + + it("invalidates PI discovery stores when implicit main auth changes without config", async () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-cache-state-")); + vi.stubEnv("OPENCLAW_STATE_DIR", rootDir); + const agentDir = path.join(rootDir, "agents", "worker", "agent"); + const mainAgentDir = path.join(rootDir, "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + fs.mkdirSync(mainAgentDir, { recursive: true }); + mockDiscoveredModel(discoverModels, { + provider: "openai", + modelId: "gpt-5.5", + templateModel: { + provider: "openai", + ...makeModel("gpt-5.5"), + }, + }); + + const first = await resolveModelAsync("openai", "gpt-5.5", agentDir, undefined, { + runtimeHooks: createRuntimeHooks(), + }); + fs.writeFileSync( + path.join(mainAgentDir, "auth-profiles.json"), + JSON.stringify({ version: 1, profiles: { openai: { type: "api_key", key: "one" } } }), + ); + const second = await resolveModelAsync("openai", "gpt-5.5", agentDir, undefined, { + runtimeHooks: createRuntimeHooks(), + }); + + expectResolvedModel(first); + expectResolvedModel(second); + expect(discoverAuthStorage).toHaveBeenCalledTimes(2); + expect(discoverModels).toHaveBeenCalledTimes(2); + }); + + it("does not cache PI discovery stores while runtime auth snapshots are active", async () => { + replaceRuntimeAuthProfileStoreSnapshots([ + { + store: { + version: 1, + profiles: { + openai: { type: "api_key", key: "one" }, + }, + } as never, + }, + ]); + mockDiscoveredModel(discoverModels, { + provider: "openai", + modelId: "gpt-5.5", + templateModel: { + provider: "openai", + ...makeModel("gpt-5.5"), + }, + }); + + const first = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + const second = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + + expectResolvedModel(first); + expectResolvedModel(second); + expect(discoverAuthStorage).toHaveBeenCalledTimes(2); + expect(discoverModels).toHaveBeenCalledTimes(2); + }); + + it("does not cache PI discovery stores while plugin auth overlays are active", async () => { + resolveRuntimeSyntheticAuthProviderRefsMock.mockReturnValue(["runtime-provider"]); + resolveRuntimeExternalAuthProviderRefsMock.mockReturnValue(["external-provider"]); + mockDiscoveredModel(discoverModels, { + provider: "openai", + modelId: "gpt-5.5", + templateModel: { + provider: "openai", + ...makeModel("gpt-5.5"), + }, + }); + + const first = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + const second = await resolveModelAsync("openai", "gpt-5.5", "/tmp/agent", undefined, { + runtimeHooks: createRuntimeHooks(), + }); + + expectResolvedModel(first); + expectResolvedModel(second); + expect(discoverAuthStorage).toHaveBeenCalledTimes(2); + expect(discoverModels).toHaveBeenCalledTimes(2); + }); + it("skips PI auth and model discovery during dynamic model resolution", async () => { const result = await resolveModelAsync( "openrouter", diff --git a/src/agents/pi-embedded-runner/model.ts b/src/agents/pi-embedded-runner/model.ts index fc9a573285a..f7d313d2595 100644 --- a/src/agents/pi-embedded-runner/model.ts +++ b/src/agents/pi-embedded-runner/model.ts @@ -34,6 +34,7 @@ import { resolveProviderRequestConfig, sanitizeConfiguredModelProviderRequest, } from "../provider-request-config.js"; +import { discoverCachedPiStores } from "./model-discovery-cache.js"; import { buildInlineProviderModels, type InlineProviderConfig, @@ -134,6 +135,19 @@ function resolveRuntimeHooks(params?: { return DEFAULT_PROVIDER_RUNTIME_HOOKS; } +function discoverCachedPiStoresForAgent( + resolvedAgentDir: string, + cfg: OpenClawConfig | undefined, +): { + authStorage: AuthStorage; + modelRegistry: ModelRegistry; +} { + return discoverCachedPiStores({ + agentDir: resolvedAgentDir, + inheritedAuthDir: resolveDefaultAgentDir(cfg ?? {}), + }); +} + function canonicalizeLegacyResolvedModel(params: { provider: string; model: Model; @@ -1031,8 +1045,16 @@ export function resolveModel( }; const resolvedAgentDir = agentDir ?? resolveDefaultAgentDir(cfg ?? {}); const workspaceDir = options?.workspaceDir ?? cfg?.agents?.defaults?.workspace; - const authStorage = options?.authStorage ?? discoverAuthStorage(resolvedAgentDir); - const modelRegistry = options?.modelRegistry ?? discoverModels(authStorage, resolvedAgentDir); + const cachedStores = + !options?.authStorage && !options?.modelRegistry + ? discoverCachedPiStoresForAgent(resolvedAgentDir, cfg) + : undefined; + const authStorage = + options?.authStorage ?? cachedStores?.authStorage ?? discoverAuthStorage(resolvedAgentDir); + const modelRegistry = + options?.modelRegistry ?? + cachedStores?.modelRegistry ?? + discoverModels(authStorage, resolvedAgentDir); const runtimeHooks = resolveRuntimeHooks(options); const model = resolveModelWithRegistry({ provider: normalizedRef.provider, @@ -1092,13 +1114,19 @@ export async function resolveModelAsync( options?.skipPiDiscovery && (!options.authStorage || !options.modelRegistry) ? createEmptyPiDiscoveryStores() : undefined; + const cachedStores = + !emptyDiscoveryStores && !options?.authStorage && !options?.modelRegistry + ? discoverCachedPiStoresForAgent(resolvedAgentDir, cfg) + : undefined; const authStorage = options?.authStorage ?? emptyDiscoveryStores?.authStorage ?? + cachedStores?.authStorage ?? discoverAuthStorage(resolvedAgentDir); const modelRegistry = options?.modelRegistry ?? emptyDiscoveryStores?.modelRegistry ?? + cachedStores?.modelRegistry ?? discoverModels(authStorage, resolvedAgentDir); const runtimeHooks = resolveRuntimeHooks(options); const explicitModel = resolveExplicitModelWithRegistry({ diff --git a/src/agents/pi-model-discovery.test.ts b/src/agents/pi-model-discovery.test.ts new file mode 100644 index 00000000000..7b1c9b2ba3d --- /dev/null +++ b/src/agents/pi-model-discovery.test.ts @@ -0,0 +1,38 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { discoverAuthStorage, discoverModels } from "./pi-model-discovery.js"; + +function writeModelsJson(agentDir: string, modelId: string): void { + fs.writeFileSync( + path.join(agentDir, "models.json"), + JSON.stringify({ + providers: { + custom: { + baseUrl: "https://example.test/v1", + apiKey: "sk-test", + api: "openai", + models: [{ id: modelId, name: modelId }], + }, + }, + }), + ); +} + +describe("discoverModels", () => { + it("clears cached find results when the PI registry refreshes", () => { + const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-pi-models-")); + writeModelsJson(agentDir, "old-model"); + const authStorage = discoverAuthStorage(agentDir, { skipCredentials: true }); + const registry = discoverModels(authStorage, agentDir, { normalizeModels: false }); + + expect(registry.find("custom", "new-model")).toBeUndefined(); + + writeModelsJson(agentDir, "new-model"); + registry.refresh(); + + expect(registry.getAll().some((model) => model.id === "new-model")).toBe(true); + expect(registry.find("custom", "new-model")?.id).toBe("new-model"); + }); +}); diff --git a/src/agents/pi-model-discovery.ts b/src/agents/pi-model-discovery.ts index 9033297b3c1..bb8e7bc7be8 100644 --- a/src/agents/pi-model-discovery.ts +++ b/src/agents/pi-model-discovery.ts @@ -147,10 +147,14 @@ function createOpenClawModelRegistry( const getAll = registry.getAll.bind(registry); const getAvailable = registry.getAvailable.bind(registry); const find = registry.find.bind(registry); + const refresh = registry.refresh.bind(registry); const providerFilter = options?.providerFilter ? normalizeProviderId(options.providerFilter) : ""; const matchesProviderFilter = (entry: Model) => !providerFilter || normalizeProviderId(entry.provider) === providerFilter; const shouldNormalize = options?.normalizeModels !== false; + const findCache = new Map | undefined>(); + const normalizeEntry = (entry: Model) => + shouldNormalize ? normalizeDiscoveredPiModel(entry, agentDir) : entry; registry.getAll = () => { const entries = getAll().filter((entry: Model) => matchesProviderFilter(entry)); @@ -164,10 +168,21 @@ function createOpenClawModelRegistry( ? entries.map((entry: Model) => normalizeDiscoveredPiModel(entry, agentDir)) : entries; }; - registry.find = (provider: string, modelId: string) => - shouldNormalize - ? normalizeDiscoveredPiModel(find(provider, modelId), agentDir) - : find(provider, modelId); + registry.find = (provider: string, modelId: string) => { + const normalizedProvider = normalizeProviderId(provider); + const key = `${normalizedProvider}\0${modelId}`; + if (findCache.has(key)) { + return findCache.get(key); + } + const fallbackEntry = find(provider, modelId); + const resolved = fallbackEntry ? normalizeEntry(fallbackEntry) : undefined; + findCache.set(key, resolved); + return resolved; + }; + registry.refresh = () => { + findCache.clear(); + return refresh(); + }; return registry; } diff --git a/src/plugins/synthetic-auth.runtime.test.ts b/src/plugins/synthetic-auth.runtime.test.ts index 61a775e7bd2..9f13aaab672 100644 --- a/src/plugins/synthetic-auth.runtime.test.ts +++ b/src/plugins/synthetic-auth.runtime.test.ts @@ -8,6 +8,11 @@ type SyntheticAuthRegistrySnapshotResult = { diagnostics: []; }; +type ExternalAuthManifestRegistryResult = { + plugins: Array<{ contracts?: { externalAuthProviders?: string[] } }>; + diagnostics: []; +}; + const getPluginRegistryState = vi.hoisted(() => vi.fn()); const pluginRegistryMocks = vi.hoisted(() => ({ loadPluginRegistrySnapshotWithMetadata: vi.fn( @@ -17,6 +22,12 @@ const pluginRegistryMocks = vi.hoisted(() => ({ diagnostics: [], }), ), + loadPluginManifestRegistryForInstalledIndex: vi.fn<() => ExternalAuthManifestRegistryResult>( + () => ({ + plugins: [], + diagnostics: [], + }), + ), })); vi.mock("./runtime-state.js", () => ({ @@ -28,7 +39,15 @@ vi.mock("./plugin-registry.js", () => ({ pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata, })); -import { resolveRuntimeSyntheticAuthProviderRefs } from "./synthetic-auth.runtime.js"; +vi.mock("./manifest-registry-installed.js", () => ({ + loadPluginManifestRegistryForInstalledIndex: + pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex, +})); + +import { + resolveRuntimeExternalAuthProviderRefs, + resolveRuntimeSyntheticAuthProviderRefs, +} from "./synthetic-auth.runtime.js"; describe("synthetic auth runtime refs", () => { beforeEach(() => { @@ -38,6 +57,10 @@ describe("synthetic auth runtime refs", () => { snapshot: { plugins: [] as Array<{ syntheticAuthRefs?: string[] }> }, diagnostics: [], }); + pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex.mockReset().mockReturnValue({ + plugins: [], + diagnostics: [], + } satisfies ExternalAuthManifestRegistryResult); }); it("uses persisted registry synthetic auth refs before the runtime registry exists", () => { @@ -61,6 +84,30 @@ describe("synthetic auth runtime refs", () => { expect(pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledWith({}); }); + it("uses persisted registry external auth provider refs before the runtime registry exists", () => { + const snapshot = { + plugins: [{ syntheticAuthRefs: [] }], + }; + pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot, + diagnostics: [], + }); + pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex.mockReturnValue({ + plugins: [ + { contracts: { externalAuthProviders: [" runtime-provider ", "runtime-provider"] } }, + { contracts: { externalAuthProviders: ["external-cli"] } }, + { contracts: {} }, + ], + diagnostics: [], + }); + + expect(resolveRuntimeExternalAuthProviderRefs()).toEqual(["runtime-provider", "external-cli"]); + expect(pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledWith({ + index: snapshot, + }); + }); + it("does not derive the registry just to resolve synthetic auth refs", () => { pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ source: "derived", @@ -77,6 +124,17 @@ describe("synthetic auth runtime refs", () => { expect(resolveRuntimeSyntheticAuthProviderRefs()).toStrictEqual([]); }); + it("does not derive the registry just to resolve external auth refs", () => { + pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "derived", + snapshot: { plugins: [] }, + diagnostics: [], + }); + + expect(resolveRuntimeExternalAuthProviderRefs()).toStrictEqual([]); + expect(pluginRegistryMocks.loadPluginManifestRegistryForInstalledIndex).not.toHaveBeenCalled(); + }); + it("prefers the active runtime registry when plugins are already loaded", () => { getPluginRegistryState.mockReturnValue({ activeRegistry: { @@ -101,10 +159,54 @@ describe("synthetic auth runtime refs", () => { }, }, ], + plugins: [ + { + contracts: { + externalAuthProviders: ["manifest-provider"], + }, + }, + ], }, }); expect(resolveRuntimeSyntheticAuthProviderRefs()).toEqual(["runtime-provider", "runtime-cli"]); expect(pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata).not.toHaveBeenCalled(); }); + + it("prefers active runtime registry external auth refs when plugins are already loaded", () => { + getPluginRegistryState.mockReturnValue({ + activeRegistry: { + plugins: [ + { + contracts: { + externalAuthProviders: ["manifest-provider"], + }, + }, + ], + providers: [ + { + provider: { + id: "runtime-provider", + resolveExternalAuthProfiles: () => [], + }, + }, + ], + cliBackends: [ + { + backend: { + id: "runtime-cli", + resolveExternalOAuthProfiles: () => [], + }, + }, + ], + }, + }); + + expect(resolveRuntimeExternalAuthProviderRefs()).toEqual([ + "manifest-provider", + "runtime-provider", + "runtime-cli", + ]); + expect(pluginRegistryMocks.loadPluginRegistrySnapshotWithMetadata).not.toHaveBeenCalled(); + }); }); diff --git a/src/plugins/synthetic-auth.runtime.ts b/src/plugins/synthetic-auth.runtime.ts index 0a5b6cc3439..1914198566b 100644 --- a/src/plugins/synthetic-auth.runtime.ts +++ b/src/plugins/synthetic-auth.runtime.ts @@ -1,4 +1,5 @@ import { normalizeProviderId } from "../agents/provider-id.js"; +import { loadPluginManifestRegistryForInstalledIndex } from "./manifest-registry-installed.js"; import { loadPluginRegistrySnapshotWithMetadata } from "./plugin-registry.js"; import { getPluginRegistryState } from "./runtime-state.js"; @@ -27,6 +28,19 @@ function resolveManifestSyntheticAuthProviderRefs(): string[] { ); } +function resolveManifestExternalAuthProviderRefs(): string[] { + const result = loadPluginRegistrySnapshotWithMetadata({}); + if (result.source !== "persisted" && result.source !== "provided") { + return []; + } + const manifestRegistry = loadPluginManifestRegistryForInstalledIndex({ + index: result.snapshot, + }); + return uniqueProviderRefs( + manifestRegistry.plugins.flatMap((plugin) => plugin.contracts?.externalAuthProviders ?? []), + ); +} + export function resolveRuntimeSyntheticAuthProviderRefs(): string[] { const registry = getPluginRegistryState()?.activeRegistry; if (registry) { @@ -49,3 +63,31 @@ export function resolveRuntimeSyntheticAuthProviderRefs(): string[] { } return resolveManifestSyntheticAuthProviderRefs(); } + +export function resolveRuntimeExternalAuthProviderRefs(): string[] { + const registry = getPluginRegistryState()?.activeRegistry; + if (registry) { + return uniqueProviderRefs([ + ...registry.plugins.flatMap((plugin) => plugin.contracts?.externalAuthProviders ?? []), + ...(registry.providers ?? []) + .filter( + (entry) => + ("resolveExternalAuthProfiles" in entry.provider && + typeof entry.provider.resolveExternalAuthProfiles === "function") || + ("resolveExternalOAuthProfiles" in entry.provider && + typeof entry.provider.resolveExternalOAuthProfiles === "function"), + ) + .map((entry) => entry.provider.id), + ...(registry.cliBackends ?? []) + .filter( + (entry) => + ("resolveExternalAuthProfiles" in entry.backend && + typeof entry.backend.resolveExternalAuthProfiles === "function") || + ("resolveExternalOAuthProfiles" in entry.backend && + typeof entry.backend.resolveExternalOAuthProfiles === "function"), + ) + .map((entry) => entry.backend.id), + ]); + } + return resolveManifestExternalAuthProviderRefs(); +}