import crypto from "node:crypto"; import fs from "node:fs/promises"; import { resolveClawdbotAgentDir } from "../../agents/agent-paths.js"; import { ensureAuthProfileStore, listProfilesForProvider, resolveAuthProfileDisplayLabel, } from "../../agents/auth-profiles.js"; import { runEmbeddedPiAgent } from "../../agents/pi-embedded.js"; import { describeFailoverError } from "../../agents/failover-error.js"; import { loadModelCatalog } from "../../agents/model-catalog.js"; import { getCustomProviderApiKey, resolveEnvApiKey } from "../../agents/model-auth.js"; import { normalizeProviderId, parseModelRef } from "../../agents/model-selection.js"; import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../../agents/agent-scope.js"; import { resolveDefaultAgentWorkspaceDir } from "../../agents/workspace.js"; import type { ClawdbotConfig } from "../../config/config.js"; import { resolveSessionTranscriptPath, resolveSessionTranscriptsDirForAgent, } from "../../config/sessions/paths.js"; import { redactSecrets } from "../status-all/format.js"; import { DEFAULT_PROVIDER, formatMs } from "./shared.js"; const PROBE_PROMPT = "Reply with OK. Do not use tools."; export type AuthProbeStatus = | "ok" | "auth" | "rate_limit" | "billing" | "timeout" | "format" | "unknown" | "no_model"; export type AuthProbeResult = { provider: string; model?: string; profileId?: string; label: string; source: "profile" | "env" | "models.json"; mode?: string; status: AuthProbeStatus; error?: string; latencyMs?: number; }; type AuthProbeTarget = { provider: string; model?: { provider: string; model: string } | null; profileId?: string; label: string; source: "profile" | "env" | "models.json"; mode?: string; }; export type AuthProbeSummary = { startedAt: number; finishedAt: number; durationMs: number; totalTargets: number; options: { provider?: string; profileIds?: string[]; timeoutMs: number; concurrency: number; maxTokens: number; }; results: AuthProbeResult[]; }; export type AuthProbeOptions = { provider?: string; profileIds?: string[]; timeoutMs: number; concurrency: number; maxTokens: number; }; const toStatus = (reason?: string | null): AuthProbeStatus => { if (!reason) return "unknown"; if (reason === "auth") return "auth"; if (reason === "rate_limit") return "rate_limit"; if (reason === "billing") return "billing"; if (reason === "timeout") return "timeout"; if (reason === "format") return "format"; return "unknown"; }; function buildCandidateMap(modelCandidates: string[]): Map { const map = new Map(); for (const raw of modelCandidates) { const parsed = parseModelRef(String(raw ?? ""), DEFAULT_PROVIDER); if (!parsed) continue; const list = map.get(parsed.provider) ?? []; if (!list.includes(parsed.model)) list.push(parsed.model); map.set(parsed.provider, list); } return map; } function selectProbeModel(params: { provider: string; candidates: Map; catalog: Array<{ provider: string; id: string }>; }): { provider: string; model: string } | null { const { provider, candidates, catalog } = params; const direct = candidates.get(provider); if (direct && direct.length > 0) { return { provider, model: direct[0] }; } const fromCatalog = catalog.find((entry) => entry.provider === provider); if (fromCatalog) return { provider: fromCatalog.provider, model: fromCatalog.id }; return null; } function buildProbeTargets(params: { cfg: ClawdbotConfig; providers: string[]; modelCandidates: string[]; options: AuthProbeOptions; }): Promise<{ targets: AuthProbeTarget[]; results: AuthProbeResult[] }> { const { cfg, providers, modelCandidates, options } = params; const store = ensureAuthProfileStore(); const providerFilter = options.provider?.trim(); const providerFilterKey = providerFilter ? normalizeProviderId(providerFilter) : null; const profileFilter = new Set((options.profileIds ?? []).map((id) => id.trim()).filter(Boolean)); return loadModelCatalog({ config: cfg }).then((catalog) => { const candidates = buildCandidateMap(modelCandidates); const targets: AuthProbeTarget[] = []; const results: AuthProbeResult[] = []; for (const provider of providers) { const providerKey = normalizeProviderId(provider); if (providerFilterKey && providerKey !== providerFilterKey) continue; const model = selectProbeModel({ provider: providerKey, candidates, catalog, }); const profileIds = listProfilesForProvider(store, providerKey); const filteredProfiles = profileFilter.size ? profileIds.filter((id) => profileFilter.has(id)) : profileIds; if (filteredProfiles.length > 0) { for (const profileId of filteredProfiles) { const profile = store.profiles[profileId]; const mode = profile?.type; const label = resolveAuthProfileDisplayLabel({ cfg, store, profileId }); if (!model) { results.push({ provider: providerKey, model: undefined, profileId, label, source: "profile", mode, status: "no_model", error: "No model available for probe", }); continue; } targets.push({ provider: providerKey, model, profileId, label, source: "profile", mode, }); } continue; } if (profileFilter.size > 0) continue; const envKey = resolveEnvApiKey(providerKey); const customKey = getCustomProviderApiKey(cfg, providerKey); if (!envKey && !customKey) continue; const label = envKey ? "env" : "models.json"; const source = envKey ? "env" : "models.json"; const mode = envKey?.source.includes("OAUTH_TOKEN") ? "oauth" : "api_key"; if (!model) { results.push({ provider: providerKey, model: undefined, label, source, mode, status: "no_model", error: "No model available for probe", }); continue; } targets.push({ provider: providerKey, model, label, source, mode, }); } return { targets, results }; }); } async function probeTarget(params: { cfg: ClawdbotConfig; agentId: string; agentDir: string; workspaceDir: string; sessionDir: string; target: AuthProbeTarget; timeoutMs: number; maxTokens: number; }): Promise { const { cfg, agentId, agentDir, workspaceDir, sessionDir, target, timeoutMs, maxTokens } = params; if (!target.model) { return { provider: target.provider, model: undefined, profileId: target.profileId, label: target.label, source: target.source, mode: target.mode, status: "no_model", error: "No model available for probe", }; } const sessionId = `probe-${target.provider}-${crypto.randomUUID()}`; const sessionFile = resolveSessionTranscriptPath(sessionId, agentId); await fs.mkdir(sessionDir, { recursive: true }); const start = Date.now(); try { await runEmbeddedPiAgent({ sessionId, sessionFile, workspaceDir, agentDir, config: cfg, prompt: PROBE_PROMPT, provider: target.model.provider, model: target.model.model, authProfileId: target.profileId, authProfileIdSource: target.profileId ? "user" : undefined, timeoutMs, runId: `probe-${crypto.randomUUID()}`, lane: `auth-probe:${target.provider}:${target.profileId ?? target.source}`, thinkLevel: "off", reasoningLevel: "off", verboseLevel: "off", streamParams: { maxTokens }, }); return { provider: target.provider, model: `${target.model.provider}/${target.model.model}`, profileId: target.profileId, label: target.label, source: target.source, mode: target.mode, status: "ok", latencyMs: Date.now() - start, }; } catch (err) { const described = describeFailoverError(err); return { provider: target.provider, model: `${target.model.provider}/${target.model.model}`, profileId: target.profileId, label: target.label, source: target.source, mode: target.mode, status: toStatus(described.reason), error: redactSecrets(described.message), latencyMs: Date.now() - start, }; } } async function runTargetsWithConcurrency(params: { cfg: ClawdbotConfig; targets: AuthProbeTarget[]; timeoutMs: number; maxTokens: number; concurrency: number; onProgress?: (update: { completed: number; total: number; label?: string }) => void; }): Promise { const { cfg, targets, timeoutMs, maxTokens, onProgress } = params; const concurrency = Math.max(1, Math.min(targets.length || 1, params.concurrency)); const agentId = resolveDefaultAgentId(cfg); const agentDir = resolveClawdbotAgentDir(); const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId) ?? resolveDefaultAgentWorkspaceDir(); const sessionDir = resolveSessionTranscriptsDirForAgent(agentId); await fs.mkdir(workspaceDir, { recursive: true }); let completed = 0; const results: Array = Array.from({ length: targets.length }); let cursor = 0; const worker = async () => { while (true) { const index = cursor; cursor += 1; if (index >= targets.length) return; const target = targets[index]; onProgress?.({ completed, total: targets.length, label: `Probing ${target.provider}${target.profileId ? ` (${target.label})` : ""}`, }); const result = await probeTarget({ cfg, agentId, agentDir, workspaceDir, sessionDir, target, timeoutMs, maxTokens, }); results[index] = result; completed += 1; onProgress?.({ completed, total: targets.length }); } }; await Promise.all(Array.from({ length: concurrency }, () => worker())); return results.filter((entry): entry is AuthProbeResult => Boolean(entry)); } export async function runAuthProbes(params: { cfg: ClawdbotConfig; providers: string[]; modelCandidates: string[]; options: AuthProbeOptions; onProgress?: (update: { completed: number; total: number; label?: string }) => void; }): Promise { const startedAt = Date.now(); const plan = await buildProbeTargets({ cfg: params.cfg, providers: params.providers, modelCandidates: params.modelCandidates, options: params.options, }); const totalTargets = plan.targets.length; params.onProgress?.({ completed: 0, total: totalTargets }); const results = totalTargets ? await runTargetsWithConcurrency({ cfg: params.cfg, targets: plan.targets, timeoutMs: params.options.timeoutMs, maxTokens: params.options.maxTokens, concurrency: params.options.concurrency, onProgress: params.onProgress, }) : []; const finishedAt = Date.now(); return { startedAt, finishedAt, durationMs: finishedAt - startedAt, totalTargets, options: params.options, results: [...plan.results, ...results], }; } export function formatProbeLatency(latencyMs?: number | null) { if (!latencyMs && latencyMs !== 0) return "-"; return formatMs(latencyMs); } export function groupProbeResults(results: AuthProbeResult[]): Map { const map = new Map(); for (const result of results) { const list = map.get(result.provider) ?? []; list.push(result); map.set(result.provider, list); } return map; } export function sortProbeResults(results: AuthProbeResult[]): AuthProbeResult[] { return results.slice().sort((a, b) => { const provider = a.provider.localeCompare(b.provider); if (provider !== 0) return provider; const aLabel = a.label || a.profileId || ""; const bLabel = b.label || b.profileId || ""; return aLabel.localeCompare(bLabel); }); } export function describeProbeSummary(summary: AuthProbeSummary): string { if (summary.totalTargets === 0) return "No probe targets."; return `Probed ${summary.totalTargets} target${summary.totalTargets === 1 ? "" : "s"} in ${formatMs(summary.durationMs)}`; }