mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 17:26:15 +00:00
2364 lines
76 KiB
TypeScript
2364 lines
76 KiB
TypeScript
// Live-sweeps discovered model profiles with optional provider/model filters and probes.
|
|
import { writeSync } from "node:fs";
|
|
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
|
import { type Api, completeSimple, type Model } from "openclaw/plugin-sdk/llm";
|
|
import { Type } from "typebox";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import { getRuntimeConfig } from "../config/config.js";
|
|
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
|
import { coerceSecretRef, type SecretInput } from "../config/types.secrets.js";
|
|
import { parseLiveCsvFilter } from "../media-generation/live-test-helpers.js";
|
|
import { withBundledPluginEnablementCompat } from "../plugins/bundled-compat.js";
|
|
import { resolveOwningPluginIdsForProviderRef } from "../plugins/providers.js";
|
|
import { runTasksWithConcurrency } from "../utils/run-with-concurrency.js";
|
|
import {
|
|
discoverAuthStorage,
|
|
discoverModels,
|
|
normalizeDiscoveredAgentModel,
|
|
} from "./agent-model-discovery.js";
|
|
import { resolveDefaultAgentDir } from "./agent-scope.js";
|
|
import { externalCliDiscoveryForProviders } from "./auth-profiles/external-cli-discovery.js";
|
|
import { ensureCustomApiRegistered } from "./custom-api-registry.js";
|
|
import { isRateLimitErrorMessage } from "./embedded-agent-helpers/errors.js";
|
|
import { extractAssistantText } from "./embedded-agent-utils.js";
|
|
import { collectAnthropicApiKeys } from "./live-auth-keys.js";
|
|
import { appendPrioritizedDynamicLiveModels } from "./live-model-dynamic-candidates.js";
|
|
import { isModelNotFoundErrorMessage } from "./live-model-errors.js";
|
|
import {
|
|
DEFAULT_SMALL_LIVE_MODEL_LIMIT,
|
|
isHighSignalLiveModelRef,
|
|
isPrioritizedHighSignalLiveModelRef,
|
|
isSmallLiveModelRef,
|
|
listPrioritizedSmallLiveModelRefs,
|
|
resolveHighSignalLiveModelLimit,
|
|
selectHighSignalLiveItems,
|
|
selectSmallLiveItems,
|
|
shouldExcludeProviderFromDefaultHighSignalLiveSweep,
|
|
} from "./live-model-filter.js";
|
|
import {
|
|
buildLiveModelFileProbeContext,
|
|
buildLiveModelFileProbeRetryContext,
|
|
buildLiveModelImageProbeContext,
|
|
fileProbeTextMatches,
|
|
imageProbeTextMatches,
|
|
isLiveModelProbeEnabled,
|
|
LIVE_MODEL_FILE_PROBE_ENV,
|
|
LIVE_MODEL_FILE_PROBE_TOKEN,
|
|
LIVE_MODEL_IMAGE_PROBE_ENV,
|
|
modelSupportsImageInput,
|
|
shouldSkipLiveModelExtraProbes,
|
|
shouldSkipLiveModelFileProbe,
|
|
shouldSkipLiveModelImageProbe,
|
|
} from "./live-model-turn-probes.js";
|
|
import { createLiveTargetMatcher } from "./live-target-matcher.js";
|
|
import {
|
|
isLiveProfileKeyModeEnabled,
|
|
isLiveTestEnabled,
|
|
requiresLiveProfileCredential,
|
|
resolveLiveCredentialPrecedence,
|
|
} from "./live-test-helpers.js";
|
|
import {
|
|
isLiveBillingDrift,
|
|
isLiveRateLimitDrift,
|
|
shouldSkipLiveProviderDrift,
|
|
} from "./live-test-provider-drift.js";
|
|
import {
|
|
getApiKeyForModel,
|
|
requireApiKey,
|
|
resolveUsableCustomProviderApiKey,
|
|
} from "./model-auth.js";
|
|
import { shouldSuppressBuiltInModel } from "./model-suppression.js";
|
|
import { ensureOpenClawModelsJson } from "./models-config.js";
|
|
import type { StreamFn } from "./runtime/index.js";
|
|
import { prepareModelForSimpleCompletion } from "./simple-completion-transport.js";
|
|
|
|
const LIVE = isLiveTestEnabled();
|
|
const DIRECT_ENABLED = Boolean(process.env.OPENCLAW_LIVE_MODELS?.trim());
|
|
const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled();
|
|
const LIVE_HEARTBEAT_MS = Math.max(1_000, toInt(process.env.OPENCLAW_LIVE_HEARTBEAT_MS, 30_000));
|
|
const LIVE_SETUP_TIMEOUT_MS = Math.max(
|
|
1_000,
|
|
toInt(process.env.OPENCLAW_LIVE_SETUP_TIMEOUT_MS, 45_000),
|
|
);
|
|
const LIVE_TEST_TIMEOUT_MS = Math.max(
|
|
1_000,
|
|
toInt(process.env.OPENCLAW_LIVE_TEST_TIMEOUT_MS, 60 * 60 * 1000),
|
|
);
|
|
const DEFAULT_LIVE_MODEL_CONCURRENCY = 20;
|
|
const LIVE_MODEL_CONCURRENCY = resolveLiveModelConcurrency(
|
|
process.env.OPENCLAW_LIVE_MODEL_CONCURRENCY,
|
|
);
|
|
const LIVE_MODELS_JSON_TIMEOUT_MS = resolveLiveModelsJsonTimeoutMs(
|
|
process.env.OPENCLAW_LIVE_MODELS_JSON_TIMEOUT_MS,
|
|
);
|
|
const LIVE_FILE_PROBE_ENABLED = isLiveModelProbeEnabled(process.env, LIVE_MODEL_FILE_PROBE_ENV);
|
|
const LIVE_IMAGE_PROBE_ENABLED = isLiveModelProbeEnabled(process.env, LIVE_MODEL_IMAGE_PROBE_ENV);
|
|
const OLLAMA_DEFAULT_BASE_URL = "http://127.0.0.1:11434";
|
|
const OLLAMA_LOCAL_API_KEY_MARKER = "ollama-local";
|
|
const OLLAMA_REMOTE_API_KEY_ENV = "OLLAMA_API_KEY";
|
|
const LOCAL_OLLAMA_HOSTNAMES = new Set([
|
|
"localhost",
|
|
"127.0.0.1",
|
|
"0.0.0.0",
|
|
"::1",
|
|
"::",
|
|
"docker.orb.internal",
|
|
"host.docker.internal",
|
|
"host.orb.internal",
|
|
]);
|
|
let activeLiveCompletionConfig: OpenClawConfig | undefined;
|
|
|
|
type OllamaRuntimeApi = {
|
|
createConfiguredOllamaStreamFn: (params: {
|
|
model: { baseUrl?: string; headers?: unknown };
|
|
providerBaseUrl?: string;
|
|
}) => StreamFn;
|
|
};
|
|
|
|
const describeLive = LIVE ? describe : describe.skip;
|
|
|
|
function parseCsvFilter(raw?: string): Set<string> | null {
|
|
return parseLiveCsvFilter(raw, { lowercase: false });
|
|
}
|
|
|
|
function parseProviderFilter(raw?: string): Set<string> | null {
|
|
return parseCsvFilter(raw);
|
|
}
|
|
|
|
function parseModelFilter(raw?: string): Set<string> | null {
|
|
return parseCsvFilter(raw);
|
|
}
|
|
|
|
function parseExplicitLiveModelRefs(
|
|
filter: Set<string> | null,
|
|
): Array<{ provider: string; id: string }> {
|
|
// Explicit refs use provider/model syntax; bare provider filters are handled elsewhere.
|
|
if (!filter) {
|
|
return [];
|
|
}
|
|
const refs: Array<{ provider: string; id: string }> = [];
|
|
const seen = new Set<string>();
|
|
for (const raw of filter) {
|
|
const trimmed = raw.trim();
|
|
const slash = trimmed.indexOf("/");
|
|
if (slash <= 0 || slash === trimmed.length - 1) {
|
|
continue;
|
|
}
|
|
const provider = normalizeProviderId(trimmed.slice(0, slash));
|
|
const id = trimmed.slice(slash + 1).trim();
|
|
if (!provider || !id) {
|
|
continue;
|
|
}
|
|
const key = `${provider}\0${id.toLowerCase()}`;
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
seen.add(key);
|
|
refs.push({ provider, id });
|
|
}
|
|
return refs;
|
|
}
|
|
|
|
function formatExplicitLiveModelRef(ref: { provider: string; id: string }): string {
|
|
return `${ref.provider}/${ref.id}`;
|
|
}
|
|
|
|
function filterLiveModelRefsByProvider(
|
|
refs: readonly { provider: string; id: string }[],
|
|
providerFilter: Set<string> | null,
|
|
): Array<{ provider: string; id: string }> {
|
|
if (!providerFilter) {
|
|
return [...refs];
|
|
}
|
|
const normalizedProviders = new Set(
|
|
[...providerFilter].map((provider) => normalizeProviderId(provider)).filter(Boolean),
|
|
);
|
|
return refs.filter((ref) => normalizedProviders.has(normalizeProviderId(ref.provider)));
|
|
}
|
|
|
|
function findUnmatchedExplicitLiveModelRefs(params: {
|
|
refs: readonly { provider: string; id: string }[];
|
|
models: readonly Pick<Model, "provider" | "id">[];
|
|
config?: OpenClawConfig;
|
|
env?: NodeJS.ProcessEnv;
|
|
}): string[] {
|
|
const unmatched: string[] = [];
|
|
for (const ref of params.refs) {
|
|
const matcher = createLiveTargetMatcher({
|
|
providerFilter: null,
|
|
modelFilter: new Set([formatExplicitLiveModelRef(ref)]),
|
|
config: params.config,
|
|
env: params.env,
|
|
});
|
|
const matched = params.models.some((model) => matcher.matchesModel(model.provider, model.id));
|
|
if (!matched) {
|
|
unmatched.push(formatExplicitLiveModelRef(ref));
|
|
}
|
|
}
|
|
return unmatched;
|
|
}
|
|
|
|
function resolveLiveProviderDiscoveryProviderIds(params: {
|
|
providerFilter: Set<string> | null;
|
|
explicitRefs: readonly { provider: string; id: string }[];
|
|
priorityRefs?: readonly { provider: string; id: string }[];
|
|
}): string[] | undefined {
|
|
// Narrow startup discovery to providers that can affect the requested live target set.
|
|
const providers = new Set<string>();
|
|
for (const provider of params.providerFilter ?? []) {
|
|
const normalized = normalizeProviderId(provider);
|
|
if (normalized) {
|
|
providers.add(normalized);
|
|
}
|
|
}
|
|
for (const ref of params.explicitRefs) {
|
|
providers.add(ref.provider);
|
|
}
|
|
for (const ref of params.priorityRefs ?? []) {
|
|
providers.add(ref.provider);
|
|
}
|
|
return providers.size > 0
|
|
? [...providers].toSorted((left, right) => left.localeCompare(right))
|
|
: undefined;
|
|
}
|
|
|
|
function resolveLiveProviderDiscoveryPluginIds(params: {
|
|
config?: OpenClawConfig;
|
|
providers: readonly string[] | undefined;
|
|
env?: NodeJS.ProcessEnv;
|
|
}): string[] {
|
|
const pluginIds = new Set<string>();
|
|
for (const provider of params.providers ?? []) {
|
|
const owners =
|
|
resolveOwningPluginIdsForProviderRef({
|
|
provider,
|
|
config: params.config,
|
|
env: params.env,
|
|
}) ?? [];
|
|
if (owners.length === 0) {
|
|
pluginIds.add(provider);
|
|
continue;
|
|
}
|
|
for (const owner of owners) {
|
|
pluginIds.add(owner);
|
|
}
|
|
}
|
|
return [...pluginIds].toSorted((left, right) => left.localeCompare(right));
|
|
}
|
|
|
|
function applyLiveProviderDiscoveryPluginCompat(params: {
|
|
config: OpenClawConfig;
|
|
providers: readonly string[] | undefined;
|
|
env?: NodeJS.ProcessEnv;
|
|
}): OpenClawConfig {
|
|
const pluginIds = resolveLiveProviderDiscoveryPluginIds(params);
|
|
const pluginConfig =
|
|
pluginIds.length > 0 ? enableLiveProviderPlugins(params.config, pluginIds) : params.config;
|
|
return applyLiveOllamaProviderEnvCompat({
|
|
config: pluginConfig,
|
|
providers: params.providers,
|
|
env: params.env,
|
|
});
|
|
}
|
|
|
|
function enableLiveProviderPlugins(
|
|
config: OpenClawConfig,
|
|
pluginIds: readonly string[],
|
|
): OpenClawConfig {
|
|
const compatConfig =
|
|
withBundledPluginEnablementCompat({
|
|
config,
|
|
pluginIds,
|
|
}) ?? config;
|
|
const entries = { ...compatConfig.plugins?.entries };
|
|
const allow = new Set(compatConfig.plugins?.allow ?? []);
|
|
for (const pluginId of pluginIds) {
|
|
allow.add(pluginId);
|
|
entries[pluginId] ??= { enabled: true };
|
|
}
|
|
return {
|
|
...compatConfig,
|
|
plugins: {
|
|
...compatConfig.plugins,
|
|
enabled: true,
|
|
allow: [...allow].toSorted((left, right) => left.localeCompare(right)),
|
|
bundledDiscovery: compatConfig.plugins?.bundledDiscovery ?? "compat",
|
|
entries,
|
|
},
|
|
};
|
|
}
|
|
|
|
function applyLiveOllamaProviderEnvCompat(params: {
|
|
config: OpenClawConfig;
|
|
providers: readonly string[] | undefined;
|
|
env?: NodeJS.ProcessEnv;
|
|
}): OpenClawConfig {
|
|
if (!params.providers?.some((provider) => normalizeProviderId(provider) === "ollama")) {
|
|
return params.config;
|
|
}
|
|
const existingProvider = params.config.models?.providers?.ollama;
|
|
const configuredBaseUrl = readConfiguredOllamaBaseUrl(existingProvider);
|
|
const liveBaseUrl = params.env?.OPENCLAW_LIVE_OLLAMA_BASE_URL?.trim();
|
|
const baseUrl = liveBaseUrl || configuredBaseUrl || OLLAMA_DEFAULT_BASE_URL;
|
|
const shouldPreserveConfiguredApiKey =
|
|
!liveBaseUrl ||
|
|
Boolean(
|
|
configuredBaseUrl &&
|
|
canonicalOllamaCredentialBaseUrl(configuredBaseUrl) ===
|
|
canonicalOllamaCredentialBaseUrl(baseUrl),
|
|
);
|
|
const apiKey = resolveLiveOllamaProviderApiKey({
|
|
baseUrl,
|
|
existingApiKey: existingProvider?.apiKey,
|
|
shouldPreserveConfiguredApiKey,
|
|
});
|
|
return {
|
|
...params.config,
|
|
models: {
|
|
...params.config.models,
|
|
providers: {
|
|
...params.config.models?.providers,
|
|
ollama: {
|
|
...existingProvider,
|
|
api: "ollama",
|
|
baseUrl,
|
|
apiKey,
|
|
models: existingProvider?.models ?? [],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
async function ensureLiveProviderApisRegistered(params: {
|
|
config: OpenClawConfig;
|
|
providers: readonly string[] | undefined;
|
|
}): Promise<void> {
|
|
if (!params.providers?.some((provider) => normalizeProviderId(provider) === "ollama")) {
|
|
return;
|
|
}
|
|
// Live Vitest setup installs a stub plugin registry for channel tests; direct
|
|
// model probes still need the public Ollama runtime registered in-process.
|
|
const runtimeApiUrl = new URL("../../extensions/ollama/runtime-api.ts", import.meta.url).href;
|
|
const { createConfiguredOllamaStreamFn } = (await import(
|
|
/* @vite-ignore */ runtimeApiUrl
|
|
)) as OllamaRuntimeApi;
|
|
const providerConfig = params.config.models?.providers?.ollama;
|
|
const providerBaseUrl = readConfiguredOllamaBaseUrl(providerConfig) || OLLAMA_DEFAULT_BASE_URL;
|
|
ensureCustomApiRegistered(
|
|
"ollama",
|
|
createLiveOllamaRuntimeStreamFn({
|
|
createConfiguredOllamaStreamFn,
|
|
providerBaseUrl,
|
|
}),
|
|
);
|
|
}
|
|
|
|
function createLiveOllamaRuntimeStreamFn(params: {
|
|
createConfiguredOllamaStreamFn: OllamaRuntimeApi["createConfiguredOllamaStreamFn"];
|
|
providerBaseUrl: string;
|
|
}): StreamFn {
|
|
return (model, context, options) => {
|
|
const modelBaseUrl = readStringProperty(model, "baseUrl");
|
|
const streamFn = params.createConfiguredOllamaStreamFn({
|
|
model,
|
|
providerBaseUrl: modelBaseUrl ? undefined : params.providerBaseUrl,
|
|
});
|
|
return streamFn(model, context, options);
|
|
};
|
|
}
|
|
|
|
function readConfiguredOllamaBaseUrl(provider: unknown): string {
|
|
return readStringProperty(provider, "baseUrl") || readStringProperty(provider, "baseURL");
|
|
}
|
|
|
|
function resolveLiveOllamaProviderApiKey(params: {
|
|
baseUrl: string;
|
|
existingApiKey: SecretInput | undefined;
|
|
shouldPreserveConfiguredApiKey: boolean;
|
|
}): SecretInput {
|
|
if (isLocalOllamaBaseUrl(params.baseUrl)) {
|
|
return params.shouldPreserveConfiguredApiKey &&
|
|
params.existingApiKey !== undefined &&
|
|
!isOllamaRemoteApiKeyReference(params.existingApiKey)
|
|
? params.existingApiKey
|
|
: OLLAMA_LOCAL_API_KEY_MARKER;
|
|
}
|
|
return params.shouldPreserveConfiguredApiKey
|
|
? (params.existingApiKey ?? OLLAMA_REMOTE_API_KEY_ENV)
|
|
: OLLAMA_REMOTE_API_KEY_ENV;
|
|
}
|
|
|
|
function isOllamaRemoteApiKeyReference(value: SecretInput | undefined): boolean {
|
|
if (value === undefined) {
|
|
return false;
|
|
}
|
|
if (typeof value === "string") {
|
|
if (value.trim() === OLLAMA_REMOTE_API_KEY_ENV) {
|
|
return true;
|
|
}
|
|
}
|
|
const ref = coerceSecretRef(value);
|
|
return ref?.source === "env" && ref.id.trim() === OLLAMA_REMOTE_API_KEY_ENV;
|
|
}
|
|
|
|
function readStringProperty(value: unknown, key: string): string {
|
|
if (!value || typeof value !== "object" || !(key in value)) {
|
|
return "";
|
|
}
|
|
const raw = (value as Record<string, unknown>)[key];
|
|
return typeof raw === "string" ? raw.trim() : "";
|
|
}
|
|
|
|
function isLocalOllamaBaseUrl(baseUrl: string): boolean {
|
|
try {
|
|
let host = new URL(baseUrl).hostname.toLowerCase();
|
|
if (host.startsWith("[") && host.endsWith("]")) {
|
|
host = host.slice(1, -1);
|
|
}
|
|
return (
|
|
LOCAL_OLLAMA_HOSTNAMES.has(host) ||
|
|
host.endsWith(".local") ||
|
|
isIpv4PrivateRange(host) ||
|
|
isIpv6LocalRange(host) ||
|
|
(!host.includes(".") && !host.includes(":"))
|
|
);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function resolveLiveOllamaBaseUrl(model: Pick<Model, "baseUrl">, config?: OpenClawConfig): string {
|
|
return (
|
|
readStringProperty(model, "baseUrl") ||
|
|
readConfiguredOllamaBaseUrl(config?.models?.providers?.ollama) ||
|
|
OLLAMA_DEFAULT_BASE_URL
|
|
);
|
|
}
|
|
|
|
function isLiveLocalOllamaModel(
|
|
model: Pick<Model, "provider" | "baseUrl">,
|
|
config?: OpenClawConfig,
|
|
): boolean {
|
|
return (
|
|
normalizeProviderId(model.provider) === "ollama" &&
|
|
isLocalOllamaBaseUrl(resolveLiveOllamaBaseUrl(model, config))
|
|
);
|
|
}
|
|
|
|
function canReuseConfiguredLocalOllamaApiKey(
|
|
model: Pick<Model, "baseUrl">,
|
|
config?: OpenClawConfig,
|
|
): boolean {
|
|
const providerConfig = config?.models?.providers?.ollama;
|
|
if (isOllamaRemoteApiKeyReference(providerConfig?.apiKey)) {
|
|
return false;
|
|
}
|
|
const modelBaseUrl = readStringProperty(model, "baseUrl");
|
|
if (!modelBaseUrl) {
|
|
return true;
|
|
}
|
|
const providerBaseUrl = readConfiguredOllamaBaseUrl(providerConfig) || OLLAMA_DEFAULT_BASE_URL;
|
|
return (
|
|
canonicalOllamaCredentialBaseUrl(providerBaseUrl) ===
|
|
canonicalOllamaCredentialBaseUrl(modelBaseUrl)
|
|
);
|
|
}
|
|
|
|
function canonicalOllamaCredentialBaseUrl(baseUrl: string): string {
|
|
try {
|
|
const parsed = new URL(baseUrl);
|
|
let pathname = parsed.pathname.replace(/\/+$/, "");
|
|
if (pathname === "/v1") {
|
|
pathname = "";
|
|
}
|
|
parsed.pathname = pathname || "/";
|
|
parsed.search = "";
|
|
parsed.hash = "";
|
|
return parsed.toString().replace(/\/$/, "");
|
|
} catch {
|
|
return baseUrl.trim().replace(/\/+$/, "");
|
|
}
|
|
}
|
|
|
|
async function resolveLiveModelApiKeyInfo(params: {
|
|
model: Model;
|
|
cfg: OpenClawConfig;
|
|
requireProfileKeys: boolean;
|
|
}): Promise<Awaited<ReturnType<typeof getApiKeyForModel>>> {
|
|
if (isLiveLocalOllamaModel(params.model, params.cfg)) {
|
|
const configuredKey = canReuseConfiguredLocalOllamaApiKey(params.model, params.cfg)
|
|
? resolveUsableCustomProviderApiKey({
|
|
cfg: params.cfg,
|
|
provider: "ollama",
|
|
})
|
|
: null;
|
|
if (configuredKey && configuredKey.apiKey !== OLLAMA_LOCAL_API_KEY_MARKER) {
|
|
return {
|
|
apiKey: configuredKey.apiKey,
|
|
source: configuredKey.source,
|
|
mode: "api-key",
|
|
};
|
|
}
|
|
return {
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
source: "live Ollama local marker",
|
|
mode: "api-key",
|
|
};
|
|
}
|
|
return await getApiKeyForModel({
|
|
model: params.model,
|
|
cfg: params.cfg,
|
|
credentialPrecedence: resolveLiveCredentialPrecedence(
|
|
params.model.provider,
|
|
params.requireProfileKeys,
|
|
),
|
|
});
|
|
}
|
|
|
|
function isIpv4PrivateRange(host: string): boolean {
|
|
if (!/^\d+\.\d+\.\d+\.\d+$/.test(host)) {
|
|
return false;
|
|
}
|
|
const octets = host.split(".").map((part) => Number.parseInt(part, 10));
|
|
if (octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
|
return false;
|
|
}
|
|
const [a, b] = octets;
|
|
return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168);
|
|
}
|
|
|
|
function isIpv6LocalRange(host: string): boolean {
|
|
const lower = host.toLowerCase();
|
|
return /^fe[89ab][0-9a-f]:/.test(lower) || /^f[cd][0-9a-f]{2}:/.test(lower);
|
|
}
|
|
|
|
function logProgress(message: string): void {
|
|
writeSync(2, `[live] ${message}\n`);
|
|
}
|
|
|
|
function formatElapsedSeconds(ms: number): string {
|
|
return `${Math.max(1, Math.round(ms / 1_000))}s`;
|
|
}
|
|
|
|
async function withLiveHeartbeat<T>(operation: Promise<T>, context: string): Promise<T> {
|
|
const startedAt = Date.now();
|
|
let heartbeatCount = 0;
|
|
const timer = setInterval(() => {
|
|
heartbeatCount += 1;
|
|
logProgress(`${context}: still running (${formatElapsedSeconds(Date.now() - startedAt)})`);
|
|
}, LIVE_HEARTBEAT_MS);
|
|
timer.unref?.();
|
|
try {
|
|
return await operation;
|
|
} finally {
|
|
clearInterval(timer);
|
|
if (heartbeatCount > 0) {
|
|
logProgress(`${context}: completed after ${formatElapsedSeconds(Date.now() - startedAt)}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function withLiveStageTimeout<T>(
|
|
operation: Promise<T>,
|
|
context: string,
|
|
timeoutMs = LIVE_SETUP_TIMEOUT_MS,
|
|
): Promise<T> {
|
|
let hardTimer: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
return await withLiveHeartbeat(
|
|
Promise.race([
|
|
operation,
|
|
new Promise<never>((_, reject) => {
|
|
hardTimer = setTimeout(() => {
|
|
reject(new Error(`${context} timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
hardTimer.unref?.();
|
|
}),
|
|
]),
|
|
context,
|
|
);
|
|
} finally {
|
|
if (hardTimer) {
|
|
clearTimeout(hardTimer);
|
|
}
|
|
}
|
|
}
|
|
|
|
function formatFailurePreview(
|
|
failures: Array<{ model: string; error: string }>,
|
|
maxItems: number,
|
|
): string {
|
|
const limit = Math.max(1, maxItems);
|
|
const lines = failures.slice(0, limit).map((failure, index) => {
|
|
const normalized = failure.error.replace(/\s+/g, " ").trim();
|
|
const clipped = normalized.length > 320 ? `${normalized.slice(0, 317)}...` : normalized;
|
|
return `${index + 1}. ${failure.model}: ${clipped}`;
|
|
});
|
|
const remaining = failures.length - limit;
|
|
if (remaining > 0) {
|
|
lines.push(`... and ${remaining} more`);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function formatSkippedPreview(
|
|
skipped: Array<{ model: string; reason: string }>,
|
|
maxItems: number,
|
|
): string {
|
|
return formatFailurePreview(
|
|
skipped.map((entry) => ({ model: entry.model, error: entry.reason })),
|
|
maxItems,
|
|
);
|
|
}
|
|
|
|
function isGoogleModelNotFoundError(err: unknown): boolean {
|
|
const msg = String(err);
|
|
if (!/not found/i.test(msg)) {
|
|
return false;
|
|
}
|
|
if (/\b404\b/.test(msg)) {
|
|
return true;
|
|
}
|
|
if (/models\/.+ is not found for api version/i.test(msg)) {
|
|
return true;
|
|
}
|
|
if (/"status"\\s*:\\s*"NOT_FOUND"/.test(msg)) {
|
|
return true;
|
|
}
|
|
if (/"code"\\s*:\\s*404/.test(msg)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
describe("isModelNotFoundErrorMessage", () => {
|
|
it("matches whitespace-separated not found errors", () => {
|
|
expect(isModelNotFoundErrorMessage("404 model not found")).toBe(true);
|
|
expect(isModelNotFoundErrorMessage("model: minimax-text-01 not found")).toBe(true);
|
|
});
|
|
|
|
it("still matches underscore and hyphen variants", () => {
|
|
expect(isModelNotFoundErrorMessage("404 model not_found")).toBe(true);
|
|
expect(isModelNotFoundErrorMessage("404 model not-found")).toBe(true);
|
|
});
|
|
|
|
it("matches deprecated free model transition messages", () => {
|
|
expect(
|
|
isModelNotFoundErrorMessage(
|
|
"404 The free model has been deprecated. Transition to qwen/qwen3.6-plus for continued paid access.",
|
|
),
|
|
).toBe(true);
|
|
});
|
|
|
|
it("matches OpenRouter no-endpoints wording", () => {
|
|
expect(
|
|
isModelNotFoundErrorMessage("404 No endpoints found for deepseek/deepseek-r1:free."),
|
|
).toBe(true);
|
|
});
|
|
});
|
|
|
|
function isChatGPTUsageLimitErrorMessage(raw: string): boolean {
|
|
const msg = raw.toLowerCase();
|
|
return msg.includes("hit your chatgpt usage limit") && msg.includes("try again in");
|
|
}
|
|
|
|
function isRefreshTokenReused(raw: string): boolean {
|
|
return /refresh_token_reused/i.test(raw);
|
|
}
|
|
|
|
function isAccountIdExtractionError(raw: string): boolean {
|
|
return /failed to extract accountid from token/i.test(raw);
|
|
}
|
|
|
|
function isInstructionsRequiredError(raw: string): boolean {
|
|
return /instructions are required/i.test(raw);
|
|
}
|
|
|
|
function isOpenAiCodexHtmlInterruption(raw: string): boolean {
|
|
const trimmed = raw.trim().replace(/^Error:\s*/i, "");
|
|
return (
|
|
/^(?:<!doctype\s+html\b|<html\b)/i.test(trimmed) &&
|
|
(/<meta\s+name=["']viewport["']/i.test(trimmed) || /<body\b/i.test(trimmed))
|
|
);
|
|
}
|
|
|
|
function isOllamaUnavailableErrorMessage(raw: string): boolean {
|
|
const msg = raw.toLowerCase();
|
|
return (
|
|
msg.includes("ollama could not be reached") ||
|
|
(msg.includes("127.0.0.1:11434") && msg.includes("econnrefused")) ||
|
|
(msg.includes("localhost:11434") && msg.includes("econnrefused"))
|
|
);
|
|
}
|
|
|
|
function isAudioOnlyModelErrorMessage(raw: string): boolean {
|
|
return /requires that either input content or output modality contain audio/i.test(raw);
|
|
}
|
|
|
|
function isUnsupportedReasoningEffortErrorMessage(raw: string): boolean {
|
|
return (
|
|
/does not support parameter reasoningeffort/i.test(raw) ||
|
|
/invalid reasoning effort/i.test(raw) ||
|
|
/unsupported value:\s*'low'.*reasoning\.effort.*supported values are:\s*'medium'/i.test(raw)
|
|
);
|
|
}
|
|
|
|
function isUnsupportedThinkingToggleErrorMessage(raw: string): boolean {
|
|
return /does not support parameter [`"]?enable_thinking[`"]?/i.test(raw);
|
|
}
|
|
|
|
function isUnsupportedPlanErrorMessage(raw: string): boolean {
|
|
return /current token plan (?:does )?not support (?:this )?model/i.test(raw);
|
|
}
|
|
|
|
function isOpenRouterOpaqueBadRequestErrorMessage(raw: string): boolean {
|
|
const msg = raw.toLowerCase();
|
|
return (
|
|
msg.includes("provider returned error") &&
|
|
msg.includes('"code":400') &&
|
|
msg.includes('"msg":"bad request"')
|
|
);
|
|
}
|
|
|
|
describe("isUnsupportedReasoningEffortErrorMessage", () => {
|
|
it("matches provider-native reasoning effort rejections", () => {
|
|
expect(isUnsupportedReasoningEffortErrorMessage('Error: 400 "Invalid reasoning effort."')).toBe(
|
|
true,
|
|
);
|
|
expect(isUnsupportedReasoningEffortErrorMessage("Error: 400 model not found")).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("isUnsupportedPlanErrorMessage", () => {
|
|
it("matches provider plan-gated models", () => {
|
|
expect(isUnsupportedPlanErrorMessage("current token plan does not support this model")).toBe(
|
|
true,
|
|
);
|
|
expect(isUnsupportedPlanErrorMessage("your current token plan not support model")).toBe(true);
|
|
expect(isUnsupportedPlanErrorMessage("model not found")).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("isOpenRouterOpaqueBadRequestErrorMessage", () => {
|
|
it("matches opaque OpenRouter upstream bad requests", () => {
|
|
expect(
|
|
isOpenRouterOpaqueBadRequestErrorMessage(
|
|
'Error: 400 Provider returned error {"code":400,"msg":"bad request","request_id":"abc"}',
|
|
),
|
|
).toBe(true);
|
|
expect(isOpenRouterOpaqueBadRequestErrorMessage("Error: 400 bad request")).toBe(false);
|
|
});
|
|
});
|
|
|
|
function toInt(value: string | undefined, fallback: number): number {
|
|
const trimmed = value?.trim();
|
|
if (!trimmed) {
|
|
return fallback;
|
|
}
|
|
const parsed = Number.parseInt(trimmed, 10);
|
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
}
|
|
|
|
function resolveLiveModelConcurrency(raw?: string): number {
|
|
return Math.max(1, toInt(raw, DEFAULT_LIVE_MODEL_CONCURRENCY));
|
|
}
|
|
|
|
describe("resolveLiveModelConcurrency", () => {
|
|
it("defaults direct-model probes to 20-way concurrency", () => {
|
|
expect(resolveLiveModelConcurrency()).toBe(20);
|
|
});
|
|
|
|
it("accepts explicit concurrency overrides", () => {
|
|
expect(resolveLiveModelConcurrency("7")).toBe(7);
|
|
expect(resolveLiveModelConcurrency("0")).toBe(1);
|
|
});
|
|
});
|
|
|
|
function resolveLiveModelsJsonTimeoutMs(
|
|
modelsJsonTimeoutRaw?: string,
|
|
setupTimeoutMs = LIVE_SETUP_TIMEOUT_MS,
|
|
): number {
|
|
return Math.max(setupTimeoutMs, toInt(modelsJsonTimeoutRaw, 180_000));
|
|
}
|
|
|
|
describe("resolveLiveModelsJsonTimeoutMs", () => {
|
|
it("defaults models.json preparation to a longer setup timeout", () => {
|
|
expect(resolveLiveModelsJsonTimeoutMs(undefined, 45_000)).toBe(180_000);
|
|
});
|
|
|
|
it("never goes below the shared live setup timeout", () => {
|
|
expect(resolveLiveModelsJsonTimeoutMs("30000", 45_000)).toBe(45_000);
|
|
});
|
|
});
|
|
|
|
describe("explicit live model discovery scope", () => {
|
|
it("derives provider ids from explicit model refs", () => {
|
|
const filter = parseModelFilter(
|
|
"zai/glm-5.1, together/Qwen/Qwen2.5-7B-Instruct-Turbo, glm-5.1",
|
|
);
|
|
const explicitRefs = parseExplicitLiveModelRefs(filter);
|
|
|
|
expect(explicitRefs).toEqual([
|
|
{ provider: "zai", id: "glm-5.1" },
|
|
{ provider: "together", id: "Qwen/Qwen2.5-7B-Instruct-Turbo" },
|
|
]);
|
|
expect(
|
|
resolveLiveProviderDiscoveryProviderIds({
|
|
providerFilter: null,
|
|
explicitRefs,
|
|
}),
|
|
).toEqual(["together", "zai"]);
|
|
});
|
|
|
|
it("merges explicit model providers with OPENCLAW_LIVE_PROVIDERS", () => {
|
|
const explicitRefs = parseExplicitLiveModelRefs(parseModelFilter("zai/glm-5.1"));
|
|
|
|
expect(
|
|
resolveLiveProviderDiscoveryProviderIds({
|
|
providerFilter: parseProviderFilter("deepseek,together"),
|
|
explicitRefs,
|
|
}),
|
|
).toEqual(["deepseek", "together", "zai"]);
|
|
});
|
|
|
|
it("includes curated small-model providers in discovery scope", () => {
|
|
expect(
|
|
resolveLiveProviderDiscoveryProviderIds({
|
|
providerFilter: null,
|
|
explicitRefs: [],
|
|
priorityRefs: listPrioritizedSmallLiveModelRefs(),
|
|
}),
|
|
).toContain("ollama");
|
|
});
|
|
|
|
it("respects provider filters for curated small-model refs", () => {
|
|
expect(
|
|
filterLiveModelRefsByProvider(
|
|
listPrioritizedSmallLiveModelRefs(),
|
|
parseProviderFilter("openrouter"),
|
|
).map((ref) => ref.provider),
|
|
).toEqual(["openrouter", "openrouter", "openrouter"]);
|
|
});
|
|
|
|
it("activates bundled provider plugins for explicit live discovery", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
allow: ["openai"],
|
|
bundledDiscovery: "compat",
|
|
entries: {
|
|
openai: { enabled: true },
|
|
},
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["deepseek"],
|
|
env: {},
|
|
});
|
|
|
|
expect(result.plugins?.enabled).toBe(true);
|
|
expect(result.plugins?.allow).toContain("deepseek");
|
|
expect(result.plugins?.bundledDiscovery).toBe("compat");
|
|
expect(result.plugins?.entries?.deepseek).toEqual({ enabled: true });
|
|
});
|
|
|
|
it("hydrates Ollama Cloud provider settings from live env when Ollama is in scope", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: "https://ollama.com",
|
|
},
|
|
});
|
|
|
|
expect(result.plugins?.entries?.ollama).toEqual({ enabled: true });
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: "https://ollama.com",
|
|
apiKey: OLLAMA_REMOTE_API_KEY_ENV,
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("defaults Ollama live provider settings to the local endpoint", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {},
|
|
});
|
|
|
|
expect(result.plugins?.entries?.ollama).toEqual({ enabled: true });
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: OLLAMA_DEFAULT_BASE_URL,
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("preserves configured Ollama provider endpoints when live env is absent", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "http://192.168.1.10:11434",
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: "http://192.168.1.10:11434",
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("honors the documented Ollama baseURL alias when live env is absent", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseURL: "http://ollama.local:11434",
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
} as unknown as OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseURL: "http://ollama.local:11434",
|
|
baseUrl: "http://ollama.local:11434",
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("uses the local Ollama auth marker for self-hosted live env URLs", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
for (const baseUrl of [
|
|
"http://127.0.0.1:11434",
|
|
"http://192.168.1.10:11434",
|
|
"http://ollama.local:11434",
|
|
"http://ollama-host:11434",
|
|
]) {
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: baseUrl,
|
|
},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl,
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
}
|
|
});
|
|
|
|
it("does not preserve the cloud env marker for local Ollama endpoints", () => {
|
|
const remoteApiKeyRefs: SecretInput[] = [
|
|
OLLAMA_REMOTE_API_KEY_ENV,
|
|
"$OLLAMA_API_KEY",
|
|
"${OLLAMA_API_KEY}",
|
|
{ source: "env", provider: "default", id: OLLAMA_REMOTE_API_KEY_ENV },
|
|
];
|
|
|
|
for (const apiKey of remoteApiKeyRefs) {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
apiKey,
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {
|
|
OLLAMA_API_KEY: "real-cloud-key",
|
|
},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
}
|
|
});
|
|
|
|
it("replaces configured Ollama auth when live env redirects to a different local endpoint", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "https://ollama.com",
|
|
apiKey: OLLAMA_REMOTE_API_KEY_ENV,
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
|
},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("preserves configured Ollama auth for equivalent live env base URLs", () => {
|
|
const cfg = {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
} satisfies OpenClawConfig;
|
|
|
|
const result = applyLiveProviderDiscoveryPluginCompat({
|
|
config: cfg,
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
|
},
|
|
});
|
|
|
|
expect(result.models?.providers?.ollama).toEqual({
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
|
models: [],
|
|
});
|
|
});
|
|
|
|
it("keeps local Ollama live auth on the non-secret marker", async () => {
|
|
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
|
config: {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
},
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
|
OLLAMA_API_KEY: "real-cloud-key",
|
|
},
|
|
});
|
|
|
|
await expect(
|
|
resolveLiveModelApiKeyInfo({
|
|
model: {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
} as Model,
|
|
cfg,
|
|
requireProfileKeys: false,
|
|
}),
|
|
).resolves.toEqual({
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
source: "live Ollama local marker",
|
|
mode: "api-key",
|
|
});
|
|
});
|
|
|
|
it("does not reuse cloud Ollama auth for model-level local endpoint overrides", async () => {
|
|
const oldEnv = process.env.OLLAMA_API_KEY;
|
|
process.env.OLLAMA_API_KEY = "real-cloud-key";
|
|
try {
|
|
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
|
config: {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
},
|
|
providers: ["ollama"],
|
|
env: {
|
|
OPENCLAW_LIVE_OLLAMA_BASE_URL: "https://ollama.com",
|
|
},
|
|
});
|
|
|
|
await expect(
|
|
resolveLiveModelApiKeyInfo({
|
|
model: {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
} as Model,
|
|
cfg,
|
|
requireProfileKeys: false,
|
|
}),
|
|
).resolves.toEqual({
|
|
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
|
source: "live Ollama local marker",
|
|
mode: "api-key",
|
|
});
|
|
} finally {
|
|
if (oldEnv === undefined) {
|
|
delete process.env.OLLAMA_API_KEY;
|
|
} else {
|
|
process.env.OLLAMA_API_KEY = oldEnv;
|
|
}
|
|
}
|
|
});
|
|
|
|
it("honors configured local Ollama credentials before the live local marker", async () => {
|
|
const oldEnv = process.env.LOCAL_OLLAMA_API_KEY;
|
|
process.env.LOCAL_OLLAMA_API_KEY = "secured-local-key";
|
|
try {
|
|
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
|
config: {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
providers: ["ollama"],
|
|
env: {},
|
|
});
|
|
|
|
await expect(
|
|
resolveLiveModelApiKeyInfo({
|
|
model: {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
} as Model,
|
|
cfg,
|
|
requireProfileKeys: false,
|
|
}),
|
|
).resolves.toEqual({
|
|
apiKey: "secured-local-key",
|
|
source: "env: LOCAL_OLLAMA_API_KEY (models.json secretref)",
|
|
mode: "api-key",
|
|
});
|
|
} finally {
|
|
if (oldEnv === undefined) {
|
|
delete process.env.LOCAL_OLLAMA_API_KEY;
|
|
} else {
|
|
process.env.LOCAL_OLLAMA_API_KEY = oldEnv;
|
|
}
|
|
}
|
|
});
|
|
|
|
it("reuses configured local Ollama credentials across canonical base URL forms", async () => {
|
|
const oldEnv = process.env.LOCAL_OLLAMA_API_KEY;
|
|
process.env.LOCAL_OLLAMA_API_KEY = "secured-local-key";
|
|
try {
|
|
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
|
config: {
|
|
plugins: {
|
|
bundledDiscovery: "compat",
|
|
},
|
|
models: {
|
|
providers: {
|
|
ollama: {
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434/v1",
|
|
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
|
models: [],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
providers: ["ollama"],
|
|
env: {},
|
|
});
|
|
|
|
await expect(
|
|
resolveLiveModelApiKeyInfo({
|
|
model: {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
baseUrl: "http://127.0.0.1:11434",
|
|
} as Model,
|
|
cfg,
|
|
requireProfileKeys: false,
|
|
}),
|
|
).resolves.toEqual({
|
|
apiKey: "secured-local-key",
|
|
source: "env: LOCAL_OLLAMA_API_KEY (models.json secretref)",
|
|
mode: "api-key",
|
|
});
|
|
} finally {
|
|
if (oldEnv === undefined) {
|
|
delete process.env.LOCAL_OLLAMA_API_KEY;
|
|
} else {
|
|
process.env.LOCAL_OLLAMA_API_KEY = oldEnv;
|
|
}
|
|
}
|
|
});
|
|
|
|
it("preserves model-level Ollama endpoint overrides when registering runtime streams", () => {
|
|
const returnedStream = {} as ReturnType<StreamFn>;
|
|
const runtimeCalls: Array<{ model: Model; context: Parameters<StreamFn>[1] }> = [];
|
|
const createConfiguredOllamaStreamFn = vi.fn<
|
|
OllamaRuntimeApi["createConfiguredOllamaStreamFn"]
|
|
>(
|
|
() =>
|
|
((model, context) => {
|
|
runtimeCalls.push({ model, context });
|
|
return returnedStream;
|
|
}) as StreamFn,
|
|
);
|
|
const streamFn = createLiveOllamaRuntimeStreamFn({
|
|
createConfiguredOllamaStreamFn,
|
|
providerBaseUrl: OLLAMA_DEFAULT_BASE_URL,
|
|
});
|
|
const model = {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
baseUrl: "http://192.168.1.10:11434",
|
|
} as Model;
|
|
const context = { systemPrompt: "system", messages: [] } as Parameters<StreamFn>[1];
|
|
|
|
expect(streamFn(model, context)).toBe(returnedStream);
|
|
expect(createConfiguredOllamaStreamFn).toHaveBeenCalledWith({
|
|
model,
|
|
providerBaseUrl: undefined,
|
|
});
|
|
expect(runtimeCalls).toEqual([{ model, context }]);
|
|
});
|
|
|
|
it("falls back to the configured Ollama provider endpoint for models without overrides", () => {
|
|
const createConfiguredOllamaStreamFn = vi.fn<
|
|
OllamaRuntimeApi["createConfiguredOllamaStreamFn"]
|
|
>(() => (() => ({}) as ReturnType<StreamFn>) as StreamFn);
|
|
const streamFn = createLiveOllamaRuntimeStreamFn({
|
|
createConfiguredOllamaStreamFn,
|
|
providerBaseUrl: "https://ollama.com",
|
|
});
|
|
const model = {
|
|
provider: "ollama",
|
|
id: "gemma3:4b",
|
|
api: "ollama",
|
|
} as Model;
|
|
|
|
void streamFn(model, { systemPrompt: "system", messages: [] } as Parameters<StreamFn>[1]);
|
|
|
|
expect(createConfiguredOllamaStreamFn).toHaveBeenCalledWith({
|
|
model,
|
|
providerBaseUrl: "https://ollama.com",
|
|
});
|
|
});
|
|
|
|
it("reports explicit refs that never become runnable candidates", () => {
|
|
expect(
|
|
findUnmatchedExplicitLiveModelRefs({
|
|
refs: [
|
|
{ provider: "deepseek", id: "deepseek-v4-flash" },
|
|
{ provider: "zai", id: "glm-5.1" },
|
|
],
|
|
models: [{ provider: "deepseek", id: "deepseek-v4-flash" }],
|
|
env: {},
|
|
}),
|
|
).toEqual(["zai/glm-5.1"]);
|
|
});
|
|
});
|
|
|
|
function resolveTestReasoning(
|
|
model: Model,
|
|
): "minimal" | "low" | "medium" | "high" | "xhigh" | undefined {
|
|
if (!model.reasoning) {
|
|
return undefined;
|
|
}
|
|
const id = model.id.toLowerCase();
|
|
if (id.includes("deep-research")) {
|
|
return "medium";
|
|
}
|
|
if (model.provider === "openrouter" && id.startsWith("qwq")) {
|
|
return undefined;
|
|
}
|
|
if (model.provider === "xai" && id.startsWith("grok-4")) {
|
|
return undefined;
|
|
}
|
|
if (model.provider === "openai") {
|
|
if (id.includes("pro")) {
|
|
return "high";
|
|
}
|
|
return "medium";
|
|
}
|
|
return "low";
|
|
}
|
|
|
|
function resolveLiveSystemPrompt(model: Model): string | undefined {
|
|
if (model.provider === "openai") {
|
|
return "You are a concise assistant. Follow the user's instruction exactly.";
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
describe("resolveLiveSystemPrompt", () => {
|
|
it("adds instructions for openai probes", () => {
|
|
expect(
|
|
resolveLiveSystemPrompt({
|
|
provider: "openai",
|
|
} as Model),
|
|
).toContain("Follow the user's instruction exactly.");
|
|
});
|
|
|
|
it("keeps other providers unchanged", () => {
|
|
expect(
|
|
resolveLiveSystemPrompt({
|
|
provider: "ollama",
|
|
} as Model),
|
|
).toBeUndefined();
|
|
});
|
|
|
|
it("matches OpenAI Codex HTML interruption pages", () => {
|
|
expect(
|
|
isOpenAiCodexHtmlInterruption(
|
|
'Error: <html><head><meta name="viewport" content="width=device-width" /></head><body>Try again</body></html>',
|
|
),
|
|
).toBe(true);
|
|
expect(isOpenAiCodexHtmlInterruption("Error: connection reset")).toBe(false);
|
|
});
|
|
});
|
|
|
|
async function completeSimpleWithTimeout<TApi extends Api>(
|
|
model: Model<TApi>,
|
|
context: Parameters<typeof completeSimple<TApi>>[1],
|
|
options: Parameters<typeof completeSimple<TApi>>[2],
|
|
timeoutMs: number,
|
|
progressContext: string,
|
|
) {
|
|
const maxTimeoutMs = Math.max(1, timeoutMs);
|
|
const controller = new AbortController();
|
|
const abortTimer = setTimeout(() => {
|
|
controller.abort();
|
|
}, maxTimeoutMs);
|
|
abortTimer.unref?.();
|
|
let hardTimer: ReturnType<typeof setTimeout> | undefined;
|
|
const timeout = new Promise<never>((_, reject) => {
|
|
hardTimer = setTimeout(() => {
|
|
reject(new Error(`model call timed out after ${maxTimeoutMs}ms`));
|
|
}, maxTimeoutMs);
|
|
hardTimer.unref?.();
|
|
});
|
|
try {
|
|
const completionModel = prepareModelForSimpleCompletion({
|
|
model,
|
|
cfg: activeLiveCompletionConfig,
|
|
});
|
|
return await withLiveHeartbeat(
|
|
Promise.race([
|
|
completeSimple(completionModel, context, {
|
|
...options,
|
|
signal: controller.signal,
|
|
}),
|
|
timeout,
|
|
]),
|
|
progressContext,
|
|
);
|
|
} finally {
|
|
clearTimeout(abortTimer);
|
|
if (hardTimer) {
|
|
clearTimeout(hardTimer);
|
|
}
|
|
}
|
|
}
|
|
|
|
function requireToolChoicePayload(payload: unknown): unknown {
|
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
|
return undefined;
|
|
}
|
|
const candidate = payload as { tools?: unknown; tool_choice?: unknown };
|
|
if (!Array.isArray(candidate.tools) || candidate.tools.length === 0) {
|
|
return undefined;
|
|
}
|
|
return {
|
|
...candidate,
|
|
tool_choice: { type: "function", name: "noop" },
|
|
};
|
|
}
|
|
|
|
describe("requireToolChoicePayload", () => {
|
|
it("requires tool use when a Responses payload has tools", () => {
|
|
expect(requireToolChoicePayload({ model: "gpt", tools: [{ name: "noop" }] })).toEqual({
|
|
model: "gpt",
|
|
tools: [{ name: "noop" }],
|
|
tool_choice: { type: "function", name: "noop" },
|
|
});
|
|
});
|
|
|
|
it("leaves payloads without tools unchanged", () => {
|
|
expect(requireToolChoicePayload({ model: "gpt", tools: [] })).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
async function completeOkWithRetry(params: {
|
|
model: Model;
|
|
apiKey: string;
|
|
timeoutMs: number;
|
|
progressLabel: string;
|
|
}) {
|
|
const runOnce = async (maxTokens: number) => {
|
|
const res = await completeSimpleWithTimeout(
|
|
params.model,
|
|
{
|
|
systemPrompt: resolveLiveSystemPrompt(params.model),
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: "Reply with the word ok.",
|
|
timestamp: Date.now(),
|
|
},
|
|
],
|
|
},
|
|
{
|
|
apiKey: params.apiKey,
|
|
reasoning: resolveTestReasoning(params.model),
|
|
maxTokens,
|
|
},
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: prompt call (maxTokens=${maxTokens})`,
|
|
);
|
|
const text = res.content
|
|
.filter((block) => block.type === "text")
|
|
.map((block) => block.text.trim())
|
|
.join(" ");
|
|
return { res, text };
|
|
};
|
|
|
|
const first = await runOnce(64);
|
|
if (first.text.length > 0) {
|
|
return first;
|
|
}
|
|
// Some providers (for example Moonshot Kimi and MiniMax M2.5) may emit
|
|
// reasoning blocks first and only return text once token budget is higher.
|
|
return await runOnce(256);
|
|
}
|
|
|
|
function isDeepSeekV4Model(model: Pick<Model, "id" | "provider">): boolean {
|
|
return (
|
|
model.provider === "deepseek" &&
|
|
(model.id === "deepseek-v4-flash" || model.id === "deepseek-v4-pro")
|
|
);
|
|
}
|
|
|
|
async function runDeepSeekV4ReplayRegression(params: {
|
|
model: Model;
|
|
apiKey: string;
|
|
timeoutMs: number;
|
|
progressLabel: string;
|
|
}) {
|
|
const noopTool = {
|
|
name: "noop",
|
|
description: "Return ok.",
|
|
parameters: Type.Object({}, { additionalProperties: false }),
|
|
};
|
|
let firstUser = {
|
|
role: "user" as const,
|
|
content: "Call the tool `noop` with {}. Do not write any other text.",
|
|
timestamp: Date.now(),
|
|
};
|
|
let first = await completeSimpleWithTimeout(
|
|
params.model,
|
|
{ messages: [firstUser], tools: [noopTool] },
|
|
{
|
|
apiKey: params.apiKey,
|
|
reasoning: resolveTestReasoning(params.model),
|
|
maxTokens: 256,
|
|
},
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: DeepSeek V4 replay first call`,
|
|
);
|
|
let toolCall = first.content.find((block) => block.type === "toolCall");
|
|
|
|
for (let i = 0; i < 2 && !toolCall; i += 1) {
|
|
firstUser = {
|
|
role: "user" as const,
|
|
content: "Call the tool `noop` with {}. IMPORTANT: respond with the tool call.",
|
|
timestamp: Date.now(),
|
|
};
|
|
first = await completeSimpleWithTimeout(
|
|
params.model,
|
|
{ messages: [firstUser], tools: [noopTool] },
|
|
{
|
|
apiKey: params.apiKey,
|
|
reasoning: resolveTestReasoning(params.model),
|
|
maxTokens: 256,
|
|
},
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: DeepSeek V4 replay retry ${i + 1}`,
|
|
);
|
|
toolCall = first.content.find((block) => block.type === "toolCall");
|
|
}
|
|
|
|
if (!toolCall || toolCall.type !== "toolCall") {
|
|
throw new Error("expected DeepSeek V4 tool call");
|
|
}
|
|
expect(toolCall.name).toBe("noop");
|
|
|
|
const second = await completeSimpleWithTimeout(
|
|
params.model,
|
|
{
|
|
messages: [
|
|
firstUser,
|
|
first,
|
|
{
|
|
role: "toolResult",
|
|
toolCallId: toolCall.id,
|
|
toolName: "noop",
|
|
content: [{ type: "text", text: "ok" }],
|
|
isError: false,
|
|
timestamp: Date.now(),
|
|
},
|
|
{
|
|
role: "user",
|
|
content: "Reply with the word ok.",
|
|
timestamp: Date.now(),
|
|
},
|
|
],
|
|
},
|
|
{
|
|
apiKey: params.apiKey,
|
|
reasoning: resolveTestReasoning(params.model),
|
|
maxTokens: 256,
|
|
},
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: DeepSeek V4 replay followup`,
|
|
);
|
|
if (second.stopReason === "error") {
|
|
throw new Error(second.errorMessage || "DeepSeek V4 replay followup returned error");
|
|
}
|
|
expect(extractAssistantText(second).length).toBeGreaterThan(0);
|
|
}
|
|
|
|
async function runExtraTurnProbes(params: {
|
|
model: Model;
|
|
apiKey: string;
|
|
timeoutMs: number;
|
|
progressLabel: string;
|
|
}) {
|
|
if (shouldSkipLiveModelExtraProbes(params.model)) {
|
|
logProgress(`${params.progressLabel}: extra probes skipped (known empty route)`);
|
|
return;
|
|
}
|
|
const options = {
|
|
apiKey: params.apiKey,
|
|
reasoning: resolveTestReasoning(params.model),
|
|
maxTokens: 128,
|
|
};
|
|
if (LIVE_FILE_PROBE_ENABLED && !shouldSkipLiveModelFileProbe(params.model)) {
|
|
logProgress(`${params.progressLabel}: file-read probe`);
|
|
const file = await completeSimpleWithTimeout(
|
|
params.model,
|
|
buildLiveModelFileProbeContext({ systemPrompt: resolveLiveSystemPrompt(params.model) }),
|
|
options,
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: file-read probe`,
|
|
);
|
|
if (file.stopReason === "error") {
|
|
throw new Error(file.errorMessage || "file-read probe returned error with no message");
|
|
}
|
|
let fileText = extractAssistantText(file);
|
|
if (!fileProbeTextMatches(fileText)) {
|
|
logProgress(`${params.progressLabel}: file-read probe retry`);
|
|
const retry = await completeSimpleWithTimeout(
|
|
params.model,
|
|
buildLiveModelFileProbeRetryContext({
|
|
systemPrompt: resolveLiveSystemPrompt(params.model),
|
|
}),
|
|
options,
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: file-read probe retry`,
|
|
);
|
|
if (retry.stopReason === "error") {
|
|
throw new Error(
|
|
retry.errorMessage || "file-read probe retry returned error with no message",
|
|
);
|
|
}
|
|
fileText = extractAssistantText(retry);
|
|
}
|
|
if (!fileProbeTextMatches(fileText)) {
|
|
if (fileText.length === 0) {
|
|
logProgress(`${params.progressLabel}: file-read probe skipped (empty response)`);
|
|
} else {
|
|
throw new Error(
|
|
`file-read probe did not return ${LIVE_MODEL_FILE_PROBE_TOKEN}: ${fileText}`,
|
|
);
|
|
}
|
|
}
|
|
} else if (LIVE_FILE_PROBE_ENABLED) {
|
|
logProgress(`${params.progressLabel}: file-read probe skipped (known empty route)`);
|
|
}
|
|
|
|
if (!LIVE_IMAGE_PROBE_ENABLED) {
|
|
return;
|
|
}
|
|
if (!modelSupportsImageInput(params.model)) {
|
|
logProgress(`${params.progressLabel}: image probe skipped (no image input)`);
|
|
return;
|
|
}
|
|
if (shouldSkipLiveModelImageProbe(params.model)) {
|
|
logProgress(`${params.progressLabel}: image probe skipped (known empty route)`);
|
|
return;
|
|
}
|
|
|
|
logProgress(`${params.progressLabel}: image probe`);
|
|
const image = await completeSimpleWithTimeout(
|
|
params.model,
|
|
buildLiveModelImageProbeContext({ systemPrompt: resolveLiveSystemPrompt(params.model) }),
|
|
options,
|
|
params.timeoutMs,
|
|
`${params.progressLabel}: image probe`,
|
|
);
|
|
if (image.stopReason === "error") {
|
|
throw new Error(image.errorMessage || "image probe returned error with no message");
|
|
}
|
|
const imageText = extractAssistantText(image);
|
|
if (!imageProbeTextMatches(imageText)) {
|
|
if (imageText.length === 0) {
|
|
logProgress(`${params.progressLabel}: image probe skipped (empty response)`);
|
|
return;
|
|
}
|
|
throw new Error(`image probe did not return ok: ${imageText}`);
|
|
}
|
|
}
|
|
|
|
describeLive("live models (profile keys)", () => {
|
|
it(
|
|
"completes across selected models",
|
|
async () => {
|
|
logProgress("[live-models] loading config");
|
|
const loadedCfg = await withLiveStageTimeout(
|
|
Promise.resolve().then(() => getRuntimeConfig()),
|
|
"[live-models] load config",
|
|
);
|
|
const rawModels = process.env.OPENCLAW_LIVE_MODELS?.trim();
|
|
const useModern = rawModels === "modern" || rawModels === "all";
|
|
const useSmall = rawModels === "small";
|
|
const useExplicit = Boolean(rawModels) && !useModern && !useSmall;
|
|
const filter = useExplicit ? parseModelFilter(rawModels) : null;
|
|
const explicitRefs = useExplicit ? parseExplicitLiveModelRefs(filter) : [];
|
|
const providers = parseProviderFilter(process.env.OPENCLAW_LIVE_PROVIDERS);
|
|
const priorityRefs = useSmall
|
|
? filterLiveModelRefsByProvider(listPrioritizedSmallLiveModelRefs(), providers)
|
|
: [];
|
|
const providerList = resolveLiveProviderDiscoveryProviderIds({
|
|
providerFilter: providers,
|
|
explicitRefs,
|
|
priorityRefs,
|
|
});
|
|
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
|
config: loadedCfg,
|
|
providers: providerList,
|
|
env: process.env,
|
|
});
|
|
await ensureLiveProviderApisRegistered({
|
|
config: cfg,
|
|
providers: providerList,
|
|
});
|
|
activeLiveCompletionConfig = cfg;
|
|
logProgress("[live-models] preparing models.json");
|
|
await withLiveStageTimeout(
|
|
ensureOpenClawModelsJson(
|
|
cfg,
|
|
undefined,
|
|
providerList ? { providerDiscoveryProviderIds: providerList } : undefined,
|
|
),
|
|
"[live-models] prepare models.json",
|
|
LIVE_MODELS_JSON_TIMEOUT_MS,
|
|
);
|
|
if (!DIRECT_ENABLED) {
|
|
logProgress(
|
|
"[live-models] skipping (set OPENCLAW_LIVE_MODELS=modern|small|all|<list>; all=modern)",
|
|
);
|
|
return;
|
|
}
|
|
const anthropicKeys = collectAnthropicApiKeys();
|
|
if (anthropicKeys.length > 0) {
|
|
process.env.ANTHROPIC_API_KEY = anthropicKeys[0];
|
|
logProgress(`[live-models] anthropic keys loaded: ${anthropicKeys.length}`);
|
|
}
|
|
|
|
logProgress("[live-models] resolving agent dir");
|
|
const agentDir = resolveDefaultAgentDir(cfg);
|
|
const useDefaultPriorityOnly = !filter && useModern && !providers;
|
|
const useSmallPriorityOnly = !filter && useSmall && !providers;
|
|
const allowNotFoundSkip = useModern || useSmall;
|
|
const models = await (async () => {
|
|
if (useDefaultPriorityOnly) {
|
|
logProgress("[live-models] loading configured prioritized model refs");
|
|
}
|
|
if (useSmallPriorityOnly) {
|
|
logProgress("[live-models] loading configured small model refs");
|
|
}
|
|
logProgress("[live-models] loading auth storage");
|
|
const authStorage = await withLiveStageTimeout(
|
|
Promise.resolve().then(() =>
|
|
discoverAuthStorage(agentDir, {
|
|
config: cfg,
|
|
env: process.env,
|
|
externalCli: externalCliDiscoveryForProviders({ cfg, providers: providerList ?? [] }),
|
|
...(providerList
|
|
? {
|
|
skipExternalAuthProfiles: true,
|
|
syntheticAuthProviderRefs: [],
|
|
}
|
|
: {}),
|
|
}),
|
|
),
|
|
"[live-models] load auth storage",
|
|
);
|
|
logProgress("[live-models] loading model registry");
|
|
const modelRegistry = await withLiveStageTimeout(
|
|
Promise.resolve().then(() =>
|
|
discoverModels(authStorage, agentDir, { normalizeModels: false }),
|
|
),
|
|
"[live-models] load model registry",
|
|
);
|
|
const configuredModels = modelRegistry.getAll();
|
|
const augmented = await appendPrioritizedDynamicLiveModels({
|
|
models: configuredModels,
|
|
config: cfg,
|
|
agentDir,
|
|
env: process.env,
|
|
modelRegistry,
|
|
...(explicitRefs.length > 0
|
|
? { refs: explicitRefs }
|
|
: useSmall
|
|
? { refs: priorityRefs }
|
|
: {}),
|
|
});
|
|
if (augmented.added.length > 0) {
|
|
logProgress(
|
|
`[live-models] loaded ${augmented.added.length} prioritized dynamic model refs`,
|
|
);
|
|
}
|
|
return augmented.models;
|
|
})();
|
|
const perModelTimeoutMs = toInt(process.env.OPENCLAW_LIVE_MODEL_TIMEOUT_MS, 30_000);
|
|
const maxModels = resolveHighSignalLiveModelLimit({
|
|
rawMaxModels: process.env.OPENCLAW_LIVE_MAX_MODELS,
|
|
useExplicitModels: useExplicit,
|
|
...(useSmall ? { defaultLimit: DEFAULT_SMALL_LIVE_MODEL_LIMIT } : {}),
|
|
});
|
|
const targetMatcher = createLiveTargetMatcher({
|
|
providerFilter: providers,
|
|
modelFilter: filter,
|
|
config: cfg,
|
|
env: process.env,
|
|
});
|
|
|
|
const failures: Array<{ model: string; error: string }> = [];
|
|
const skipped: Array<{ model: string; reason: string }> = [];
|
|
const candidates: Array<{
|
|
model: Model;
|
|
apiKeyInfo: Awaited<ReturnType<typeof getApiKeyForModel>>;
|
|
}> = [];
|
|
|
|
for (const model of models) {
|
|
if (shouldSuppressBuiltInModel({ provider: model.provider, id: model.id })) {
|
|
continue;
|
|
}
|
|
if (!targetMatcher.matchesProvider(model.provider)) {
|
|
continue;
|
|
}
|
|
const id = `${model.provider}/${model.id}`;
|
|
if (!targetMatcher.matchesModel(model.provider, model.id)) {
|
|
continue;
|
|
}
|
|
if (!filter && useSmall) {
|
|
if (!isSmallLiveModelRef({ provider: model.provider, id: model.id })) {
|
|
continue;
|
|
}
|
|
} else if (!filter && useModern) {
|
|
if (
|
|
useDefaultPriorityOnly &&
|
|
!isPrioritizedHighSignalLiveModelRef({ provider: model.provider, id: model.id })
|
|
) {
|
|
continue;
|
|
}
|
|
if (
|
|
shouldExcludeProviderFromDefaultHighSignalLiveSweep({
|
|
provider: model.provider,
|
|
useExplicitModels: useExplicit,
|
|
providerFilter: providers,
|
|
config: cfg,
|
|
env: process.env,
|
|
})
|
|
) {
|
|
continue;
|
|
}
|
|
if (!isHighSignalLiveModelRef({ provider: model.provider, id: model.id })) {
|
|
continue;
|
|
}
|
|
}
|
|
try {
|
|
const apiKeyInfo = await resolveLiveModelApiKeyInfo({
|
|
model,
|
|
cfg,
|
|
requireProfileKeys: REQUIRE_PROFILE_KEYS,
|
|
});
|
|
if (
|
|
requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) &&
|
|
!apiKeyInfo.source.startsWith("profile:")
|
|
) {
|
|
skipped.push({
|
|
model: id,
|
|
reason: `non-profile credential source: ${apiKeyInfo.source}`,
|
|
});
|
|
continue;
|
|
}
|
|
candidates.push({
|
|
model: normalizeDiscoveredAgentModel(model, agentDir),
|
|
apiKeyInfo,
|
|
});
|
|
} catch (err) {
|
|
skipped.push({ model: id, reason: String(err) });
|
|
}
|
|
}
|
|
|
|
if (candidates.length === 0) {
|
|
if (useExplicit) {
|
|
const skippedPreview =
|
|
skipped.length > 0 ? `\nSkipped candidates:\n${formatSkippedPreview(skipped, 8)}` : "";
|
|
throw new Error(
|
|
`[live-models] explicit model selection matched no runnable models.${skippedPreview}`,
|
|
);
|
|
}
|
|
logProgress("[live-models] no API keys found; skipping");
|
|
return;
|
|
}
|
|
if (useExplicit && explicitRefs.length > 0) {
|
|
const unmatched = findUnmatchedExplicitLiveModelRefs({
|
|
refs: explicitRefs,
|
|
models: candidates.map((entry) => entry.model),
|
|
config: cfg,
|
|
env: process.env,
|
|
});
|
|
if (unmatched.length > 0) {
|
|
const skippedPreview =
|
|
skipped.length > 0 ? `\nSkipped candidates:\n${formatSkippedPreview(skipped, 8)}` : "";
|
|
throw new Error(
|
|
`[live-models] explicit model selection missed requested models: ${unmatched.join(", ")}.${skippedPreview}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const selectCandidates = useSmall ? selectSmallLiveItems : selectHighSignalLiveItems;
|
|
const selectedCandidates = selectCandidates(
|
|
candidates,
|
|
maxModels > 0 ? maxModels : candidates.length,
|
|
(entry) => ({ provider: entry.model.provider, id: entry.model.id }),
|
|
(entry) => entry.model.provider,
|
|
);
|
|
const selectionLabel = useExplicit ? "explicit" : useSmall ? "small" : "high-signal";
|
|
logProgress(`[live-models] selection=${selectionLabel}`);
|
|
if (selectedCandidates.length < candidates.length) {
|
|
logProgress(
|
|
`[live-models] capped to ${selectedCandidates.length}/${candidates.length} via OPENCLAW_LIVE_MAX_MODELS=${maxModels}`,
|
|
);
|
|
}
|
|
logProgress(`[live-models] running ${selectedCandidates.length} models`);
|
|
logProgress(
|
|
`[live-models] heartbeat=${formatElapsedSeconds(LIVE_HEARTBEAT_MS)} timeout=${formatElapsedSeconds(perModelTimeoutMs)} concurrency=${LIVE_MODEL_CONCURRENCY}`,
|
|
);
|
|
const total = selectedCandidates.length;
|
|
|
|
const tasks = selectedCandidates.map((entry, index) => async () => {
|
|
const { model, apiKeyInfo } = entry;
|
|
const id = `${model.provider}/${model.id}`;
|
|
const progressLabel = `[live-models] ${index + 1}/${total} ${id}`;
|
|
const attemptMax =
|
|
model.provider === "anthropic" && anthropicKeys.length > 0 ? anthropicKeys.length : 1;
|
|
for (let attempt = 0; attempt < attemptMax; attempt += 1) {
|
|
if (model.provider === "anthropic" && anthropicKeys.length > 0) {
|
|
process.env.ANTHROPIC_API_KEY = anthropicKeys[attempt];
|
|
}
|
|
const apiKey =
|
|
model.provider === "anthropic" && anthropicKeys.length > 0
|
|
? anthropicKeys[attempt]
|
|
: requireApiKey(apiKeyInfo, model.provider);
|
|
try {
|
|
// Special regression: OpenAI requires replayed `reasoning` items for tool-only turns.
|
|
if (
|
|
model.provider === "openai" &&
|
|
model.api === "openai-responses" &&
|
|
model.id === "gpt-5.2"
|
|
) {
|
|
logProgress(`${progressLabel}: tool-only regression`);
|
|
const noopTool = {
|
|
name: "noop",
|
|
description: "Return ok.",
|
|
parameters: Type.Object({}, { additionalProperties: false }),
|
|
};
|
|
|
|
let firstUserContent = "Call the tool `noop` with {}. Do not write any other text.";
|
|
let firstUser = {
|
|
role: "user" as const,
|
|
content: firstUserContent,
|
|
timestamp: Date.now(),
|
|
};
|
|
|
|
let first = await completeSimpleWithTimeout(
|
|
model,
|
|
{ messages: [firstUser], tools: [noopTool] },
|
|
{
|
|
apiKey,
|
|
reasoning: resolveTestReasoning(model),
|
|
maxTokens: 128,
|
|
onPayload: requireToolChoicePayload,
|
|
},
|
|
perModelTimeoutMs,
|
|
`${progressLabel}: tool-only regression first call`,
|
|
);
|
|
|
|
let toolCall = first.content.find((b) => b.type === "toolCall");
|
|
let firstText = first.content
|
|
.filter((b) => b.type === "text")
|
|
.map((b) => b.text.trim())
|
|
.join(" ")
|
|
.trim();
|
|
|
|
// Occasional flake: model answers in text instead of tool call (or adds text).
|
|
// Retry a couple times with a stronger instruction so we still exercise the tool-only replay path.
|
|
for (let i = 0; i < 2 && (!toolCall || firstText.length > 0); i += 1) {
|
|
firstUserContent =
|
|
"Call the tool `noop` with {}. IMPORTANT: respond ONLY with the tool call; no other text.";
|
|
firstUser = {
|
|
role: "user" as const,
|
|
content: firstUserContent,
|
|
timestamp: Date.now(),
|
|
};
|
|
|
|
first = await completeSimpleWithTimeout(
|
|
model,
|
|
{ messages: [firstUser], tools: [noopTool] },
|
|
{
|
|
apiKey,
|
|
reasoning: resolveTestReasoning(model),
|
|
maxTokens: 128,
|
|
onPayload: requireToolChoicePayload,
|
|
},
|
|
perModelTimeoutMs,
|
|
`${progressLabel}: tool-only regression retry ${i + 1}`,
|
|
);
|
|
|
|
toolCall = first.content.find((b) => b.type === "toolCall");
|
|
firstText = first.content
|
|
.filter((b) => b.type === "text")
|
|
.map((b) => b.text.trim())
|
|
.join(" ")
|
|
.trim();
|
|
}
|
|
|
|
if (first.stopReason === "error") {
|
|
throw new Error(
|
|
first.errorMessage || "tool-only regression returned error with no message",
|
|
);
|
|
}
|
|
expect(firstText.length).toBe(0);
|
|
if (!toolCall || toolCall.type !== "toolCall") {
|
|
throw new Error("expected tool call");
|
|
}
|
|
expect(toolCall.name).toBe("noop");
|
|
|
|
const second = await completeSimpleWithTimeout(
|
|
model,
|
|
{
|
|
messages: [
|
|
firstUser,
|
|
first,
|
|
{
|
|
role: "toolResult",
|
|
toolCallId: toolCall.id,
|
|
toolName: "noop",
|
|
content: [{ type: "text", text: "ok" }],
|
|
isError: false,
|
|
timestamp: Date.now(),
|
|
},
|
|
{
|
|
role: "user",
|
|
content: "Reply with the word ok.",
|
|
timestamp: Date.now(),
|
|
},
|
|
],
|
|
},
|
|
{
|
|
apiKey,
|
|
reasoning: resolveTestReasoning(model),
|
|
// Headroom: reasoning summary can consume most of the output budget.
|
|
maxTokens: 256,
|
|
},
|
|
perModelTimeoutMs,
|
|
`${progressLabel}: tool-only regression followup`,
|
|
);
|
|
|
|
const secondText = second.content
|
|
.filter((b) => b.type === "text")
|
|
.map((b) => b.text.trim())
|
|
.join(" ");
|
|
expect(secondText.length).toBeGreaterThan(0);
|
|
await runExtraTurnProbes({
|
|
model,
|
|
apiKey,
|
|
timeoutMs: perModelTimeoutMs,
|
|
progressLabel,
|
|
});
|
|
logProgress(`${progressLabel}: done`);
|
|
break;
|
|
}
|
|
|
|
if (isDeepSeekV4Model(model)) {
|
|
logProgress(`${progressLabel}: DeepSeek V4 replay regression`);
|
|
await runDeepSeekV4ReplayRegression({
|
|
model,
|
|
apiKey,
|
|
timeoutMs: perModelTimeoutMs,
|
|
progressLabel,
|
|
});
|
|
await runExtraTurnProbes({
|
|
model,
|
|
apiKey,
|
|
timeoutMs: perModelTimeoutMs,
|
|
progressLabel,
|
|
});
|
|
logProgress(`${progressLabel}: done`);
|
|
break;
|
|
}
|
|
|
|
logProgress(`${progressLabel}: prompt`);
|
|
const ok = await completeOkWithRetry({
|
|
model,
|
|
apiKey,
|
|
timeoutMs: perModelTimeoutMs,
|
|
progressLabel,
|
|
});
|
|
|
|
if (ok.res.stopReason === "error") {
|
|
const msg = ok.res.errorMessage ?? "";
|
|
if (allowNotFoundSkip && isModelNotFoundErrorMessage(msg)) {
|
|
skipped.push({ model: id, reason: msg });
|
|
logProgress(`${progressLabel}: skip (model not found)`);
|
|
break;
|
|
}
|
|
throw new Error(msg || "model returned error with no message");
|
|
}
|
|
|
|
if (
|
|
ok.text.length === 0 &&
|
|
(model.provider === "google" || model.provider === "google-gemini-cli")
|
|
) {
|
|
skipped.push({
|
|
model: id,
|
|
reason: "no text returned (likely unavailable model id)",
|
|
});
|
|
logProgress(`${progressLabel}: skip (google model not found)`);
|
|
break;
|
|
}
|
|
if (
|
|
ok.text.length === 0 &&
|
|
(model.provider === "openrouter" ||
|
|
model.provider === "opencode" ||
|
|
model.provider === "opencode-go")
|
|
) {
|
|
skipped.push({
|
|
model: id,
|
|
reason: "no text returned (provider returned empty content)",
|
|
});
|
|
logProgress(`${progressLabel}: skip (empty response)`);
|
|
break;
|
|
}
|
|
if (
|
|
ok.text.length === 0 &&
|
|
allowNotFoundSkip &&
|
|
(model.provider === "fireworks" ||
|
|
model.provider === "google-antigravity" ||
|
|
model.provider === "minimax" ||
|
|
model.provider === "openai" ||
|
|
model.provider === "xai" ||
|
|
model.provider === "zai")
|
|
) {
|
|
skipped.push({
|
|
model: id,
|
|
reason: "no text returned (provider returned empty content)",
|
|
});
|
|
logProgress(`${progressLabel}: skip (empty response)`);
|
|
break;
|
|
}
|
|
expect(ok.text.length).toBeGreaterThan(0);
|
|
await runExtraTurnProbes({
|
|
model,
|
|
apiKey,
|
|
timeoutMs: perModelTimeoutMs,
|
|
progressLabel,
|
|
});
|
|
logProgress(`${progressLabel}: done`);
|
|
break;
|
|
} catch (err) {
|
|
const message = String(err);
|
|
if (
|
|
model.provider === "anthropic" &&
|
|
isLiveRateLimitDrift(message) &&
|
|
attempt + 1 < attemptMax
|
|
) {
|
|
logProgress(`${progressLabel}: rate limit, retrying with next key`);
|
|
continue;
|
|
}
|
|
if (model.provider === "anthropic" && isLiveRateLimitDrift(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (anthropic rate limit)`);
|
|
break;
|
|
}
|
|
if (model.provider === "anthropic" && isLiveBillingDrift(message)) {
|
|
if (attempt + 1 < attemptMax) {
|
|
logProgress(`${progressLabel}: billing issue, retrying with next key`);
|
|
continue;
|
|
}
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (anthropic billing)`);
|
|
break;
|
|
}
|
|
if (
|
|
(model.provider === "google" || model.provider === "google-gemini-cli") &&
|
|
isGoogleModelNotFoundError(err)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (google model not found)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "minimax" &&
|
|
message.includes("request ended without sending any chunks")
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (minimax empty response)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
(model.provider === "minimax" ||
|
|
model.provider === "zai" ||
|
|
model.provider === "openrouter") &&
|
|
isRateLimitErrorMessage(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (rate limit)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
(model.provider === "opencode" || model.provider === "opencode-go") &&
|
|
isRateLimitErrorMessage(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (rate limit)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && model.provider === "openai" && isRefreshTokenReused(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (codex refresh token reused)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "openai" &&
|
|
isAccountIdExtractionError(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (codex account id extraction)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "openai" &&
|
|
isChatGPTUsageLimitErrorMessage(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (chatgpt usage limit)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "openai" &&
|
|
isInstructionsRequiredError(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (instructions required)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "openai" &&
|
|
isOpenAiCodexHtmlInterruption(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (codex html interruption)`);
|
|
break;
|
|
}
|
|
const driftSkip = shouldSkipLiveProviderDrift({
|
|
error: message,
|
|
allowAuth: allowNotFoundSkip,
|
|
allowModelNotFound: false,
|
|
allowProviderUnavailable: allowNotFoundSkip,
|
|
allowTimeout: allowNotFoundSkip,
|
|
});
|
|
if (driftSkip) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (${driftSkip.label})`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "openrouter" &&
|
|
isOpenRouterOpaqueBadRequestErrorMessage(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (openrouter upstream bad request)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && isModelNotFoundErrorMessage(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (model not found)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && isAudioOnlyModelErrorMessage(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (audio-only model)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && isUnsupportedReasoningEffortErrorMessage(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (reasoning unsupported)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && isUnsupportedThinkingToggleErrorMessage(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (thinking toggle unsupported)`);
|
|
break;
|
|
}
|
|
if (allowNotFoundSkip && isUnsupportedPlanErrorMessage(message)) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (plan unsupported)`);
|
|
break;
|
|
}
|
|
if (
|
|
allowNotFoundSkip &&
|
|
model.provider === "ollama" &&
|
|
isOllamaUnavailableErrorMessage(message)
|
|
) {
|
|
skipped.push({ model: id, reason: message });
|
|
logProgress(`${progressLabel}: skip (ollama unavailable)`);
|
|
break;
|
|
}
|
|
logProgress(`${progressLabel}: failed`);
|
|
failures.push({ model: id, error: message });
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
|
|
await runTasksWithConcurrency({
|
|
tasks,
|
|
limit: LIVE_MODEL_CONCURRENCY,
|
|
});
|
|
|
|
if (failures.length > 0) {
|
|
const preview = formatFailurePreview(failures, 20);
|
|
throw new Error(
|
|
`live model failures (${failures.length}, showing ${Math.min(failures.length, 20)}):\n${preview}`,
|
|
);
|
|
}
|
|
|
|
void skipped;
|
|
},
|
|
LIVE_TEST_TIMEOUT_MS,
|
|
);
|
|
});
|