mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-03 18:14:06 +00:00
test(agents): include Ollama in small live model matrix (#87838)
* test(agents): include Ollama in small live model matrix * test: avoid Ollama cloud key in local live runs * test: recognize Ollama env secret refs * test: type Ollama live key fixtures * test: prevent Ollama cloud auth in local live probes * test: preserve equivalent Ollama live credentials --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -74,9 +74,10 @@ Live tests are split into two layers so we can isolate failures:
|
||||
- Set `OPENCLAW_LIVE_MODELS=modern`, `small`, or `all` (alias for modern) to actually run this suite; otherwise it skips to keep `pnpm test:live` focused on gateway smoke
|
||||
- How to select models:
|
||||
- `OPENCLAW_LIVE_MODELS=modern` to run the modern allowlist (Opus/Sonnet 4.6+, GPT-5.2 + Codex, Gemini 3, DeepSeek V4, GLM 4.7, MiniMax M2.7, Grok 4.3)
|
||||
- `OPENCLAW_LIVE_MODELS=small` to run the constrained small-model allowlist (Qwen 8B/9B local-compatible routes, OpenRouter Qwen/GLM, and Z.AI GLM)
|
||||
- `OPENCLAW_LIVE_MODELS=small` to run the constrained small-model allowlist (Qwen 8B/9B local-compatible routes, Ollama Gemma, OpenRouter Qwen/GLM, and Z.AI GLM)
|
||||
- `OPENCLAW_LIVE_MODELS=all` is an alias for the modern allowlist
|
||||
- or `OPENCLAW_LIVE_MODELS="openai/gpt-5.5,anthropic/claude-opus-4-6,..."` (comma allowlist)
|
||||
- Local Ollama small-model runs default to `http://127.0.0.1:11434`; set `OPENCLAW_LIVE_OLLAMA_BASE_URL` only for LAN, custom, or Ollama Cloud endpoints.
|
||||
- Modern/all and small sweeps default to their curated caps; set `OPENCLAW_LIVE_MAX_MODELS=0` for an exhaustive selected-profile sweep or a positive number for a smaller cap.
|
||||
- Exhaustive sweeps use `OPENCLAW_LIVE_TEST_TIMEOUT_MS` for the whole direct-model test timeout. Default: 60 minutes.
|
||||
- Direct-model probes run with 20-way parallelism by default; set `OPENCLAW_LIVE_MODEL_CONCURRENCY` to override.
|
||||
|
||||
@@ -35,6 +35,7 @@ const SMALL_LIVE_MODEL_PRIORITY = [
|
||||
"lmstudio/qwen/qwen3.5-9b",
|
||||
"vllm/qwen/qwen3-8b",
|
||||
"sglang/qwen/qwen3-8b",
|
||||
"ollama/gemma3:4b",
|
||||
"openrouter/qwen/qwen3.5-9b",
|
||||
"openrouter/z-ai/glm-5.1",
|
||||
"openrouter/z-ai/glm-5",
|
||||
|
||||
@@ -673,6 +673,7 @@ describe("isPrioritizedHighSignalLiveModelRef", () => {
|
||||
describe("isSmallLiveModelRef", () => {
|
||||
it("matches the small-model live matrix without requiring provider modern hooks", () => {
|
||||
expect(isSmallLiveModelRef({ provider: "lmstudio", id: "Qwen/Qwen3.5-9B" })).toBe(true);
|
||||
expect(isSmallLiveModelRef({ provider: "ollama", id: "gemma3:4b" })).toBe(true);
|
||||
expect(isSmallLiveModelRef({ provider: "openrouter", id: "qwen/qwen3.5-9b" })).toBe(true);
|
||||
expect(isSmallLiveModelRef({ provider: "openrouter", id: "z-ai/glm-5.1" })).toBe(true);
|
||||
expect(isSmallLiveModelRef({ provider: "openai", id: "gpt-5.5" })).toBe(false);
|
||||
@@ -689,6 +690,7 @@ describe("isPrioritizedSmallLiveModelRef", () => {
|
||||
{ provider: "lmstudio", id: "qwen/qwen3.5-9b" },
|
||||
{ provider: "vllm", id: "qwen/qwen3-8b" },
|
||||
{ provider: "sglang", id: "qwen/qwen3-8b" },
|
||||
{ provider: "ollama", id: "gemma3:4b" },
|
||||
{ provider: "openrouter", id: "qwen/qwen3.5-9b" },
|
||||
{ provider: "openrouter", id: "z-ai/glm-5.1" },
|
||||
{ provider: "openrouter", id: "z-ai/glm-5" },
|
||||
@@ -775,6 +777,7 @@ describe("selectSmallLiveItems", () => {
|
||||
{ provider: "openai", id: "gpt-5.5" },
|
||||
{ provider: "vllm", id: "qwen/qwen3-8b" },
|
||||
{ provider: "lmstudio", id: "qwen/qwen3.5-9b" },
|
||||
{ provider: "ollama", id: "gemma3:4b" },
|
||||
{ provider: "openrouter", id: "qwen/qwen3.5-9b" },
|
||||
];
|
||||
|
||||
@@ -788,7 +791,7 @@ describe("selectSmallLiveItems", () => {
|
||||
).toEqual([
|
||||
{ provider: "lmstudio", id: "qwen/qwen3.5-9b" },
|
||||
{ provider: "vllm", id: "qwen/qwen3-8b" },
|
||||
{ provider: "openrouter", id: "qwen/qwen3.5-9b" },
|
||||
{ provider: "ollama", id: "gemma3:4b" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,9 +2,10 @@ import { writeSync } from "node:fs";
|
||||
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
||||
import { type Api, completeSimple, type Model } from "openclaw/plugin-sdk/llm";
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { coerceSecretRef, type SecretInput } from "../config/types.secrets.js";
|
||||
import { parseLiveCsvFilter } from "../media-generation/live-test-helpers.js";
|
||||
import { withBundledPluginEnablementCompat } from "../plugins/bundled-compat.js";
|
||||
import { resolveOwningPluginIdsForProviderRef } from "../plugins/providers.js";
|
||||
@@ -16,6 +17,7 @@ import {
|
||||
} from "./agent-model-discovery.js";
|
||||
import { resolveDefaultAgentDir } from "./agent-scope.js";
|
||||
import { externalCliDiscoveryForProviders } from "./auth-profiles/external-cli-discovery.js";
|
||||
import { ensureCustomApiRegistered } from "./custom-api-registry.js";
|
||||
import { isRateLimitErrorMessage } from "./embedded-agent-helpers/errors.js";
|
||||
import { collectAnthropicApiKeys } from "./live-auth-keys.js";
|
||||
import { appendPrioritizedDynamicLiveModels } from "./live-model-dynamic-candidates.js";
|
||||
@@ -60,9 +62,14 @@ import {
|
||||
isLiveRateLimitDrift,
|
||||
shouldSkipLiveProviderDrift,
|
||||
} from "./live-test-provider-drift.js";
|
||||
import { getApiKeyForModel, requireApiKey } from "./model-auth.js";
|
||||
import {
|
||||
getApiKeyForModel,
|
||||
requireApiKey,
|
||||
resolveUsableCustomProviderApiKey,
|
||||
} from "./model-auth.js";
|
||||
import { shouldSuppressBuiltInModel } from "./model-suppression.js";
|
||||
import { ensureOpenClawModelsJson } from "./models-config.js";
|
||||
import type { StreamFn } from "./runtime/index.js";
|
||||
import { prepareModelForSimpleCompletion } from "./simple-completion-transport.js";
|
||||
|
||||
const LIVE = isLiveTestEnabled();
|
||||
@@ -86,8 +93,28 @@ const LIVE_MODELS_JSON_TIMEOUT_MS = resolveLiveModelsJsonTimeoutMs(
|
||||
);
|
||||
const LIVE_FILE_PROBE_ENABLED = isLiveModelProbeEnabled(process.env, LIVE_MODEL_FILE_PROBE_ENV);
|
||||
const LIVE_IMAGE_PROBE_ENABLED = isLiveModelProbeEnabled(process.env, LIVE_MODEL_IMAGE_PROBE_ENV);
|
||||
const OLLAMA_DEFAULT_BASE_URL = "http://127.0.0.1:11434";
|
||||
const OLLAMA_LOCAL_API_KEY_MARKER = "ollama-local";
|
||||
const OLLAMA_REMOTE_API_KEY_ENV = "OLLAMA_API_KEY";
|
||||
const LOCAL_OLLAMA_HOSTNAMES = new Set([
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"0.0.0.0",
|
||||
"::1",
|
||||
"::",
|
||||
"docker.orb.internal",
|
||||
"host.docker.internal",
|
||||
"host.orb.internal",
|
||||
]);
|
||||
let activeLiveCompletionConfig: OpenClawConfig | undefined;
|
||||
|
||||
type OllamaRuntimeApi = {
|
||||
createConfiguredOllamaStreamFn: (params: {
|
||||
model: { baseUrl?: string; headers?: unknown };
|
||||
providerBaseUrl?: string;
|
||||
}) => StreamFn;
|
||||
};
|
||||
|
||||
const describeLive = LIVE ? describe : describe.skip;
|
||||
|
||||
function parseCsvFilter(raw?: string): Set<string> | null {
|
||||
@@ -135,6 +162,19 @@ function formatExplicitLiveModelRef(ref: { provider: string; id: string }): stri
|
||||
return `${ref.provider}/${ref.id}`;
|
||||
}
|
||||
|
||||
function filterLiveModelRefsByProvider(
|
||||
refs: readonly { provider: string; id: string }[],
|
||||
providerFilter: Set<string> | null,
|
||||
): Array<{ provider: string; id: string }> {
|
||||
if (!providerFilter) {
|
||||
return [...refs];
|
||||
}
|
||||
const normalizedProviders = new Set(
|
||||
[...providerFilter].map((provider) => normalizeProviderId(provider)).filter(Boolean),
|
||||
);
|
||||
return refs.filter((ref) => normalizedProviders.has(normalizeProviderId(ref.provider)));
|
||||
}
|
||||
|
||||
function findUnmatchedExplicitLiveModelRefs(params: {
|
||||
refs: readonly { provider: string; id: string }[];
|
||||
models: readonly Pick<Model, "provider" | "id">[];
|
||||
@@ -160,6 +200,7 @@ function findUnmatchedExplicitLiveModelRefs(params: {
|
||||
function resolveLiveProviderDiscoveryProviderIds(params: {
|
||||
providerFilter: Set<string> | null;
|
||||
explicitRefs: readonly { provider: string; id: string }[];
|
||||
priorityRefs?: readonly { provider: string; id: string }[];
|
||||
}): string[] | undefined {
|
||||
const providers = new Set<string>();
|
||||
for (const provider of params.providerFilter ?? []) {
|
||||
@@ -171,6 +212,9 @@ function resolveLiveProviderDiscoveryProviderIds(params: {
|
||||
for (const ref of params.explicitRefs) {
|
||||
providers.add(ref.provider);
|
||||
}
|
||||
for (const ref of params.priorityRefs ?? []) {
|
||||
providers.add(ref.provider);
|
||||
}
|
||||
return providers.size > 0
|
||||
? [...providers].toSorted((left, right) => left.localeCompare(right))
|
||||
: undefined;
|
||||
@@ -206,12 +250,285 @@ function applyLiveProviderDiscoveryPluginCompat(params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): OpenClawConfig {
|
||||
const pluginIds = resolveLiveProviderDiscoveryPluginIds(params);
|
||||
return pluginIds.length > 0
|
||||
? (withBundledPluginEnablementCompat({
|
||||
config: params.config,
|
||||
pluginIds,
|
||||
}) ?? params.config)
|
||||
: params.config;
|
||||
const pluginConfig =
|
||||
pluginIds.length > 0 ? enableLiveProviderPlugins(params.config, pluginIds) : params.config;
|
||||
return applyLiveOllamaProviderEnvCompat({
|
||||
config: pluginConfig,
|
||||
providers: params.providers,
|
||||
env: params.env,
|
||||
});
|
||||
}
|
||||
|
||||
function enableLiveProviderPlugins(
|
||||
config: OpenClawConfig,
|
||||
pluginIds: readonly string[],
|
||||
): OpenClawConfig {
|
||||
const compatConfig =
|
||||
withBundledPluginEnablementCompat({
|
||||
config,
|
||||
pluginIds,
|
||||
}) ?? config;
|
||||
const entries = { ...compatConfig.plugins?.entries };
|
||||
const allow = new Set(compatConfig.plugins?.allow ?? []);
|
||||
for (const pluginId of pluginIds) {
|
||||
allow.add(pluginId);
|
||||
entries[pluginId] ??= { enabled: true };
|
||||
}
|
||||
return {
|
||||
...compatConfig,
|
||||
plugins: {
|
||||
...compatConfig.plugins,
|
||||
enabled: true,
|
||||
allow: [...allow].toSorted((left, right) => left.localeCompare(right)),
|
||||
bundledDiscovery: compatConfig.plugins?.bundledDiscovery ?? "compat",
|
||||
entries,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function applyLiveOllamaProviderEnvCompat(params: {
|
||||
config: OpenClawConfig;
|
||||
providers: readonly string[] | undefined;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): OpenClawConfig {
|
||||
if (!params.providers?.some((provider) => normalizeProviderId(provider) === "ollama")) {
|
||||
return params.config;
|
||||
}
|
||||
const existingProvider = params.config.models?.providers?.ollama;
|
||||
const configuredBaseUrl = readConfiguredOllamaBaseUrl(existingProvider);
|
||||
const liveBaseUrl = params.env?.OPENCLAW_LIVE_OLLAMA_BASE_URL?.trim();
|
||||
const baseUrl = liveBaseUrl || configuredBaseUrl || OLLAMA_DEFAULT_BASE_URL;
|
||||
const shouldPreserveConfiguredApiKey =
|
||||
!liveBaseUrl ||
|
||||
Boolean(
|
||||
configuredBaseUrl &&
|
||||
canonicalOllamaCredentialBaseUrl(configuredBaseUrl) ===
|
||||
canonicalOllamaCredentialBaseUrl(baseUrl),
|
||||
);
|
||||
const apiKey = resolveLiveOllamaProviderApiKey({
|
||||
baseUrl,
|
||||
existingApiKey: existingProvider?.apiKey,
|
||||
shouldPreserveConfiguredApiKey,
|
||||
});
|
||||
return {
|
||||
...params.config,
|
||||
models: {
|
||||
...params.config.models,
|
||||
providers: {
|
||||
...params.config.models?.providers,
|
||||
ollama: {
|
||||
...existingProvider,
|
||||
api: "ollama",
|
||||
baseUrl,
|
||||
apiKey,
|
||||
models: existingProvider?.models ?? [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureLiveProviderApisRegistered(params: {
|
||||
config: OpenClawConfig;
|
||||
providers: readonly string[] | undefined;
|
||||
}): Promise<void> {
|
||||
if (!params.providers?.some((provider) => normalizeProviderId(provider) === "ollama")) {
|
||||
return;
|
||||
}
|
||||
// Live Vitest setup installs a stub plugin registry for channel tests; direct
|
||||
// model probes still need the public Ollama runtime registered in-process.
|
||||
const runtimeApiUrl = new URL("../../extensions/ollama/runtime-api.ts", import.meta.url).href;
|
||||
const { createConfiguredOllamaStreamFn } = (await import(
|
||||
/* @vite-ignore */ runtimeApiUrl
|
||||
)) as OllamaRuntimeApi;
|
||||
const providerConfig = params.config.models?.providers?.ollama;
|
||||
const providerBaseUrl = readConfiguredOllamaBaseUrl(providerConfig) || OLLAMA_DEFAULT_BASE_URL;
|
||||
ensureCustomApiRegistered(
|
||||
"ollama",
|
||||
createLiveOllamaRuntimeStreamFn({
|
||||
createConfiguredOllamaStreamFn,
|
||||
providerBaseUrl,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function createLiveOllamaRuntimeStreamFn(params: {
|
||||
createConfiguredOllamaStreamFn: OllamaRuntimeApi["createConfiguredOllamaStreamFn"];
|
||||
providerBaseUrl: string;
|
||||
}): StreamFn {
|
||||
return (model, context, options) => {
|
||||
const modelBaseUrl = readStringProperty(model, "baseUrl");
|
||||
const streamFn = params.createConfiguredOllamaStreamFn({
|
||||
model,
|
||||
providerBaseUrl: modelBaseUrl ? undefined : params.providerBaseUrl,
|
||||
});
|
||||
return streamFn(model, context, options);
|
||||
};
|
||||
}
|
||||
|
||||
function readConfiguredOllamaBaseUrl(provider: unknown): string {
|
||||
return readStringProperty(provider, "baseUrl") || readStringProperty(provider, "baseURL");
|
||||
}
|
||||
|
||||
function resolveLiveOllamaProviderApiKey(params: {
|
||||
baseUrl: string;
|
||||
existingApiKey: SecretInput | undefined;
|
||||
shouldPreserveConfiguredApiKey: boolean;
|
||||
}): SecretInput {
|
||||
if (isLocalOllamaBaseUrl(params.baseUrl)) {
|
||||
return params.shouldPreserveConfiguredApiKey &&
|
||||
params.existingApiKey !== undefined &&
|
||||
!isOllamaRemoteApiKeyReference(params.existingApiKey)
|
||||
? params.existingApiKey
|
||||
: OLLAMA_LOCAL_API_KEY_MARKER;
|
||||
}
|
||||
return params.shouldPreserveConfiguredApiKey
|
||||
? (params.existingApiKey ?? OLLAMA_REMOTE_API_KEY_ENV)
|
||||
: OLLAMA_REMOTE_API_KEY_ENV;
|
||||
}
|
||||
|
||||
function isOllamaRemoteApiKeyReference(value: SecretInput | undefined): boolean {
|
||||
if (value === undefined) {
|
||||
return false;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
if (value.trim() === OLLAMA_REMOTE_API_KEY_ENV) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const ref = coerceSecretRef(value);
|
||||
return ref?.source === "env" && ref.id.trim() === OLLAMA_REMOTE_API_KEY_ENV;
|
||||
}
|
||||
|
||||
function readStringProperty(value: unknown, key: string): string {
|
||||
if (!value || typeof value !== "object" || !(key in value)) {
|
||||
return "";
|
||||
}
|
||||
const raw = (value as Record<string, unknown>)[key];
|
||||
return typeof raw === "string" ? raw.trim() : "";
|
||||
}
|
||||
|
||||
function isLocalOllamaBaseUrl(baseUrl: string): boolean {
|
||||
try {
|
||||
let host = new URL(baseUrl).hostname.toLowerCase();
|
||||
if (host.startsWith("[") && host.endsWith("]")) {
|
||||
host = host.slice(1, -1);
|
||||
}
|
||||
return (
|
||||
LOCAL_OLLAMA_HOSTNAMES.has(host) ||
|
||||
host.endsWith(".local") ||
|
||||
isIpv4PrivateRange(host) ||
|
||||
isIpv6LocalRange(host) ||
|
||||
(!host.includes(".") && !host.includes(":"))
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveLiveOllamaBaseUrl(model: Pick<Model, "baseUrl">, config?: OpenClawConfig): string {
|
||||
return (
|
||||
readStringProperty(model, "baseUrl") ||
|
||||
readConfiguredOllamaBaseUrl(config?.models?.providers?.ollama) ||
|
||||
OLLAMA_DEFAULT_BASE_URL
|
||||
);
|
||||
}
|
||||
|
||||
function isLiveLocalOllamaModel(
|
||||
model: Pick<Model, "provider" | "baseUrl">,
|
||||
config?: OpenClawConfig,
|
||||
): boolean {
|
||||
return (
|
||||
normalizeProviderId(model.provider) === "ollama" &&
|
||||
isLocalOllamaBaseUrl(resolveLiveOllamaBaseUrl(model, config))
|
||||
);
|
||||
}
|
||||
|
||||
function canReuseConfiguredLocalOllamaApiKey(
|
||||
model: Pick<Model, "baseUrl">,
|
||||
config?: OpenClawConfig,
|
||||
): boolean {
|
||||
const providerConfig = config?.models?.providers?.ollama;
|
||||
if (isOllamaRemoteApiKeyReference(providerConfig?.apiKey)) {
|
||||
return false;
|
||||
}
|
||||
const modelBaseUrl = readStringProperty(model, "baseUrl");
|
||||
if (!modelBaseUrl) {
|
||||
return true;
|
||||
}
|
||||
const providerBaseUrl = readConfiguredOllamaBaseUrl(providerConfig) || OLLAMA_DEFAULT_BASE_URL;
|
||||
return (
|
||||
canonicalOllamaCredentialBaseUrl(providerBaseUrl) ===
|
||||
canonicalOllamaCredentialBaseUrl(modelBaseUrl)
|
||||
);
|
||||
}
|
||||
|
||||
function canonicalOllamaCredentialBaseUrl(baseUrl: string): string {
|
||||
try {
|
||||
const parsed = new URL(baseUrl);
|
||||
let pathname = parsed.pathname.replace(/\/+$/, "");
|
||||
if (pathname === "/v1") {
|
||||
pathname = "";
|
||||
}
|
||||
parsed.pathname = pathname || "/";
|
||||
parsed.search = "";
|
||||
parsed.hash = "";
|
||||
return parsed.toString().replace(/\/$/, "");
|
||||
} catch {
|
||||
return baseUrl.trim().replace(/\/+$/, "");
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveLiveModelApiKeyInfo(params: {
|
||||
model: Model;
|
||||
cfg: OpenClawConfig;
|
||||
requireProfileKeys: boolean;
|
||||
}): Promise<Awaited<ReturnType<typeof getApiKeyForModel>>> {
|
||||
if (isLiveLocalOllamaModel(params.model, params.cfg)) {
|
||||
const configuredKey = canReuseConfiguredLocalOllamaApiKey(params.model, params.cfg)
|
||||
? resolveUsableCustomProviderApiKey({
|
||||
cfg: params.cfg,
|
||||
provider: "ollama",
|
||||
})
|
||||
: null;
|
||||
if (configuredKey && configuredKey.apiKey !== OLLAMA_LOCAL_API_KEY_MARKER) {
|
||||
return {
|
||||
apiKey: configuredKey.apiKey,
|
||||
source: configuredKey.source,
|
||||
mode: "api-key",
|
||||
};
|
||||
}
|
||||
return {
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
source: "live Ollama local marker",
|
||||
mode: "api-key",
|
||||
};
|
||||
}
|
||||
return await getApiKeyForModel({
|
||||
model: params.model,
|
||||
cfg: params.cfg,
|
||||
credentialPrecedence: resolveLiveCredentialPrecedence(
|
||||
params.model.provider,
|
||||
params.requireProfileKeys,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function isIpv4PrivateRange(host: string): boolean {
|
||||
if (!/^\d+\.\d+\.\d+\.\d+$/.test(host)) {
|
||||
return false;
|
||||
}
|
||||
const octets = host.split(".").map((part) => Number.parseInt(part, 10));
|
||||
if (octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
||||
return false;
|
||||
}
|
||||
const [a, b] = octets;
|
||||
return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168);
|
||||
}
|
||||
|
||||
function isIpv6LocalRange(host: string): boolean {
|
||||
const lower = host.toLowerCase();
|
||||
return /^fe[89ab][0-9a-f]:/.test(lower) || /^f[cd][0-9a-f]{2}:/.test(lower);
|
||||
}
|
||||
|
||||
function logProgress(message: string): void {
|
||||
@@ -503,6 +820,25 @@ describe("explicit live model discovery scope", () => {
|
||||
).toEqual(["deepseek", "together", "zai"]);
|
||||
});
|
||||
|
||||
it("includes curated small-model providers in discovery scope", () => {
|
||||
expect(
|
||||
resolveLiveProviderDiscoveryProviderIds({
|
||||
providerFilter: null,
|
||||
explicitRefs: [],
|
||||
priorityRefs: listPrioritizedSmallLiveModelRefs(),
|
||||
}),
|
||||
).toContain("ollama");
|
||||
});
|
||||
|
||||
it("respects provider filters for curated small-model refs", () => {
|
||||
expect(
|
||||
filterLiveModelRefsByProvider(
|
||||
listPrioritizedSmallLiveModelRefs(),
|
||||
parseProviderFilter("openrouter"),
|
||||
).map((ref) => ref.provider),
|
||||
).toEqual(["openrouter", "openrouter", "openrouter"]);
|
||||
});
|
||||
|
||||
it("activates bundled provider plugins for explicit live discovery", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
@@ -514,13 +850,486 @@ describe("explicit live model discovery scope", () => {
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
expect(
|
||||
applyLiveProviderDiscoveryPluginCompat({
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["deepseek"],
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(result.plugins?.enabled).toBe(true);
|
||||
expect(result.plugins?.allow).toContain("deepseek");
|
||||
expect(result.plugins?.bundledDiscovery).toBe("compat");
|
||||
expect(result.plugins?.entries?.deepseek).toEqual({ enabled: true });
|
||||
});
|
||||
|
||||
it("hydrates Ollama Cloud provider settings from live env when Ollama is in scope", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: "https://ollama.com",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.plugins?.entries?.ollama).toEqual({ enabled: true });
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: "https://ollama.com",
|
||||
apiKey: OLLAMA_REMOTE_API_KEY_ENV,
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("defaults Ollama live provider settings to the local endpoint", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(result.plugins?.entries?.ollama).toEqual({ enabled: true });
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: OLLAMA_DEFAULT_BASE_URL,
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves configured Ollama provider endpoints when live env is absent", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "http://192.168.1.10:11434",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: "http://192.168.1.10:11434",
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("honors the documented Ollama baseURL alias when live env is absent", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseURL: "http://ollama.local:11434",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseURL: "http://ollama.local:11434",
|
||||
baseUrl: "http://ollama.local:11434",
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the local Ollama auth marker for self-hosted live env URLs", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
for (const baseUrl of [
|
||||
"http://127.0.0.1:11434",
|
||||
"http://192.168.1.10:11434",
|
||||
"http://ollama.local:11434",
|
||||
"http://ollama-host:11434",
|
||||
]) {
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["deepseek"],
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: baseUrl,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl,
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("does not preserve the cloud env marker for local Ollama endpoints", () => {
|
||||
const remoteApiKeyRefs: SecretInput[] = [
|
||||
OLLAMA_REMOTE_API_KEY_ENV,
|
||||
"$OLLAMA_API_KEY",
|
||||
"${OLLAMA_API_KEY}",
|
||||
{ source: "env", provider: "default", id: OLLAMA_REMOTE_API_KEY_ENV },
|
||||
];
|
||||
|
||||
for (const apiKey of remoteApiKeyRefs) {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
apiKey,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OLLAMA_API_KEY: "real-cloud-key",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("replaces configured Ollama auth when live env redirects to a different local endpoint", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "https://ollama.com",
|
||||
apiKey: OLLAMA_REMOTE_API_KEY_ENV,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves configured Ollama auth for equivalent live env base URLs", () => {
|
||||
const cfg = {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434/v1",
|
||||
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies OpenClawConfig;
|
||||
|
||||
const result = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: cfg,
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.models?.providers?.ollama).toEqual({
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
||||
models: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps local Ollama live auth on the non-secret marker", async () => {
|
||||
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
},
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: "http://127.0.0.1:11434",
|
||||
OLLAMA_API_KEY: "real-cloud-key",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLiveModelApiKeyInfo({
|
||||
model: {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
} as Model,
|
||||
cfg,
|
||||
requireProfileKeys: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
source: "live Ollama local marker",
|
||||
mode: "api-key",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not reuse cloud Ollama auth for model-level local endpoint overrides", async () => {
|
||||
const oldEnv = process.env.OLLAMA_API_KEY;
|
||||
process.env.OLLAMA_API_KEY = "real-cloud-key";
|
||||
try {
|
||||
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
},
|
||||
providers: ["ollama"],
|
||||
env: {
|
||||
OPENCLAW_LIVE_OLLAMA_BASE_URL: "https://ollama.com",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLiveModelApiKeyInfo({
|
||||
model: {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
} as Model,
|
||||
cfg,
|
||||
requireProfileKeys: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
apiKey: OLLAMA_LOCAL_API_KEY_MARKER,
|
||||
source: "live Ollama local marker",
|
||||
mode: "api-key",
|
||||
});
|
||||
} finally {
|
||||
if (oldEnv === undefined) {
|
||||
delete process.env.OLLAMA_API_KEY;
|
||||
} else {
|
||||
process.env.OLLAMA_API_KEY = oldEnv;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("honors configured local Ollama credentials before the live local marker", async () => {
|
||||
const oldEnv = process.env.LOCAL_OLLAMA_API_KEY;
|
||||
process.env.LOCAL_OLLAMA_API_KEY = "secured-local-key";
|
||||
try {
|
||||
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
providers: ["ollama"],
|
||||
env: {},
|
||||
}).plugins?.entries?.deepseek,
|
||||
).toEqual({ enabled: true });
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLiveModelApiKeyInfo({
|
||||
model: {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
} as Model,
|
||||
cfg,
|
||||
requireProfileKeys: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
apiKey: "secured-local-key",
|
||||
source: "env: LOCAL_OLLAMA_API_KEY (models.json secretref)",
|
||||
mode: "api-key",
|
||||
});
|
||||
} finally {
|
||||
if (oldEnv === undefined) {
|
||||
delete process.env.LOCAL_OLLAMA_API_KEY;
|
||||
} else {
|
||||
process.env.LOCAL_OLLAMA_API_KEY = oldEnv;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses configured local Ollama credentials across canonical base URL forms", async () => {
|
||||
const oldEnv = process.env.LOCAL_OLLAMA_API_KEY;
|
||||
process.env.LOCAL_OLLAMA_API_KEY = "secured-local-key";
|
||||
try {
|
||||
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: {
|
||||
plugins: {
|
||||
bundledDiscovery: "compat",
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: {
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434/v1",
|
||||
apiKey: { source: "env", provider: "default", id: "LOCAL_OLLAMA_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
providers: ["ollama"],
|
||||
env: {},
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolveLiveModelApiKeyInfo({
|
||||
model: {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434",
|
||||
} as Model,
|
||||
cfg,
|
||||
requireProfileKeys: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
apiKey: "secured-local-key",
|
||||
source: "env: LOCAL_OLLAMA_API_KEY (models.json secretref)",
|
||||
mode: "api-key",
|
||||
});
|
||||
} finally {
|
||||
if (oldEnv === undefined) {
|
||||
delete process.env.LOCAL_OLLAMA_API_KEY;
|
||||
} else {
|
||||
process.env.LOCAL_OLLAMA_API_KEY = oldEnv;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves model-level Ollama endpoint overrides when registering runtime streams", () => {
|
||||
const returnedStream = {} as ReturnType<StreamFn>;
|
||||
const runtimeCalls: Array<{ model: Model; context: Parameters<StreamFn>[1] }> = [];
|
||||
const createConfiguredOllamaStreamFn = vi.fn<
|
||||
OllamaRuntimeApi["createConfiguredOllamaStreamFn"]
|
||||
>(
|
||||
() =>
|
||||
((model, context) => {
|
||||
runtimeCalls.push({ model, context });
|
||||
return returnedStream;
|
||||
}) as StreamFn,
|
||||
);
|
||||
const streamFn = createLiveOllamaRuntimeStreamFn({
|
||||
createConfiguredOllamaStreamFn,
|
||||
providerBaseUrl: OLLAMA_DEFAULT_BASE_URL,
|
||||
});
|
||||
const model = {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
baseUrl: "http://192.168.1.10:11434",
|
||||
} as Model;
|
||||
const context = { systemPrompt: "system", messages: [] } as Parameters<StreamFn>[1];
|
||||
|
||||
expect(streamFn(model, context)).toBe(returnedStream);
|
||||
expect(createConfiguredOllamaStreamFn).toHaveBeenCalledWith({
|
||||
model,
|
||||
providerBaseUrl: undefined,
|
||||
});
|
||||
expect(runtimeCalls).toEqual([{ model, context }]);
|
||||
});
|
||||
|
||||
it("falls back to the configured Ollama provider endpoint for models without overrides", () => {
|
||||
const createConfiguredOllamaStreamFn = vi.fn<
|
||||
OllamaRuntimeApi["createConfiguredOllamaStreamFn"]
|
||||
>(() => (() => ({}) as ReturnType<StreamFn>) as StreamFn);
|
||||
const streamFn = createLiveOllamaRuntimeStreamFn({
|
||||
createConfiguredOllamaStreamFn,
|
||||
providerBaseUrl: "https://ollama.com",
|
||||
});
|
||||
const model = {
|
||||
provider: "ollama",
|
||||
id: "gemma3:4b",
|
||||
api: "ollama",
|
||||
} as Model;
|
||||
|
||||
void streamFn(model, { systemPrompt: "system", messages: [] } as Parameters<StreamFn>[1]);
|
||||
|
||||
expect(createConfiguredOllamaStreamFn).toHaveBeenCalledWith({
|
||||
model,
|
||||
providerBaseUrl: "https://ollama.com",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports explicit refs that never become runnable candidates", () => {
|
||||
@@ -581,7 +1390,7 @@ describe("resolveLiveSystemPrompt", () => {
|
||||
it("keeps other providers unchanged", () => {
|
||||
expect(
|
||||
resolveLiveSystemPrompt({
|
||||
provider: "anthropic",
|
||||
provider: "ollama",
|
||||
} as Model),
|
||||
).toBeUndefined();
|
||||
});
|
||||
@@ -914,15 +1723,23 @@ describeLive("live models (profile keys)", () => {
|
||||
const filter = useExplicit ? parseModelFilter(rawModels) : null;
|
||||
const explicitRefs = useExplicit ? parseExplicitLiveModelRefs(filter) : [];
|
||||
const providers = parseProviderFilter(process.env.OPENCLAW_LIVE_PROVIDERS);
|
||||
const priorityRefs = useSmall
|
||||
? filterLiveModelRefsByProvider(listPrioritizedSmallLiveModelRefs(), providers)
|
||||
: [];
|
||||
const providerList = resolveLiveProviderDiscoveryProviderIds({
|
||||
providerFilter: providers,
|
||||
explicitRefs,
|
||||
priorityRefs,
|
||||
});
|
||||
const cfg = applyLiveProviderDiscoveryPluginCompat({
|
||||
config: loadedCfg,
|
||||
providers: providerList,
|
||||
env: process.env,
|
||||
});
|
||||
await ensureLiveProviderApisRegistered({
|
||||
config: cfg,
|
||||
providers: providerList,
|
||||
});
|
||||
activeLiveCompletionConfig = cfg;
|
||||
logProgress("[live-models] preparing models.json");
|
||||
await withLiveStageTimeout(
|
||||
@@ -992,7 +1809,7 @@ describeLive("live models (profile keys)", () => {
|
||||
...(explicitRefs.length > 0
|
||||
? { refs: explicitRefs }
|
||||
: useSmall
|
||||
? { refs: listPrioritizedSmallLiveModelRefs() }
|
||||
? { refs: priorityRefs }
|
||||
: {}),
|
||||
});
|
||||
if (augmented.added.length > 0) {
|
||||
@@ -1066,13 +1883,10 @@ describeLive("live models (profile keys)", () => {
|
||||
}
|
||||
}
|
||||
try {
|
||||
const apiKeyInfo = await getApiKeyForModel({
|
||||
const apiKeyInfo = await resolveLiveModelApiKeyInfo({
|
||||
model,
|
||||
cfg,
|
||||
credentialPrecedence: resolveLiveCredentialPrecedence(
|
||||
model.provider,
|
||||
REQUIRE_PROFILE_KEYS,
|
||||
),
|
||||
requireProfileKeys: REQUIRE_PROFILE_KEYS,
|
||||
});
|
||||
if (
|
||||
requiresLiveProfileCredential(model.provider, REQUIRE_PROFILE_KEYS) &&
|
||||
|
||||
Reference in New Issue
Block a user