test: trim provider compatibility cold starts

This commit is contained in:
Peter Steinberger
2026-04-05 15:44:03 +01:00
parent 3d3ef6f65f
commit 575371b6f7
7 changed files with 212 additions and 277 deletions

View File

@@ -5,6 +5,7 @@ export {
CLOUDFLARE_AI_GATEWAY_PROVIDER_ID,
resolveCloudflareAiGatewayBaseUrl,
} from "./models.js";
export { buildCloudflareAiGatewayCatalogProvider } from "./catalog-provider.js";
export {
applyCloudflareAiGatewayConfig,

View File

@@ -0,0 +1,67 @@
import {
coerceSecretRef,
ensureAuthProfileStore,
resolveNonEnvSecretRefApiKeyMarker,
} from "openclaw/plugin-sdk/provider-auth";
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import {
buildCloudflareAiGatewayModelDefinition,
resolveCloudflareAiGatewayBaseUrl,
} from "./models.js";
export type CloudflareAiGatewayCredential =
| ReturnType<typeof ensureAuthProfileStore>["profiles"][string]
| undefined;
export function resolveCloudflareAiGatewayApiKey(
cred: CloudflareAiGatewayCredential,
): string | undefined {
if (!cred || cred.type !== "api_key") {
return undefined;
}
const keyRef = coerceSecretRef(cred.keyRef);
if (keyRef && keyRef.id.trim()) {
return keyRef.source === "env"
? keyRef.id.trim()
: resolveNonEnvSecretRefApiKeyMarker(keyRef.source);
}
return cred.key?.trim() || undefined;
}
export function resolveCloudflareAiGatewayMetadata(cred: CloudflareAiGatewayCredential): {
accountId?: string;
gatewayId?: string;
} {
if (!cred || cred.type !== "api_key") {
return {};
}
return {
accountId: cred.metadata?.accountId?.trim() || undefined,
gatewayId: cred.metadata?.gatewayId?.trim() || undefined,
};
}
export function buildCloudflareAiGatewayCatalogProvider(params: {
credential: CloudflareAiGatewayCredential;
envApiKey?: string;
}): ModelProviderConfig | null {
const apiKey = params.envApiKey?.trim() || resolveCloudflareAiGatewayApiKey(params.credential);
if (!apiKey) {
return null;
}
const { accountId, gatewayId } = resolveCloudflareAiGatewayMetadata(params.credential);
if (!accountId || !gatewayId) {
return null;
}
const baseUrl = resolveCloudflareAiGatewayBaseUrl({ accountId, gatewayId });
if (!baseUrl) {
return null;
}
return {
baseUrl,
api: "anthropic-messages",
apiKey,
models: [buildCloudflareAiGatewayModelDefinition()],
};
}

View File

@@ -2,17 +2,16 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import {
applyAuthProfileConfig,
buildApiKeyCredential,
coerceSecretRef,
ensureApiKeyFromOptionEnvOrPrompt,
ensureAuthProfileStore,
listProfilesForProvider,
normalizeApiKeyInput,
normalizeOptionalSecretInput,
resolveNonEnvSecretRefApiKeyMarker,
type SecretInput,
upsertAuthProfile,
validateApiKeyInput,
} from "openclaw/plugin-sdk/provider-auth";
import { buildCloudflareAiGatewayCatalogProvider } from "./catalog-provider.js";
import {
buildCloudflareAiGatewayModelDefinition,
CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF,
@@ -24,34 +23,6 @@ const PROVIDER_ID = "cloudflare-ai-gateway";
const PROVIDER_ENV_VAR = "CLOUDFLARE_AI_GATEWAY_API_KEY";
const PROFILE_ID = "cloudflare-ai-gateway:default";
function resolveApiKeyFromCredential(
cred: ReturnType<typeof ensureAuthProfileStore>["profiles"][string] | undefined,
): string | undefined {
if (!cred || cred.type !== "api_key") {
return undefined;
}
const keyRef = coerceSecretRef(cred.keyRef);
if (keyRef && keyRef.id.trim()) {
return keyRef.source === "env"
? keyRef.id.trim()
: resolveNonEnvSecretRefApiKeyMarker(keyRef.source);
}
return cred.key?.trim() || undefined;
}
function resolveMetadataFromCredential(
cred: ReturnType<typeof ensureAuthProfileStore>["profiles"][string] | undefined,
): { accountId?: string; gatewayId?: string } {
if (!cred || cred.type !== "api_key") {
return {};
}
return {
accountId: cred?.metadata?.accountId?.trim() || undefined,
gatewayId: cred?.metadata?.gatewayId?.trim() || undefined,
};
}
async function resolveCloudflareGatewayMetadataInteractive(ctx: {
accountId?: string;
gatewayId?: string;
@@ -162,7 +133,15 @@ export default definePluginEntry({
const authStore = ensureAuthProfileStore(ctx.agentDir, {
allowKeychainPrompt: false,
});
const storedMetadata = resolveMetadataFromCredential(authStore.profiles[PROFILE_ID]);
const storedMetadata =
authStore.profiles[PROFILE_ID]?.type === "api_key"
? {
accountId:
authStore.profiles[PROFILE_ID]?.metadata?.accountId?.trim() || undefined,
gatewayId:
authStore.profiles[PROFILE_ID]?.metadata?.gatewayId?.trim() || undefined,
}
: {};
const accountId =
normalizeOptionalSecretInput(ctx.opts.cloudflareAiGatewayAccountId) ??
storedMetadata.accountId;
@@ -217,30 +196,15 @@ export default definePluginEntry({
});
const envManagedApiKey = ctx.env[PROVIDER_ENV_VAR]?.trim() ? PROVIDER_ENV_VAR : undefined;
for (const profileId of listProfilesForProvider(authStore, PROVIDER_ID)) {
const cred = authStore.profiles[profileId];
if (!cred || cred.type !== "api_key") {
continue;
}
const apiKey = envManagedApiKey ?? resolveApiKeyFromCredential(cred);
if (!apiKey) {
continue;
}
const accountId = cred.metadata?.accountId?.trim();
const gatewayId = cred.metadata?.gatewayId?.trim();
if (!accountId || !gatewayId) {
continue;
}
const baseUrl = resolveCloudflareAiGatewayBaseUrl({ accountId, gatewayId });
if (!baseUrl) {
const provider = buildCloudflareAiGatewayCatalogProvider({
credential: authStore.profiles[profileId],
envApiKey: envManagedApiKey,
});
if (!provider) {
continue;
}
return {
provider: {
baseUrl,
api: "anthropic-messages",
apiKey,
models: [buildCloudflareAiGatewayModelDefinition()],
},
provider,
};
}
return null;

View File

@@ -125,6 +125,8 @@ export const MODELS_CONFIG_IMPLICIT_ENV_VARS = [
"TOGETHER_API_KEY",
"VOLCANO_ENGINE_API_KEY",
"BYTEPLUS_API_KEY",
"CHUTES_API_KEY",
"CHUTES_OAUTH_TOKEN",
"KILOCODE_API_KEY",
"KIMI_API_KEY",
"KIMICODE_API_KEY",
@@ -166,6 +168,8 @@ const TEST_PROVIDER_ENV_TO_PROVIDER_IDS: Record<string, string[]> = {
AWS_SESSION_TOKEN: ["amazon-bedrock"],
AWS_SHARED_CREDENTIALS_FILE: ["amazon-bedrock"],
BYTEPLUS_API_KEY: ["byteplus"],
CHUTES_API_KEY: ["chutes"],
CHUTES_OAUTH_TOKEN: ["chutes"],
CLOUD_ML_REGION: ["anthropic-vertex"],
CLOUDFLARE_AI_GATEWAY_API_KEY: ["cloudflare-ai-gateway"],
COPILOT_GITHUB_TOKEN: ["github-copilot"],

View File

@@ -1,154 +1,86 @@
import { mkdtempSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { buildCloudflareAiGatewayCatalogProvider } from "../../extensions/cloudflare-ai-gateway/api.js";
import { captureEnv } from "../test-utils/env.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import { resolveImplicitProvidersForTest } from "./models-config.e2e-harness.js";
function expectedCloudflareGatewayBaseUrl(accountId: string, gatewayId: string): string {
return `https://gateway.ai.cloudflare.com/v1/${accountId}/${gatewayId}/anthropic`;
}
describe("cloudflare-ai-gateway profile provenance", () => {
it("prefers env keyRef marker over runtime plaintext for persistence", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
it("prefers env keyRef marker over runtime plaintext for persistence", () => {
const envSnapshot = captureEnv(["CLOUDFLARE_AI_GATEWAY_API_KEY"]);
delete process.env.CLOUDFLARE_AI_GATEWAY_API_KEY;
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:default": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
try {
const provider = buildCloudflareAiGatewayCatalogProvider({
credential: {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
null,
2,
),
"utf8",
);
try {
const providers = await resolveImplicitProvidersForTest({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
});
expect(provider?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
} finally {
envSnapshot.restore();
}
});
it("uses non-env marker for non-env keyRef cloudflare profiles", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:default": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "file", provider: "vault", id: "/cloudflare/apiKey" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
},
const provider = buildCloudflareAiGatewayCatalogProvider({
credential: {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "file", provider: "vault", id: "/cloudflare/apiKey" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProvidersForTest({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
},
});
expect(provider?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
});
it("keeps Cloudflare gateway metadata and apiKey from the same auth profile", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:key-only": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-first",
},
"cloudflare-ai-gateway:gateway": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-second",
metadata: {
accountId: "acct_456",
gatewayId: "gateway_789",
},
},
},
const provider = buildCloudflareAiGatewayCatalogProvider({
credential: {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-second",
metadata: {
accountId: "acct_456",
gatewayId: "gateway_789",
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProvidersForTest({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe("sk-second");
expect(providers?.["cloudflare-ai-gateway"]?.baseUrl).toBe(
expectedCloudflareGatewayBaseUrl("acct_456", "gateway_789"),
);
},
});
expect(provider?.apiKey).toBe("sk-second");
expect(provider?.baseUrl).toBe(expectedCloudflareGatewayBaseUrl("acct_456", "gateway_789"));
});
it("prefers the runtime env marker over stored profile secrets", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
it("prefers the runtime env marker over stored profile secrets", () => {
const envSnapshot = captureEnv(["CLOUDFLARE_AI_GATEWAY_API_KEY"]);
process.env.CLOUDFLARE_AI_GATEWAY_API_KEY = "rotated-secret"; // pragma: allowlist secret
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:default": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "stale-stored-secret",
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
try {
const provider = buildCloudflareAiGatewayCatalogProvider({
credential: {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "stale-stored-secret",
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
null,
2,
),
"utf8",
);
try {
const providers = await resolveImplicitProvidersForTest({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
expect(providers?.["cloudflare-ai-gateway"]?.baseUrl).toBe(
expectedCloudflareGatewayBaseUrl("acct_123", "gateway_456"),
);
envApiKey: "CLOUDFLARE_AI_GATEWAY_API_KEY",
});
expect(provider?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
expect(provider?.baseUrl).toBe(expectedCloudflareGatewayBaseUrl("acct_123", "gateway_456"));
} finally {
envSnapshot.restore();
}

View File

@@ -1,82 +1,49 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { captureEnv } from "../test-utils/env.js";
import { resolveImplicitProvidersForTest } from "./models-config.e2e-harness.js";
import {
applyMoonshotNativeStreamingUsageCompat,
buildMoonshotProvider,
MOONSHOT_CN_BASE_URL,
} from "../../extensions/moonshot/api.js";
import { resolveMissingProviderApiKey } from "./models-config.providers.secrets.js";
describe("moonshot implicit provider (#33637)", () => {
it("uses explicit CN baseUrl when provided", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["MOONSHOT_API_KEY"]);
process.env.MOONSHOT_API_KEY = "sk-test-cn";
it("uses explicit CN baseUrl when provided", () => {
const provider = {
...buildMoonshotProvider(),
baseUrl: MOONSHOT_CN_BASE_URL,
};
try {
const providers = await resolveImplicitProvidersForTest({
agentDir,
explicitProviders: {
moonshot: {
baseUrl: "https://api.moonshot.cn/v1",
api: "openai-completions",
models: [
{
id: "kimi-k2.5",
name: "Kimi K2.5",
reasoning: false,
input: ["text", "image"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 262144,
maxTokens: 262144,
},
],
},
},
});
expect(providers?.moonshot).toBeDefined();
expect(providers?.moonshot?.baseUrl).toBe("https://api.moonshot.cn/v1");
expect(providers?.moonshot?.apiKey).toBeDefined();
expect(providers?.moonshot?.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
} finally {
envSnapshot.restore();
}
expect(provider.baseUrl).toBe(MOONSHOT_CN_BASE_URL);
expect(provider.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
expect(
applyMoonshotNativeStreamingUsageCompat(provider).models?.[0]?.compat
?.supportsUsageInStreaming,
).toBe(true);
});
it("keeps streaming usage opt-in unset before the final compat pass", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["MOONSHOT_API_KEY"]);
process.env.MOONSHOT_API_KEY = "sk-test-custom";
it("keeps streaming usage opt-in unset before the final compat pass", () => {
const provider = {
...buildMoonshotProvider(),
baseUrl: "https://proxy.example.com/v1",
};
try {
const providers = await resolveImplicitProvidersForTest({
agentDir,
explicitProviders: {
moonshot: {
baseUrl: "https://proxy.example.com/v1",
api: "openai-completions",
models: [],
},
},
});
expect(providers?.moonshot).toBeDefined();
expect(providers?.moonshot?.baseUrl).toBe("https://proxy.example.com/v1");
expect(providers?.moonshot?.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
} finally {
envSnapshot.restore();
}
expect(provider.baseUrl).toBe("https://proxy.example.com/v1");
expect(provider.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
expect(
applyMoonshotNativeStreamingUsageCompat(provider).models?.[0]?.compat
?.supportsUsageInStreaming,
).toBeUndefined();
});
it("includes moonshot when MOONSHOT_API_KEY is configured", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["MOONSHOT_API_KEY"]);
process.env.MOONSHOT_API_KEY = "sk-test";
it("includes moonshot when MOONSHOT_API_KEY is configured", () => {
const provider = resolveMissingProviderApiKey({
providerKey: "moonshot",
provider: buildMoonshotProvider(),
env: { MOONSHOT_API_KEY: "sk-test" } as NodeJS.ProcessEnv,
profileApiKey: undefined,
});
try {
const providers = await resolveImplicitProvidersForTest({ agentDir });
expect(providers?.moonshot).toBeDefined();
expect(providers?.moonshot?.apiKey).toBeDefined();
expect(providers?.moonshot?.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
} finally {
envSnapshot.restore();
}
expect(provider.apiKey).toBe("MOONSHOT_API_KEY");
expect(provider.models?.[0]?.compat?.supportsUsageInStreaming).toBeUndefined();
});
});

View File

@@ -1,53 +1,53 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { captureEnv } from "../test-utils/env.js";
import { resolveImplicitProvidersForTest } from "./models-config.e2e-harness.js";
import {
withBundledPluginAllowlistCompat,
withBundledPluginEnablementCompat,
} from "../plugins/bundled-compat.js";
import { resolveEnabledProviderPluginIds } from "../plugins/providers.js";
describe("implicit provider plugin allowlist compatibility", () => {
it("keeps bundled implicit providers discoverable when plugins.allow is set", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["KILOCODE_API_KEY", "MOONSHOT_API_KEY"]);
process.env.KILOCODE_API_KEY = "test-kilo-key"; // pragma: allowlist secret
process.env.MOONSHOT_API_KEY = "test-moonshot-key"; // pragma: allowlist secret
try {
const providers = await resolveImplicitProvidersForTest({
agentDir,
it("keeps bundled implicit providers discoverable when plugins.allow is set", () => {
const config = withBundledPluginEnablementCompat({
config: withBundledPluginAllowlistCompat({
config: {
plugins: {
allow: ["openrouter"],
},
},
});
expect(providers?.kilocode).toBeDefined();
expect(providers?.moonshot).toBeDefined();
} finally {
envSnapshot.restore();
}
pluginIds: ["kilocode", "moonshot"],
}),
pluginIds: ["kilocode", "moonshot"],
});
expect(
resolveEnabledProviderPluginIds({
config,
env: { VITEST: "1" } as NodeJS.ProcessEnv,
onlyPluginIds: ["kilocode", "moonshot", "openrouter"],
}),
).toEqual(["kilocode", "moonshot", "openrouter"]);
});
it("still honors explicit plugin denies over compat allowlist injection", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["KILOCODE_API_KEY", "MOONSHOT_API_KEY"]);
process.env.KILOCODE_API_KEY = "test-kilo-key"; // pragma: allowlist secret
process.env.MOONSHOT_API_KEY = "test-moonshot-key"; // pragma: allowlist secret
try {
const providers = await resolveImplicitProvidersForTest({
agentDir,
it("still honors explicit plugin denies over compat allowlist injection", () => {
const config = withBundledPluginEnablementCompat({
config: withBundledPluginAllowlistCompat({
config: {
plugins: {
allow: ["openrouter"],
deny: ["kilocode"],
},
},
});
expect(providers?.kilocode).toBeUndefined();
expect(providers?.moonshot).toBeDefined();
} finally {
envSnapshot.restore();
}
pluginIds: ["kilocode", "moonshot"],
}),
pluginIds: ["kilocode", "moonshot"],
});
expect(
resolveEnabledProviderPluginIds({
config,
env: { VITEST: "1" } as NodeJS.ProcessEnv,
onlyPluginIds: ["kilocode", "moonshot", "openrouter"],
}),
).toEqual(["moonshot", "openrouter"]);
});
});