From d5df4cd4e56a090c0f6787a43ce1da3eae661d45 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Apr 2026 17:23:25 +0100 Subject: [PATCH] test: add Anthropic Opus QA smokes --- extensions/qa-lab/src/gateway-child.test.ts | 49 +++++++++- extensions/qa-lab/src/gateway-child.ts | 71 ++++++++++++++- qa/scenarios/anthropic-opus-api-key-smoke.md | 85 ++++++++++++++++++ .../anthropic-opus-setup-token-smoke.md | 90 +++++++++++++++++++ 4 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 qa/scenarios/anthropic-opus-api-key-smoke.md create mode 100644 qa/scenarios/anthropic-opus-setup-token-smoke.md diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index a4a6e029f5a..571767b40e0 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -1,4 +1,4 @@ -import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -151,6 +151,19 @@ describe("buildQaRuntimeEnv", () => { expect(env.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE).toBe("subscription"); }); + it("does not pass QA setup-token values to the gateway child env", () => { + const env = buildQaRuntimeEnv({ + ...createParams({ + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: `sk-ant-oat01-${"a".repeat(80)}`, + OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN: `sk-ant-oat01-${"b".repeat(80)}`, + }), + providerMode: "live-frontier", + }); + + expect(env.OPENCLAW_LIVE_SETUP_TOKEN_VALUE).toBeUndefined(); + expect(env.OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN).toBeUndefined(); + }); + it("requires an Anthropic key for live Claude CLI API-key mode", async () => { const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-")); cleanups.push(async () => { @@ -224,6 +237,40 @@ describe("buildQaRuntimeEnv", () => { expect(__testing.isRetryableGatewayCallError("service restart in progress")).toBe(true); expect(__testing.isRetryableGatewayCallError("permission denied")).toBe(false); }); + + it("stages a live Anthropic setup-token profile for isolated QA workers", async () => { + const stateDir = await mkdtemp(path.join(os.tmpdir(), "qa-setup-token-state-")); + cleanups.push(async () => { + await rm(stateDir, { recursive: true, force: true }); + }); + const token = `sk-ant-oat01-${"c".repeat(80)}`; + + const cfg = await __testing.stageQaLiveAnthropicSetupToken({ + cfg: {}, + stateDir, + env: { + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: token, + }, + }); + + expect(cfg.auth?.profiles?.["anthropic:qa-setup-token"]).toMatchObject({ + provider: "anthropic", + mode: "token", + }); + const storeRaw = await readFile( + path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), + "utf8", + ); + expect(JSON.parse(storeRaw)).toMatchObject({ + profiles: { + "anthropic:qa-setup-token": { + type: "token", + provider: "anthropic", + token, + }, + }, + }); + }); }); describe("resolveQaControlUiRoot", () => { diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index 22ecb314265..fd9e3dd846a 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -8,6 +8,11 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + applyAuthProfileConfig, + upsertAuthProfile, + validateAnthropicSetupToken, +} from "openclaw/plugin-sdk/provider-auth"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; @@ -69,6 +74,10 @@ const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([ ]); const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH"; +const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN"; +const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE"; +const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE"; +const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token"; const QA_OPENAI_PLUGIN_ID = "openai"; const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV"; const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE"; @@ -235,7 +244,57 @@ export function buildQaRuntimeEnv(params: { ? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion } : {}), }; - return normalizeQaProviderModeEnv(env, params.providerMode); + const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode); + delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]; + delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV]; + return normalizedEnv; +} + +function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) { + const token = ( + env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() || + env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() || + "" + ).replaceAll(/\s+/g, ""); + if (!token) { + return null; + } + const tokenError = validateAnthropicSetupToken(token); + if (tokenError) { + throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`); + } + const profileId = + env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() || + QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID; + return { token, profileId }; +} + +export async function stageQaLiveAnthropicSetupToken(params: { + cfg: OpenClawConfig; + stateDir: string; + env?: NodeJS.ProcessEnv; +}): Promise { + const resolved = resolveQaLiveAnthropicSetupToken(params.env); + if (!resolved) { + return params.cfg; + } + const agentDir = path.join(params.stateDir, "agents", "main", "agent"); + await fs.mkdir(agentDir, { recursive: true }); + upsertAuthProfile({ + profileId: resolved.profileId, + credential: { + type: "token", + provider: "anthropic", + token: resolved.token, + }, + agentDir, + }); + return applyAuthProfileConfig(params.cfg, { + profileId: resolved.profileId, + provider: "anthropic", + mode: "token", + displayName: "QA setup-token", + }); } function isRetryableGatewayCallError(details: string): boolean { @@ -253,6 +312,8 @@ export const __testing = { buildQaRuntimeEnv, isRetryableGatewayCallError, readQaLiveProviderConfigOverrides, + resolveQaLiveAnthropicSetupToken, + stageQaLiveAnthropicSetupToken, resolveQaLiveCliAuthEnv, resolveQaOwnerPluginIdsForProviderIds, resolveQaBundledPluginsSourceRoot, @@ -656,7 +717,7 @@ export async function startQaGatewayChild(params: { providerConfigs: liveProviderConfigs, }) : undefined; - const baseCfg = buildQaGatewayConfig({ + let cfg = buildQaGatewayConfig({ bind: "loopback", gatewayPort, gatewayToken, @@ -677,7 +738,11 @@ export async function startQaGatewayChild(params: { thinkingDefault: params.thinkingDefault, controlUiEnabled: params.controlUiEnabled, }); - const cfg = params.mutateConfig ? params.mutateConfig(baseCfg) : baseCfg; + cfg = await stageQaLiveAnthropicSetupToken({ + cfg, + stateDir, + }); + cfg = params.mutateConfig ? params.mutateConfig(cfg) : cfg; await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, { encoding: "utf8", mode: 0o600, diff --git a/qa/scenarios/anthropic-opus-api-key-smoke.md b/qa/scenarios/anthropic-opus-api-key-smoke.md new file mode 100644 index 00000000000..b530620e50b --- /dev/null +++ b/qa/scenarios/anthropic-opus-api-key-smoke.md @@ -0,0 +1,85 @@ +# Anthropic Opus API key smoke + +```yaml qa-scenario +id: anthropic-opus-api-key-smoke +title: Anthropic Opus API key smoke +surface: model-provider +objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using API-key auth. +successCriteria: + - A live-frontier run fails fast unless the selected primary provider is anthropic. + - The selected primary model is Anthropic Opus 4.6. + - The QA gateway worker has an Anthropic API key available through environment auth. + - The agent replies through the regular Anthropic provider. +docsRefs: + - docs/concepts/model-providers.md + - docs/help/testing.md +codeRefs: + - extensions/anthropic/register.runtime.ts + - extensions/qa-lab/src/gateway-child.ts + - extensions/qa-lab/src/suite.ts +execution: + kind: flow + summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-6 --alt-model anthropic/claude-opus-4-6 --scenario anthropic-opus-api-key-smoke`. + config: + requiredProvider: anthropic + requiredModel: claude-opus-4-6 + chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK" + chatExpected: ANTHROPIC-OPUS-API-KEY-OK +``` + +```yaml qa-flow +steps: + - name: confirms regular Anthropic API-key lane + actions: + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - assert: + expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + message: + expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" + - assert: + expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + message: + expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" + - assert: + expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())" + message: expected ANTHROPIC_API_KEY to be available for API-key QA mode + detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`" + - name: talks through regular Anthropic Opus + actions: + - if: + expr: "env.providerMode !== 'live-frontier'" + then: + - assert: "true" + else: + - call: reset + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:anthropic-opus-api-key + message: + expr: config.chatPrompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: chatOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator'" + - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) + - assert: + expr: "chatOutbound.text.includes(config.chatExpected)" + message: + expr: "`chat marker missing: ${chatOutbound.text}`" + detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text" +``` diff --git a/qa/scenarios/anthropic-opus-setup-token-smoke.md b/qa/scenarios/anthropic-opus-setup-token-smoke.md new file mode 100644 index 00000000000..df3a2ae6a06 --- /dev/null +++ b/qa/scenarios/anthropic-opus-setup-token-smoke.md @@ -0,0 +1,90 @@ +# Anthropic Opus setup-token smoke + +```yaml qa-scenario +id: anthropic-opus-setup-token-smoke +title: Anthropic Opus setup-token smoke +surface: model-provider +objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using setup-token auth. +successCriteria: + - A live-frontier run fails fast unless the selected primary provider is anthropic. + - The selected primary model is Anthropic Opus 4.6. + - The QA gateway worker stages a token auth profile in the isolated agent store. + - The agent replies through the regular Anthropic provider. +docsRefs: + - docs/concepts/model-providers.md + - docs/help/testing.md +codeRefs: + - extensions/anthropic/register.runtime.ts + - extensions/qa-lab/src/gateway-child.ts + - extensions/qa-lab/src/suite.ts +execution: + kind: flow + summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE= pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-6 --alt-model anthropic/claude-opus-4-6 --scenario anthropic-opus-setup-token-smoke`. + config: + requiredProvider: anthropic + requiredModel: claude-opus-4-6 + profileId: "anthropic:qa-setup-token" + chatPrompt: "Anthropic Opus setup-token smoke. Reply exactly: ANTHROPIC-OPUS-SETUP-TOKEN-OK" + chatExpected: ANTHROPIC-OPUS-SETUP-TOKEN-OK +``` + +```yaml qa-flow +steps: + - name: confirms regular Anthropic setup-token lane + actions: + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - assert: + expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider" + message: + expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`" + - assert: + expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel" + message: + expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`" + - assert: + expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'" + message: + expr: "`expected token profile ${config.profileId} in QA config`" + - assert: + expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE" + message: setup-token value should not be passed to the gateway child env + detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`" + - name: talks through regular Anthropic Opus + actions: + - if: + expr: "env.providerMode !== 'live-frontier'" + then: + - assert: "true" + else: + - call: reset + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:anthropic-opus-setup-token + message: + expr: config.chatPrompt + provider: + expr: selected?.provider + model: + expr: selected?.model + timeoutMs: + expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel) + - call: waitForOutboundMessage + saveAs: chatOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator'" + - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel) + - assert: + expr: "chatOutbound.text.includes(config.chatExpected)" + message: + expr: "`chat marker missing: ${chatOutbound.text}`" + detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text" +```