test: add Anthropic Opus QA smokes

2026-04-12 01:31:08 +00:00 · 2026-04-10 17:23:25 +01:00
parent 5df09052e0
commit d5df4cd4e5
4 changed files with 291 additions and 4 deletions
--- a/extensions/qa-lab/src/gateway-child.test.ts
+++ b/extensions/qa-lab/src/gateway-child.test.ts
@@ -1,4 +1,4 @@
-import { lstat, mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import { lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
@@ -151,6 +151,19 @@ describe("buildQaRuntimeEnv", () => {
    expect(env.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE).toBe("subscription");
  });

+  it("does not pass QA setup-token values to the gateway child env", () => {
+    const env = buildQaRuntimeEnv({
+      ...createParams({
+        OPENCLAW_LIVE_SETUP_TOKEN_VALUE: `sk-ant-oat01-${"a".repeat(80)}`,
+        OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN: `sk-ant-oat01-${"b".repeat(80)}`,
+      }),
+      providerMode: "live-frontier",
+    });
+
+    expect(env.OPENCLAW_LIVE_SETUP_TOKEN_VALUE).toBeUndefined();
+    expect(env.OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN).toBeUndefined();
+  });
+
  it("requires an Anthropic key for live Claude CLI API-key mode", async () => {
    const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-"));
    cleanups.push(async () => {
@@ -224,6 +237,40 @@ describe("buildQaRuntimeEnv", () => {
    expect(__testing.isRetryableGatewayCallError("service restart in progress")).toBe(true);
    expect(__testing.isRetryableGatewayCallError("permission denied")).toBe(false);
  });
+
+  it("stages a live Anthropic setup-token profile for isolated QA workers", async () => {
+    const stateDir = await mkdtemp(path.join(os.tmpdir(), "qa-setup-token-state-"));
+    cleanups.push(async () => {
+      await rm(stateDir, { recursive: true, force: true });
+    });
+    const token = `sk-ant-oat01-${"c".repeat(80)}`;
+
+    const cfg = await __testing.stageQaLiveAnthropicSetupToken({
+      cfg: {},
+      stateDir,
+      env: {
+        OPENCLAW_LIVE_SETUP_TOKEN_VALUE: token,
+      },
+    });
+
+    expect(cfg.auth?.profiles?.["anthropic:qa-setup-token"]).toMatchObject({
+      provider: "anthropic",
+      mode: "token",
+    });
+    const storeRaw = await readFile(
+      path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"),
+      "utf8",
+    );
+    expect(JSON.parse(storeRaw)).toMatchObject({
+      profiles: {
+        "anthropic:qa-setup-token": {
+          type: "token",
+          provider: "anthropic",
+          token,
+        },
+      },
+    });
+  });
 });

 describe("resolveQaControlUiRoot", () => {
--- a/extensions/qa-lab/src/gateway-child.ts
+++ b/extensions/qa-lab/src/gateway-child.ts
@@ -8,6 +8,11 @@ import path from "node:path";
 import { setTimeout as sleep } from "node:timers/promises";
 import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
 import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+import {
+  applyAuthProfileConfig,
+  upsertAuthProfile,
+  validateAnthropicSetupToken,
+} from "openclaw/plugin-sdk/provider-auth";
 import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
 import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
 import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
@@ -69,6 +74,10 @@ const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
 ]);

 const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
+const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
+const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
+const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
+const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
 const QA_OPENAI_PLUGIN_ID = "openai";
 const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
 const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
@@ -235,7 +244,57 @@ export function buildQaRuntimeEnv(params: {
      ? { OPENCLAW_COMPATIBILITY_HOST_VERSION: params.compatibilityHostVersion }
      : {}),
  };
-  return normalizeQaProviderModeEnv(env, params.providerMode);
+  const normalizedEnv = normalizeQaProviderModeEnv(env, params.providerMode);
+  delete normalizedEnv[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV];
+  delete normalizedEnv[QA_LIVE_SETUP_TOKEN_VALUE_ENV];
+  return normalizedEnv;
+}
+
+function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
+  const token = (
+    env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
+    env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
+    ""
+  ).replaceAll(/\s+/g, "");
+  if (!token) {
+    return null;
+  }
+  const tokenError = validateAnthropicSetupToken(token);
+  if (tokenError) {
+    throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
+  }
+  const profileId =
+    env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
+    QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
+  return { token, profileId };
+}
+
+export async function stageQaLiveAnthropicSetupToken(params: {
+  cfg: OpenClawConfig;
+  stateDir: string;
+  env?: NodeJS.ProcessEnv;
+}): Promise<OpenClawConfig> {
+  const resolved = resolveQaLiveAnthropicSetupToken(params.env);
+  if (!resolved) {
+    return params.cfg;
+  }
+  const agentDir = path.join(params.stateDir, "agents", "main", "agent");
+  await fs.mkdir(agentDir, { recursive: true });
+  upsertAuthProfile({
+    profileId: resolved.profileId,
+    credential: {
+      type: "token",
+      provider: "anthropic",
+      token: resolved.token,
+    },
+    agentDir,
+  });
+  return applyAuthProfileConfig(params.cfg, {
+    profileId: resolved.profileId,
+    provider: "anthropic",
+    mode: "token",
+    displayName: "QA setup-token",
+  });
 }

 function isRetryableGatewayCallError(details: string): boolean {
@@ -253,6 +312,8 @@ export const __testing = {
  buildQaRuntimeEnv,
  isRetryableGatewayCallError,
  readQaLiveProviderConfigOverrides,
+  resolveQaLiveAnthropicSetupToken,
+  stageQaLiveAnthropicSetupToken,
  resolveQaLiveCliAuthEnv,
  resolveQaOwnerPluginIdsForProviderIds,
  resolveQaBundledPluginsSourceRoot,
@@ -656,7 +717,7 @@ export async function startQaGatewayChild(params: {
          providerConfigs: liveProviderConfigs,
        })
      : undefined;
-  const baseCfg = buildQaGatewayConfig({
+  let cfg = buildQaGatewayConfig({
    bind: "loopback",
    gatewayPort,
    gatewayToken,
@@ -677,7 +738,11 @@ export async function startQaGatewayChild(params: {
    thinkingDefault: params.thinkingDefault,
    controlUiEnabled: params.controlUiEnabled,
  });
-  const cfg = params.mutateConfig ? params.mutateConfig(baseCfg) : baseCfg;
+  cfg = await stageQaLiveAnthropicSetupToken({
+    cfg,
+    stateDir,
+  });
+  cfg = params.mutateConfig ? params.mutateConfig(cfg) : cfg;
  await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, {
    encoding: "utf8",
    mode: 0o600,
--- a/qa/scenarios/anthropic-opus-api-key-smoke.md
+++ b/qa/scenarios/anthropic-opus-api-key-smoke.md
@@ -0,0 +1,85 @@
+# Anthropic Opus API key smoke
+
+```yaml qa-scenario
+id: anthropic-opus-api-key-smoke
+title: Anthropic Opus API key smoke
+surface: model-provider
+objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using API-key auth.
+successCriteria:
+  - A live-frontier run fails fast unless the selected primary provider is anthropic.
+  - The selected primary model is Anthropic Opus 4.6.
+  - The QA gateway worker has an Anthropic API key available through environment auth.
+  - The agent replies through the regular Anthropic provider.
+docsRefs:
+  - docs/concepts/model-providers.md
+  - docs/help/testing.md
+codeRefs:
+  - extensions/anthropic/register.runtime.ts
+  - extensions/qa-lab/src/gateway-child.ts
+  - extensions/qa-lab/src/suite.ts
+execution:
+  kind: flow
+  summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-6 --alt-model anthropic/claude-opus-4-6 --scenario anthropic-opus-api-key-smoke`.
+  config:
+    requiredProvider: anthropic
+    requiredModel: claude-opus-4-6
+    chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK"
+    chatExpected: ANTHROPIC-OPUS-API-KEY-OK
+```
+
+```yaml qa-flow
+steps:
+  - name: confirms regular Anthropic API-key lane
+    actions:
+      - set: selected
+        value:
+          expr: splitModelRef(env.primaryModel)
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+          message:
+            expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+          message:
+            expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
+          message: expected ANTHROPIC_API_KEY to be available for API-key QA mode
+    detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`"
+  - name: talks through regular Anthropic Opus
+    actions:
+      - if:
+          expr: "env.providerMode !== 'live-frontier'"
+          then:
+            - assert: "true"
+          else:
+            - call: reset
+            - set: selected
+              value:
+                expr: splitModelRef(env.primaryModel)
+            - call: runAgentPrompt
+              args:
+                - ref: env
+                - sessionKey: agent:qa:anthropic-opus-api-key
+                  message:
+                    expr: config.chatPrompt
+                  provider:
+                    expr: selected?.provider
+                  model:
+                    expr: selected?.model
+                  timeoutMs:
+                    expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+            - call: waitForOutboundMessage
+              saveAs: chatOutbound
+              args:
+                - ref: state
+                - lambda:
+                    params: [candidate]
+                    expr: "candidate.conversation.id === 'qa-operator'"
+                - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+            - assert:
+                expr: "chatOutbound.text.includes(config.chatExpected)"
+                message:
+                  expr: "`chat marker missing: ${chatOutbound.text}`"
+    detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
+```
--- a/qa/scenarios/anthropic-opus-setup-token-smoke.md
+++ b/qa/scenarios/anthropic-opus-setup-token-smoke.md
@@ -0,0 +1,90 @@
+# Anthropic Opus setup-token smoke
+
+```yaml qa-scenario
+id: anthropic-opus-setup-token-smoke
+title: Anthropic Opus setup-token smoke
+surface: model-provider
+objective: Verify the regular Anthropic Opus lane can complete a quick chat turn using setup-token auth.
+successCriteria:
+  - A live-frontier run fails fast unless the selected primary provider is anthropic.
+  - The selected primary model is Anthropic Opus 4.6.
+  - The QA gateway worker stages a token auth profile in the isolated agent store.
+  - The agent replies through the regular Anthropic provider.
+docsRefs:
+  - docs/concepts/model-providers.md
+  - docs/help/testing.md
+codeRefs:
+  - extensions/anthropic/register.runtime.ts
+  - extensions/qa-lab/src/gateway-child.ts
+  - extensions/qa-lab/src/suite.ts
+execution:
+  kind: flow
+  summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE=<setup-token> pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-4-6 --alt-model anthropic/claude-opus-4-6 --scenario anthropic-opus-setup-token-smoke`.
+  config:
+    requiredProvider: anthropic
+    requiredModel: claude-opus-4-6
+    profileId: "anthropic:qa-setup-token"
+    chatPrompt: "Anthropic Opus setup-token smoke. Reply exactly: ANTHROPIC-OPUS-SETUP-TOKEN-OK"
+    chatExpected: ANTHROPIC-OPUS-SETUP-TOKEN-OK
+```
+
+```yaml qa-flow
+steps:
+  - name: confirms regular Anthropic setup-token lane
+    actions:
+      - set: selected
+        value:
+          expr: splitModelRef(env.primaryModel)
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
+          message:
+            expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
+          message:
+            expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
+          message:
+            expr: "`expected token profile ${config.profileId} in QA config`"
+      - assert:
+          expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
+          message: setup-token value should not be passed to the gateway child env
+    detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`"
+  - name: talks through regular Anthropic Opus
+    actions:
+      - if:
+          expr: "env.providerMode !== 'live-frontier'"
+          then:
+            - assert: "true"
+          else:
+            - call: reset
+            - set: selected
+              value:
+                expr: splitModelRef(env.primaryModel)
+            - call: runAgentPrompt
+              args:
+                - ref: env
+                - sessionKey: agent:qa:anthropic-opus-setup-token
+                  message:
+                    expr: config.chatPrompt
+                  provider:
+                    expr: selected?.provider
+                  model:
+                    expr: selected?.model
+                  timeoutMs:
+                    expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
+            - call: waitForOutboundMessage
+              saveAs: chatOutbound
+              args:
+                - ref: state
+                - lambda:
+                    params: [candidate]
+                    expr: "candidate.conversation.id === 'qa-operator'"
+                - expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
+            - assert:
+                expr: "chatOutbound.text.includes(config.chatExpected)"
+                message:
+                  expr: "`chat marker missing: ${chatOutbound.text}`"
+    detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
+```