test(plugins): harden kitchen sink live gauntlet

extensions/qa-lab/src/gateway-child.test.ts

@@ -407,6 +407,44 @@ describe("buildQaRuntimeEnv", () => {
     });
   });
 
+  it("stages live env API-key profiles for isolated QA workers", async () => {
+    const stateDir = await mkdtemp(path.join(os.tmpdir(), "qa-live-api-key-state-"));
+    cleanups.push(async () => {
+      await rm(stateDir, { recursive: true, force: true });
+    });
+
+    const cfg = await __testing.stageQaLiveApiKeyProfiles({
+      cfg: {},
+      stateDir,
+      providerIds: ["openai"],
+      env: {
+        OPENAI_API_KEY: "qa-live-not-a-real-key",
+      },
+    });
+
+    expect(cfg.auth?.profiles?.["qa-live-openai-env"]).toMatchObject({
+      provider: "openai",
+      mode: "api_key",
+      displayName: "QA live openai env credential",
+    });
+
+    for (const agentId of ["main", "qa"]) {
+      const storeRaw = await readFile(
+        path.join(stateDir, "agents", agentId, "agent", "auth-profiles.json"),
+        "utf8",
+      );
+      expect(JSON.parse(storeRaw)).toMatchObject({
+        profiles: {
+          "qa-live-openai-env": {
+            type: "api_key",
+            provider: "openai",
+            key: "qa-live-not-a-real-key",
+          },
+        },
+      });
+    }
+  });
+
   it("stages placeholder mock auth profiles per agent dir so mock-openai runs can resolve credentials", async () => {
     const stateDir = await mkdtemp(path.join(os.tmpdir(), "qa-mock-auth-"));
     cleanups.push(async () => {

extensions/qa-lab/src/gateway-child.ts

@@ -34,6 +34,7 @@ import { DEFAULT_QA_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
 import {
   QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
   QA_LIVE_SETUP_TOKEN_VALUE_ENV,
+  stageQaLiveApiKeyProfiles,
   stageQaLiveAnthropicSetupToken,
 } from "./providers/live-frontier/auth.js";
 import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
@@ -314,6 +315,7 @@ export const __testing = {
   redactQaGatewayDebugText,
   readQaLiveProviderConfigOverrides,
   resolveQaGatewayChildProviderMode,
+  stageQaLiveApiKeyProfiles,
   stageQaLiveAnthropicSetupToken,
   stageQaMockAuthProfiles,
   resolveQaLiveCliAuthEnv,
@@ -573,6 +575,11 @@ export async function startQaGatewayChild(params: {
     });
   const buildStagedGatewayConfig = async (gatewayPort: number) => {
     let cfg = buildGatewayConfig(gatewayPort);
+    cfg = await stageQaLiveApiKeyProfiles({
+      cfg,
+      stateDir,
+      providerIds: liveProviderIds,
+    });
     cfg = await stageQaLiveAnthropicSetupToken({
       cfg,
       stateDir,

extensions/qa-lab/src/providers/live-frontier/auth.ts

@@ -1,6 +1,7 @@
 import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
 import {
   applyAuthProfileConfig,
+  resolveEnvApiKey,
   validateAnthropicSetupToken,
 } from "openclaw/plugin-sdk/provider-auth";
 import { resolveQaAgentAuthDir, writeQaAuthProfiles } from "../shared/auth-store.js";
@@ -9,6 +10,11 @@ export const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SET
 export const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
 const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
 const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
+const QA_LIVE_API_KEY_AGENT_IDS = Object.freeze(["main", "qa"] as const);
+
+function buildQaLiveApiKeyProfileId(provider: string): string {
+  return `qa-live-${provider.replaceAll(/[^a-z0-9_-]/giu, "-")}-env`;
+}
 
 function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
   const token = (
@@ -55,3 +61,59 @@ export async function stageQaLiveAnthropicSetupToken(params: {
     displayName: "QA setup-token",
   });
 }
+
+export async function stageQaLiveApiKeyProfiles(params: {
+  cfg: OpenClawConfig;
+  stateDir: string;
+  providerIds: readonly string[];
+  env?: NodeJS.ProcessEnv;
+  agentIds?: readonly string[];
+}): Promise<OpenClawConfig> {
+  const env = params.env ?? process.env;
+  const providerIds = [...new Set(params.providerIds.map((providerId) => providerId.trim()))]
+    .filter((providerId) => providerId.length > 0)
+    .toSorted();
+  const profiles: Record<
+    string,
+    {
+      type: "api_key";
+      provider: string;
+      key: string;
+      displayName: string;
+    }
+  > = {};
+  let next = params.cfg;
+  for (const providerId of providerIds) {
+    const resolved = resolveEnvApiKey(providerId, env, { config: next });
+    if (!resolved?.apiKey) {
+      continue;
+    }
+    const profileId = buildQaLiveApiKeyProfileId(providerId);
+    const displayName = `QA live ${providerId} env credential`;
+    profiles[profileId] = {
+      type: "api_key",
+      provider: providerId,
+      key: resolved.apiKey,
+      displayName,
+    };
+    next = applyAuthProfileConfig(next, {
+      profileId,
+      provider: providerId,
+      mode: "api_key",
+      displayName,
+    });
+  }
+  if (Object.keys(profiles).length === 0) {
+    return next;
+  }
+  const agentIds = [...new Set(params.agentIds ?? QA_LIVE_API_KEY_AGENT_IDS)];
+  await Promise.all(
+    agentIds.map((agentId) =>
+      writeQaAuthProfiles({
+        agentDir: resolveQaAgentAuthDir({ stateDir: params.stateDir, agentId }),
+        profiles,
+      }),
+    ),
+  );
+  return next;
+}

extensions/qa-lab/src/providers/shared/auth-store.ts

@@ -22,10 +22,15 @@ export async function writeQaAuthProfiles(params: {
   agentDir: string;
   profiles: Record<string, QaAuthProfileCredential>;
 }): Promise<void> {
+  const authPath = path.join(params.agentDir, "auth-profiles.json");
+  const existing = await fs
+    .readFile(authPath, "utf8")
+    .then((raw) => JSON.parse(raw) as { profiles?: Record<string, QaAuthProfileCredential> })
+    .catch(() => ({ profiles: {} }));
   await fs.mkdir(params.agentDir, { recursive: true });
   await fs.writeFile(
-    path.join(params.agentDir, "auth-profiles.json"),
-    `${JSON.stringify({ version: 1, profiles: params.profiles }, null, 2)}\n`,
+    authPath,
+    `${JSON.stringify({ version: 1, profiles: { ...existing.profiles, ...params.profiles } }, null, 2)}\n`,
     "utf8",
   );
 }

extensions/qa-lab/src/scenario-catalog.test.ts

@@ -187,6 +187,7 @@ describe("qa scenario catalog", () => {
           pluginId?: string;
           pluginPersonality?: string;
           adversarialPersonality?: string;
+          expectedSurfaceIds?: Record<string, string[]>;
           expectedAdversarialDiagnostics?: string[];
         }
       | undefined;
@@ -198,9 +199,22 @@ describe("qa scenario catalog", () => {
     expect(config?.pluginId).toBe("openclaw-kitchen-sink-fixture");
     expect(config?.pluginPersonality).toBe("conformance");
     expect(config?.adversarialPersonality).toBe("adversarial");
+    expect(config?.expectedSurfaceIds?.webSearchProviderIds).toContain(
+      "kitchen-sink-web-search-provider",
+    );
+    expect(config?.expectedSurfaceIds?.realtimeVoiceProviderIds).toContain(
+      "kitchen-sink-realtime-voice-provider",
+    );
     expect(config?.expectedAdversarialDiagnostics).toContain(
       "only bundled plugins can register agent tool result middleware",
     );
+    expect(config?.expectedAdversarialDiagnostics).toContain(
+      "control UI descriptor registration requires id, surface, label, and valid optional fields",
+    );
+    expect(
+      config?.expectedAdversarialDiagnostics?.every((entry) => typeof entry === "string"),
+    ).toBe(true);
+    expect(JSON.stringify(scenario.execution.flow)).toContain("--runtime");
     expect(scenario.execution.flow?.steps.map((step) => step.name)).toEqual([
       "installs and inspects the Kitchen Sink plugin",
       "restarts gateway with Kitchen Sink configured",

extensions/qa-lab/src/suite-runtime-agent-tools.test.ts

@@ -51,6 +51,8 @@ import {
 import { createTempDirHarness } from "./temp-dir.test-helper.js";
 
 const { cleanup, makeTempDir } = createTempDirHarness();
+const repoRoot = "/repo/openclaw";
+const gatewayTempRoot = "/tmp/openclaw-qa-runtime";
 
 afterEach(cleanup);
 
@@ -111,12 +113,14 @@ describe("qa suite runtime agent tools helpers", () => {
       callPluginToolsMcp({
         env: {
           gateway: {
+            tempRoot: gatewayTempRoot,
             runtimeEnv: {
               PATH: "/usr/bin",
               OPENCLAW_KEY: "1",
               EMPTY: undefined,
             },
           },
+          repoRoot,
         } as never,
         toolName: "plugin.echo",
         args: { text: "hello" },
@@ -127,8 +131,13 @@ describe("qa suite runtime agent tools helpers", () => {
 
     expect(stdioTransportMock).toHaveBeenCalledWith({
       command: "/usr/bin/node",
-      args: ["--import", "tsx", "src/mcp/plugin-tools-serve.ts"],
+      args: [
+        "--import",
+        expect.stringContaining(path.join("node_modules", "tsx")),
+        path.join(repoRoot, "src", "mcp", "plugin-tools-serve.ts"),
+      ],
       stderr: "pipe",
+      cwd: gatewayTempRoot,
       env: {
         PATH: "/usr/bin",
         OPENCLAW_KEY: "1",
@@ -140,4 +149,31 @@ describe("qa suite runtime agent tools helpers", () => {
     });
     expect(closeMock).toHaveBeenCalled();
   });
+
+  it("reports available plugin-tools MCP names when the requested tool is missing", async () => {
+    listToolsMock.mockResolvedValueOnce({
+      tools: [{ name: "plugin.beta" }, { name: "plugin.alpha" }] as never[],
+    });
+
+    await expect(
+      callPluginToolsMcp({
+        env: {
+          gateway: {
+            tempRoot: gatewayTempRoot,
+            runtimeEnv: {
+              PATH: "/usr/bin",
+            },
+          },
+          repoRoot,
+        } as never,
+        toolName: "plugin.missing",
+        args: {},
+      }),
+    ).rejects.toThrow(
+      "MCP tool missing: plugin.missing; available tools: plugin.alpha, plugin.beta",
+    );
+
+    expect(callToolMock).not.toHaveBeenCalled();
+    expect(closeMock).toHaveBeenCalled();
+  });
 });

extensions/qa-lab/src/suite-runtime-agent-tools.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import { createRequire } from "node:module";
 import path from "node:path";
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
@@ -11,6 +12,8 @@ import type {
   QaTransportActionName,
 } from "./suite-runtime-types.js";
 
+const requireFromHere = createRequire(import.meta.url);
+
 function findSkill(skills: QaSkillStatusEntry[], name: string) {
   return skills.find((skill) => skill.name === name);
 }
@@ -28,7 +31,7 @@ async function writeWorkspaceSkill(params: {
 }
 
 async function callPluginToolsMcp(params: {
-  env: Pick<QaSuiteRuntimeEnv, "gateway">;
+  env: Pick<QaSuiteRuntimeEnv, "gateway" | "repoRoot">;
   toolName: string;
   args: Record<string, unknown>;
 }) {
@@ -40,8 +43,13 @@ async function callPluginToolsMcp(params: {
   const nodeExecPath = await resolveQaNodeExecPath();
   const transport = new StdioClientTransport({
     command: nodeExecPath,
-    args: ["--import", "tsx", "src/mcp/plugin-tools-serve.ts"],
+    args: [
+      "--import",
+      requireFromHere.resolve("tsx"),
+      path.join(params.env.repoRoot, "src/mcp/plugin-tools-serve.ts"),
+    ],
     stderr: "pipe",
+    cwd: params.env.gateway.tempRoot,
     env: transportEnv,
   });
   const client = new Client({ name: "openclaw-qa-suite", version: "0.0.0" }, {});
@@ -50,7 +58,13 @@ async function callPluginToolsMcp(params: {
     const listed = await client.listTools();
     const tool = listed.tools.find((entry) => entry.name === params.toolName);
     if (!tool) {
-      throw new Error(`MCP tool missing: ${params.toolName}`);
+      const availableTools = listed.tools
+        .map((entry) => entry.name)
+        .filter((name): name is string => typeof name === "string" && name.length > 0)
+        .toSorted();
+      throw new Error(
+        `MCP tool missing: ${params.toolName}; available tools: ${availableTools.join(", ") || "<none>"}`,
+      );
     }
     return await client.callTool({
       name: params.toolName,