QA: organize scenarios by theme

extensions/qa-lab/src/bundled-plugin-staging.ts

@@ -10,6 +10,12 @@ const QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS = Object.freeze([
 ]);
 const QA_OPENAI_PLUGIN_ID = "openai";
 const QA_BUNDLED_PLUGIN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const QA_CLI_METADATA_ENTRY_BASENAMES = Object.freeze([
+  "cli-metadata.ts",
+  "cli-metadata.js",
+  "cli-metadata.mjs",
+  "cli-metadata.cjs",
+]);
 
 function assertSafeQaBundledPluginId(pluginId: string) {
   if (!QA_BUNDLED_PLUGIN_ID_PATTERN.test(pluginId)) {
@@ -69,12 +75,17 @@ export function resolveQaBundledPluginSourceDir(params: { repoRoot: string; plug
     path.join(params.repoRoot, "dist-runtime", "extensions", params.pluginId),
     path.join(params.repoRoot, "extensions", params.pluginId),
   ];
-  for (const candidate of candidates) {
-    if (existsSync(candidate)) {
-      return candidate;
-    }
+  const existingCandidates = candidates.filter((candidate) => existsSync(candidate));
+  if (existingCandidates.length === 0) {
+    return null;
   }
-  return null;
+  const cliMetadataCandidate = existingCandidates.find((candidate) =>
+    QA_CLI_METADATA_ENTRY_BASENAMES.some((basename) => existsSync(path.join(candidate, basename))),
+  );
+  if (cliMetadataCandidate) {
+    return cliMetadataCandidate;
+  }
+  return existingCandidates[0] ?? null;
 }
 
 function resolveQaBundledPluginScanRoots(repoRoot: string) {

extensions/qa-lab/src/gateway-child.test.ts

@@ -714,6 +714,43 @@ describe("qa bundled plugin dir", () => {
     ).toBe(path.join(repoRoot, "extensions", "qa-channel"));
   });
 
+  it("uses a source bundled plugin when the built copy is missing CLI metadata", async () => {
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-bundled-cli-metadata-root-"));
+    cleanups.push(async () => {
+      await rm(repoRoot, { recursive: true, force: true });
+    });
+    await mkdir(path.join(repoRoot, "dist", "extensions", "memory-core"), { recursive: true });
+    await writeFile(
+      path.join(repoRoot, "dist", "extensions", "memory-core", "package.json"),
+      "{}",
+      "utf8",
+    );
+    await writeFile(
+      path.join(repoRoot, "dist", "extensions", "memory-core", "openclaw.plugin.json"),
+      JSON.stringify({ id: "memory-core", kind: "memory" }),
+      "utf8",
+    );
+    await mkdir(path.join(repoRoot, "extensions", "memory-core"), { recursive: true });
+    await writeFile(path.join(repoRoot, "extensions", "memory-core", "package.json"), "{}", "utf8");
+    await writeFile(
+      path.join(repoRoot, "extensions", "memory-core", "openclaw.plugin.json"),
+      JSON.stringify({ id: "memory-core", kind: "memory" }),
+      "utf8",
+    );
+    await writeFile(
+      path.join(repoRoot, "extensions", "memory-core", "cli-metadata.ts"),
+      "export default { id: 'memory-core' };\n",
+      "utf8",
+    );
+
+    expect(
+      __testing.resolveQaBundledPluginSourceDir({
+        repoRoot,
+        pluginId: "memory-core",
+      }),
+    ).toBe(path.join(repoRoot, "extensions", "memory-core"));
+  });
+
   it("creates a scoped bundled plugin tree for allowed plugins plus always-allowed runtime facades", async () => {
     const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-bundled-scope-"));
     cleanups.push(async () => {

extensions/qa-lab/src/scenario-catalog.test.ts

@@ -17,6 +17,9 @@ describe("qa scenario catalog", () => {
     expect(pack.agent.identityMarkdown).toContain("Dev C-3PO");
     expect(pack.kickoffTask).toContain("Lobster Invaders");
     expect(listQaScenarioMarkdownPaths().length).toBe(pack.scenarios.length);
+    expect(listQaScenarioMarkdownPaths()).toContain(
+      "qa/scenarios/media/image-generation-roundtrip.md",
+    );
     expect(pack.scenarios.some((scenario) => scenario.id === "image-generation-roundtrip")).toBe(
       true,
     );
@@ -112,7 +115,7 @@ describe("qa scenario catalog", () => {
       (candidate) => candidate.id === "codex-harness-no-meta-leak",
     );
 
-    expect(scenario?.sourcePath).toBe("qa/scenarios/codex-harness-no-meta-leak.md");
+    expect(scenario?.sourcePath).toBe("qa/scenarios/models/codex-harness-no-meta-leak.md");
     expect(scenario?.execution.flow?.steps.map((step) => step.name)).toContain(
       "keeps codex coordination chatter out of the visible reply",
     );
@@ -135,7 +138,7 @@ describe("qa scenario catalog", () => {
           }
         | undefined;
 
-      expect(scenario.sourcePath).toBe(`qa/scenarios/${scenarioId}.md`);
+      expect(scenario.sourcePath).toBe(`qa/scenarios/runtime/${scenarioId}.md`);
       expect(config?.requiredProvider).toBe("mock-openai");
       expect(config?.prompt).toContain("check");
       expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0);

extensions/qa-lab/src/scenario-catalog.ts

@@ -137,6 +137,10 @@ const qaSeedScenarioSchema = z.object({
   id: z.string().trim().min(1),
   title: z.string().trim().min(1),
   surface: z.string().trim().min(1),
+  category: z.string().trim().min(1).optional(),
+  capabilities: z.array(z.string().trim().min(1)).optional(),
+  lane: z.record(z.string(), z.union([z.boolean(), z.string()])).optional(),
+  riskLevel: z.string().trim().min(1).optional(),
   objective: z.string().trim().min(1),
   successCriteria: z.array(z.string().trim().min(1)).min(1),
   plugins: z.array(z.string().trim().min(1)).optional(),
@@ -225,14 +229,6 @@ function readTextFile(relativePath: string): string {
   return fs.readFileSync(resolved, "utf8");
 }
 
-function readDirEntries(relativePath: string): string[] {
-  const resolved = resolveRepoPath(relativePath, "directory");
-  if (!resolved) {
-    return [];
-  }
-  return fs.readdirSync(resolved);
-}
-
 function extractQaPackYaml(content: string) {
   const match = content.match(QA_PACK_FENCE_RE);
   if (!match?.[1]) {
@@ -324,6 +320,13 @@ export function readQaScenarioPack(): QaScenarioPack {
       } satisfies QaSeedScenarioWithSource;
     })(),
   );
+  const seenScenarioIds = new Set<string>();
+  for (const scenario of scenarios) {
+    if (seenScenarioIds.has(scenario.id)) {
+      throw new Error(`duplicate qa scenario id: ${scenario.id}`);
+    }
+    seenScenarioIds.add(scenario.id);
+  }
   return {
     ...parsedPack,
     scenarios,
@@ -331,10 +334,37 @@ export function readQaScenarioPack(): QaScenarioPack {
 }
 
 export function listQaScenarioMarkdownPaths(): string[] {
-  return readDirEntries(QA_SCENARIO_DIR_PATH)
-    .filter((entry) => entry.endsWith(".md") && entry !== "index.md")
-    .map((entry) => `${QA_SCENARIO_DIR_PATH}/${entry}`)
-    .toSorted();
+  const resolved = resolveRepoPath(QA_SCENARIO_DIR_PATH, "directory");
+  if (!resolved) {
+    return [];
+  }
+  return listQaScenarioMarkdownPathsInDirectory(resolved, QA_SCENARIO_DIR_PATH).toSorted();
+}
+
+function listQaScenarioMarkdownPathsInDirectory(
+  absoluteDir: string,
+  relativeDir: string,
+): string[] {
+  const paths: string[] = [];
+  const entries = fs
+    .readdirSync(absoluteDir, { withFileTypes: true })
+    .toSorted((left, right) => left.name.localeCompare(right.name));
+  for (const entry of entries) {
+    if (entry.name.startsWith(".")) {
+      continue;
+    }
+    const relativePath = `${relativeDir}/${entry.name}`;
+    if (entry.isDirectory()) {
+      paths.push(
+        ...listQaScenarioMarkdownPathsInDirectory(path.join(absoluteDir, entry.name), relativePath),
+      );
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".md") && entry.name !== "index.md") {
+      paths.push(relativePath);
+    }
+  }
+  return paths;
 }
 
 export function readQaScenarioOverviewMarkdown(): string {

extensions/qa-lab/src/suite-runtime-agent-process.integration.test.ts

@@ -0,0 +1,74 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { runQaCli } from "./suite-runtime-agent-process.js";
+
+const cleanups: Array<() => Promise<void>> = [];
+
+afterEach(async () => {
+  while (cleanups.length > 0) {
+    await cleanups.pop()?.();
+  }
+});
+
+describe("qa suite runtime CLI integration", () => {
+  it("runs the plugin-owned memory status command with staged CLI metadata", async () => {
+    const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-memory-repo-"));
+    const tempRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-memory-runtime-"));
+    cleanups.push(async () => {
+      await rm(repoRoot, { recursive: true, force: true });
+      await rm(tempRoot, { recursive: true, force: true });
+    });
+    const distDir = path.join(repoRoot, "dist");
+    const bundledPluginsDir = path.join(tempRoot, "dist", "extensions");
+    await mkdir(path.join(distDir), { recursive: true });
+    await mkdir(path.join(bundledPluginsDir, "memory-core"), { recursive: true });
+    await writeFile(
+      path.join(bundledPluginsDir, "memory-core", "cli-metadata.js"),
+      "export default { id: 'memory-core' };\n",
+      "utf8",
+    );
+    await writeFile(
+      path.join(distDir, "index.js"),
+      [
+        "import fs from 'node:fs';",
+        "import path from 'node:path';",
+        "const [command, subcommand] = process.argv.slice(2);",
+        "const metadataPath = path.join(process.env.OPENCLAW_BUNDLED_PLUGINS_DIR ?? '', 'memory-core', 'cli-metadata.js');",
+        "if (command === 'memory' && subcommand === 'status' && fs.existsSync(metadataPath)) {",
+        "  console.log(JSON.stringify({ command, subcommand, status: 'ok' }));",
+        "  process.exit(0);",
+        "}",
+        "console.error(\"error: unknown command 'memory'\");",
+        "process.exit(1);",
+        "",
+      ].join("\n"),
+      "utf8",
+    );
+
+    await expect(
+      runQaCli(
+        {
+          repoRoot,
+          gateway: {
+            tempRoot,
+            runtimeEnv: {
+              ...process.env,
+              OPENCLAW_BUNDLED_PLUGINS_DIR: bundledPluginsDir,
+            },
+          },
+          primaryModel: "openai/gpt-5.4",
+          alternateModel: "openai/gpt-5.4",
+          providerMode: "mock-openai",
+        } as never,
+        ["memory", "status", "--json"],
+        { json: true },
+      ),
+    ).resolves.toEqual({
+      command: "memory",
+      subcommand: "status",
+      status: "ok",
+    });
+  });
+});