refactor(qa): split Matrix QA into optional plugin (#66723)

Merged via squash.

Prepared head SHA: 27241bd089
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras

src/plugin-sdk/qa-lab-runtime.ts

@@ -0,0 +1,39 @@
+import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
+
+type QaLabRuntimeSurface = {
+  defaultQaRuntimeModelForMode: (
+    mode: string,
+    options?: {
+      alternate?: boolean;
+      preferredLiveModel?: string;
+    },
+  ) => string;
+  startQaLiveLaneGateway: (...args: unknown[]) => Promise<unknown>;
+};
+
+function isMissingQaLabRuntimeError(error: unknown) {
+  return (
+    error instanceof Error &&
+    (error.message === "Unable to resolve bundled plugin public surface qa-lab/runtime-api.js" ||
+      error.message.startsWith("Unable to open bundled plugin public surface "))
+  );
+}
+
+export function loadQaLabRuntimeModule(): QaLabRuntimeSurface {
+  return loadBundledPluginPublicSurfaceModuleSync<QaLabRuntimeSurface>({
+    dirName: "qa-lab",
+    artifactBasename: "runtime-api.js",
+  });
+}
+
+export function isQaLabRuntimeAvailable(): boolean {
+  try {
+    loadQaLabRuntimeModule();
+    return true;
+  } catch (error) {
+    if (isMissingQaLabRuntimeError(error)) {
+      return false;
+    }
+    throw error;
+  }
+}

src/plugin-sdk/qa-runner-runtime.integration.test.ts

@@ -0,0 +1,143 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { clearPluginDiscoveryCache } from "../plugins/discovery.js";
+import { clearPluginManifestRegistryCache } from "../plugins/manifest-registry.js";
+import { resetFacadeRuntimeStateForTest } from "./facade-runtime.js";
+
+const ORIGINAL_ENV = {
+  OPENCLAW_DISABLE_BUNDLED_PLUGINS: process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS,
+  OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH,
+  OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE: process.env.OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE,
+  OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE: process.env.OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE,
+  OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS: process.env.OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS,
+  OPENCLAW_PLUGIN_MANIFEST_CACHE_MS: process.env.OPENCLAW_PLUGIN_MANIFEST_CACHE_MS,
+  OPENCLAW_TEST_FAST: process.env.OPENCLAW_TEST_FAST,
+} as const;
+
+const tempDirs: string[] = [];
+
+function makeTempDir(prefix: string): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function resetQaRunnerRuntimeState() {
+  clearPluginDiscoveryCache();
+  clearPluginManifestRegistryCache();
+  resetFacadeRuntimeStateForTest();
+}
+
+describe("plugin-sdk qa-runner-runtime linked plugin smoke", () => {
+  beforeEach(() => {
+    resetQaRunnerRuntimeState();
+    process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS = "1";
+    process.env.OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE = "1";
+    process.env.OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE = "1";
+    process.env.OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS = "0";
+    process.env.OPENCLAW_PLUGIN_MANIFEST_CACHE_MS = "0";
+    process.env.OPENCLAW_TEST_FAST = "1";
+  });
+
+  afterEach(() => {
+    resetQaRunnerRuntimeState();
+    for (const dir of tempDirs.splice(0)) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+    for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
+      if (value === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = value;
+      }
+    }
+  });
+
+  it("loads an activated qa runner from a linked plugin path", async () => {
+    const stateDir = makeTempDir("openclaw-qa-runner-state-");
+    const pluginDir = path.join(stateDir, "extensions", "qa-linked");
+    const configPath = path.join(stateDir, "openclaw.json");
+
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify({
+        plugins: {},
+      }),
+      "utf8",
+    );
+    process.env.OPENCLAW_CONFIG_PATH = configPath;
+
+    fs.mkdirSync(pluginDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(pluginDir, "openclaw.plugin.json"),
+      JSON.stringify({
+        id: "qa-linked",
+        qaRunners: [
+          {
+            commandName: "linked",
+            description: "Run the linked QA lane",
+          },
+        ],
+        configSchema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {},
+        },
+      }),
+      "utf8",
+    );
+    fs.writeFileSync(
+      path.join(pluginDir, "package.json"),
+      JSON.stringify({
+        name: "@openclaw/qa-linked",
+        type: "module",
+        openclaw: {
+          extensions: ["./index.js"],
+          install: {
+            npmSpec: "@openclaw/qa-linked",
+          },
+        },
+      }),
+      "utf8",
+    );
+    fs.writeFileSync(path.join(pluginDir, "index.js"), 'export default {};\n', "utf8");
+    fs.writeFileSync(
+      path.join(pluginDir, "runtime-api.js"),
+      [
+        "export const qaRunnerCliRegistrations = [",
+        "  {",
+        '    commandName: "linked",',
+        "    register() {}",
+        "  }",
+        "];",
+      ].join("\n"),
+      "utf8",
+    );
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual(
+      expect.arrayContaining([
+        {
+          pluginId: "qa-linked",
+          commandName: "linked",
+          description: "Run the linked QA lane",
+          status: "available",
+          registration: {
+            commandName: "linked",
+            register: expect.any(Function),
+          },
+        },
+        {
+          pluginId: "qa-matrix",
+          commandName: "matrix",
+          description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver",
+          status: "missing",
+          npmSpec: "@openclaw/qa-matrix",
+        },
+      ]),
+    );
+  });
+});

src/plugin-sdk/qa-runner-runtime.test.ts

@@ -0,0 +1,183 @@
+import type { Command } from "commander";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const loadPluginManifestRegistry = vi.hoisted(() => vi.fn());
+const tryLoadActivatedBundledPluginPublicSurfaceModuleSync = vi.hoisted(() => vi.fn());
+const listBundledQaRunnerCatalog = vi.hoisted(() =>
+  vi.fn<
+    () => Array<{
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      npmSpec: string;
+    }>
+  >(() => []),
+);
+
+vi.mock("../plugins/manifest-registry.js", () => ({
+  loadPluginManifestRegistry,
+}));
+
+vi.mock("../plugins/qa-runner-catalog.js", () => ({
+  listBundledQaRunnerCatalog,
+}));
+
+vi.mock("./facade-runtime.js", () => ({
+  tryLoadActivatedBundledPluginPublicSurfaceModuleSync,
+}));
+
+describe("plugin-sdk qa-runner-runtime", () => {
+  beforeEach(() => {
+    loadPluginManifestRegistry.mockReset().mockReturnValue({
+      plugins: [],
+      diagnostics: [],
+    });
+    listBundledQaRunnerCatalog.mockReset().mockReturnValue([]);
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReset();
+  });
+
+  it("stays cold until runner discovery is requested", async () => {
+    await import("./qa-runner-runtime.js");
+
+    expect(loadPluginManifestRegistry).not.toHaveBeenCalled();
+    expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).not.toHaveBeenCalled();
+  });
+
+  it("returns activated runner registrations declared in plugin manifests", async () => {
+    const register = vi.fn((qa: Command) => qa);
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [
+            {
+              commandName: "matrix",
+              description: "Run the Matrix live QA lane",
+            },
+          ],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue({
+      qaRunnerCliRegistrations: [{ commandName: "matrix", register }],
+    });
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        status: "available",
+        registration: {
+          commandName: "matrix",
+          register,
+        },
+      },
+    ]);
+    expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).toHaveBeenCalledWith({
+      dirName: "qa-matrix",
+      artifactBasename: "runtime-api.js",
+    });
+  });
+
+  it("reports declared runners as blocked when the plugin is present but not activated", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        status: "blocked",
+      },
+    ]);
+  });
+
+  it("reports missing optional runners from the generated catalog", async () => {
+    listBundledQaRunnerCatalog.mockReturnValue([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        npmSpec: "@openclaw/qa-matrix",
+      },
+    ]);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        status: "missing",
+        npmSpec: "@openclaw/qa-matrix",
+      },
+    ]);
+  });
+
+  it("fails fast when two plugins declare the same qa runner command", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "alpha",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/alpha",
+        },
+        {
+          id: "beta",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/beta",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(() => module.listQaRunnerCliContributions()).toThrow(
+      'QA runner command "matrix" declared by both "alpha" and "beta"',
+    );
+  });
+
+  it("fails when runtime registrations include an undeclared command", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue({
+      qaRunnerCliRegistrations: [
+        { commandName: "matrix", register: vi.fn() },
+        { commandName: "extra", register: vi.fn() },
+      ],
+    });
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(() => module.listQaRunnerCliContributions()).toThrow(
+      'QA runner plugin "qa-matrix" exported "extra" from runtime-api.js but did not declare it in openclaw.plugin.json',
+    );
+  });
+});

src/plugin-sdk/qa-runner-runtime.ts

@@ -0,0 +1,161 @@
+import type { Command } from "commander";
+import type { PluginManifestRecord } from "../plugins/manifest-registry.js";
+import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js";
+import { listBundledQaRunnerCatalog } from "../plugins/qa-runner-catalog.js";
+import { tryLoadActivatedBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
+
+export type QaRunnerCliRegistration = {
+  commandName: string;
+  register(qa: Command): void;
+};
+
+type QaRunnerRuntimeSurface = {
+  qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[];
+};
+
+export type QaRunnerCliContribution =
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "available";
+      registration: QaRunnerCliRegistration;
+    }
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "blocked";
+    }
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "missing";
+      npmSpec: string;
+    };
+
+function listDeclaredQaRunnerPlugins(): Array<
+  PluginManifestRecord & {
+    qaRunners: NonNullable<PluginManifestRecord["qaRunners"]>;
+  }
+> {
+  return loadPluginManifestRegistry({ cache: true })
+    .plugins.filter(
+      (
+        plugin,
+      ): plugin is PluginManifestRecord & {
+        qaRunners: NonNullable<PluginManifestRecord["qaRunners"]>;
+      } => Array.isArray(plugin.qaRunners) && plugin.qaRunners.length > 0,
+    )
+    .toSorted((left, right) => {
+      const idCompare = left.id.localeCompare(right.id);
+      if (idCompare !== 0) {
+        return idCompare;
+      }
+      return left.rootDir.localeCompare(right.rootDir);
+    });
+}
+
+function indexRuntimeRegistrations(
+  pluginId: string,
+  surface: QaRunnerRuntimeSurface,
+): ReadonlyMap<string, QaRunnerCliRegistration> {
+  const registrations = surface.qaRunnerCliRegistrations ?? [];
+  const registrationByCommandName = new Map<string, QaRunnerCliRegistration>();
+  for (const registration of registrations) {
+    if (!registration?.commandName || typeof registration.register !== "function") {
+      throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`);
+    }
+    if (registrationByCommandName.has(registration.commandName)) {
+      throw new Error(
+        `QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`,
+      );
+    }
+    registrationByCommandName.set(registration.commandName, registration);
+  }
+  return registrationByCommandName;
+}
+
+function buildKnownQaRunnerCatalog(): readonly QaRunnerCliContribution[] {
+  const knownRunners = listBundledQaRunnerCatalog();
+  const seenCommandNames = new Map<string, string>();
+  return knownRunners.map((runner) => {
+    const previousOwner = seenCommandNames.get(runner.commandName);
+    if (previousOwner) {
+      throw new Error(
+        `QA runner command "${runner.commandName}" declared by both "${previousOwner}" and "${runner.pluginId}"`,
+      );
+    }
+    seenCommandNames.set(runner.commandName, runner.pluginId);
+    return {
+      pluginId: runner.pluginId,
+      commandName: runner.commandName,
+      ...(runner.description ? { description: runner.description } : {}),
+      status: "missing" as const,
+      npmSpec: runner.npmSpec,
+    };
+  });
+}
+
+export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution[] {
+  const contributions = new Map<string, QaRunnerCliContribution>();
+
+  for (const runner of buildKnownQaRunnerCatalog()) {
+    contributions.set(runner.commandName, runner);
+  }
+
+  for (const plugin of listDeclaredQaRunnerPlugins()) {
+    const runtimeSurface =
+      tryLoadActivatedBundledPluginPublicSurfaceModuleSync<QaRunnerRuntimeSurface>({
+        dirName: plugin.id,
+        artifactBasename: "runtime-api.js",
+      });
+    const runtimeRegistrationByCommandName = runtimeSurface
+      ? indexRuntimeRegistrations(plugin.id, runtimeSurface)
+      : null;
+    const declaredCommandNames = new Set(plugin.qaRunners.map((runner) => runner.commandName));
+
+    for (const runner of plugin.qaRunners) {
+      const previous = contributions.get(runner.commandName);
+      if (previous && previous.pluginId !== plugin.id) {
+        throw new Error(
+          `QA runner command "${runner.commandName}" declared by both "${previous.pluginId}" and "${plugin.id}"`,
+        );
+      }
+
+      const registration = runtimeRegistrationByCommandName?.get(runner.commandName);
+      if (!runtimeSurface) {
+        contributions.set(runner.commandName, {
+          pluginId: plugin.id,
+          commandName: runner.commandName,
+          ...(runner.description ? { description: runner.description } : {}),
+          status: "blocked",
+        });
+        continue;
+      }
+      if (!registration) {
+        throw new Error(
+          `QA runner plugin "${plugin.id}" declared "${runner.commandName}" in openclaw.plugin.json but did not export a matching CLI registration`,
+        );
+      }
+      contributions.set(runner.commandName, {
+        pluginId: plugin.id,
+        commandName: runner.commandName,
+        ...(runner.description ? { description: runner.description } : {}),
+        status: "available",
+        registration,
+      });
+    }
+
+    for (const commandName of runtimeRegistrationByCommandName?.keys() ?? []) {
+      if (!declaredCommandNames.has(commandName)) {
+        throw new Error(
+          `QA runner plugin "${plugin.id}" exported "${commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`,
+        );
+      }
+    }
+  }
+
+  return [...contributions.values()];
+}

src/plugins/bundled-plugin-metadata.test.ts

@@ -131,11 +131,12 @@ describe("bundled plugin metadata", () => {
     },
   );
 
-  it("excludes private QA sidecars from the packaged runtime sidecar baseline", () => {
+  it("excludes non-packaged QA sidecars from the packaged runtime sidecar baseline", () => {
     expect(BUNDLED_RUNTIME_SIDECAR_PATHS).not.toContain(
       "dist/extensions/qa-channel/runtime-api.js",
     );
     expect(BUNDLED_RUNTIME_SIDECAR_PATHS).not.toContain("dist/extensions/qa-lab/runtime-api.js");
+    expect(BUNDLED_RUNTIME_SIDECAR_PATHS).not.toContain("dist/extensions/qa-matrix/runtime-api.js");
   });
 
   it("captures setup-entry metadata for bundled channel plugins", () => {

src/plugins/install.test.ts

@@ -1427,6 +1427,21 @@ describe("installPluginFromArchive", () => {
     ).toBe(true);
   });
 
+  it("does not flag the real qa-matrix plugin as dangerous install code", async () => {
+    const pluginDir = path.resolve(process.cwd(), "extensions", "qa-matrix");
+
+    const scanResult = await installSecurityScan.scanPackageInstallSource({
+      extensions: ["./index.ts"],
+      logger: { warn: vi.fn() },
+      packageDir: pluginDir,
+      pluginId: "qa-matrix",
+      packageName: "@openclaw/qa-matrix",
+      manifestId: "qa-matrix",
+    });
+
+    expect(scanResult?.blocked).toBeUndefined();
+  });
+
   it("keeps blocked dependency package checks active when forced unsafe install is set", async () => {
     const { pluginDir, extensionsDir } = setupPluginInstallDirs();
 

src/plugins/manifest-registry.test.ts

@@ -499,6 +499,33 @@ describe("loadPluginManifestRegistry", () => {
     });
   });
 
+  it("preserves qa runner descriptors from plugin manifests", () => {
+    const dir = makeTempDir();
+    writeManifest(dir, {
+      id: "qa-matrix",
+      qaRunners: [
+        {
+          commandName: "matrix",
+          description: "Run the Matrix live QA lane",
+        },
+      ],
+      configSchema: { type: "object" },
+    });
+
+    const registry = loadSingleCandidateRegistry({
+      idHint: "qa-matrix",
+      rootDir: dir,
+      origin: "bundled",
+    });
+
+    expect(registry.plugins[0]?.qaRunners).toEqual([
+      {
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+      },
+    ]);
+  });
+
   it("preserves channel config metadata from plugin manifests", () => {
     const dir = makeTempDir();
     writeManifest(dir, {

src/plugins/manifest-registry.ts

@@ -34,6 +34,7 @@ import {
   type PluginManifestChannelConfig,
   type PluginManifestContracts,
   type PluginManifestModelSupport,
+  type PluginManifestQaRunner,
   type PluginManifestSetup,
 } from "./manifest.js";
 import { checkMinHostVersion } from "./min-host-version.js";
@@ -92,6 +93,7 @@ export type PluginManifestRecord = {
   providerAuthChoices?: PluginManifest["providerAuthChoices"];
   activation?: PluginManifestActivation;
   setup?: PluginManifestSetup;
+  qaRunners?: PluginManifestQaRunner[];
   skills: string[];
   settingsFiles?: string[];
   hooks: string[];
@@ -333,6 +335,7 @@ function buildRecord(params: {
     providerAuthChoices: params.manifest.providerAuthChoices,
     activation: params.manifest.activation,
     setup: params.manifest.setup,
+    qaRunners: params.manifest.qaRunners,
     skills: params.manifest.skills ?? [],
     settingsFiles: [],
     hooks: [],

src/plugins/manifest.ts

@@ -80,6 +80,13 @@ export type PluginManifestSetup = {
   requiresRuntime?: boolean;
 };
 
+export type PluginManifestQaRunner = {
+  /** Subcommand mounted beneath `openclaw qa`, for example `matrix`. */
+  commandName: string;
+  /** Optional user-facing help text for fallback host stubs. */
+  description?: string;
+};
+
 export type PluginManifestConfigLiteral = string | number | boolean | null;
 
 export type PluginManifestDangerousConfigFlag = {
@@ -174,6 +181,8 @@ export type PluginManifest = {
   activation?: PluginManifestActivation;
   /** Cheap setup/onboarding metadata exposed before plugin runtime loads. */
   setup?: PluginManifestSetup;
+  /** Cheap QA runner metadata exposed before plugin runtime loads. */
+  qaRunners?: PluginManifestQaRunner[];
   skills?: string[];
   name?: string;
   description?: string;
@@ -484,6 +493,28 @@ function normalizeManifestSetup(value: unknown): PluginManifestSetup | undefined
   return Object.keys(setup).length > 0 ? setup : undefined;
 }
 
+function normalizeManifestQaRunners(value: unknown): PluginManifestQaRunner[] | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+  const normalized: PluginManifestQaRunner[] = [];
+  for (const entry of value) {
+    if (!isRecord(entry)) {
+      continue;
+    }
+    const commandName = normalizeOptionalString(entry.commandName) ?? "";
+    if (!commandName) {
+      continue;
+    }
+    const description = normalizeOptionalString(entry.description) ?? "";
+    normalized.push({
+      commandName,
+      ...(description ? { description } : {}),
+    });
+  }
+  return normalized.length > 0 ? normalized : undefined;
+}
+
 function normalizeProviderAuthChoices(
   value: unknown,
 ): PluginManifestProviderAuthChoice[] | undefined {
@@ -673,6 +704,7 @@ export function loadPluginManifest(
   const providerAuthChoices = normalizeProviderAuthChoices(raw.providerAuthChoices);
   const activation = normalizeManifestActivation(raw.activation);
   const setup = normalizeManifestSetup(raw.setup);
+  const qaRunners = normalizeManifestQaRunners(raw.qaRunners);
   const skills = normalizeTrimmedStringList(raw.skills);
   const contracts = normalizeManifestContracts(raw.contracts);
   const configContracts = normalizeManifestConfigContracts(raw.configContracts);
@@ -706,6 +738,7 @@ export function loadPluginManifest(
       providerAuthChoices,
       activation,
       setup,
+      qaRunners,
       skills,
       name,
       description,

src/plugins/qa-runner-catalog.ts

@@ -0,0 +1,74 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { listBundledPluginMetadata } from "./bundled-plugin-metadata.js";
+
+export type QaRunnerCatalogEntry = {
+  pluginId: string;
+  commandName: string;
+  description?: string;
+  npmSpec: string;
+};
+
+const QA_RUNNER_CATALOG_JSON_PATH = fileURLToPath(
+  new URL("../../scripts/lib/qa-runner-catalog.json", import.meta.url),
+);
+
+export function listBundledQaRunnerCatalog(): readonly QaRunnerCatalogEntry[] {
+  if (!fs.existsSync(QA_RUNNER_CATALOG_JSON_PATH)) {
+    return [];
+  }
+  return JSON.parse(fs.readFileSync(QA_RUNNER_CATALOG_JSON_PATH, "utf8")) as QaRunnerCatalogEntry[];
+}
+
+export function collectBundledQaRunnerCatalog(params?: {
+  rootDir?: string;
+}): readonly QaRunnerCatalogEntry[] {
+  const catalog: QaRunnerCatalogEntry[] = [];
+  const seenCommandNames = new Map<string, string>();
+
+  for (const entry of listBundledPluginMetadata({
+    rootDir: params?.rootDir,
+    includeChannelConfigs: false,
+  })) {
+    const qaRunners = entry.manifest.qaRunners ?? [];
+    const npmSpec = entry.packageManifest?.install?.npmSpec?.trim() || entry.packageName?.trim();
+    if (!npmSpec) {
+      continue;
+    }
+    for (const runner of qaRunners) {
+      const previousOwner = seenCommandNames.get(runner.commandName);
+      if (previousOwner) {
+        throw new Error(
+          `QA runner command "${runner.commandName}" declared by both "${previousOwner}" and "${entry.manifest.id}"`,
+        );
+      }
+      seenCommandNames.set(runner.commandName, entry.manifest.id);
+      catalog.push({
+        pluginId: entry.manifest.id,
+        commandName: runner.commandName,
+        ...(runner.description ? { description: runner.description } : {}),
+        npmSpec,
+      });
+    }
+  }
+
+  return catalog.toSorted((left, right) => left.commandName.localeCompare(right.commandName));
+}
+
+export async function writeBundledQaRunnerCatalog(params: {
+  repoRoot: string;
+  check: boolean;
+}): Promise<{ changed: boolean; jsonPath: string }> {
+  const jsonPath = path.join(params.repoRoot, "scripts", "lib", "qa-runner-catalog.json");
+  const expectedJson = `${JSON.stringify(collectBundledQaRunnerCatalog({ rootDir: params.repoRoot }), null, 2)}\n`;
+  const currentJson = fs.existsSync(jsonPath) ? fs.readFileSync(jsonPath, "utf8") : "";
+  const changed = currentJson !== expectedJson;
+
+  if (!params.check && changed) {
+    fs.mkdirSync(path.dirname(jsonPath), { recursive: true });
+    fs.writeFileSync(jsonPath, expectedJson, "utf8");
+  }
+
+  return { changed, jsonPath };
+}

src/plugins/runtime-sidecar-paths-baseline.ts

@@ -2,7 +2,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { listBundledPluginMetadata } from "./bundled-plugin-metadata.js";
 
-const NON_PACKAGED_RUNTIME_SIDECAR_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab"]);
+const NON_PACKAGED_RUNTIME_SIDECAR_PLUGIN_DIRS = new Set(["qa-channel", "qa-lab", "qa-matrix"]);
 
 function buildBundledDistArtifactPath(dirName: string, artifact: string): string {
   return ["dist", "extensions", dirName, artifact].join("/");