refactor(qa): split Matrix QA into optional plugin (#66723)

Merged via squash.

Prepared head SHA: 27241bd089
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras

src/plugin-sdk/qa-lab-runtime.ts

@@ -0,0 +1,39 @@
+import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
+
+type QaLabRuntimeSurface = {
+  defaultQaRuntimeModelForMode: (
+    mode: string,
+    options?: {
+      alternate?: boolean;
+      preferredLiveModel?: string;
+    },
+  ) => string;
+  startQaLiveLaneGateway: (...args: unknown[]) => Promise<unknown>;
+};
+
+function isMissingQaLabRuntimeError(error: unknown) {
+  return (
+    error instanceof Error &&
+    (error.message === "Unable to resolve bundled plugin public surface qa-lab/runtime-api.js" ||
+      error.message.startsWith("Unable to open bundled plugin public surface "))
+  );
+}
+
+export function loadQaLabRuntimeModule(): QaLabRuntimeSurface {
+  return loadBundledPluginPublicSurfaceModuleSync<QaLabRuntimeSurface>({
+    dirName: "qa-lab",
+    artifactBasename: "runtime-api.js",
+  });
+}
+
+export function isQaLabRuntimeAvailable(): boolean {
+  try {
+    loadQaLabRuntimeModule();
+    return true;
+  } catch (error) {
+    if (isMissingQaLabRuntimeError(error)) {
+      return false;
+    }
+    throw error;
+  }
+}

src/plugin-sdk/qa-runner-runtime.integration.test.ts

@@ -0,0 +1,143 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { clearPluginDiscoveryCache } from "../plugins/discovery.js";
+import { clearPluginManifestRegistryCache } from "../plugins/manifest-registry.js";
+import { resetFacadeRuntimeStateForTest } from "./facade-runtime.js";
+
+const ORIGINAL_ENV = {
+  OPENCLAW_DISABLE_BUNDLED_PLUGINS: process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS,
+  OPENCLAW_CONFIG_PATH: process.env.OPENCLAW_CONFIG_PATH,
+  OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE: process.env.OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE,
+  OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE: process.env.OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE,
+  OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS: process.env.OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS,
+  OPENCLAW_PLUGIN_MANIFEST_CACHE_MS: process.env.OPENCLAW_PLUGIN_MANIFEST_CACHE_MS,
+  OPENCLAW_TEST_FAST: process.env.OPENCLAW_TEST_FAST,
+} as const;
+
+const tempDirs: string[] = [];
+
+function makeTempDir(prefix: string): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function resetQaRunnerRuntimeState() {
+  clearPluginDiscoveryCache();
+  clearPluginManifestRegistryCache();
+  resetFacadeRuntimeStateForTest();
+}
+
+describe("plugin-sdk qa-runner-runtime linked plugin smoke", () => {
+  beforeEach(() => {
+    resetQaRunnerRuntimeState();
+    process.env.OPENCLAW_DISABLE_BUNDLED_PLUGINS = "1";
+    process.env.OPENCLAW_DISABLE_PLUGIN_DISCOVERY_CACHE = "1";
+    process.env.OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE = "1";
+    process.env.OPENCLAW_PLUGIN_DISCOVERY_CACHE_MS = "0";
+    process.env.OPENCLAW_PLUGIN_MANIFEST_CACHE_MS = "0";
+    process.env.OPENCLAW_TEST_FAST = "1";
+  });
+
+  afterEach(() => {
+    resetQaRunnerRuntimeState();
+    for (const dir of tempDirs.splice(0)) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+    for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
+      if (value === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = value;
+      }
+    }
+  });
+
+  it("loads an activated qa runner from a linked plugin path", async () => {
+    const stateDir = makeTempDir("openclaw-qa-runner-state-");
+    const pluginDir = path.join(stateDir, "extensions", "qa-linked");
+    const configPath = path.join(stateDir, "openclaw.json");
+
+    fs.writeFileSync(
+      configPath,
+      JSON.stringify({
+        plugins: {},
+      }),
+      "utf8",
+    );
+    process.env.OPENCLAW_CONFIG_PATH = configPath;
+
+    fs.mkdirSync(pluginDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(pluginDir, "openclaw.plugin.json"),
+      JSON.stringify({
+        id: "qa-linked",
+        qaRunners: [
+          {
+            commandName: "linked",
+            description: "Run the linked QA lane",
+          },
+        ],
+        configSchema: {
+          type: "object",
+          additionalProperties: false,
+          properties: {},
+        },
+      }),
+      "utf8",
+    );
+    fs.writeFileSync(
+      path.join(pluginDir, "package.json"),
+      JSON.stringify({
+        name: "@openclaw/qa-linked",
+        type: "module",
+        openclaw: {
+          extensions: ["./index.js"],
+          install: {
+            npmSpec: "@openclaw/qa-linked",
+          },
+        },
+      }),
+      "utf8",
+    );
+    fs.writeFileSync(path.join(pluginDir, "index.js"), 'export default {};\n', "utf8");
+    fs.writeFileSync(
+      path.join(pluginDir, "runtime-api.js"),
+      [
+        "export const qaRunnerCliRegistrations = [",
+        "  {",
+        '    commandName: "linked",',
+        "    register() {}",
+        "  }",
+        "];",
+      ].join("\n"),
+      "utf8",
+    );
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual(
+      expect.arrayContaining([
+        {
+          pluginId: "qa-linked",
+          commandName: "linked",
+          description: "Run the linked QA lane",
+          status: "available",
+          registration: {
+            commandName: "linked",
+            register: expect.any(Function),
+          },
+        },
+        {
+          pluginId: "qa-matrix",
+          commandName: "matrix",
+          description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver",
+          status: "missing",
+          npmSpec: "@openclaw/qa-matrix",
+        },
+      ]),
+    );
+  });
+});

src/plugin-sdk/qa-runner-runtime.test.ts

@@ -0,0 +1,183 @@
+import type { Command } from "commander";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const loadPluginManifestRegistry = vi.hoisted(() => vi.fn());
+const tryLoadActivatedBundledPluginPublicSurfaceModuleSync = vi.hoisted(() => vi.fn());
+const listBundledQaRunnerCatalog = vi.hoisted(() =>
+  vi.fn<
+    () => Array<{
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      npmSpec: string;
+    }>
+  >(() => []),
+);
+
+vi.mock("../plugins/manifest-registry.js", () => ({
+  loadPluginManifestRegistry,
+}));
+
+vi.mock("../plugins/qa-runner-catalog.js", () => ({
+  listBundledQaRunnerCatalog,
+}));
+
+vi.mock("./facade-runtime.js", () => ({
+  tryLoadActivatedBundledPluginPublicSurfaceModuleSync,
+}));
+
+describe("plugin-sdk qa-runner-runtime", () => {
+  beforeEach(() => {
+    loadPluginManifestRegistry.mockReset().mockReturnValue({
+      plugins: [],
+      diagnostics: [],
+    });
+    listBundledQaRunnerCatalog.mockReset().mockReturnValue([]);
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReset();
+  });
+
+  it("stays cold until runner discovery is requested", async () => {
+    await import("./qa-runner-runtime.js");
+
+    expect(loadPluginManifestRegistry).not.toHaveBeenCalled();
+    expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).not.toHaveBeenCalled();
+  });
+
+  it("returns activated runner registrations declared in plugin manifests", async () => {
+    const register = vi.fn((qa: Command) => qa);
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [
+            {
+              commandName: "matrix",
+              description: "Run the Matrix live QA lane",
+            },
+          ],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue({
+      qaRunnerCliRegistrations: [{ commandName: "matrix", register }],
+    });
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        status: "available",
+        registration: {
+          commandName: "matrix",
+          register,
+        },
+      },
+    ]);
+    expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).toHaveBeenCalledWith({
+      dirName: "qa-matrix",
+      artifactBasename: "runtime-api.js",
+    });
+  });
+
+  it("reports declared runners as blocked when the plugin is present but not activated", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        status: "blocked",
+      },
+    ]);
+  });
+
+  it("reports missing optional runners from the generated catalog", async () => {
+    listBundledQaRunnerCatalog.mockReturnValue([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        npmSpec: "@openclaw/qa-matrix",
+      },
+    ]);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(module.listQaRunnerCliContributions()).toEqual([
+      {
+        pluginId: "qa-matrix",
+        commandName: "matrix",
+        description: "Run the Matrix live QA lane",
+        status: "missing",
+        npmSpec: "@openclaw/qa-matrix",
+      },
+    ]);
+  });
+
+  it("fails fast when two plugins declare the same qa runner command", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "alpha",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/alpha",
+        },
+        {
+          id: "beta",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/beta",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null);
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(() => module.listQaRunnerCliContributions()).toThrow(
+      'QA runner command "matrix" declared by both "alpha" and "beta"',
+    );
+  });
+
+  it("fails when runtime registrations include an undeclared command", async () => {
+    loadPluginManifestRegistry.mockReturnValue({
+      plugins: [
+        {
+          id: "qa-matrix",
+          qaRunners: [{ commandName: "matrix" }],
+          rootDir: "/tmp/qa-matrix",
+        },
+      ],
+      diagnostics: [],
+    });
+    tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue({
+      qaRunnerCliRegistrations: [
+        { commandName: "matrix", register: vi.fn() },
+        { commandName: "extra", register: vi.fn() },
+      ],
+    });
+
+    const module = await import("./qa-runner-runtime.js");
+
+    expect(() => module.listQaRunnerCliContributions()).toThrow(
+      'QA runner plugin "qa-matrix" exported "extra" from runtime-api.js but did not declare it in openclaw.plugin.json',
+    );
+  });
+});

src/plugin-sdk/qa-runner-runtime.ts

@@ -0,0 +1,161 @@
+import type { Command } from "commander";
+import type { PluginManifestRecord } from "../plugins/manifest-registry.js";
+import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js";
+import { listBundledQaRunnerCatalog } from "../plugins/qa-runner-catalog.js";
+import { tryLoadActivatedBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
+
+export type QaRunnerCliRegistration = {
+  commandName: string;
+  register(qa: Command): void;
+};
+
+type QaRunnerRuntimeSurface = {
+  qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[];
+};
+
+export type QaRunnerCliContribution =
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "available";
+      registration: QaRunnerCliRegistration;
+    }
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "blocked";
+    }
+  | {
+      pluginId: string;
+      commandName: string;
+      description?: string;
+      status: "missing";
+      npmSpec: string;
+    };
+
+function listDeclaredQaRunnerPlugins(): Array<
+  PluginManifestRecord & {
+    qaRunners: NonNullable<PluginManifestRecord["qaRunners"]>;
+  }
+> {
+  return loadPluginManifestRegistry({ cache: true })
+    .plugins.filter(
+      (
+        plugin,
+      ): plugin is PluginManifestRecord & {
+        qaRunners: NonNullable<PluginManifestRecord["qaRunners"]>;
+      } => Array.isArray(plugin.qaRunners) && plugin.qaRunners.length > 0,
+    )
+    .toSorted((left, right) => {
+      const idCompare = left.id.localeCompare(right.id);
+      if (idCompare !== 0) {
+        return idCompare;
+      }
+      return left.rootDir.localeCompare(right.rootDir);
+    });
+}
+
+function indexRuntimeRegistrations(
+  pluginId: string,
+  surface: QaRunnerRuntimeSurface,
+): ReadonlyMap<string, QaRunnerCliRegistration> {
+  const registrations = surface.qaRunnerCliRegistrations ?? [];
+  const registrationByCommandName = new Map<string, QaRunnerCliRegistration>();
+  for (const registration of registrations) {
+    if (!registration?.commandName || typeof registration.register !== "function") {
+      throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`);
+    }
+    if (registrationByCommandName.has(registration.commandName)) {
+      throw new Error(
+        `QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`,
+      );
+    }
+    registrationByCommandName.set(registration.commandName, registration);
+  }
+  return registrationByCommandName;
+}
+
+function buildKnownQaRunnerCatalog(): readonly QaRunnerCliContribution[] {
+  const knownRunners = listBundledQaRunnerCatalog();
+  const seenCommandNames = new Map<string, string>();
+  return knownRunners.map((runner) => {
+    const previousOwner = seenCommandNames.get(runner.commandName);
+    if (previousOwner) {
+      throw new Error(
+        `QA runner command "${runner.commandName}" declared by both "${previousOwner}" and "${runner.pluginId}"`,
+      );
+    }
+    seenCommandNames.set(runner.commandName, runner.pluginId);
+    return {
+      pluginId: runner.pluginId,
+      commandName: runner.commandName,
+      ...(runner.description ? { description: runner.description } : {}),
+      status: "missing" as const,
+      npmSpec: runner.npmSpec,
+    };
+  });
+}
+
+export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution[] {
+  const contributions = new Map<string, QaRunnerCliContribution>();
+
+  for (const runner of buildKnownQaRunnerCatalog()) {
+    contributions.set(runner.commandName, runner);
+  }
+
+  for (const plugin of listDeclaredQaRunnerPlugins()) {
+    const runtimeSurface =
+      tryLoadActivatedBundledPluginPublicSurfaceModuleSync<QaRunnerRuntimeSurface>({
+        dirName: plugin.id,
+        artifactBasename: "runtime-api.js",
+      });
+    const runtimeRegistrationByCommandName = runtimeSurface
+      ? indexRuntimeRegistrations(plugin.id, runtimeSurface)
+      : null;
+    const declaredCommandNames = new Set(plugin.qaRunners.map((runner) => runner.commandName));
+
+    for (const runner of plugin.qaRunners) {
+      const previous = contributions.get(runner.commandName);
+      if (previous && previous.pluginId !== plugin.id) {
+        throw new Error(
+          `QA runner command "${runner.commandName}" declared by both "${previous.pluginId}" and "${plugin.id}"`,
+        );
+      }
+
+      const registration = runtimeRegistrationByCommandName?.get(runner.commandName);
+      if (!runtimeSurface) {
+        contributions.set(runner.commandName, {
+          pluginId: plugin.id,
+          commandName: runner.commandName,
+          ...(runner.description ? { description: runner.description } : {}),
+          status: "blocked",
+        });
+        continue;
+      }
+      if (!registration) {
+        throw new Error(
+          `QA runner plugin "${plugin.id}" declared "${runner.commandName}" in openclaw.plugin.json but did not export a matching CLI registration`,
+        );
+      }
+      contributions.set(runner.commandName, {
+        pluginId: plugin.id,
+        commandName: runner.commandName,
+        ...(runner.description ? { description: runner.description } : {}),
+        status: "available",
+        registration,
+      });
+    }
+
+    for (const commandName of runtimeRegistrationByCommandName?.keys() ?? []) {
+      if (!declaredCommandNames.has(commandName)) {
+        throw new Error(
+          `QA runner plugin "${plugin.id}" exported "${commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`,
+        );
+      }
+    }
+  }
+
+  return [...contributions.values()];
+}