refactor(qa): split Matrix QA into optional plugin (#66723)

Merged via squash.

Prepared head SHA: 27241bd089
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras

extensions/qa-matrix/src/shared/live-lane-helpers.ts

@@ -0,0 +1,18 @@
+import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
+
+export function appendLiveLaneIssue(issues: string[], label: string, error: unknown) {
+  issues.push(`${label}: ${formatErrorMessage(error)}`);
+}
+
+export function buildLiveLaneArtifactsError(params: {
+  heading: string;
+  artifacts: Record<string, string>;
+  details?: string[];
+}) {
+  return [
+    params.heading,
+    ...(params.details ?? []),
+    "Artifacts:",
+    ...Object.entries(params.artifacts).map(([label, filePath]) => `- ${label}: ${filePath}`),
+  ].join("\n");
+}

extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts

@@ -0,0 +1,40 @@
+import path from "node:path";
+import { resolveRepoRelativeOutputDir } from "../cli-paths.js";
+import type { QaProviderMode } from "../run-config.js";
+import { normalizeQaProviderMode } from "../run-config.js";
+import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
+
+export function resolveLiveTransportQaRunOptions(
+  opts: LiveTransportQaCommandOptions,
+): LiveTransportQaCommandOptions & {
+  repoRoot: string;
+  providerMode: QaProviderMode;
+} {
+  return {
+    repoRoot: path.resolve(opts.repoRoot ?? process.cwd()),
+    outputDir: resolveRepoRelativeOutputDir(
+      path.resolve(opts.repoRoot ?? process.cwd()),
+      opts.outputDir,
+    ),
+    providerMode:
+      opts.providerMode === undefined
+        ? "live-frontier"
+        : normalizeQaProviderMode(opts.providerMode),
+    primaryModel: opts.primaryModel,
+    alternateModel: opts.alternateModel,
+    fastMode: opts.fastMode,
+    scenarioIds: opts.scenarioIds,
+    sutAccountId: opts.sutAccountId,
+    credentialSource: opts.credentialSource?.trim(),
+    credentialRole: opts.credentialRole?.trim(),
+  };
+}
+
+export function printLiveTransportQaArtifacts(
+  laneLabel: string,
+  artifacts: Record<string, string>,
+) {
+  for (const [label, filePath] of Object.entries(artifacts)) {
+    process.stdout.write(`${laneLabel} ${label}: ${filePath}\n`);
+  }
+}

extensions/qa-matrix/src/shared/live-transport-cli.ts

@@ -0,0 +1,132 @@
+import type { Command } from "commander";
+import { collectString } from "../cli-options.js";
+import type { QaProviderModeInput } from "../run-config.js";
+
+export type LiveTransportQaCommandOptions = {
+  repoRoot?: string;
+  outputDir?: string;
+  providerMode?: QaProviderModeInput;
+  primaryModel?: string;
+  alternateModel?: string;
+  fastMode?: boolean;
+  scenarioIds?: string[];
+  sutAccountId?: string;
+  credentialSource?: string;
+  credentialRole?: string;
+};
+
+type LiveTransportQaCommanderOptions = {
+  repoRoot?: string;
+  outputDir?: string;
+  providerMode?: QaProviderModeInput;
+  model?: string;
+  altModel?: string;
+  scenario?: string[];
+  fast?: boolean;
+  sutAccount?: string;
+  credentialSource?: string;
+  credentialRole?: string;
+};
+
+export type LiveTransportQaCliRegistration = {
+  commandName: string;
+  register(qa: Command): void;
+};
+
+export type LiveTransportQaCredentialCliOptions = {
+  sourceDescription?: string;
+  roleDescription?: string;
+};
+
+export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
+  let promise: Promise<T> | null = null;
+  return async () => {
+    promise ??= load();
+    return await promise;
+  };
+}
+
+export function mapLiveTransportQaCommanderOptions(
+  opts: LiveTransportQaCommanderOptions,
+): LiveTransportQaCommandOptions {
+  return {
+    repoRoot: opts.repoRoot,
+    outputDir: opts.outputDir,
+    providerMode: opts.providerMode,
+    primaryModel: opts.model,
+    alternateModel: opts.altModel,
+    fastMode: opts.fast,
+    scenarioIds: opts.scenario,
+    sutAccountId: opts.sutAccount,
+    credentialSource: opts.credentialSource,
+    credentialRole: opts.credentialRole,
+  };
+}
+
+export function registerLiveTransportQaCli(params: {
+  qa: Command;
+  commandName: string;
+  credentialOptions?: LiveTransportQaCredentialCliOptions;
+  description: string;
+  outputDirHelp: string;
+  scenarioHelp: string;
+  sutAccountHelp: string;
+  run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
+}) {
+  const command = params.qa
+    .command(params.commandName)
+    .description(params.description)
+    .option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
+    .option("--output-dir <path>", params.outputDirHelp)
+    .option(
+      "--provider-mode <mode>",
+      "Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
+      "live-frontier",
+    )
+    .option("--model <ref>", "Primary provider/model ref")
+    .option("--alt-model <ref>", "Alternate provider/model ref")
+    .option("--scenario <id>", params.scenarioHelp, collectString, [])
+    .option("--fast", "Enable provider fast mode where supported", false)
+    .option("--sut-account <id>", params.sutAccountHelp, "sut");
+
+  if (params.credentialOptions) {
+    command.option(
+      "--credential-source <source>",
+      params.credentialOptions.sourceDescription ??
+        "Credential source for live lanes: env or convex (default: env)",
+    );
+    if (params.credentialOptions.roleDescription) {
+      command.option("--credential-role <role>", params.credentialOptions.roleDescription);
+    }
+  }
+
+  command.action(async (opts: LiveTransportQaCommanderOptions) => {
+    await params.run(mapLiveTransportQaCommanderOptions(opts));
+  });
+}
+
+export function createLiveTransportQaCliRegistration(params: {
+  commandName: string;
+  credentialOptions?: LiveTransportQaCredentialCliOptions;
+  description: string;
+  outputDirHelp: string;
+  scenarioHelp: string;
+  sutAccountHelp: string;
+  run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
+}): LiveTransportQaCliRegistration {
+  return {
+    commandName: params.commandName,
+    register(qa: Command) {
+      registerLiveTransportQaCli({
+        qa,
+        commandName: params.commandName,
+        credentialOptions: params.credentialOptions,
+        description: params.description,
+        outputDirHelp: params.outputDirHelp,
+        scenarioHelp: params.scenarioHelp,
+        sutAccountHelp: params.sutAccountHelp,
+        run: params.run,
+      });
+    },
+  };
+}

extensions/qa-matrix/src/shared/live-transport-scenarios.ts

@@ -0,0 +1,149 @@
+export type LiveTransportStandardScenarioId =
+  | "canary"
+  | "mention-gating"
+  | "allowlist-block"
+  | "top-level-reply-shape"
+  | "restart-resume"
+  | "thread-follow-up"
+  | "thread-isolation"
+  | "reaction-observation"
+  | "help-command";
+
+export type LiveTransportScenarioDefinition<TId extends string = string> = {
+  id: TId;
+  standardId?: LiveTransportStandardScenarioId;
+  timeoutMs: number;
+  title: string;
+};
+
+export type LiveTransportStandardScenarioDefinition = {
+  description: string;
+  id: LiveTransportStandardScenarioId;
+  title: string;
+};
+
+export const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] =
+  [
+    {
+      id: "canary",
+      title: "Transport canary",
+      description: "The lane can trigger one known-good reply on the real transport.",
+    },
+    {
+      id: "mention-gating",
+      title: "Mention gating",
+      description: "Messages without the required mention do not trigger a reply.",
+    },
+    {
+      id: "allowlist-block",
+      title: "Sender allowlist block",
+      description: "Non-allowlisted senders do not trigger a reply.",
+    },
+    {
+      id: "top-level-reply-shape",
+      title: "Top-level reply shape",
+      description: "Top-level replies stay top-level when the lane is configured that way.",
+    },
+    {
+      id: "restart-resume",
+      title: "Restart resume",
+      description: "The lane still responds after a gateway restart.",
+    },
+    {
+      id: "thread-follow-up",
+      title: "Thread follow-up",
+      description: "Threaded prompts receive threaded replies with the expected relation metadata.",
+    },
+    {
+      id: "thread-isolation",
+      title: "Thread isolation",
+      description: "Fresh top-level prompts stay out of prior threads.",
+    },
+    {
+      id: "reaction-observation",
+      title: "Reaction observation",
+      description: "Reaction events are observed and normalized correctly.",
+    },
+    {
+      id: "help-command",
+      title: "Help command",
+      description: "The transport-specific help command path replies successfully.",
+    },
+  ] as const;
+
+export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
+  [
+    "canary",
+    "mention-gating",
+    "allowlist-block",
+    "top-level-reply-shape",
+    "restart-resume",
+  ] as const;
+
+const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
+  LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
+);
+
+function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) {
+  for (const id of ids) {
+    if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) {
+      throw new Error(`unknown live transport standard scenario id: ${id}`);
+    }
+  }
+}
+
+export function selectLiveTransportScenarios<TDefinition extends { id: string }>(params: {
+  ids?: string[];
+  laneLabel: string;
+  scenarios: readonly TDefinition[];
+}) {
+  if (!params.ids || params.ids.length === 0) {
+    return [...params.scenarios];
+  }
+  const requested = new Set(params.ids);
+  const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id));
+  const missingIds = [...requested].filter(
+    (id) => !selected.some((scenario) => scenario.id === id),
+  );
+  if (missingIds.length > 0) {
+    throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
+  }
+  return selected;
+}
+
+export function collectLiveTransportStandardScenarioCoverage<TId extends string>(params: {
+  alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[];
+  scenarios: readonly LiveTransportScenarioDefinition<TId>[];
+}) {
+  const coverage: LiveTransportStandardScenarioId[] = [];
+  const seen = new Set<LiveTransportStandardScenarioId>();
+  const append = (id: LiveTransportStandardScenarioId | undefined) => {
+    if (!id || seen.has(id)) {
+      return;
+    }
+    seen.add(id);
+    coverage.push(id);
+  };
+
+  assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []);
+  for (const id of params.alwaysOnStandardScenarioIds ?? []) {
+    append(id);
+  }
+  for (const scenario of params.scenarios) {
+    if (scenario.standardId) {
+      assertKnownStandardScenarioIds([scenario.standardId]);
+    }
+    append(scenario.standardId);
+  }
+  return coverage;
+}
+
+export function findMissingLiveTransportStandardScenarios(params: {
+  coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
+  expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
+}) {
+  assertKnownStandardScenarioIds(params.coveredStandardScenarioIds);
+  assertKnownStandardScenarioIds(params.expectedStandardScenarioIds);
+  const covered = new Set(params.coveredStandardScenarioIds);
+  return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
+}