ci: shard release live validation

.agents/skills/openclaw-testing/SKILL.md

@@ -244,6 +244,19 @@ Useful knobs:
   targeted Docker live model job instead of the full provider matrix.
 - blank `live_model_providers`: run the full live-model provider matrix.
 
+When live suites are enabled, the workflow shards broad native `pnpm test:live`
+coverage through `scripts/test-live-shard.mjs` instead of one serial `live-all`
+job:
+
+- `native-live-src-agents`
+- `native-live-src-gateway`
+- `native-live-test`
+- `native-live-extensions-a-k`
+- `native-live-extensions-l-z`
+
+Use `node scripts/test-live-shard.mjs <shard> --list` to see the exact files
+before rerunning a failed native live shard.
+
 For model-list or provider-selection fixes, use `live_models_only=true` plus the
 specific `live_model_providers` allowlist. Confirm logs show the expected
 `OPENCLAW_LIVE_PROVIDERS` and selected model ids before declaring proof.

.github/workflows/openclaw-live-and-e2e-checks-reusable.yml

@@ -1425,30 +1425,59 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - suite_id: live-all
-            label: pnpm test:live
-            command: pnpm test:live
-            timeout_minutes: 180
+          - suite_id: native-live-src-agents
+            label: Native live agents
+            command: node scripts/test-live-shard.mjs native-live-src-agents
+            timeout_minutes: 90
+            needs_ffmpeg: false
+            profile_env_only: false
+          - suite_id: native-live-src-gateway
+            label: Native live gateway
+            command: node scripts/test-live-shard.mjs native-live-src-gateway
+            timeout_minutes: 90
+            needs_ffmpeg: false
+            profile_env_only: false
+          - suite_id: native-live-test
+            label: Native live test harnesses
+            command: node scripts/test-live-shard.mjs native-live-test
+            timeout_minutes: 90
+            needs_ffmpeg: false
+            profile_env_only: false
+          - suite_id: native-live-extensions-a-k
+            label: Native live plugins A-K
+            command: node scripts/test-live-shard.mjs native-live-extensions-a-k
+            timeout_minutes: 90
+            needs_ffmpeg: true
+            profile_env_only: false
+          - suite_id: native-live-extensions-l-z
+            label: Native live plugins L-Z
+            command: node scripts/test-live-shard.mjs native-live-extensions-l-z
+            timeout_minutes: 90
+            needs_ffmpeg: true
             profile_env_only: false
           - suite_id: live-gateway-docker
             label: Docker live gateway
             command: pnpm test:docker:live-gateway
             timeout_minutes: 120
+            needs_ffmpeg: false
             profile_env_only: false
           - suite_id: live-cli-backend-docker
             label: Docker live CLI backend
             command: pnpm test:docker:live-cli-backend
             timeout_minutes: 120
+            needs_ffmpeg: false
             profile_env_only: false
           - suite_id: live-acp-bind-docker
             label: Docker live ACP bind
             command: pnpm test:docker:live-acp-bind
             timeout_minutes: 120
+            needs_ffmpeg: false
             profile_env_only: false
           - suite_id: live-codex-harness-docker
             label: Docker live Codex harness
             command: pnpm test:docker:live-codex-harness
             timeout_minutes: 120
+            needs_ffmpeg: false
             profile_env_only: false
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -1516,7 +1545,7 @@ jobs:
         run: bash scripts/ci-hydrate-live-auth.sh
 
       - name: Install live media dependencies
-        if: matrix.suite_id == 'live-all'
+        if: matrix.needs_ffmpeg
         shell: bash
         run: |
           set -euo pipefail

scripts/lib/docker-e2e-plan.mjs

@@ -37,6 +37,7 @@ export function parseLaneSelection(raw) {
   }
   const laneAliases = new Map([
     ["bundled-channel-deps", ["bundled-channel-deps-compat"]],
+    ["install-e2e", ["install-e2e-openai", "install-e2e-anthropic"]],
     [
       "bundled-plugin-install-uninstall",
       Array.from(
@@ -145,8 +146,11 @@ export function findLaneByName(name) {
 
 export function laneCredentialRequirements(poolLane) {
   const credentials = [];
-  if (poolLane.name === "install-e2e") {
-    credentials.push("openai", "anthropic");
+  if (poolLane.name === "install-e2e-openai") {
+    credentials.push("openai");
+  }
+  if (poolLane.name === "install-e2e-anthropic") {
+    credentials.push("anthropic");
   }
   if (poolLane.name === "openwebui" || poolLane.name === "openai-web-search-minimal") {
     credentials.push("openai");

scripts/lib/docker-e2e-scenarios.mjs

@@ -400,11 +400,19 @@ const releasePathChunks = {
   ],
   "package-update": [
     npmLane(
-      "install-e2e",
-      "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=both pnpm test:install:e2e",
+      "install-e2e-openai",
+      "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=openai OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-openai:local pnpm test:install:e2e",
       {
         resources: ["service"],
-        weight: 4,
+        weight: 3,
+      },
+    ),
+    npmLane(
+      "install-e2e-anthropic",
+      "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=anthropic OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-anthropic:local pnpm test:install:e2e",
+      {
+        resources: ["service"],
+        weight: 3,
       },
     ),
     npmLane(

scripts/test-live-shard.mjs

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnPnpmRunner } from "./pnpm-runner.mjs";
+
+const LIVE_TEST_SUFFIX = ".live.test.ts";
+
+export const LIVE_TEST_SHARDS = Object.freeze([
+  "native-live-src-agents",
+  "native-live-src-gateway",
+  "native-live-test",
+  "native-live-extensions-a-k",
+  "native-live-extensions-l-z",
+]);
+
+function walkFiles(rootDir) {
+  const files = [];
+  if (!fs.existsSync(rootDir)) {
+    return files;
+  }
+  const stack = [rootDir];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    const entries = fs.readdirSync(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        if (
+          entry.name === "node_modules" ||
+          entry.name === "dist" ||
+          entry.name === "vendor" ||
+          entry.name === "fixtures"
+        ) {
+          continue;
+        }
+        stack.push(fullPath);
+        continue;
+      }
+      if (entry.isFile()) {
+        files.push(fullPath);
+      }
+    }
+  }
+  return files;
+}
+
+export function collectAllLiveTestFiles(repoRoot = process.cwd()) {
+  return ["src", "test", "extensions"]
+    .flatMap((dir) => walkFiles(path.join(repoRoot, dir)))
+    .map((file) => path.relative(repoRoot, file).split(path.sep).join("/"))
+    .filter((file) => file.endsWith(LIVE_TEST_SUFFIX))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+function extensionKey(file) {
+  const relative = file.slice("extensions/".length);
+  return relative.split("/", 1)[0]?.toLowerCase() ?? "";
+}
+
+function isExtensionInRange(file, start, end) {
+  if (!file.startsWith("extensions/")) {
+    return false;
+  }
+  const key = extensionKey(file);
+  if (!key) {
+    return false;
+  }
+  const first = key[0];
+  return first >= start && first <= end;
+}
+
+export function selectLiveShardFiles(shard, files = collectAllLiveTestFiles()) {
+  switch (shard) {
+    case "native-live-src-agents":
+      return files.filter((file) => file.startsWith("src/agents/"));
+    case "native-live-src-gateway":
+      return files.filter(
+        (file) => file.startsWith("src/gateway/") || file.startsWith("src/crestodian/"),
+      );
+    case "native-live-test":
+      return files.filter((file) => file.startsWith("test/"));
+    case "native-live-extensions-a-k":
+      return files.filter((file) => isExtensionInRange(file, "a", "k"));
+    case "native-live-extensions-l-z":
+      return files.filter((file) => isExtensionInRange(file, "l", "z"));
+    default:
+      throw new Error(
+        `Unknown live test shard '${shard}'. Expected one of: ${LIVE_TEST_SHARDS.join(", ")}`,
+      );
+  }
+}
+
+function usage() {
+  console.error(`Usage: node scripts/test-live-shard.mjs <${LIVE_TEST_SHARDS.join("|")}> [--list]`);
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const args = process.argv.slice(2);
+  const shard = args.find((arg) => !arg.startsWith("-"));
+  const listOnly = args.includes("--list");
+  if (!shard) {
+    usage();
+    process.exit(2);
+  }
+
+  let files;
+  try {
+    files = selectLiveShardFiles(shard);
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    usage();
+    process.exit(2);
+  }
+  if (files.length === 0) {
+    console.error(`Live test shard '${shard}' selected no files.`);
+    process.exit(2);
+  }
+
+  if (listOnly) {
+    for (const file of files) {
+      console.log(file);
+    }
+    process.exit(0);
+  }
+
+  console.log(`[test:live:shard] ${shard}: ${files.length} file(s)`);
+  const child = spawnPnpmRunner({
+    stdio: "inherit",
+    pnpmArgs: ["test:live", "--", ...files],
+    env: process.env,
+  });
+  child.on("exit", (code, signal) => {
+    if (signal) {
+      process.kill(process.pid, signal);
+      return;
+    }
+    process.exit(code ?? 1);
+  });
+  child.on("error", (error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}

test/scripts/docker-all-scheduler.test.ts

@@ -66,6 +66,36 @@ describe("scripts/test-docker-all scheduler", () => {
     ).toBe(false);
   });
 
+  it("can co-schedule the split installer provider lanes", () => {
+    expect(
+      canStartSchedulerLane(
+        {
+          name: "install-e2e-anthropic",
+          resources: ["npm", "service"],
+          weight: 3,
+        },
+        activePool({
+          count: 1,
+          resources: {
+            docker: 3,
+            npm: 3,
+            service: 3,
+          },
+          weight: 3,
+        }),
+        10,
+        {
+          resourceLimits: {
+            docker: 10,
+            npm: 10,
+            service: 7,
+          },
+          weightLimit: 10,
+        },
+      ),
+    ).toBe(true);
+  });
+
   it("preserves the parallelism count cap", () => {
     expect(
       canStartSchedulerLane(

test/scripts/docker-build-helper.test.ts

@@ -84,7 +84,10 @@ describe("docker build helper", () => {
     const scenarios = readFileSync(DOCKER_E2E_SCENARIOS_PATH, "utf8");
 
     expect(scenarios).toContain(
-      '"OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=both pnpm test:install:e2e"',
+      '"OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=openai OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-openai:local pnpm test:install:e2e"',
+    );
+    expect(scenarios).toContain(
+      '"OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=anthropic OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-anthropic:local pnpm test:install:e2e"',
     );
   });
 

test/scripts/docker-e2e-plan.test.ts

@@ -41,7 +41,8 @@ describe("scripts/lib/docker-e2e-plan", () => {
       package: true,
     });
     expect(plan.credentials).toEqual(["anthropic", "openai"]);
-    expect(plan.lanes.map((lane) => lane.name)).toContain("install-e2e");
+    expect(plan.lanes.map((lane) => lane.name)).toContain("install-e2e-openai");
+    expect(plan.lanes.map((lane) => lane.name)).toContain("install-e2e-anthropic");
     expect(plan.lanes.map((lane) => lane.name)).toContain("mcp-channels");
     expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-channel-feishu");
     expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-channel-update-acpx");
@@ -166,6 +167,24 @@ describe("scripts/lib/docker-e2e-plan", () => {
     ]);
   });
 
+  it("maps installer E2E to provider-specific package install lanes", () => {
+    const selectedLaneNames = parseLaneSelection("install-e2e");
+    const plan = planFor({ selectedLaneNames });
+
+    expect(selectedLaneNames).toEqual(["install-e2e-openai", "install-e2e-anthropic"]);
+    expect(plan.lanes).toEqual([
+      expect.objectContaining({
+        command: expect.stringContaining("OPENCLAW_E2E_MODELS=openai"),
+        name: "install-e2e-openai",
+      }),
+      expect.objectContaining({
+        command: expect.stringContaining("OPENCLAW_E2E_MODELS=anthropic"),
+        name: "install-e2e-anthropic",
+      }),
+    ]);
+    expect(plan.credentials).toEqual(["anthropic", "openai"]);
+  });
+
   it("maps bundled plugin install/uninstall to package-backed shards", () => {
     const selectedLaneNames = parseLaneSelection("bundled-plugin-install-uninstall");
     const plan = planFor({ selectedLaneNames });

test/scripts/package-acceptance-workflow.test.ts

@@ -90,6 +90,19 @@ describe("package artifact reuse", () => {
     expect(workflow).not.toContain("cache-to: type=gha,mode=max,scope=docker-e2e");
   });
 
+  it("shards broad native live tests instead of one serial live-all job", () => {
+    const workflow = readFileSync(LIVE_E2E_WORKFLOW, "utf8");
+
+    expect(workflow).not.toContain("suite_id: live-all");
+    expect(workflow).not.toContain("command: pnpm test:live\n");
+    expect(workflow).toContain("suite_id: native-live-src-agents");
+    expect(workflow).toContain("command: node scripts/test-live-shard.mjs native-live-src-agents");
+    expect(workflow).toContain("suite_id: native-live-src-gateway");
+    expect(workflow).toContain("suite_id: native-live-extensions-a-k");
+    expect(workflow).toContain("suite_id: native-live-extensions-l-z");
+    expect(workflow).toContain("if: matrix.needs_ffmpeg");
+  });
+
   it("allows the Telegram lane to run from reusable package acceptance artifacts", () => {
     const workflow = readFileSync(NPM_TELEGRAM_WORKFLOW, "utf8");
 

test/scripts/test-live-shard.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import {
+  LIVE_TEST_SHARDS,
+  collectAllLiveTestFiles,
+  selectLiveShardFiles,
+} from "../../scripts/test-live-shard.mjs";
+
+describe("scripts/test-live-shard", () => {
+  it("partitions every native live test into exactly one release shard", () => {
+    const allFiles = collectAllLiveTestFiles();
+    const selected = LIVE_TEST_SHARDS.flatMap((shard) =>
+      selectLiveShardFiles(shard, allFiles).map((file) => ({ file, shard })),
+    );
+    const selectedFiles = selected.map(({ file }) => file);
+
+    expect(allFiles.length).toBeGreaterThan(0);
+    expect(selectedFiles.toSorted()).toEqual(allFiles);
+    expect(new Set(selectedFiles).size).toBe(selectedFiles.length);
+  });
+
+  it("keeps media-capable extension and test harness files in their own shards", () => {
+    const allFiles = collectAllLiveTestFiles();
+
+    expect(selectLiveShardFiles("native-live-test", allFiles)).toEqual(
+      expect.arrayContaining([
+        "test/image-generation.infer-cli.live.test.ts",
+        "test/image-generation.runtime.live.test.ts",
+      ]),
+    );
+    expect(selectLiveShardFiles("native-live-extensions-l-z", allFiles)).toEqual(
+      expect.arrayContaining([
+        "extensions/music-generation-providers.live.test.ts",
+        "extensions/video-generation-providers.live.test.ts",
+      ]),
+    );
+  });
+
+  it("rejects unknown shard names", () => {
+    expect(() => selectLiveShardFiles("native-live-missing")).toThrow(/Unknown live test shard/u);
+  });
+});