test: add docker e2e rerun helpers

package.json

@@ -1335,6 +1335,7 @@
     "check:timed": "node scripts/check-timed.mjs",
     "check:timed:all-types": "node scripts/check-timed.mjs --include-test-types",
     "check:timed:architecture": "node scripts/check-timed.mjs --include-architecture",
+    "check:workflows": "node scripts/check-workflows.mjs",
     "ci:timings": "node scripts/ci-run-timings.mjs --latest-main",
     "ci:timings:recent": "node scripts/ci-run-timings.mjs --recent 10",
     "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts",
@@ -1542,7 +1543,9 @@
     "test:docker:plugin-update": "bash scripts/e2e/plugin-update-unchanged-docker.sh",
     "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh",
     "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh",
+    "test:docker:rerun": "node scripts/docker-e2e-rerun.mjs",
     "test:docker:session-runtime-context": "bash scripts/e2e/session-runtime-context-docker.sh",
+    "test:docker:timings": "node scripts/docker-e2e-timings.mjs",
     "test:docker:update-channel-switch": "bash scripts/e2e/update-channel-switch-docker.sh",
     "test:e2e": "node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts",
     "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts extensions/openshell/src/backend.e2e.test.ts",

scripts/check-openclaw-package-tarball.mjs

@@ -38,6 +38,20 @@ const normalized = entries.map((entry) => entry.replace(/^package\//u, ""));
 const entrySet = new Set(normalized);
 const errors = [];
 
+function readTarEntry(entryPath) {
+  const candidates = [entryPath, `package/${entryPath}`];
+  for (const candidate of candidates) {
+    const result = spawnSync("tar", ["-xOf", tarball, candidate], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    if (result.status === 0) {
+      return result.stdout;
+    }
+  }
+  return "";
+}
+
 for (const entry of normalized) {
   if (entry.startsWith("/") || entry.split("/").includes("..")) {
     errors.push(`unsafe tar entry: ${entry}`);
@@ -53,6 +67,27 @@ if (!normalized.some((entry) => entry.startsWith("dist/"))) {
 if (!entrySet.has("dist/postinstall-inventory.json")) {
   errors.push("missing dist/postinstall-inventory.json");
 }
+if (entrySet.has("dist/postinstall-inventory.json")) {
+  try {
+    const inventory = JSON.parse(readTarEntry("dist/postinstall-inventory.json"));
+    if (!Array.isArray(inventory) || inventory.some((entry) => typeof entry !== "string")) {
+      errors.push("invalid dist/postinstall-inventory.json");
+    } else {
+      for (const inventoryEntry of inventory) {
+        const normalizedEntry = inventoryEntry.replace(/\\/gu, "/");
+        if (!entrySet.has(normalizedEntry)) {
+          errors.push(`inventory references missing tar entry ${normalizedEntry}`);
+        }
+      }
+    }
+  } catch (error) {
+    errors.push(
+      `unreadable dist/postinstall-inventory.json: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
+}
 
 if (errors.length > 0) {
   fail(`OpenClaw package tarball integrity failed:\n${errors.join("\n")}`);

scripts/check-workflows.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+// Runs local workflow sanity checks.
+// Uses an installed actionlint when present, otherwise falls back to `go run`
+// for the pinned version used by CI, then runs repo-specific composite guards.
+import { spawnSync } from "node:child_process";
+
+const ACTIONLINT_VERSION = "1.7.11";
+
+function commandExists(command) {
+  return spawnSync("bash", ["-lc", `command -v ${command}`], { stdio: "ignore" }).status === 0;
+}
+
+function run(command, args) {
+  const result = spawnSync(command, args, { stdio: "inherit" });
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1);
+  }
+}
+
+if (commandExists("actionlint")) {
+  run("actionlint", []);
+} else {
+  run("go", ["run", `github.com/rhysd/actionlint/cmd/actionlint@v${ACTIONLINT_VERSION}`]);
+}
+
+run("python3", ["scripts/check-composite-action-input-interpolation.py"]);
+run("node", ["scripts/check-no-conflict-markers.mjs"]);

scripts/docker-e2e-rerun.mjs

@@ -0,0 +1,259 @@
+#!/usr/bin/env node
+// Builds cheap rerun commands from a Docker E2E GitHub run or local summary.
+// For GitHub runs, the script downloads Docker E2E artifacts, reads
+// summary/failures JSON, and prints targeted workflow commands that prepare a
+// fresh OpenClaw tarball for the same ref before running only failed lanes.
+import { spawnSync } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";
+
+function usage() {
+  return [
+    "Usage:",
+    "  node scripts/docker-e2e-rerun.mjs <run-id|summary.json|failures.json> [--repo owner/repo] [--dir output-dir] [--workflow workflow.yml] [--ref ref]",
+  ].join("\n");
+}
+
+function parseArgs(argv) {
+  const options = {
+    dir: "",
+    input: "",
+    ref: "",
+    repo: "",
+    workflow: DEFAULT_WORKFLOW,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--repo") {
+      options.repo = argv[(index += 1)] ?? "";
+    } else if (arg?.startsWith("--repo=")) {
+      options.repo = arg.slice("--repo=".length);
+    } else if (arg === "--dir") {
+      options.dir = argv[(index += 1)] ?? "";
+    } else if (arg?.startsWith("--dir=")) {
+      options.dir = arg.slice("--dir=".length);
+    } else if (arg === "--workflow") {
+      options.workflow = argv[(index += 1)] ?? "";
+    } else if (arg?.startsWith("--workflow=")) {
+      options.workflow = arg.slice("--workflow=".length);
+    } else if (arg === "--ref") {
+      options.ref = argv[(index += 1)] ?? "";
+    } else if (arg?.startsWith("--ref=")) {
+      options.ref = arg.slice("--ref=".length);
+    } else if (!options.input) {
+      options.input = arg;
+    } else {
+      throw new Error(`unknown argument: ${arg}\n${usage()}`);
+    }
+  }
+  if (!options.input || !options.workflow) {
+    throw new Error(usage());
+  }
+  return options;
+}
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    encoding: "utf8",
+    stdio: options.stdio ?? ["ignore", "pipe", "pipe"],
+  });
+  if (result.status !== 0) {
+    throw new Error(
+      `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${result.stderr}`,
+    );
+  }
+  return result.stdout;
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function shellQuote(value) {
+  return `'${String(value).replaceAll("'", "'\\''")}'`;
+}
+
+function ghWorkflowCommand(lanes, ref, workflow) {
+  return [
+    "gh workflow run",
+    shellQuote(workflow),
+    "-f",
+    `ref=${shellQuote(ref)}`,
+    "-f",
+    "include_repo_e2e=false",
+    "-f",
+    "include_release_path_suites=false",
+    "-f",
+    "include_openwebui=false",
+    "-f",
+    `docker_lanes=${shellQuote(lanes.join(" "))}`,
+    "-f",
+    "include_live_suites=false",
+    "-f",
+    "live_models_only=false",
+  ].join(" ");
+}
+
+function detectRepo() {
+  return JSON.parse(run("gh", ["repo", "view", "--json", "nameWithOwner"])).nameWithOwner;
+}
+
+function findFiles(rootDir, basenames, out = []) {
+  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) {
+    const file = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      findFiles(file, basenames, out);
+    } else if (basenames.has(entry.name)) {
+      out.push(file);
+    }
+  }
+  return out;
+}
+
+function failedLaneEntriesFromJson(file, ref, workflow) {
+  const parsed = readJson(file);
+  const source = path.basename(file);
+  if (source === "failures.json" && Array.isArray(parsed.lanes)) {
+    return parsed.lanes
+      .filter((lane) => lane.name)
+      .map((lane) => ({
+        ghWorkflowCommand: lane.ghWorkflowCommand,
+        lane: lane.name,
+        localRerunCommand: lane.rerunCommand,
+        logFile: lane.logFile,
+        source: file,
+        status: lane.status,
+      }));
+  }
+
+  const lanes = Array.isArray(parsed.lanes) ? parsed.lanes : [];
+  return lanes
+    .filter((lane) => lane.status !== 0 && lane.name)
+    .map((lane) => ({
+      ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow),
+      lane: lane.name,
+      localRerunCommand: lane.rerunCommand,
+      logFile: lane.logFile,
+      source: file,
+      status: lane.status,
+    }));
+}
+
+function mergeByLane(entries) {
+  const byLane = new Map();
+  for (const entry of entries) {
+    if (!byLane.has(entry.lane)) {
+      byLane.set(entry.lane, entry);
+    }
+  }
+  return [...byLane.values()].toSorted((left, right) => left.lane.localeCompare(right.lane));
+}
+
+function downloadDockerArtifacts(runId, repo, outputDir) {
+  fs.mkdirSync(outputDir, { recursive: true });
+  const artifacts = JSON.parse(
+    run("gh", [
+      "api",
+      `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`,
+      "--jq",
+      ".artifacts",
+    ]),
+  );
+  const names = artifacts
+    .filter((artifact) => !artifact.expired && artifact.name.startsWith("docker-e2e-"))
+    .map((artifact) => artifact.name);
+  if (names.length === 0) {
+    throw new Error(`No docker-e2e-* artifacts found for run ${runId}`);
+  }
+  for (const name of names) {
+    run(
+      "gh",
+      ["run", "download", String(runId), "--repo", repo, "--name", name, "--dir", outputDir],
+      {
+        stdio: "inherit",
+      },
+    );
+  }
+  return names;
+}
+
+function runInfo(runId, repo) {
+  return JSON.parse(
+    run("gh", [
+      "run",
+      "view",
+      String(runId),
+      "--repo",
+      repo,
+      "--json",
+      "databaseId,headSha,headBranch,status,conclusion,url,workflowName",
+    ]),
+  );
+}
+
+function printEntries(entries, ref, workflow, run) {
+  if (run) {
+    console.log(`Run: ${run.url}`);
+    console.log(`Workflow: ${run.workflowName}`);
+  }
+  console.log(`Ref: ${ref}`);
+  console.log(
+    "Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for that ref before lane execution.",
+  );
+  if (entries.length === 0) {
+    console.log("No failed Docker E2E lanes found.");
+    return;
+  }
+  console.log(`Failed lanes: ${entries.map((entry) => entry.lane).join(", ")}`);
+  console.log("");
+  console.log("Combined GitHub rerun:");
+  console.log(
+    ghWorkflowCommand(
+      entries.map((entry) => entry.lane),
+      ref,
+      workflow,
+    ),
+  );
+  console.log("");
+  console.log("Per-lane GitHub reruns:");
+  for (const entry of entries) {
+    console.log(
+      `- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`,
+    );
+  }
+  console.log("");
+  console.log("Local rerun starting points:");
+  for (const entry of entries) {
+    if (entry.localRerunCommand) {
+      console.log(`- ${entry.lane}: ${entry.localRerunCommand}`);
+    }
+  }
+}
+
+const options = parseArgs(process.argv.slice(2));
+const isLocalJson = fs.existsSync(options.input) && fs.statSync(options.input).isFile();
+if (isLocalJson) {
+  const ref = options.ref || process.env.GITHUB_SHA || "HEAD";
+  printEntries(
+    mergeByLane(failedLaneEntriesFromJson(options.input, ref, options.workflow)),
+    ref,
+    options.workflow,
+  );
+} else {
+  const repo = options.repo || detectRepo();
+  const run = runInfo(options.input, repo);
+  const ref = options.ref || run.headSha || run.headBranch;
+  const outputDir =
+    options.dir || path.join(os.tmpdir(), `openclaw-docker-e2e-rerun-${options.input}`);
+  const artifactNames = downloadDockerArtifacts(options.input, repo, outputDir);
+  const files = findFiles(outputDir, new Set(["failures.json", "summary.json"]));
+  const entries = mergeByLane(
+    files.flatMap((file) => failedLaneEntriesFromJson(file, ref, options.workflow)),
+  );
+  console.log(`Artifacts: ${artifactNames.join(", ")}`);
+  console.log(`Downloaded: ${outputDir}`);
+  printEntries(entries, ref, options.workflow, run);
+}

scripts/docker-e2e-timings.mjs

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+// Summarizes Docker E2E timing artifacts.
+// Accepts scheduler summary.json or lane-timings.json so agents can see the
+// slowest lanes and phase critical path before deciding what to rerun.
+import fs from "node:fs";
+
+function usage() {
+  return "Usage: node scripts/docker-e2e-timings.mjs <summary.json|lane-timings.json> [--limit N]";
+}
+
+function parseArgs(argv) {
+  const options = { file: "", limit: 12 };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--limit") {
+      options.limit = Number(argv[(index += 1)] ?? "");
+    } else if (arg?.startsWith("--limit=")) {
+      options.limit = Number(arg.slice("--limit=".length));
+    } else if (!options.file) {
+      options.file = arg;
+    } else {
+      throw new Error(`unknown argument: ${arg}\n${usage()}`);
+    }
+  }
+  if (!options.file || !Number.isInteger(options.limit) || options.limit < 1) {
+    throw new Error(usage());
+  }
+  return options;
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function seconds(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function durationBetween(startedAt, finishedAt) {
+  if (!startedAt || !finishedAt) {
+    return 0;
+  }
+  const started = Date.parse(startedAt);
+  const finished = Date.parse(finishedAt);
+  if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started) {
+    return 0;
+  }
+  return Math.round((finished - started) / 1000);
+}
+
+function summarizeSummary(summary, limit) {
+  const lanes = (Array.isArray(summary.lanes) ? summary.lanes : [])
+    .map((lane) => ({
+      imageKind: lane.imageKind ?? "",
+      name: lane.name,
+      seconds: seconds(lane.elapsedSeconds),
+      status: lane.status === 0 ? "pass" : `fail ${lane.status}`,
+      timedOut: lane.timedOut === true,
+    }))
+    .filter((lane) => lane.name)
+    .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name));
+  const phases = (Array.isArray(summary.phases) ? summary.phases : [])
+    .map((phase) => ({
+      name: phase.name,
+      seconds: seconds(phase.elapsedSeconds),
+      status: phase.status ?? "",
+    }))
+    .filter((phase) => phase.name);
+  const wallSeconds = durationBetween(summary.startedAt, summary.finishedAt);
+  const totalLaneSeconds = lanes.reduce((total, lane) => total + lane.seconds, 0);
+  const criticalPathSeconds =
+    phases.reduce((total, phase) => total + phase.seconds, 0) ||
+    wallSeconds ||
+    lanes[0]?.seconds ||
+    0;
+
+  console.log(`Status: ${summary.status ?? "unknown"}`);
+  if (wallSeconds > 0) {
+    console.log(`Wall seconds: ${wallSeconds}`);
+  }
+  console.log(`Lane seconds total: ${totalLaneSeconds}`);
+  console.log(`Approx critical path seconds: ${criticalPathSeconds}`);
+  if (wallSeconds > 0 && totalLaneSeconds > 0) {
+    console.log(`Approx parallelism: ${(totalLaneSeconds / wallSeconds).toFixed(1)}x`);
+  }
+  if (phases.length > 0) {
+    console.log("");
+    console.log("Phases:");
+    for (const phase of phases.toSorted((left, right) => right.seconds - left.seconds)) {
+      console.log(`- ${phase.name}: ${phase.seconds}s ${phase.status}`);
+    }
+  }
+  console.log("");
+  console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`);
+  for (const lane of lanes.slice(0, limit)) {
+    console.log(
+      `- ${lane.name}: ${lane.seconds}s ${lane.status}${lane.timedOut ? " timeout" : ""}${
+        lane.imageKind ? ` image=${lane.imageKind}` : ""
+      }`,
+    );
+  }
+}
+
+function summarizeTimingStore(store, limit) {
+  const lanes = Object.entries(store.lanes ?? {})
+    .map(([name, lane]) => ({
+      name,
+      seconds: seconds(lane.durationSeconds),
+      status: lane.status === 0 ? "pass" : `fail ${lane.status}`,
+      updatedAt: lane.updatedAt ?? "",
+    }))
+    .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name));
+  console.log(`Updated: ${store.updatedAt ?? "unknown"}`);
+  console.log(`Known lanes: ${lanes.length}`);
+  console.log("");
+  console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`);
+  for (const lane of lanes.slice(0, limit)) {
+    console.log(`- ${lane.name}: ${lane.seconds}s ${lane.status} ${lane.updatedAt}`.trim());
+  }
+}
+
+const options = parseArgs(process.argv.slice(2));
+const payload = readJson(options.file);
+if (Array.isArray(payload.lanes)) {
+  summarizeSummary(payload, options.limit);
+} else if (payload.lanes && typeof payload.lanes === "object") {
+  summarizeTimingStore(payload, options.limit);
+} else {
+  throw new Error(`Unsupported Docker E2E timing artifact: ${options.file}`);
+}

scripts/test-docker-all.mjs

@@ -35,6 +35,7 @@ const DEFAULT_LANE_START_STAGGER_MS = 2_000;
 const DEFAULT_STATUS_INTERVAL_MS = 30_000;
 const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
 const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
+const DEFAULT_GITHUB_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";
 const cliArgs = new Set(process.argv.slice(2));
 for (const arg of cliArgs) {
   if (arg !== "--plan-json") {
@@ -151,6 +152,27 @@ function shellQuote(value) {
   return `'${String(value).replaceAll("'", "'\\''")}'`;
 }
 
+function githubWorkflowRerunCommand(laneNames, ref) {
+  return [
+    "gh workflow run",
+    shellQuote(process.env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW),
+    "-f",
+    `ref=${shellQuote(ref)}`,
+    "-f",
+    "include_repo_e2e=false",
+    "-f",
+    "include_release_path_suites=false",
+    "-f",
+    "include_openwebui=false",
+    "-f",
+    `docker_lanes=${shellQuote(laneNames.join(" "))}`,
+    "-f",
+    "include_live_suites=false",
+    "-f",
+    "live_models_only=false",
+  ].join(" ");
+}
+
 function buildLaneRerunCommand(name, baseEnv) {
   const poolLane = findLaneByName(name);
   const build = name.startsWith("live-") ? "1" : "0";
@@ -228,12 +250,63 @@ async function writeRunSummary(logDir, summary) {
   const payload = {
     ...summary,
     finishedAt: new Date().toISOString(),
+    github: {
+      ref: process.env.GITHUB_REF_NAME || undefined,
+      repository: process.env.GITHUB_REPOSITORY || undefined,
+      runId: process.env.GITHUB_RUN_ID || undefined,
+      runUrl:
+        process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID
+          ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
+          : undefined,
+      sha: process.env.GITHUB_SHA || undefined,
+      workflow: process.env.GITHUB_WORKFLOW || undefined,
+    },
     version: 1,
   };
   await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`);
+  await writeFailureIndex(logDir, payload);
   console.log(`==> Docker run summary: ${file}`);
 }
 
+async function writeFailureIndex(logDir, summary) {
+  const ref = summary.github?.sha || summary.github?.ref || process.env.GITHUB_SHA || "HEAD";
+  const failures = Array.isArray(summary.failures)
+    ? summary.failures
+    : (summary.lanes ?? []).filter((lane) => lane.status !== 0);
+  const lanes = failures.map((failure) => ({
+    ghWorkflowCommand: githubWorkflowRerunCommand([failure.name], ref),
+    image: failure.image,
+    imageKind: failure.imageKind,
+    lane: failure.name,
+    logFile: failure.logFile,
+    name: failure.name,
+    rerunCommand: failure.rerunCommand,
+    status: failure.status,
+    timedOut: failure.timedOut,
+  }));
+  const failureIndex = {
+    combinedGhWorkflowCommand:
+      lanes.length > 0
+        ? githubWorkflowRerunCommand(
+            lanes.map((lane) => lane.lane),
+            ref,
+          )
+        : undefined,
+    generatedAt: new Date().toISOString(),
+    lanes,
+    note: "Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for the selected ref before lane execution.",
+    ref,
+    runUrl: summary.github?.runUrl,
+    status: summary.status,
+    version: 1,
+    workflow: process.env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW,
+  };
+  await fs.promises.writeFile(
+    path.join(logDir, "failures.json"),
+    `${JSON.stringify(failureIndex, null, 2)}\n`,
+  );
+}
+
 function phaseElapsedSeconds(startedAtMs) {
   return Math.round((Date.now() - startedAtMs) / 1000);
 }