From 1a02d00eb4501cc00218f123d7d125a27f747dc4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 26 Apr 2026 23:55:57 +0100 Subject: [PATCH] test: add docker e2e rerun helpers --- package.json | 3 + scripts/check-openclaw-package-tarball.mjs | 35 +++ scripts/check-workflows.mjs | 27 +++ scripts/docker-e2e-rerun.mjs | 259 +++++++++++++++++++++ scripts/docker-e2e-timings.mjs | 130 +++++++++++ scripts/test-docker-all.mjs | 73 ++++++ 6 files changed, 527 insertions(+) create mode 100644 scripts/check-workflows.mjs create mode 100644 scripts/docker-e2e-rerun.mjs create mode 100644 scripts/docker-e2e-timings.mjs diff --git a/package.json b/package.json index c62824334d5..607c298b14d 100644 --- a/package.json +++ b/package.json @@ -1335,6 +1335,7 @@ "check:timed": "node scripts/check-timed.mjs", "check:timed:all-types": "node scripts/check-timed.mjs --include-test-types", "check:timed:architecture": "node scripts/check-timed.mjs --include-architecture", + "check:workflows": "node scripts/check-workflows.mjs", "ci:timings": "node scripts/ci-run-timings.mjs --latest-main", "ci:timings:recent": "node scripts/ci-run-timings.mjs --recent 10", "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts", @@ -1542,7 +1543,9 @@ "test:docker:plugin-update": "bash scripts/e2e/plugin-update-unchanged-docker.sh", "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh", "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh", + "test:docker:rerun": "node scripts/docker-e2e-rerun.mjs", "test:docker:session-runtime-context": "bash scripts/e2e/session-runtime-context-docker.sh", + "test:docker:timings": "node scripts/docker-e2e-timings.mjs", "test:docker:update-channel-switch": "bash scripts/e2e/update-channel-switch-docker.sh", "test:e2e": "node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts", "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 node scripts/run-vitest.mjs run --config test/vitest/vitest.e2e.config.ts extensions/openshell/src/backend.e2e.test.ts", diff --git a/scripts/check-openclaw-package-tarball.mjs b/scripts/check-openclaw-package-tarball.mjs index e7275e1e61c..bdf62b00ded 100644 --- a/scripts/check-openclaw-package-tarball.mjs +++ b/scripts/check-openclaw-package-tarball.mjs @@ -38,6 +38,20 @@ const normalized = entries.map((entry) => entry.replace(/^package\//u, "")); const entrySet = new Set(normalized); const errors = []; +function readTarEntry(entryPath) { + const candidates = [entryPath, `package/${entryPath}`]; + for (const candidate of candidates) { + const result = spawnSync("tar", ["-xOf", tarball, candidate], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status === 0) { + return result.stdout; + } + } + return ""; +} + for (const entry of normalized) { if (entry.startsWith("/") || entry.split("/").includes("..")) { errors.push(`unsafe tar entry: ${entry}`); @@ -53,6 +67,27 @@ if (!normalized.some((entry) => entry.startsWith("dist/"))) { if (!entrySet.has("dist/postinstall-inventory.json")) { errors.push("missing dist/postinstall-inventory.json"); } +if (entrySet.has("dist/postinstall-inventory.json")) { + try { + const inventory = JSON.parse(readTarEntry("dist/postinstall-inventory.json")); + if (!Array.isArray(inventory) || inventory.some((entry) => typeof entry !== "string")) { + errors.push("invalid dist/postinstall-inventory.json"); + } else { + for (const inventoryEntry of inventory) { + const normalizedEntry = inventoryEntry.replace(/\\/gu, "/"); + if (!entrySet.has(normalizedEntry)) { + errors.push(`inventory references missing tar entry ${normalizedEntry}`); + } + } + } + } catch (error) { + errors.push( + `unreadable dist/postinstall-inventory.json: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} if (errors.length > 0) { fail(`OpenClaw package tarball integrity failed:\n${errors.join("\n")}`); diff --git a/scripts/check-workflows.mjs b/scripts/check-workflows.mjs new file mode 100644 index 00000000000..36a321e3416 --- /dev/null +++ b/scripts/check-workflows.mjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node +// Runs local workflow sanity checks. +// Uses an installed actionlint when present, otherwise falls back to `go run` +// for the pinned version used by CI, then runs repo-specific composite guards. +import { spawnSync } from "node:child_process"; + +const ACTIONLINT_VERSION = "1.7.11"; + +function commandExists(command) { + return spawnSync("bash", ["-lc", `command -v ${command}`], { stdio: "ignore" }).status === 0; +} + +function run(command, args) { + const result = spawnSync(command, args, { stdio: "inherit" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +if (commandExists("actionlint")) { + run("actionlint", []); +} else { + run("go", ["run", `github.com/rhysd/actionlint/cmd/actionlint@v${ACTIONLINT_VERSION}`]); +} + +run("python3", ["scripts/check-composite-action-input-interpolation.py"]); +run("node", ["scripts/check-no-conflict-markers.mjs"]); diff --git a/scripts/docker-e2e-rerun.mjs b/scripts/docker-e2e-rerun.mjs new file mode 100644 index 00000000000..710cb8194a2 --- /dev/null +++ b/scripts/docker-e2e-rerun.mjs @@ -0,0 +1,259 @@ +#!/usr/bin/env node +// Builds cheap rerun commands from a Docker E2E GitHub run or local summary. +// For GitHub runs, the script downloads Docker E2E artifacts, reads +// summary/failures JSON, and prints targeted workflow commands that prepare a +// fresh OpenClaw tarball for the same ref before running only failed lanes. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const DEFAULT_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml"; + +function usage() { + return [ + "Usage:", + " node scripts/docker-e2e-rerun.mjs [--repo owner/repo] [--dir output-dir] [--workflow workflow.yml] [--ref ref]", + ].join("\n"); +} + +function parseArgs(argv) { + const options = { + dir: "", + input: "", + ref: "", + repo: "", + workflow: DEFAULT_WORKFLOW, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--repo") { + options.repo = argv[(index += 1)] ?? ""; + } else if (arg?.startsWith("--repo=")) { + options.repo = arg.slice("--repo=".length); + } else if (arg === "--dir") { + options.dir = argv[(index += 1)] ?? ""; + } else if (arg?.startsWith("--dir=")) { + options.dir = arg.slice("--dir=".length); + } else if (arg === "--workflow") { + options.workflow = argv[(index += 1)] ?? ""; + } else if (arg?.startsWith("--workflow=")) { + options.workflow = arg.slice("--workflow=".length); + } else if (arg === "--ref") { + options.ref = argv[(index += 1)] ?? ""; + } else if (arg?.startsWith("--ref=")) { + options.ref = arg.slice("--ref=".length); + } else if (!options.input) { + options.input = arg; + } else { + throw new Error(`unknown argument: ${arg}\n${usage()}`); + } + } + if (!options.input || !options.workflow) { + throw new Error(usage()); + } + return options; +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + encoding: "utf8", + stdio: options.stdio ?? ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${result.stderr}`, + ); + } + return result.stdout; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +function ghWorkflowCommand(lanes, ref, workflow) { + return [ + "gh workflow run", + shellQuote(workflow), + "-f", + `ref=${shellQuote(ref)}`, + "-f", + "include_repo_e2e=false", + "-f", + "include_release_path_suites=false", + "-f", + "include_openwebui=false", + "-f", + `docker_lanes=${shellQuote(lanes.join(" "))}`, + "-f", + "include_live_suites=false", + "-f", + "live_models_only=false", + ].join(" "); +} + +function detectRepo() { + return JSON.parse(run("gh", ["repo", "view", "--json", "nameWithOwner"])).nameWithOwner; +} + +function findFiles(rootDir, basenames, out = []) { + for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) { + const file = path.join(rootDir, entry.name); + if (entry.isDirectory()) { + findFiles(file, basenames, out); + } else if (basenames.has(entry.name)) { + out.push(file); + } + } + return out; +} + +function failedLaneEntriesFromJson(file, ref, workflow) { + const parsed = readJson(file); + const source = path.basename(file); + if (source === "failures.json" && Array.isArray(parsed.lanes)) { + return parsed.lanes + .filter((lane) => lane.name) + .map((lane) => ({ + ghWorkflowCommand: lane.ghWorkflowCommand, + lane: lane.name, + localRerunCommand: lane.rerunCommand, + logFile: lane.logFile, + source: file, + status: lane.status, + })); + } + + const lanes = Array.isArray(parsed.lanes) ? parsed.lanes : []; + return lanes + .filter((lane) => lane.status !== 0 && lane.name) + .map((lane) => ({ + ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow), + lane: lane.name, + localRerunCommand: lane.rerunCommand, + logFile: lane.logFile, + source: file, + status: lane.status, + })); +} + +function mergeByLane(entries) { + const byLane = new Map(); + for (const entry of entries) { + if (!byLane.has(entry.lane)) { + byLane.set(entry.lane, entry); + } + } + return [...byLane.values()].toSorted((left, right) => left.lane.localeCompare(right.lane)); +} + +function downloadDockerArtifacts(runId, repo, outputDir) { + fs.mkdirSync(outputDir, { recursive: true }); + const artifacts = JSON.parse( + run("gh", [ + "api", + `repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`, + "--jq", + ".artifacts", + ]), + ); + const names = artifacts + .filter((artifact) => !artifact.expired && artifact.name.startsWith("docker-e2e-")) + .map((artifact) => artifact.name); + if (names.length === 0) { + throw new Error(`No docker-e2e-* artifacts found for run ${runId}`); + } + for (const name of names) { + run( + "gh", + ["run", "download", String(runId), "--repo", repo, "--name", name, "--dir", outputDir], + { + stdio: "inherit", + }, + ); + } + return names; +} + +function runInfo(runId, repo) { + return JSON.parse( + run("gh", [ + "run", + "view", + String(runId), + "--repo", + repo, + "--json", + "databaseId,headSha,headBranch,status,conclusion,url,workflowName", + ]), + ); +} + +function printEntries(entries, ref, workflow, run) { + if (run) { + console.log(`Run: ${run.url}`); + console.log(`Workflow: ${run.workflowName}`); + } + console.log(`Ref: ${ref}`); + console.log( + "Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for that ref before lane execution.", + ); + if (entries.length === 0) { + console.log("No failed Docker E2E lanes found."); + return; + } + console.log(`Failed lanes: ${entries.map((entry) => entry.lane).join(", ")}`); + console.log(""); + console.log("Combined GitHub rerun:"); + console.log( + ghWorkflowCommand( + entries.map((entry) => entry.lane), + ref, + workflow, + ), + ); + console.log(""); + console.log("Per-lane GitHub reruns:"); + for (const entry of entries) { + console.log( + `- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`, + ); + } + console.log(""); + console.log("Local rerun starting points:"); + for (const entry of entries) { + if (entry.localRerunCommand) { + console.log(`- ${entry.lane}: ${entry.localRerunCommand}`); + } + } +} + +const options = parseArgs(process.argv.slice(2)); +const isLocalJson = fs.existsSync(options.input) && fs.statSync(options.input).isFile(); +if (isLocalJson) { + const ref = options.ref || process.env.GITHUB_SHA || "HEAD"; + printEntries( + mergeByLane(failedLaneEntriesFromJson(options.input, ref, options.workflow)), + ref, + options.workflow, + ); +} else { + const repo = options.repo || detectRepo(); + const run = runInfo(options.input, repo); + const ref = options.ref || run.headSha || run.headBranch; + const outputDir = + options.dir || path.join(os.tmpdir(), `openclaw-docker-e2e-rerun-${options.input}`); + const artifactNames = downloadDockerArtifacts(options.input, repo, outputDir); + const files = findFiles(outputDir, new Set(["failures.json", "summary.json"])); + const entries = mergeByLane( + files.flatMap((file) => failedLaneEntriesFromJson(file, ref, options.workflow)), + ); + console.log(`Artifacts: ${artifactNames.join(", ")}`); + console.log(`Downloaded: ${outputDir}`); + printEntries(entries, ref, options.workflow, run); +} diff --git a/scripts/docker-e2e-timings.mjs b/scripts/docker-e2e-timings.mjs new file mode 100644 index 00000000000..69babad99be --- /dev/null +++ b/scripts/docker-e2e-timings.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node +// Summarizes Docker E2E timing artifacts. +// Accepts scheduler summary.json or lane-timings.json so agents can see the +// slowest lanes and phase critical path before deciding what to rerun. +import fs from "node:fs"; + +function usage() { + return "Usage: node scripts/docker-e2e-timings.mjs [--limit N]"; +} + +function parseArgs(argv) { + const options = { file: "", limit: 12 }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--limit") { + options.limit = Number(argv[(index += 1)] ?? ""); + } else if (arg?.startsWith("--limit=")) { + options.limit = Number(arg.slice("--limit=".length)); + } else if (!options.file) { + options.file = arg; + } else { + throw new Error(`unknown argument: ${arg}\n${usage()}`); + } + } + if (!options.file || !Number.isInteger(options.limit) || options.limit < 1) { + throw new Error(usage()); + } + return options; +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function seconds(value) { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function durationBetween(startedAt, finishedAt) { + if (!startedAt || !finishedAt) { + return 0; + } + const started = Date.parse(startedAt); + const finished = Date.parse(finishedAt); + if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started) { + return 0; + } + return Math.round((finished - started) / 1000); +} + +function summarizeSummary(summary, limit) { + const lanes = (Array.isArray(summary.lanes) ? summary.lanes : []) + .map((lane) => ({ + imageKind: lane.imageKind ?? "", + name: lane.name, + seconds: seconds(lane.elapsedSeconds), + status: lane.status === 0 ? "pass" : `fail ${lane.status}`, + timedOut: lane.timedOut === true, + })) + .filter((lane) => lane.name) + .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name)); + const phases = (Array.isArray(summary.phases) ? summary.phases : []) + .map((phase) => ({ + name: phase.name, + seconds: seconds(phase.elapsedSeconds), + status: phase.status ?? "", + })) + .filter((phase) => phase.name); + const wallSeconds = durationBetween(summary.startedAt, summary.finishedAt); + const totalLaneSeconds = lanes.reduce((total, lane) => total + lane.seconds, 0); + const criticalPathSeconds = + phases.reduce((total, phase) => total + phase.seconds, 0) || + wallSeconds || + lanes[0]?.seconds || + 0; + + console.log(`Status: ${summary.status ?? "unknown"}`); + if (wallSeconds > 0) { + console.log(`Wall seconds: ${wallSeconds}`); + } + console.log(`Lane seconds total: ${totalLaneSeconds}`); + console.log(`Approx critical path seconds: ${criticalPathSeconds}`); + if (wallSeconds > 0 && totalLaneSeconds > 0) { + console.log(`Approx parallelism: ${(totalLaneSeconds / wallSeconds).toFixed(1)}x`); + } + if (phases.length > 0) { + console.log(""); + console.log("Phases:"); + for (const phase of phases.toSorted((left, right) => right.seconds - left.seconds)) { + console.log(`- ${phase.name}: ${phase.seconds}s ${phase.status}`); + } + } + console.log(""); + console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`); + for (const lane of lanes.slice(0, limit)) { + console.log( + `- ${lane.name}: ${lane.seconds}s ${lane.status}${lane.timedOut ? " timeout" : ""}${ + lane.imageKind ? ` image=${lane.imageKind}` : "" + }`, + ); + } +} + +function summarizeTimingStore(store, limit) { + const lanes = Object.entries(store.lanes ?? {}) + .map(([name, lane]) => ({ + name, + seconds: seconds(lane.durationSeconds), + status: lane.status === 0 ? "pass" : `fail ${lane.status}`, + updatedAt: lane.updatedAt ?? "", + })) + .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name)); + console.log(`Updated: ${store.updatedAt ?? "unknown"}`); + console.log(`Known lanes: ${lanes.length}`); + console.log(""); + console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`); + for (const lane of lanes.slice(0, limit)) { + console.log(`- ${lane.name}: ${lane.seconds}s ${lane.status} ${lane.updatedAt}`.trim()); + } +} + +const options = parseArgs(process.argv.slice(2)); +const payload = readJson(options.file); +if (Array.isArray(payload.lanes)) { + summarizeSummary(payload, options.limit); +} else if (payload.lanes && typeof payload.lanes === "object") { + summarizeTimingStore(payload, options.limit); +} else { + throw new Error(`Unsupported Docker E2E timing artifact: ${options.file}`); +} diff --git a/scripts/test-docker-all.mjs b/scripts/test-docker-all.mjs index 4ef804a43e2..c678718284e 100644 --- a/scripts/test-docker-all.mjs +++ b/scripts/test-docker-all.mjs @@ -35,6 +35,7 @@ const DEFAULT_LANE_START_STAGGER_MS = 2_000; const DEFAULT_STATUS_INTERVAL_MS = 30_000; const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000; const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json"); +const DEFAULT_GITHUB_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml"; const cliArgs = new Set(process.argv.slice(2)); for (const arg of cliArgs) { if (arg !== "--plan-json") { @@ -151,6 +152,27 @@ function shellQuote(value) { return `'${String(value).replaceAll("'", "'\\''")}'`; } +function githubWorkflowRerunCommand(laneNames, ref) { + return [ + "gh workflow run", + shellQuote(process.env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW), + "-f", + `ref=${shellQuote(ref)}`, + "-f", + "include_repo_e2e=false", + "-f", + "include_release_path_suites=false", + "-f", + "include_openwebui=false", + "-f", + `docker_lanes=${shellQuote(laneNames.join(" "))}`, + "-f", + "include_live_suites=false", + "-f", + "live_models_only=false", + ].join(" "); +} + function buildLaneRerunCommand(name, baseEnv) { const poolLane = findLaneByName(name); const build = name.startsWith("live-") ? "1" : "0"; @@ -228,12 +250,63 @@ async function writeRunSummary(logDir, summary) { const payload = { ...summary, finishedAt: new Date().toISOString(), + github: { + ref: process.env.GITHUB_REF_NAME || undefined, + repository: process.env.GITHUB_REPOSITORY || undefined, + runId: process.env.GITHUB_RUN_ID || undefined, + runUrl: + process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : undefined, + sha: process.env.GITHUB_SHA || undefined, + workflow: process.env.GITHUB_WORKFLOW || undefined, + }, version: 1, }; await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`); + await writeFailureIndex(logDir, payload); console.log(`==> Docker run summary: ${file}`); } +async function writeFailureIndex(logDir, summary) { + const ref = summary.github?.sha || summary.github?.ref || process.env.GITHUB_SHA || "HEAD"; + const failures = Array.isArray(summary.failures) + ? summary.failures + : (summary.lanes ?? []).filter((lane) => lane.status !== 0); + const lanes = failures.map((failure) => ({ + ghWorkflowCommand: githubWorkflowRerunCommand([failure.name], ref), + image: failure.image, + imageKind: failure.imageKind, + lane: failure.name, + logFile: failure.logFile, + name: failure.name, + rerunCommand: failure.rerunCommand, + status: failure.status, + timedOut: failure.timedOut, + })); + const failureIndex = { + combinedGhWorkflowCommand: + lanes.length > 0 + ? githubWorkflowRerunCommand( + lanes.map((lane) => lane.lane), + ref, + ) + : undefined, + generatedAt: new Date().toISOString(), + lanes, + note: "Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for the selected ref before lane execution.", + ref, + runUrl: summary.github?.runUrl, + status: summary.status, + version: 1, + workflow: process.env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW, + }; + await fs.promises.writeFile( + path.join(logDir, "failures.json"), + `${JSON.stringify(failureIndex, null, 2)}\n`, + ); +} + function phaseElapsedSeconds(startedAtMs) { return Math.round((Date.now() - startedAtMs) / 1000); }