vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 09:50:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(docker): reuse cached e2e images for reruns

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-27 06:29:03 +01:00
parent 679e476183
commit 5e9a96fafb
11 changed files with 319 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
scripts/docker-e2e-rerun.mjs
View File

@@ -1,8 +1,9 @@
#!/usr/bin/env node
// Builds cheap rerun commands from a Docker E2E GitHub run or local summary.
// For GitHub runs, the script downloads Docker E2E artifacts, reads
// summary/failures JSON, and prints targeted workflow commands that prepare a
// fresh OpenClaw tarball for the same ref before running only failed lanes.
// summary/failures JSON, and prints targeted workflow commands for failed
// lanes, reusing package artifacts and prepared GHCR images when artifacts
// expose them.
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
@@ -76,8 +77,44 @@ function shellQuote(value) {
return `'${String(value).replaceAll("'", "'\\''")}'`;
}
function ghWorkflowCommand(lanes, ref, workflow) {
return [
function maybeGhcrImage(value) {
return typeof value === "string" && value.startsWith("ghcr.io/") ? value : "";
}
function reuseInputsFromJson(parsed) {
const packageArtifactRunId = parsed.github?.runId || "";
if (!packageArtifactRunId) {
return {};
}
return {
bareImage: maybeGhcrImage(parsed.images?.bare),
functionalImage: maybeGhcrImage(parsed.images?.functional),
packageArtifactName:
parsed.packageArtifactName || parsed.artifacts?.packageName || "docker-e2e-package",
packageArtifactRunId,
};
}
function sameReuseInputs(left, right) {
return (
(left?.packageArtifactRunId || "") === (right?.packageArtifactRunId || "") &&
(left?.packageArtifactName || "") === (right?.packageArtifactName || "") &&
(left?.bareImage || "") === (right?.bareImage || "") &&
(left?.functionalImage || "") === (right?.functionalImage || "")
);
}
function commonReuseInputs(entries) {
const inputs = entries.map((entry) => entry.reuseInputs).filter(Boolean);
if (inputs.length === 0) {
return {};
}
const [first] = inputs;
return inputs.every((input) => sameReuseInputs(first, input)) ? first : {};
}
function ghWorkflowCommand(lanes, ref, workflow, reuseInputs = {}) {
const fields = [
"gh workflow run",
shellQuote(workflow),
"-f",
@@ -94,7 +131,21 @@ function ghWorkflowCommand(lanes, ref, workflow) {
"include_live_suites=false",
"-f",
"live_models_only=false",
].join(" ");
];
if (reuseInputs.packageArtifactRunId) {
fields.push("-f", `package_artifact_run_id=${shellQuote(reuseInputs.packageArtifactRunId)}`);
fields.push(
"-f",
`package_artifact_name=${shellQuote(reuseInputs.packageArtifactName || "docker-e2e-package")}`,
);
}
if (reuseInputs.bareImage) {
fields.push("-f", `docker_e2e_bare_image=${shellQuote(reuseInputs.bareImage)}`);
}
if (reuseInputs.functionalImage) {
fields.push("-f", `docker_e2e_functional_image=${shellQuote(reuseInputs.functionalImage)}`);
}
return fields.join(" ");
}
function detectRepo() {
@@ -115,15 +166,18 @@ function findFiles(rootDir, basenames, out = []) {
function failedLaneEntriesFromJson(file, ref, workflow) {
const parsed = readJson(file);
const reuseInputs = reuseInputsFromJson(parsed);
const source = path.basename(file);
if (source === "failures.json" && Array.isArray(parsed.lanes)) {
return parsed.lanes
.filter((lane) => lane.name)
.map((lane) => ({
ghWorkflowCommand: lane.ghWorkflowCommand,
ghWorkflowCommand:
lane.ghWorkflowCommand || ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
lane: lane.name,
localRerunCommand: lane.rerunCommand,
logFile: lane.logFile,
reuseInputs,
source: file,
status: lane.status,
}));
@@ -133,10 +187,11 @@ function failedLaneEntriesFromJson(file, ref, workflow) {
return lanes
.filter((lane) => lane.status !== 0 && lane.name)
.map((lane) => ({
ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow),
ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
lane: lane.name,
localRerunCommand: lane.rerunCommand,
logFile: lane.logFile,
reuseInputs,
source: file,
status: lane.status,
}));
@@ -201,7 +256,7 @@ function printEntries(entries, ref, workflow, run) {
}
console.log(`Ref: ${ref}`);
console.log(
"Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for that ref before lane execution.",
"Targeted GitHub reruns reuse package artifacts and prepared GHCR images when the downloaded artifacts expose them.",
);
if (entries.length === 0) {
console.log("No failed Docker E2E lanes found.");
@@ -215,6 +270,7 @@ function printEntries(entries, ref, workflow, run) {
entries.map((entry) => entry.lane),
ref,
workflow,
commonReuseInputs(entries),
),
);
console.log("");

29
scripts/docker-e2e.mjs
View File

@@ -40,8 +40,23 @@ function inlineCode(value) {
return `\`${String(value ?? "").replaceAll("`", "\\`")}\``;
}
function formatSeconds(value) {
const seconds = Number(value);
if (!Number.isFinite(seconds) || seconds < 0) {
return "";
}
const rounded = Math.round(seconds);
const minutes = Math.floor(rounded / 60);
const rest = rounded % 60;
return minutes > 0 ? `${minutes}m ${rest}s` : `${rest}s`;
}
function summaryMarkdown(summary, title) {
const lanes = Array.isArray(summary.lanes) ? summary.lanes : [];
const slowest = lanes
.filter((lane) => Number.isFinite(Number(lane.elapsedSeconds)))
.toSorted((a, b) => Number(b.elapsedSeconds) - Number(a.elapsedSeconds))
.slice(0, 8);
const lines = [
`### ${title}`,
"",
@@ -57,12 +72,22 @@ function summaryMarkdown(summary, title) {
);
}
if (slowest.length > 0) {
lines.push("", "| Slowest lane | Duration | Status |", "| --- | ---: | --- |");
for (const lane of slowest) {
const status = lane.status === 0 ? "pass" : `fail ${lane.status}`;
lines.push(
`| ${inlineCode(lane.name)} | ${markdownCell(formatSeconds(lane.elapsedSeconds))} | ${markdownCell(status)} |`,
);
}
}
const phases = Array.isArray(summary.phases) ? summary.phases : [];
if (phases.length > 0) {
lines.push("", "| Phase | Seconds | Status | Image kind |", "| --- | ---: | --- | --- |");
lines.push("", "| Phase | Duration | Status | Image kind |", "| --- | ---: | --- | --- |");
for (const phase of phases) {
lines.push(
`| ${inlineCode(phase.name)} | ${markdownCell(phase.elapsedSeconds)} | ${markdownCell(phase.status)} | ${markdownCell(phase.imageKind)} |`,
`| ${inlineCode(phase.name)} | ${markdownCell(formatSeconds(phase.elapsedSeconds))} | ${markdownCell(phase.status)} | ${markdownCell(phase.imageKind)} |`,
);
}
}

30
scripts/lib/docker-e2e-scenarios.mjs
View File

@@ -356,7 +356,7 @@ const releasePathChunks = {
weight: 3,
}),
],
"package-update": [
"package-install": [
npmLane(
"install-e2e",
"OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=both pnpm test:install:e2e",
@@ -370,6 +370,8 @@ const releasePathChunks = {
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent",
{ resources: ["service"], weight: 3 },
),
],
"package-update": [
npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", {
weight: 3,
}),
@@ -382,17 +384,21 @@ const releasePathChunks = {
},
),
],
"plugins-integrations": [
plugins: [
lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", {
resources: ["npm", "service"],
weight: 6,
}),
npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"),
],
"bundled-channel-deps": [
npmLane(
"bundled-channel-deps",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:bundled-channel-deps",
{ resources: ["service"], weight: 3 },
),
],
"service-integrations": [
serviceLane(
"cron-mcp-cleanup",
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup",
@@ -407,6 +413,12 @@ const releasePathChunks = {
{ timeoutMs: 8 * 60 * 1000 },
),
],
openwebui: [
serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", {
timeoutMs: OPENWEBUI_TIMEOUT_MS,
weight: 5,
}),
],
};
export function releasePathChunkLanes(chunk, options = {}) {
@@ -416,22 +428,16 @@ export function releasePathChunkLanes(chunk, options = {}) {
`OPENCLAW_DOCKER_ALL_CHUNK must be one of: ${Object.keys(releasePathChunks).join(", ")}. Got: ${JSON.stringify(chunk)}`,
);
}
if (chunk !== "plugins-integrations" || !options.includeOpenWebUI) {
return base;
if (chunk === "openwebui" && !options.includeOpenWebUI) {
return [];
}
return [
...base,
serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", {
timeoutMs: OPENWEBUI_TIMEOUT_MS,
weight: 5,
}),
];
return base;
}
export function allReleasePathLanes(options = {}) {
return Object.keys(releasePathChunks).flatMap((chunk) =>
releasePathChunkLanes(chunk, {
includeOpenWebUI: chunk === "plugins-integrations" && options.includeOpenWebUI,
includeOpenWebUI: options.includeOpenWebUI,
}),
);
}

31
scripts/test-docker-all.mjs
View File

@@ -194,7 +194,7 @@ function shellQuote(value) {
}
function githubWorkflowRerunCommand(laneNames, ref) {
return [
const fields = [
"gh workflow run",
shellQuote(process.env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW),
"-f",
@@ -211,7 +211,29 @@ function githubWorkflowRerunCommand(laneNames, ref) {
"include_live_suites=false",
"-f",
"live_models_only=false",
].join(" ");
];
if (process.env.GITHUB_RUN_ID) {
fields.push("-f", `package_artifact_run_id=${shellQuote(process.env.GITHUB_RUN_ID)}`);
fields.push(
"-f",
`package_artifact_name=${shellQuote(
process.env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || "docker-e2e-package",
)}`,
);
}
if (process.env.OPENCLAW_DOCKER_E2E_BARE_IMAGE) {
fields.push(
"-f",
`docker_e2e_bare_image=${shellQuote(process.env.OPENCLAW_DOCKER_E2E_BARE_IMAGE)}`,
);
}
if (process.env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE) {
fields.push(
"-f",
`docker_e2e_functional_image=${shellQuote(process.env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE)}`,
);
}
return fields.join(" ");
}
function buildLaneRerunCommand(name, baseEnv) {
@@ -301,6 +323,7 @@ async function writeRunSummary(logDir, summary) {
const file = path.join(logDir, "summary.json");
const payload = {
...summary,
packageArtifactName: process.env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
finishedAt: new Date().toISOString(),
github: {
ref: process.env.GITHUB_REF_NAME || undefined,
@@ -346,7 +369,9 @@ async function writeFailureIndex(logDir, summary) {
: undefined,
generatedAt: new Date().toISOString(),
lanes,
note: "Targeted GitHub reruns prepare a fresh OpenClaw npm tarball for the selected ref before lane execution.",
note: "Targeted GitHub reruns reuse this run's package artifact and shared Docker images when the generated command includes package_artifact_run_id and docker_e2e_*_image inputs.",
images: summary.images,
packageArtifactName: process.env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
ref,
runUrl: summary.github?.runUrl,
status: summary.status,