mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 12:40:43 +00:00
test: parallelize docker aggregate
This commit is contained in:
Peter Steinberger
294
scripts/test-docker-all.mjs
Normal file
294
scripts/test-docker-all.mjs
Normal file
@@ -0,0 +1,294 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import { mkdir, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const DEFAULT_E2E_IMAGE = "openclaw-docker-e2e:local";
|
||||
const DEFAULT_PARALLELISM = 4;
|
||||
const DEFAULT_FAILURE_TAIL_LINES = 80;
|
||||
|
||||
const lanes = [
|
||||
["live-models", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models"],
|
||||
["live-gateway", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway"],
|
||||
[
|
||||
"live-cli-backend-claude",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:claude",
|
||||
],
|
||||
[
|
||||
"live-cli-backend-gemini",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:gemini",
|
||||
],
|
||||
["openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui"],
|
||||
["onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard"],
|
||||
[
|
||||
"npm-onboard-channel-agent",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent",
|
||||
],
|
||||
["gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"],
|
||||
["mcp-channels", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:mcp-channels"],
|
||||
["pi-bundle-mcp-tools", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:pi-bundle-mcp-tools"],
|
||||
["cron-mcp-cleanup", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup"],
|
||||
["doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch"],
|
||||
["plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins"],
|
||||
["plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"],
|
||||
["config-reload", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:config-reload"],
|
||||
["bundled-channel-deps", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:bundled-channel-deps"],
|
||||
["qr", "pnpm test:docker:qr"],
|
||||
];
|
||||
|
||||
const exclusiveLanes = [
|
||||
[
|
||||
"openai-web-search-minimal",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal",
|
||||
],
|
||||
["live-codex-harness", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-harness"],
|
||||
[
|
||||
"live-cli-backend-codex",
|
||||
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:codex",
|
||||
],
|
||||
["live-acp-bind", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind"],
|
||||
];
|
||||
|
||||
function parsePositiveInt(raw, fallback, label) {
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function utcStampForPath() {
|
||||
return new Date().toISOString().replaceAll("-", "").replaceAll(":", "").replace(/\..*$/, "Z");
|
||||
}
|
||||
|
||||
function utcStamp() {
|
||||
return new Date().toISOString().replace(/\..*$/, "Z");
|
||||
}
|
||||
|
||||
function appendExtension(env, extension) {
|
||||
const current = env.OPENCLAW_DOCKER_BUILD_EXTENSIONS ?? env.OPENCLAW_EXTENSIONS ?? "";
|
||||
const tokens = current.split(/\s+/).filter(Boolean);
|
||||
if (!tokens.includes(extension)) {
|
||||
tokens.push(extension);
|
||||
}
|
||||
env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = tokens.join(" ");
|
||||
}
|
||||
|
||||
function commandEnv(extra = {}) {
|
||||
return {
|
||||
...process.env,
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function runShellCommand({ command, env, label, logFile }) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("bash", ["-lc", command], {
|
||||
cwd: ROOT_DIR,
|
||||
env,
|
||||
stdio: logFile ? ["ignore", "pipe", "pipe"] : "inherit",
|
||||
});
|
||||
activeChildren.add(child);
|
||||
|
||||
let stream;
|
||||
if (logFile) {
|
||||
stream = fs.createWriteStream(logFile, { flags: "a" });
|
||||
stream.write(`==> [${label}] command: ${command}\n`);
|
||||
stream.write(`==> [${label}] started: ${utcStamp()}\n`);
|
||||
child.stdout.pipe(stream, { end: false });
|
||||
child.stderr.pipe(stream, { end: false });
|
||||
}
|
||||
|
||||
child.on("close", (status, signal) => {
|
||||
activeChildren.delete(child);
|
||||
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
||||
if (stream) {
|
||||
stream.write(`\n==> [${label}] finished: ${utcStamp()} status=${exitCode}\n`);
|
||||
stream.end();
|
||||
}
|
||||
resolve({ status: exitCode, signal });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runForeground(label, command, env) {
|
||||
console.log(`==> ${label}`);
|
||||
const result = await runShellCommand({ command, env, label });
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`${label} failed with status ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function laneEnv(name, baseEnv, logDir) {
|
||||
const env = {
|
||||
...baseEnv,
|
||||
};
|
||||
if (!process.env.OPENCLAW_DOCKER_CLI_TOOLS_DIR) {
|
||||
env.OPENCLAW_DOCKER_CLI_TOOLS_DIR = path.join(logDir, `${name}-cli-tools`);
|
||||
}
|
||||
if (!process.env.OPENCLAW_DOCKER_CACHE_HOME_DIR) {
|
||||
env.OPENCLAW_DOCKER_CACHE_HOME_DIR = path.join(logDir, `${name}-cache`);
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
async function runLane(lane, baseEnv, logDir) {
|
||||
const [name, command] = lane;
|
||||
const logFile = path.join(logDir, `${name}.log`);
|
||||
const env = laneEnv(name, baseEnv, logDir);
|
||||
await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true });
|
||||
await mkdir(env.OPENCLAW_DOCKER_CACHE_HOME_DIR, { recursive: true });
|
||||
await fs.promises.writeFile(
|
||||
logFile,
|
||||
[
|
||||
`==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`,
|
||||
`==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`,
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
console.log(`==> [${name}] start`);
|
||||
const startedAt = Date.now();
|
||||
const result = await runShellCommand({ command, env, label: name, logFile });
|
||||
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
||||
if (result.status === 0) {
|
||||
console.log(`==> [${name}] pass ${elapsedSeconds}s`);
|
||||
} else {
|
||||
console.error(`==> [${name}] fail status=${result.status} ${elapsedSeconds}s log=${logFile}`);
|
||||
}
|
||||
return {
|
||||
command,
|
||||
logFile,
|
||||
name,
|
||||
status: result.status,
|
||||
};
|
||||
}
|
||||
|
||||
async function runLanePool(poolLanes, baseEnv, logDir, parallelism) {
|
||||
const failures = [];
|
||||
let nextIndex = 0;
|
||||
|
||||
async function worker() {
|
||||
while (nextIndex < poolLanes.length) {
|
||||
const lane = poolLanes[nextIndex++];
|
||||
const result = await runLane(lane, baseEnv, logDir);
|
||||
if (result.status !== 0) {
|
||||
failures.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerCount = Math.min(parallelism, poolLanes.length);
|
||||
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function runLanesSerial(serialLanes, baseEnv, logDir) {
|
||||
const failures = [];
|
||||
for (const lane of serialLanes) {
|
||||
const result = await runLane(lane, baseEnv, logDir);
|
||||
if (result.status !== 0) {
|
||||
failures.push(result);
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function tailFile(file, lines) {
|
||||
const content = await readFile(file, "utf8").catch(() => "");
|
||||
const tail = content.split(/\r?\n/).slice(-lines).join("\n");
|
||||
return tail.trimEnd();
|
||||
}
|
||||
|
||||
async function printFailureSummary(failures, tailLines) {
|
||||
console.error(`ERROR: ${failures.length} Docker lane(s) failed.`);
|
||||
for (const failure of failures) {
|
||||
console.error(`---- ${failure.name} failed (status=${failure.status}): ${failure.logFile}`);
|
||||
const tail = await tailFile(failure.logFile, tailLines);
|
||||
if (tail) {
|
||||
console.error(tail);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const activeChildren = new Set();
|
||||
function terminateActiveChildren(signal) {
|
||||
for (const child of activeChildren) {
|
||||
child.kill(signal);
|
||||
}
|
||||
}
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
terminateActiveChildren("SIGINT");
|
||||
process.exit(130);
|
||||
});
|
||||
process.on("SIGTERM", () => {
|
||||
terminateActiveChildren("SIGTERM");
|
||||
process.exit(143);
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const parallelism = parsePositiveInt(
|
||||
process.env.OPENCLAW_DOCKER_ALL_PARALLELISM,
|
||||
DEFAULT_PARALLELISM,
|
||||
"OPENCLAW_DOCKER_ALL_PARALLELISM",
|
||||
);
|
||||
const tailLines = parsePositiveInt(
|
||||
process.env.OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES,
|
||||
DEFAULT_FAILURE_TAIL_LINES,
|
||||
"OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES",
|
||||
);
|
||||
const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath();
|
||||
const logDir = path.resolve(
|
||||
process.env.OPENCLAW_DOCKER_ALL_LOG_DIR ||
|
||||
path.join(ROOT_DIR, ".artifacts/docker-tests", runId),
|
||||
);
|
||||
await mkdir(logDir, { recursive: true });
|
||||
|
||||
const baseEnv = commandEnv({
|
||||
OPENCLAW_DOCKER_E2E_IMAGE: process.env.OPENCLAW_DOCKER_E2E_IMAGE || DEFAULT_E2E_IMAGE,
|
||||
});
|
||||
appendExtension(baseEnv, "matrix");
|
||||
appendExtension(baseEnv, "acpx");
|
||||
appendExtension(baseEnv, "codex");
|
||||
|
||||
console.log(`==> Docker test logs: ${logDir}`);
|
||||
console.log(`==> Parallelism: ${parallelism}`);
|
||||
console.log(`==> Live-test bundled plugin deps: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`);
|
||||
|
||||
await runForeground("Build shared live-test image once", "pnpm test:docker:live-build", baseEnv);
|
||||
await runForeground(
|
||||
`Build shared Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_IMAGE}`,
|
||||
"pnpm test:docker:e2e-build",
|
||||
baseEnv,
|
||||
);
|
||||
|
||||
const failures = await runLanePool(lanes, baseEnv, logDir, parallelism);
|
||||
if (failures.length > 0) {
|
||||
await printFailureSummary(failures, tailLines);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("==> Running provider-sensitive Docker lanes exclusively");
|
||||
failures.push(...(await runLanesSerial(exclusiveLanes, baseEnv, logDir)));
|
||||
if (failures.length > 0) {
|
||||
await printFailureSummary(failures, tailLines);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
await runForeground(
|
||||
"Run cleanup smoke after parallel lanes",
|
||||
"pnpm test:docker:cleanup",
|
||||
baseEnv,
|
||||
);
|
||||
console.log("==> Docker test suite passed");
|
||||
}
|
||||
|
||||
await main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user