vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 06:20:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(release): speed up focused release reruns

Browse Source
This commit is contained in:
Peter Steinberger
2026-05-05 02:28:22 +01:00
parent 0131343db8
commit c84b7cbffc
11 changed files with 378 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

334
scripts/openclaw-cross-os-release-checks.ts
View File

@@ -33,6 +33,7 @@ const SUPPORTED_SUITES = new Set([
"packaged-upgrade",
"dev-update",
]);
const SUPPORTED_OS_IDS = new Set(["ubuntu", "windows", "macos"]);
export const CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS = parsePositiveIntegerEnv(
"OPENCLAW_CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS",
@@ -122,6 +123,10 @@ export const CROSS_OS_RELEASE_SMOKE_TOOLS_PROFILE = "minimal";
export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS = 25 * 60;
export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_WRAPPER_TIMEOUT_MS =
(CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS + 5 * 60) * 1000;
export const CROSS_OS_COMMAND_HEARTBEAT_SECONDS = parsePositiveIntegerEnv(
"OPENCLAW_CROSS_OS_COMMAND_HEARTBEAT_SECONDS",
60,
);
if (isMainModule()) {
try {
@@ -232,6 +237,7 @@ export function resolveRunnerMatrix(params) {
const pick = (...values) =>
values.find((value) => typeof value === "string" && value.trim().length > 0)?.trim();
const suites = resolveRequestedSuites(params.mode, params.ref);
const suiteFilter = parseCrossOsSuiteFilter(params.suiteFilter ?? "");
const runners = [
{
os_id: "ubuntu",
@@ -252,19 +258,84 @@ export function resolveRunnerMatrix(params) {
artifact_name: "macos",
},
];
return {
include: runners.flatMap((runner) =>
suites.map((suite) =>
const include = runners.flatMap((runner) =>
suites
.filter((suite) => suiteFilter.matches(runner.os_id, suite))
.map((suite) =>
Object.assign({}, runner, {
suite,
suite_label: formatSuiteLabel(suite),
lane: suite.includes(`upgrade`) || suite === `dev-update` ? `upgrade` : `fresh`,
}),
),
),
);
if (include.length === 0) {
throw new Error(
`cross_os_suite_filter ${JSON.stringify(params.suiteFilter ?? "")} did not match any ${params.mode} suite.`,
);
}
return {
include,
};
}
export function parseCrossOsSuiteFilter(rawFilter) {
const tokens = String(rawFilter ?? "")
.split(/[, ]+/u)
.map((token) => normalizeCrossOsSuiteFilterToken(token))
.filter(Boolean);
if (tokens.length === 0) {
return {
matches: () => true,
tokens,
};
}
const matchers = tokens.map((token) => {
if (SUPPORTED_SUITES.has(token)) {
return { osId: "", suite: token };
}
if (SUPPORTED_OS_IDS.has(token)) {
return { osId: token, suite: "" };
}
for (const separator of ["/", ":", "-"]) {
const matchedOs = [...SUPPORTED_OS_IDS].find((osId) =>
token.startsWith(`${osId}${separator}`),
);
if (!matchedOs) {
continue;
}
const suite = token.slice(matchedOs.length + separator.length);
if (!SUPPORTED_SUITES.has(suite)) {
break;
}
return { osId: matchedOs, suite };
}
throw new Error(
`Unsupported cross_os_suite_filter token ${JSON.stringify(token)}. Use an OS id, suite id, or os/suite pair such as windows/packaged-upgrade.`,
);
});
return {
matches: (osId, suite) =>
matchers.some((matcher) => {
const osMatches = !matcher.osId || matcher.osId === osId;
const suiteMatches = !matcher.suite || matcher.suite === suite;
return osMatches && suiteMatches;
}),
tokens,
};
}
function normalizeCrossOsSuiteFilterToken(token) {
return token
.trim()
.toLowerCase()
.replace(/_/gu, "-")
.replace(/\s*[/:-]\s*/gu, (separator) => separator.trim())
.replace(/\s+/gu, "-");
}
export function readRunnerOverrideEnv(env = process.env) {
const preferNonEmptyEnv = (primary: string | undefined, legacy: string | undefined) => {
const primaryValue = primary?.trim();
@@ -319,6 +390,7 @@ async function main(argv) {
ubuntuRunner: args["ubuntu-runner"],
windowsRunner: args["windows-runner"],
macosRunner: args["macos-runner"],
suiteFilter: args["suite-filter"],
...runnerOverrideEnv,
}),
)}\n`,
@@ -733,76 +805,80 @@ async function runUpgradeLane(params) {
const cleanup = [];
try {
const env = buildLaneEnv(lane, params.providerConfig, params.providerSecretValue);
logLanePhase(lane, "install-baseline");
if (!params.baselineTgz && params.baselineSpec) {
await installPackageSpec({
await runTimedLanePhase(lane, "install-baseline", async () => {
if (!params.baselineTgz && params.baselineSpec) {
await installPackageSpec({
lane,
env,
packageSpec: params.baselineSpec,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
});
} else {
await installTarballPackage({
lane,
env,
tgzPath: params.baselineTgz,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
restoreBundledPluginPostinstall: false,
});
}
});
await runTimedLanePhase(lane, "run-baseline-bundled-plugin-postinstall", async () => {
await runBundledPluginPostinstall({
lane,
env,
packageSpec: params.baselineSpec,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
});
} else {
await installTarballPackage({
lane,
env,
tgzPath: params.baselineTgz,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
restoreBundledPluginPostinstall: false,
});
}
logLanePhase(lane, "run-baseline-bundled-plugin-postinstall");
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
});
const baseline = {
version: readInstalledVersion(lane.prefixDir),
};
logLanePhase(lane, "update");
const updateEnv = buildRealUpdateEnv(env);
const updateArgs = buildPackagedUpgradeUpdateArgs(params.candidateUrl);
const updateLogPath = join(params.logsDir, "upgrade-update.log");
let updateResult;
let usedWindowsPackagedUpgradeTimeoutFallback = false;
try {
updateResult = await runOpenClaw({
lane,
env: updateEnv,
args: updateArgs,
logPath: updateLogPath,
timeoutMs: updateTimeoutMs(),
check: false,
});
} catch (error) {
if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) {
throw error;
await runTimedLanePhase(lane, "update", async () => {
try {
updateResult = await runOpenClaw({
lane,
env: updateEnv,
args: updateArgs,
logPath: updateLogPath,
timeoutMs: updateTimeoutMs(),
check: false,
});
} catch (error) {
if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) {
throw error;
}
usedWindowsPackagedUpgradeTimeoutFallback = true;
appendFileSync(
updateLogPath,
`\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`,
);
updateResult = {
exitCode: 124,
stdout: "",
stderr: formatError(error),
};
}
usedWindowsPackagedUpgradeTimeoutFallback = true;
appendFileSync(
updateLogPath,
`\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`,
);
updateResult = {
exitCode: 124,
stdout: "",
stderr: formatError(error),
};
}
});
const usedWindowsPackagedUpgradeFallback =
usedWindowsPackagedUpgradeTimeoutFallback ||
isRecoverableWindowsPackagedUpgradeSwapCleanupFailure(updateResult, process.platform);
if (usedWindowsPackagedUpgradeFallback) {
logLanePhase(lane, "update-fallback-install");
await installPackageSpec({
lane,
env,
packageSpec: params.candidateUrl,
logPath: join(params.logsDir, "upgrade-update-fallback-install.log"),
await runTimedLanePhase(lane, "update-fallback-install", async () => {
await installPackageSpec({
lane,
env,
packageSpec: params.candidateUrl,
logPath: join(params.logsDir, "upgrade-update-fallback-install.log"),
});
});
} else {
verifyPackagedUpgradeUpdateResult(updateResult, {
@@ -816,69 +892,77 @@ async function runUpgradeLane(params) {
usedWindowsPackagedUpgradeFallback,
})
) {
logLanePhase(lane, "update-status");
await runOpenClaw({
lane,
env: updateEnv,
args: ["update", "status", "--json"],
logPath: join(params.logsDir, "upgrade-update-status.log"),
timeoutMs: 2 * 60 * 1000,
await runTimedLanePhase(lane, "update-status", async () => {
await runOpenClaw({
lane,
env: updateEnv,
args: ["update", "status", "--json"],
logPath: join(params.logsDir, "upgrade-update-status.log"),
timeoutMs: 2 * 60 * 1000,
});
});
}
logLanePhase(lane, "run-bundled-plugin-postinstall");
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"),
await runTimedLanePhase(lane, "run-bundled-plugin-postinstall", async () => {
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"),
});
});
const installed = readInstalledMetadata(lane.prefixDir);
verifyInstalledCandidate(installed, params.build);
logLanePhase(lane, "onboard");
await runOnboard({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-onboard.log"),
await runTimedLanePhase(lane, "onboard", async () => {
await runOnboard({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-onboard.log"),
});
});
logLanePhase(lane, "models-set");
await runModelsSet({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-models-set.log"),
await runTimedLanePhase(lane, "models-set", async () => {
await runModelsSet({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-models-set.log"),
});
});
logLanePhase(lane, "start-gateway");
const gateway = await startGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway.log"),
});
const gateway = await runTimedLanePhase(lane, "start-gateway", async () =>
startGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway.log"),
}),
);
cleanup.push(() => stopGateway(gateway));
logLanePhase(lane, "wait-gateway");
await waitForGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway-status.log"),
await runTimedLanePhase(lane, "wait-gateway", async () => {
await waitForGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway-status.log"),
});
});
logLanePhase(lane, "dashboard");
await runDashboardSmoke({
lane,
logPath: join(params.logsDir, "upgrade-dashboard.log"),
await runTimedLanePhase(lane, "dashboard", async () => {
await runDashboardSmoke({
lane,
logPath: join(params.logsDir, "upgrade-dashboard.log"),
});
});
logLanePhase(lane, "agent-turn");
const agent = await runAgentTurn({
lane,
env,
label: "upgrade",
logPath: join(params.logsDir, "upgrade-agent.log"),
});
const agent = await runTimedLanePhase(lane, "agent-turn", async () =>
runAgentTurn({
lane,
env,
label: "upgrade",
logPath: join(params.logsDir, "upgrade-agent.log"),
}),
);
return {
status: "pass",
@@ -888,6 +972,7 @@ async function runUpgradeLane(params) {
dashboardStatus: "pass",
gatewayPort: lane.gatewayPort,
agentOutput: trimForSummary(agent.stdout),
phaseTimings: lane.phaseTimings,
};
} finally {
await runCleanup(cleanup);
@@ -1245,6 +1330,7 @@ function createLaneState(name) {
stateDir,
appDataDir,
gatewayPort: 0,
phaseTimings: [],
};
}
@@ -3268,6 +3354,10 @@ async function runCommand(command, args, options) {
let stderr = "";
let timedOut = false;
let settled = false;
const startedAt = Date.now();
let killWaitTimer = null;
let timer = null;
let heartbeatTimer = null;
const clearTimers = () => {
if (timer) {
@@ -3276,6 +3366,9 @@ async function runCommand(command, args, options) {
if (killWaitTimer) {
clearTimeout(killWaitTimer);
}
if (heartbeatTimer) {
clearInterval(heartbeatTimer);
}
};
const finalize = (callback) => {
@@ -3307,8 +3400,7 @@ async function runCommand(command, args, options) {
child.kill(process.platform === "win32" ? undefined : "SIGKILL");
};
let killWaitTimer = null;
const timer =
timer =
options.timeoutMs && Number.isFinite(options.timeoutMs)
? setTimeout(() => {
timedOut = true;
@@ -3327,6 +3419,20 @@ async function runCommand(command, args, options) {
}, 15_000);
}, options.timeoutMs)
: null;
heartbeatTimer =
CROSS_OS_COMMAND_HEARTBEAT_SECONDS > 0
? setInterval(() => {
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1000);
const message = `${new Date().toISOString()} still running after ${elapsedSeconds}s: ${command} ${args.join(" ")}\n`;
logStream.write(message);
process.stdout.write(`[release-checks] ${message}`);
}, CROSS_OS_COMMAND_HEARTBEAT_SECONDS * 1000)
: null;
heartbeatTimer?.unref?.();
logStream.write(
`${new Date().toISOString()} start command=${command} args=${args.join(" ")}\n`,
);
child.stdout?.on("data", (chunk) => {
const text = chunk.toString();
@@ -3454,6 +3560,13 @@ function writeSummary(baseDir, summaryPayload) {
result.agentOutput ? `- Agent output: \`${trimForSummary(result.agentOutput)}\`` : "",
result.error ? `- Error: \`${trimForSummary(result.error)}\`` : "",
].filter(Boolean);
if (Array.isArray(result.phaseTimings) && result.phaseTimings.length > 0) {
lines.push("", "### Phase timings");
for (const phase of result.phaseTimings) {
const suffix = phase.status === "pass" ? "" : ` (${phase.status})`;
lines.push(`- \`${phase.name}\`: ${Math.round(phase.durationMs / 1000)}s${suffix}`);
}
}
writeFileSync(summaryMarkdownPath, `${lines.join("\n")}\n`, "utf8");
}
@@ -3522,6 +3635,23 @@ function logLanePhase(lane, phase) {
logPhase(`lane.${lane.name}`, phase);
}
async function runTimedLanePhase(lane, phase, callback) {
const startedAt = Date.now();
logLanePhase(lane, phase);
try {
const result = await callback();
const durationMs = Date.now() - startedAt;
lane.phaseTimings.push({ name: phase, status: "pass", durationMs });
logPhase(`lane.${lane.name}`, `${phase}: done in ${Math.round(durationMs / 1000)}s`);
return result;
} catch (error) {
const durationMs = Date.now() - startedAt;
lane.phaseTimings.push({ name: phase, status: "fail", durationMs });
logPhase(`lane.${lane.name}`, `${phase}: failed in ${Math.round(durationMs / 1000)}s`);
throw error;
}
}
function trimForSummary(value) {
const trimmed = value.trim();
if (trimmed.length <= 600) {