fix(scripts): wait for tsdown process groups

This commit is contained in:
Vincent Koc
2026-06-17 21:19:12 +02:00
parent abb6f04e0c
commit 2c71e71833
2 changed files with 165 additions and 13 deletions

View File

@@ -32,6 +32,8 @@ const CGROUP_MEMORY_LIMIT_PATHS = [
];
const PROC_MEMINFO_PATH = "/proc/meminfo";
const TERMINATION_GRACE_MS = 5_000;
const PROCESS_GROUP_EXIT_POLL_MS = 25;
const POST_FORCE_KILL_WAIT_MS = 1_000;
const ROOT_TSDOWN_OUTPUT_ROOTS = ["dist", "dist-runtime"];
const PRESERVED_TSDOWN_OUTPUT_FILES = ["dist/cli-startup-metadata.json"];
const PRESERVE_CLI_STARTUP_METADATA_ENV = "OPENCLAW_PRESERVE_CLI_STARTUP_METADATA";
@@ -607,13 +609,65 @@ export async function runTsdownBuildInvocation(invocation, params = {}) {
let settled = false;
let lastOutputAt = Date.now();
const child = spawn(invocation.command, invocation.args, invocation.options);
const useProcessGroup = timeoutMs !== null && process.platform !== "win32";
const child = spawn(invocation.command, invocation.args, {
...invocation.options,
detached: useProcessGroup,
});
const pidText = child.pid ? ` pid=${child.pid}` : "";
function markOutput() {
lastOutputAt = Date.now();
}
function signalChild(signal) {
if (useProcessGroup && child.pid) {
try {
process.kill(-child.pid, signal);
return;
} catch {
// The group may already be gone; fall back to the direct child handle.
}
}
child.kill(signal);
}
function processTreeAlive() {
if (!child.pid) {
return false;
}
if (!useProcessGroup) {
return child.exitCode === null && child.signalCode === null;
}
try {
process.kill(-child.pid, 0);
return true;
} catch (error) {
return error?.code === "EPERM";
}
}
async function waitForProcessTreeExit(timeoutMsToWait) {
const deadlineAt = Date.now() + timeoutMsToWait;
while (Date.now() < deadlineAt) {
if (!processTreeAlive()) {
return true;
}
await new Promise((resolvePoll) => {
setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
});
}
return !processTreeAlive();
}
async function finishTimedOutProcessTree() {
if (!processTreeAlive()) {
return;
}
signalChild("SIGKILL");
await waitForProcessTreeExit(POST_FORCE_KILL_WAIT_MS);
}
child.stdout?.on("data", (chunk) => {
markOutput();
scanner.append(chunk);
@@ -649,11 +703,11 @@ export async function runTsdownBuildInvocation(invocation, params = {}) {
? setTimeout(() => {
timedOut = true;
stderr.write(`[tsdown-build] timeout after ${timeoutMs}ms${pidText}; sending SIGTERM\n`);
child.kill("SIGTERM");
signalChild("SIGTERM");
setTimeout(() => {
if (!settled) {
stderr.write(`[tsdown-build] forcing SIGKILL${pidText}\n`);
child.kill("SIGKILL");
signalChild("SIGKILL");
}
}, TERMINATION_GRACE_MS).unref();
}, timeoutMs).unref()
@@ -674,16 +728,25 @@ export async function runTsdownBuildInvocation(invocation, params = {}) {
});
});
child.once("close", (status, signal) => {
settled = true;
clearInterval(heartbeat);
clearTimeout(timeout);
resolve({
status,
signal,
timedOut,
error: null,
...scanner.finish(),
});
function finish() {
settled = true;
clearInterval(heartbeat);
clearTimeout(timeout);
resolve({
status,
signal,
timedOut,
error: null,
...scanner.finish(),
});
}
if (timedOut) {
void finishTimedOutProcessTree().then(finish, finish);
return;
}
finish();
});
});
}

View File

@@ -37,6 +37,41 @@ async function expectPathMissing(targetPath: string) {
expect(Reflect.get(statError, "code")).toBe("ENOENT");
}
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function waitForFile(filePath: string, timeoutMs: number): Promise<void> {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (fs.existsSync(filePath)) {
return;
}
await sleep(25);
}
throw new Error(`timed out waiting for ${filePath}`);
}
async function waitForDead(pid: number, timeoutMs: number): Promise<void> {
const deadlineAt = Date.now() + timeoutMs;
while (Date.now() < deadlineAt) {
if (!isProcessAlive(pid)) {
return;
}
await sleep(25);
}
throw new Error(`timed out waiting for pid ${pid} to exit`);
}
describe("resolveTsdownBuildInvocation", () => {
it("parses wrapper help before any tsdown work", () => {
expect(parseTsdownBuildArgs(["--help"])).toEqual({ forwardedArgs: [], help: true });
@@ -610,4 +645,58 @@ describe("runTsdownBuildInvocation", () => {
expect(result.signal).toBe("SIGTERM");
expect(output.chunks.join("")).toContain("timeout after 50ms");
});
it.skipIf(process.platform === "win32")(
"kills timed-out tsdown process groups when the wrapper exits first",
async () => {
const rootDir = createTempDir("openclaw-tsdown-timeout-");
const childPidPath = path.join(rootDir, "child.pid");
let childPid = 0;
const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
const parentScript = [
"const { spawn } = require('node:child_process');",
"const fs = require('node:fs');",
`const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
`fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`,
"process.on('SIGTERM', () => process.exit(0));",
"setInterval(() => {}, 1000);",
].join("");
try {
const output = createWriteSink();
const runPromise = runTsdownBuildInvocation(
{
command: process.execPath,
args: ["-e", parentScript],
options: {
stdio: ["ignore", "pipe", "pipe"],
shell: false,
env: process.env,
},
},
{
stdout: output.sink,
stderr: output.sink,
env: {
...process.env,
OPENCLAW_TSDOWN_HEARTBEAT_MS: "0",
OPENCLAW_TSDOWN_TIMEOUT_MS: "50",
},
},
);
await waitForFile(childPidPath, 2_000);
childPid = Number.parseInt(fs.readFileSync(childPidPath, "utf8"), 10);
expect(isProcessAlive(childPid)).toBe(true);
const result = await runPromise;
expect(result.timedOut).toBe(true);
await waitForDead(childPid, 2_000);
} finally {
if (childPid && isProcessAlive(childPid)) {
process.kill(childPid, "SIGKILL");
}
}
},
);
});