fix(e2e): stop multi-node health checks hanging past deadline (#108065)

* fix(e2e): bound multi-node health probes

* test(e2e): track multi-node timeout temp files
This commit is contained in:
Alix-007
2026-07-16 13:47:43 +08:00
committed by GitHub
parent da59d6ade9
commit 3009338a9a
2 changed files with 65 additions and 3 deletions

View File

@@ -438,7 +438,9 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let last = "timeout";
while (Date.now() < deadline) {
try {
const response = await fetch(url);
// Keep each request inside the outer probe budget so a stalled fetch cannot outlive it.
const remainingMs = Math.max(1, deadline - Date.now());
const response = await fetch(url, { signal: AbortSignal.timeout(remainingMs) });
if (response.ok) {
process.exit(0);
}
@@ -446,7 +448,11 @@ while (Date.now() < deadline) {
} catch (error) {
last = error instanceof Error ? error.message : String(error);
}
await sleep(500);
const remainingDelayMs = deadline - Date.now();
if (remainingDelayMs <= 0) {
break;
}
await sleep(Math.min(500, remainingDelayMs));
}
console.error(last);
process.exit(1);

View File

@@ -13,7 +13,11 @@ import {
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { describe, expect, it } from "vitest";
import { pathToFileURL } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const HELPER_PATH = "scripts/lib/docker-build.sh";
const DOCKER_ALL_SCHEDULER_PATH = "scripts/test-docker-all.mjs";
@@ -4041,6 +4045,58 @@ heartbeat_elapsed="\${BASH_REMATCH[1]}"
expect(runner).not.toContain("WARNING: Gateway status probe failed");
});
it("keeps a stalled multi-node health request inside the probe deadline", () => {
const runner = readFileSync(MULTI_NODE_UPDATE_DOCKER_E2E_PATH, "utf8");
const startMarker = "if PORT=18789 node <<NODE\n";
const endMarker = "\nNODE\n then";
const start = runner.indexOf(startMarker);
const end = runner.indexOf(endMarker, start + startMarker.length);
expect(start).toBeGreaterThanOrEqual(0);
expect(end).toBeGreaterThan(start);
const probe = runner.slice(start + startMarker.length, end);
const workDir = tempDirs.make("openclaw-multi-node-health-timeout-");
const preloadPath = join(workDir, "stalling-fetch.mjs");
writeFileSync(
preloadPath,
[
"// Advance the deadline when the request aborts without waiting 30 wall-clock seconds.",
"const realSetTimeout = globalThis.setTimeout;",
"let now = 0;",
"Date.now = () => now;",
"Object.defineProperty(AbortSignal, 'timeout', { value(delayMs) {",
" const controller = new AbortController();",
" realSetTimeout(() => {",
" now += delayMs;",
" controller.abort(new DOMException('health deadline elapsed', 'TimeoutError'));",
" }, 0);",
" return controller.signal;",
"} });",
"globalThis.fetch = async (_url, init = {}) => await new Promise((_resolve, reject) => {",
" init.signal.addEventListener('abort', () => {",
" process.stderr.write('hung fetch aborted\\n');",
" reject(init.signal.reason);",
" }, { once: true });",
"});",
].join("\n"),
);
const result = spawnSync(
process.execPath,
["--import", pathToFileURL(preloadPath).href, "--input-type=module", "--eval", probe],
{
encoding: "utf8",
env: { ...process.env, PORT: "18789" },
timeout: 5_000,
},
);
expect(result.error).toBeUndefined();
expect(result.status).toBe(1);
expect(result.stderr.match(/hung fetch aborted/gu)).toHaveLength(1);
expect(result.stderr).toContain("health deadline elapsed");
});
it("caps package acceptance legacy compatibility at 2026.4.25", () => {
const doctorScenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8");
const updateChannel = readFileSync(UPDATE_CHANNEL_SWITCH_DOCKER_E2E_PATH, "utf8");