mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 00:41:35 +00:00
fix(e2e): stop multi-node health checks hanging past deadline (#108065)
* fix(e2e): bound multi-node health probes * test(e2e): track multi-node timeout temp files
This commit is contained in:
@@ -438,7 +438,9 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
let last = "timeout";
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
// Keep each request inside the outer probe budget so a stalled fetch cannot outlive it.
|
||||
const remainingMs = Math.max(1, deadline - Date.now());
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(remainingMs) });
|
||||
if (response.ok) {
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -446,7 +448,11 @@ while (Date.now() < deadline) {
|
||||
} catch (error) {
|
||||
last = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
await sleep(500);
|
||||
const remainingDelayMs = deadline - Date.now();
|
||||
if (remainingDelayMs <= 0) {
|
||||
break;
|
||||
}
|
||||
await sleep(Math.min(500, remainingDelayMs));
|
||||
}
|
||||
console.error(last);
|
||||
process.exit(1);
|
||||
|
||||
@@ -13,7 +13,11 @@ import {
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
const HELPER_PATH = "scripts/lib/docker-build.sh";
|
||||
const DOCKER_ALL_SCHEDULER_PATH = "scripts/test-docker-all.mjs";
|
||||
@@ -4041,6 +4045,58 @@ heartbeat_elapsed="\${BASH_REMATCH[1]}"
|
||||
expect(runner).not.toContain("WARNING: Gateway status probe failed");
|
||||
});
|
||||
|
||||
it("keeps a stalled multi-node health request inside the probe deadline", () => {
|
||||
const runner = readFileSync(MULTI_NODE_UPDATE_DOCKER_E2E_PATH, "utf8");
|
||||
const startMarker = "if PORT=18789 node <<NODE\n";
|
||||
const endMarker = "\nNODE\n then";
|
||||
const start = runner.indexOf(startMarker);
|
||||
const end = runner.indexOf(endMarker, start + startMarker.length);
|
||||
expect(start).toBeGreaterThanOrEqual(0);
|
||||
expect(end).toBeGreaterThan(start);
|
||||
const probe = runner.slice(start + startMarker.length, end);
|
||||
const workDir = tempDirs.make("openclaw-multi-node-health-timeout-");
|
||||
const preloadPath = join(workDir, "stalling-fetch.mjs");
|
||||
|
||||
writeFileSync(
|
||||
preloadPath,
|
||||
[
|
||||
"// Advance the deadline when the request aborts without waiting 30 wall-clock seconds.",
|
||||
"const realSetTimeout = globalThis.setTimeout;",
|
||||
"let now = 0;",
|
||||
"Date.now = () => now;",
|
||||
"Object.defineProperty(AbortSignal, 'timeout', { value(delayMs) {",
|
||||
" const controller = new AbortController();",
|
||||
" realSetTimeout(() => {",
|
||||
" now += delayMs;",
|
||||
" controller.abort(new DOMException('health deadline elapsed', 'TimeoutError'));",
|
||||
" }, 0);",
|
||||
" return controller.signal;",
|
||||
"} });",
|
||||
"globalThis.fetch = async (_url, init = {}) => await new Promise((_resolve, reject) => {",
|
||||
" init.signal.addEventListener('abort', () => {",
|
||||
" process.stderr.write('hung fetch aborted\\n');",
|
||||
" reject(init.signal.reason);",
|
||||
" }, { once: true });",
|
||||
"});",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", pathToFileURL(preloadPath).href, "--input-type=module", "--eval", probe],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, PORT: "18789" },
|
||||
timeout: 5_000,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr.match(/hung fetch aborted/gu)).toHaveLength(1);
|
||||
expect(result.stderr).toContain("health deadline elapsed");
|
||||
});
|
||||
|
||||
it("caps package acceptance legacy compatibility at 2026.4.25", () => {
|
||||
const doctorScenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8");
|
||||
const updateChannel = readFileSync(UPDATE_CHANNEL_SWITCH_DOCKER_E2E_PATH, "utf8");
|
||||
|
||||
Reference in New Issue
Block a user