From 3009338a9a48ebb3f5dbcdb2bdc0a2fc87dfd93b Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Thu, 16 Jul 2026 13:47:43 +0800 Subject: [PATCH] fix(e2e): stop multi-node health checks hanging past deadline (#108065) * fix(e2e): bound multi-node health probes * test(e2e): track multi-node timeout temp files --- scripts/e2e/multi-node-update-docker.sh | 10 +++- test/scripts/docker-build-helper.test.ts | 58 +++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/scripts/e2e/multi-node-update-docker.sh b/scripts/e2e/multi-node-update-docker.sh index badcb0d97cfe..806d931ff8cd 100755 --- a/scripts/e2e/multi-node-update-docker.sh +++ b/scripts/e2e/multi-node-update-docker.sh @@ -438,7 +438,9 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); let last = "timeout"; while (Date.now() < deadline) { try { - const response = await fetch(url); + // Keep each request inside the outer probe budget so a stalled fetch cannot outlive it. + const remainingMs = Math.max(1, deadline - Date.now()); + const response = await fetch(url, { signal: AbortSignal.timeout(remainingMs) }); if (response.ok) { process.exit(0); } @@ -446,7 +448,11 @@ while (Date.now() < deadline) { } catch (error) { last = error instanceof Error ? error.message : String(error); } - await sleep(500); + const remainingDelayMs = deadline - Date.now(); + if (remainingDelayMs <= 0) { + break; + } + await sleep(Math.min(500, remainingDelayMs)); } console.error(last); process.exit(1); diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 09701fa9a11b..fb077580f449 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -13,7 +13,11 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; -import { describe, expect, it } from "vitest"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); const HELPER_PATH = "scripts/lib/docker-build.sh"; const DOCKER_ALL_SCHEDULER_PATH = "scripts/test-docker-all.mjs"; @@ -4041,6 +4045,58 @@ heartbeat_elapsed="\${BASH_REMATCH[1]}" expect(runner).not.toContain("WARNING: Gateway status probe failed"); }); + it("keeps a stalled multi-node health request inside the probe deadline", () => { + const runner = readFileSync(MULTI_NODE_UPDATE_DOCKER_E2E_PATH, "utf8"); + const startMarker = "if PORT=18789 node < now;", + "Object.defineProperty(AbortSignal, 'timeout', { value(delayMs) {", + " const controller = new AbortController();", + " realSetTimeout(() => {", + " now += delayMs;", + " controller.abort(new DOMException('health deadline elapsed', 'TimeoutError'));", + " }, 0);", + " return controller.signal;", + "} });", + "globalThis.fetch = async (_url, init = {}) => await new Promise((_resolve, reject) => {", + " init.signal.addEventListener('abort', () => {", + " process.stderr.write('hung fetch aborted\\n');", + " reject(init.signal.reason);", + " }, { once: true });", + "});", + ].join("\n"), + ); + + const result = spawnSync( + process.execPath, + ["--import", pathToFileURL(preloadPath).href, "--input-type=module", "--eval", probe], + { + encoding: "utf8", + env: { ...process.env, PORT: "18789" }, + timeout: 5_000, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr.match(/hung fetch aborted/gu)).toHaveLength(1); + expect(result.stderr).toContain("health deadline elapsed"); + }); + it("caps package acceptance legacy compatibility at 2026.4.25", () => { const doctorScenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8"); const updateChannel = readFileSync(UPDATE_CHANNEL_SWITCH_DOCKER_E2E_PATH, "utf8");