|
|
|
|
@@ -321,7 +321,9 @@ async function runLoopWithStart(params: {
|
|
|
|
|
|
|
|
|
|
async function waitForStart(started: Promise<void>) {
|
|
|
|
|
await started;
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function waitForLoopCondition(predicate: () => boolean, message: string) {
|
|
|
|
|
@@ -330,7 +332,9 @@ async function waitForLoopCondition(predicate: () => boolean, message: string) {
|
|
|
|
|
if (predicate()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
throw new Error(message);
|
|
|
|
|
}
|
|
|
|
|
@@ -412,13 +416,19 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(consumeGatewayRestartIntentPayloadSync).toHaveBeenCalledOnce();
|
|
|
|
|
expect(markGatewayDraining).toHaveBeenCalledOnce();
|
|
|
|
|
@@ -426,7 +436,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
expectRestartCloseCall(closeFirst, 90_000);
|
|
|
|
|
await startedSecond;
|
|
|
|
|
expect(start).toHaveBeenCalledTimes(2);
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
sigint();
|
|
|
|
|
await expect(exited).resolves.toBe(0);
|
|
|
|
|
@@ -449,8 +461,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveTasks).toHaveBeenCalledWith(2_500);
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(2_500);
|
|
|
|
|
@@ -471,8 +487,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expectRestartCloseCall(close, 15_000);
|
|
|
|
|
expect(start).toHaveBeenCalledTimes(2);
|
|
|
|
|
@@ -494,8 +514,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(undefined);
|
|
|
|
|
expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, { mode: "compacting" });
|
|
|
|
|
@@ -526,8 +550,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(undefined);
|
|
|
|
|
expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, { mode: "compacting" });
|
|
|
|
|
@@ -557,8 +585,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveTasks).toHaveBeenCalledWith(90_000);
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(90_000);
|
|
|
|
|
@@ -606,8 +638,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveTasks).not.toHaveBeenCalled();
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).not.toHaveBeenCalled();
|
|
|
|
|
@@ -661,8 +697,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigterm();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveTasks).not.toHaveBeenCalled();
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).not.toHaveBeenCalled();
|
|
|
|
|
@@ -763,13 +803,17 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
expect(start).toHaveBeenCalledTimes(1);
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
|
|
|
|
|
await startedSecond;
|
|
|
|
|
expect(start).toHaveBeenCalledTimes(2);
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(abortEmbeddedAgentRun).toHaveBeenCalledWith(undefined, { mode: "compacting" });
|
|
|
|
|
expect(waitForActiveTasks).toHaveBeenCalledWith(1_234);
|
|
|
|
|
@@ -799,7 +843,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
sigusr1();
|
|
|
|
|
|
|
|
|
|
await startedThird;
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
expectRestartCloseCall(closeSecond, 1_234);
|
|
|
|
|
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(2);
|
|
|
|
|
expect(markGatewayDraining).toHaveBeenCalledTimes(2);
|
|
|
|
|
@@ -863,7 +909,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
@@ -950,7 +998,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigint();
|
|
|
|
|
@@ -980,7 +1030,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
@@ -1040,7 +1092,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
@@ -1098,7 +1152,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
@@ -1111,7 +1167,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
),
|
|
|
|
|
"expected failed restart startup to be logged",
|
|
|
|
|
);
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
expect(start).toHaveBeenCalledTimes(2);
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
@@ -1151,8 +1209,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForActiveTasks).toHaveBeenCalledWith(DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS);
|
|
|
|
|
expect(waitForActiveEmbeddedRuns).toHaveBeenCalledWith(DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS);
|
|
|
|
|
@@ -1171,7 +1233,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(scheduleGatewaySigusr1Restart).toHaveBeenCalledWith({
|
|
|
|
|
delayMs: 0,
|
|
|
|
|
@@ -1196,7 +1260,9 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(1);
|
|
|
|
|
expect(scheduleGatewaySigusr1Restart).not.toHaveBeenCalled();
|
|
|
|
|
@@ -1238,8 +1304,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigint = captureSignal("SIGINT");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(consumeGatewaySigusr1RestartAuthorization).toHaveBeenCalledOnce();
|
|
|
|
|
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledOnce();
|
|
|
|
|
@@ -1400,15 +1470,21 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
|
|
|
|
lockPort: 18789,
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
sigusr1();
|
|
|
|
|
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
expect(acquireGatewayLock).toHaveBeenNthCalledWith(1, { port: 18789 });
|
|
|
|
|
expect(acquireGatewayLock).toHaveBeenNthCalledWith(2, { port: 18789 });
|
|
|
|
|
expect(acquireGatewayLock).toHaveBeenNthCalledWith(3, { port: 18789 });
|
|
|
|
|
@@ -1557,12 +1633,16 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
.mockResolvedValueOnce({ close: closeSecond });
|
|
|
|
|
|
|
|
|
|
await runLoopWithStart({ start, runtime, lockPort: 18789, waitForHealthyChild });
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
const sigusr1 = captureSignal("SIGUSR1");
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(waitForHealthyChild).toHaveBeenCalledWith(18789, 8888, "127.0.0.1");
|
|
|
|
|
expect(kill).toHaveBeenCalledTimes(1);
|
|
|
|
|
@@ -1613,8 +1693,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
const sigterm = captureSignal("SIGTERM");
|
|
|
|
|
|
|
|
|
|
sigusr1();
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(gatewayLog.error).toHaveBeenCalledWith(
|
|
|
|
|
"SIGUSR1 handler failed: lifecycle module corrupted",
|
|
|
|
|
@@ -1642,8 +1726,12 @@ describe("runGatewayLoop", () => {
|
|
|
|
|
sigusr1();
|
|
|
|
|
// The catch handler clears the restart token from the eagerly-loaded
|
|
|
|
|
// lifecycle runtime, so wait for the async signal body to reject.
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve) => {
|
|
|
|
|
setImmediate(resolve);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(gatewayLog.error).toHaveBeenCalledWith(
|
|
|
|
|
"SIGUSR1 handler failed: sigusr1 lifecycle import failed",
|
|
|
|
|
|