mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-19 23:01:40 +00:00
fix(tasks): fail closed on registry restore errors (#105031)
This commit is contained in:
@@ -1365,6 +1365,83 @@ describe("runGatewayLoop", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the process alive and retries after task-registry restore fails during restart", async () => {
|
||||
vi.clearAllMocks();
|
||||
reloadTaskRegistryFromStore.mockReset();
|
||||
reloadTaskRegistryFromStore.mockImplementationOnce(() => {
|
||||
throw new Error("task registry restore failed");
|
||||
});
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue(undefined);
|
||||
respawnGatewayProcessForUpdate.mockReturnValue({
|
||||
mode: "disabled",
|
||||
detail: "OPENCLAW_NO_RESPAWN",
|
||||
});
|
||||
|
||||
try {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const closeFirst = createCloseMock();
|
||||
const closeSecond = createCloseMock();
|
||||
const { runtime, exited } = createRuntimeWithExitSignal();
|
||||
let resolveSecondStart: (() => void) | null = null;
|
||||
const startedSecond = new Promise<void>((resolve) => {
|
||||
resolveSecondStart = resolve;
|
||||
});
|
||||
const start = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ close: closeFirst })
|
||||
.mockImplementationOnce(async () => {
|
||||
resolveSecondStart?.();
|
||||
return { close: closeSecond };
|
||||
});
|
||||
|
||||
const { runGatewayLoop } = await import("./run-loop.js");
|
||||
void runGatewayLoop({
|
||||
start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
|
||||
runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
|
||||
});
|
||||
await waitForLoopCondition(
|
||||
() => start.mock.calls.length === 1,
|
||||
"expected initial gateway start",
|
||||
);
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
const sigterm = captureSignal("SIGTERM");
|
||||
|
||||
try {
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() =>
|
||||
gatewayLog.error.mock.calls.some(([message]) =>
|
||||
String(message).includes("gateway startup failed: task registry restore failed."),
|
||||
),
|
||||
"expected failed task-registry restore to be logged",
|
||||
);
|
||||
|
||||
expectRestartCloseCall(closeFirst, 90_000);
|
||||
expect(reloadTaskRegistryFromStore).toHaveBeenCalledTimes(1);
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(runtime.exit).not.toHaveBeenCalled();
|
||||
|
||||
sigusr1();
|
||||
await startedSecond;
|
||||
|
||||
expect(reloadTaskRegistryFromStore).toHaveBeenCalledTimes(2);
|
||||
expect(start).toHaveBeenCalledTimes(2);
|
||||
expect(runtime.exit).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
sigterm();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
}
|
||||
|
||||
expect(closeSecond).toHaveBeenCalledWith({
|
||||
reason: "gateway stopping",
|
||||
restartExpectedMs: null,
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
reloadTaskRegistryFromStore.mockReset();
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the default restart drain timeout when config omits deferralTimeoutMs", async () => {
|
||||
vi.clearAllMocks();
|
||||
loadConfig.mockReturnValue({ gateway: { reload: {} } });
|
||||
|
||||
@@ -918,11 +918,13 @@ export async function runGatewayLoop(params: {
|
||||
// SIGTERM/SIGINT still exit after a graceful shutdown.
|
||||
let isFirstStart = true;
|
||||
for (;;) {
|
||||
await onIteration();
|
||||
// The restart hook reopens admission before reloading durable state. Clear
|
||||
// its local mirror first so a failed reload cannot skip the next drain.
|
||||
restartDrainingMarked = false;
|
||||
startupStartedAt = Date.now();
|
||||
let startupFailedBeforeServerHandle = false;
|
||||
try {
|
||||
await onIteration();
|
||||
startupStartedAt = Date.now();
|
||||
await params.beginBoot?.(startupStartedAt);
|
||||
server = await params.start({ startupStartedAt });
|
||||
startupFailedWithoutServerHandle = false;
|
||||
|
||||
Reference in New Issue
Block a user