From 514b3365b54c8b3493eaf8a94198b7c04ea34aec Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 21 Jun 2026 23:28:49 +0800 Subject: [PATCH] fix(deadcode): move restart sentinels to sqlite --- src/agents/openclaw-gateway-tool.test.ts | 23 +- src/agents/tools/gateway-tool.test.ts | 14 +- src/agents/tools/gateway-tool.ts | 11 +- .../reply/commands-session-restart.test.ts | 18 +- src/auto-reply/reply/commands-session.ts | 20 +- src/cli/update-cli.test.ts | 40 ++-- .../server-methods/config-write-flow.ts | 15 +- .../server-methods/config.shared-auth.test.ts | 4 +- src/gateway/server-methods/config.ts | 4 +- src/gateway/server-methods/update.test.ts | 5 +- src/gateway/server-methods/update.ts | 9 +- src/gateway/server-restart-sentinel.test.ts | 12 +- src/gateway/server-restart-sentinel.ts | 8 +- .../server-startup-post-attach.test.ts | 89 +++++--- src/gateway/server-startup-post-attach.ts | 68 +----- .../server.roles-allowlist-update.test.ts | 12 +- src/infra/restart-sentinel.test.ts | 160 +++++++++++--- src/infra/restart-sentinel.ts | 174 +++++++++++++--- src/infra/update-control-plane-sentinel.ts | 4 +- .../update-managed-service-handoff.test.ts | 177 ++++++++++++++-- src/infra/update-managed-service-handoff.ts | 196 +++++++++++++++--- 21 files changed, 757 insertions(+), 306 deletions(-) diff --git a/src/agents/openclaw-gateway-tool.test.ts b/src/agents/openclaw-gateway-tool.test.ts index 1a764d64e95..39602625a91 100644 --- a/src/agents/openclaw-gateway-tool.test.ts +++ b/src/agents/openclaw-gateway-tool.test.ts @@ -3,10 +3,11 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js"; import { GatewayClientRequestError } from "../gateway/client.js"; +import { readRestartSentinel } from "../infra/restart-sentinel.js"; import { testing as restartTesting } from "../infra/restart.js"; import { withEnvAsync } from "../test-utils/env.js"; -import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js"; import { createGatewayTool } from "./tools/gateway-tool.js"; import { callGatewayTool } from "./tools/gateway.js"; @@ -332,13 +333,9 @@ describe("gateway tool", () => { }); expect(restartSignalKillCalls()).toHaveLength(0); - const sentinelPath = path.join(stateDir, "restart-sentinel.json"); - const raw = await fs.readFile(sentinelPath, "utf-8"); - const parsed = JSON.parse(raw) as { - payload?: { kind?: string; doctorHint?: string | null }; - }; - expect(parsed.payload?.kind).toBe("restart"); - expect(parsed.payload?.doctorHint).toBe( + const sentinel = await readRestartSentinel(); + expect(sentinel?.payload.kind).toBe("restart"); + expect(sentinel?.payload.doctorHint).toBe( "Recommended follow-up: run openclaw --profile isolated doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.", ); }, @@ -507,15 +504,13 @@ describe("gateway tool", () => { it("distinguishes explicit terminal array consent from indexed consent", () => { expect(normalizeConfigPatchReplacePath("bindings[]")).toBe("bindings"); expect(normalizeConfigPatchReplacePath("bindings[0]")).toBe("bindings[0]"); - expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe( - "agents.list[].skills", - ); + expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe("agents.list[].skills"); expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[]"))).toBe( "bindings", ); - expect( - normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]")), - ).toBe("bindings[0]"); + expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]"))).toBe( + "bindings[0]", + ); }); it("rejects config.patch when it changes safe bin approval paths", async () => { diff --git a/src/agents/tools/gateway-tool.test.ts b/src/agents/tools/gateway-tool.test.ts index 9b3ecc814eb..9ea7c79abbf 100644 --- a/src/agents/tools/gateway-tool.test.ts +++ b/src/agents/tools/gateway-tool.test.ts @@ -12,7 +12,7 @@ const { formatDoctorNonInteractiveHintMock, isRestartEnabledMock, callGatewayToolMock, - removeRestartSentinelFileMock, + clearRestartSentinelMock, scheduleGatewaySigusr1RestartMock, writeRestartSentinelMock, } = vi.hoisted(() => ({ @@ -30,8 +30,8 @@ const { () => "Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.", ), - writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/restart"), - removeRestartSentinelFileMock: vi.fn(async (_path: string | null | undefined) => undefined), + writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => undefined), + clearRestartSentinelMock: vi.fn(async () => undefined), scheduleGatewaySigusr1RestartMock: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({ ok: true, pid: 123, @@ -59,7 +59,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => { return { ...actual, formatDoctorNonInteractiveHint: formatDoctorNonInteractiveHintMock, - removeRestartSentinelFile: removeRestartSentinelFileMock, + clearRestartSentinel: clearRestartSentinelMock, writeRestartSentinel: writeRestartSentinelMock, }; }); @@ -115,8 +115,8 @@ describe("gateway tool restart continuation", () => { "Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.", ); writeRestartSentinelMock.mockReset(); - writeRestartSentinelMock.mockResolvedValue("/tmp/restart"); - removeRestartSentinelFileMock.mockClear(); + writeRestartSentinelMock.mockResolvedValue(undefined); + clearRestartSentinelMock.mockClear(); scheduleGatewaySigusr1RestartMock.mockReset(); scheduleGatewaySigusr1RestartMock.mockReturnValue({ ok: true, @@ -354,7 +354,7 @@ describe("gateway tool restart continuation", () => { await scheduledArgs.emitHooks?.beforeEmit?.(); await scheduledArgs.emitHooks?.afterEmitRejected?.(); - expect(removeRestartSentinelFileMock).toHaveBeenCalledWith("/tmp/restart"); + expect(clearRestartSentinelMock).toHaveBeenCalledOnce(); }); it("uses the runtime session for update.run continuation routing (#86742)", async () => { diff --git a/src/agents/tools/gateway-tool.ts b/src/agents/tools/gateway-tool.ts index 1f4f0b5a652..c38fb0f49ac 100644 --- a/src/agents/tools/gateway-tool.ts +++ b/src/agents/tools/gateway-tool.ts @@ -19,8 +19,8 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { GatewayClientRequestError } from "../../gateway/client.js"; import { buildRestartSuccessContinuation, + clearRestartSentinel, formatDoctorNonInteractiveHint, - removeRestartSentinelFile, type RestartSentinelPayload, writeRestartSentinel, } from "../../infra/restart-sentinel.js"; @@ -493,7 +493,7 @@ export function createGatewayTool(opts?: { log.info( `gateway tool: restart requested (delayMs=${delayMs ?? "default"}, reason=${reason ?? "none"})`, ); - let sentinelPath: string | null = null; + let sentinelWritten = false; const scheduled = scheduleGatewaySigusr1Restart({ delayMs, reason, @@ -502,10 +502,13 @@ export function createGatewayTool(opts?: { sessionKey, emitHooks: { beforeEmit: async () => { - sentinelPath = await writeRestartSentinel(payload); + await writeRestartSentinel(payload); + sentinelWritten = true; }, afterEmitRejected: async () => { - await removeRestartSentinelFile(sentinelPath); + if (sentinelWritten) { + await clearRestartSentinel(); + } }, }, }); diff --git a/src/auto-reply/reply/commands-session-restart.test.ts b/src/auto-reply/reply/commands-session-restart.test.ts index 40f44efefcb..4e72991f6d1 100644 --- a/src/auto-reply/reply/commands-session-restart.test.ts +++ b/src/auto-reply/reply/commands-session-restart.test.ts @@ -7,7 +7,7 @@ import type { HandleCommandsParams } from "./commands-types.js"; type ScheduleGatewayRestartArgs = Parameters[0]; const mocks = vi.hoisted(() => ({ - unlink: vi.fn(async (_path: string) => undefined), + clearRestartSentinel: vi.fn(async () => undefined), isRestartEnabled: vi.fn(() => true), extractDeliveryInfo: vi.fn(() => ({ deliveryContext: { @@ -21,20 +21,13 @@ const mocks = vi.hoisted(() => ({ () => "Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.", ), - writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/sentinel.json"), + writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined), scheduleGatewaySigusr1Restart: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({ scheduled: true, })), triggerOpenClawRestart: vi.fn(() => ({ ok: true, method: "launchctl" })), })); -vi.mock("node:fs/promises", () => ({ - default: { - unlink: mocks.unlink, - }, - unlink: mocks.unlink, -})); - vi.mock("../../config/commands.flags.js", () => ({ isRestartEnabled: mocks.isRestartEnabled, })); @@ -67,6 +60,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => { ); return { ...actual, + clearRestartSentinel: mocks.clearRestartSentinel, formatDoctorNonInteractiveHint: mocks.formatDoctorNonInteractiveHint, writeRestartSentinel: mocks.writeRestartSentinel, }; @@ -119,7 +113,7 @@ describe("handleRestartCommand", () => { beforeEach(() => { mocks.isRestartEnabled.mockReset(); mocks.isRestartEnabled.mockReturnValue(true); - mocks.unlink.mockClear(); + mocks.clearRestartSentinel.mockClear(); mocks.extractDeliveryInfo.mockClear(); mocks.formatDoctorNonInteractiveHint.mockClear(); mocks.writeRestartSentinel.mockClear(); @@ -218,7 +212,7 @@ describe("handleRestartCommand", () => { expect(mocks.triggerOpenClawRestart).not.toHaveBeenCalled(); }); - it("removes the success sentinel when fallback restart fails", async () => { + it("clears the success sentinel when fallback restart fails", async () => { mocks.triggerOpenClawRestart.mockReturnValueOnce({ ok: false, method: "launchctl", @@ -227,6 +221,6 @@ describe("handleRestartCommand", () => { const result = await handleRestartCommand(restartCommandParams(), true); expect(result?.reply?.text).toContain("Restart failed"); - expect(mocks.unlink).toHaveBeenCalledWith("/tmp/sentinel.json"); + expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce(); }); }); diff --git a/src/auto-reply/reply/commands-session.ts b/src/auto-reply/reply/commands-session.ts index 10a27451c4d..4abbe79af75 100644 --- a/src/auto-reply/reply/commands-session.ts +++ b/src/auto-reply/reply/commands-session.ts @@ -21,8 +21,8 @@ import { getSessionBindingService } from "../../infra/outbound/session-binding-s import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js"; import { buildRestartSuccessContinuation, + clearRestartSentinel, formatDoctorNonInteractiveHint, - removeRestartSentinelFile, type RestartSentinelPayload, writeRestartSentinel, } from "../../infra/restart-sentinel.js"; @@ -703,7 +703,7 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0; const sentinelPayload = buildRestartCommandSentinel(params); if (hasSigusr1Listener) { - let sentinelPath: string | null = null; + let sentinelWritten = false; scheduleGatewaySigusr1Restart({ reason: "/restart", // Sibling session-routing guard: /restart writes a session-scoped sentinel @@ -713,10 +713,13 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm emitHooks: sentinelPayload ? { beforeEmit: async () => { - sentinelPath = await writeRestartSentinel(sentinelPayload); + await writeRestartSentinel(sentinelPayload); + sentinelWritten = true; }, afterEmitRejected: async () => { - await removeRestartSentinelFile(sentinelPath); + if (sentinelWritten) { + await clearRestartSentinel(); + } }, } : undefined, @@ -728,10 +731,11 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm }, }; } - let sentinelPath: string | null = null; + let sentinelWritten = false; try { if (sentinelPayload) { - sentinelPath = await writeRestartSentinel(sentinelPayload); + await writeRestartSentinel(sentinelPayload); + sentinelWritten = true; } } catch (err) { logVerbose(`failed to write /restart sentinel: ${String(err)}`); @@ -744,7 +748,9 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm } const restartMethod = triggerOpenClawRestart(); if (!restartMethod.ok) { - await removeRestartSentinelFile(sentinelPath); + if (sentinelWritten) { + await clearRestartSentinel(); + } const detail = restartMethod.detail ? ` Details: ${restartMethod.detail}` : ""; return { shouldContinue: false, diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index 96dcbfc972a..95d7864d9b7 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -366,6 +366,7 @@ const { updateCommand, updateFinalizeCommand, updateStatusCommand, updateWizardC const updateCliShared = await import("./update-cli/shared.js"); const { ensureGitCheckout, resolveGitInstallDir } = updateCliShared; const { spawnSync } = await import("node:child_process"); +const { readRestartSentinel } = await import("../infra/restart-sentinel.js"); function requireValue(value: T | undefined, label: string): T { if (value === undefined) { @@ -5891,23 +5892,17 @@ describe("update-cli", () => { }, ); - const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8"); - const sentinel = JSON.parse(raw) as { - payload?: { - status?: string; - message?: string | null; - continuation?: { kind?: string; message?: string }; - stats?: { mode?: string; after?: { version?: string | null } }; - }; - }; - expect(sentinel.payload?.status).toBe("ok"); - expect(sentinel.payload?.message).toBe("Update requested from the agent."); - expect(sentinel.payload?.continuation).toEqual({ + const sentinel = await readRestartSentinel({ + OPENCLAW_STATE_DIR: stateDir, + } as NodeJS.ProcessEnv); + expect(sentinel?.payload.status).toBe("ok"); + expect(sentinel?.payload.message).toBe("Update requested from the agent."); + expect(sentinel?.payload.continuation).toEqual({ kind: "agentTurn", message: "Check the running version and finish the update report.", }); - expect(sentinel.payload?.stats?.mode).toBe("npm"); - expect(sentinel.payload?.stats?.after?.version).toBe("2026.4.24"); + expect(sentinel?.payload.stats?.mode).toBe("npm"); + expect(sentinel?.payload.stats?.after?.version).toBe("2026.4.24"); }); it("marks the control-plane update sentinel failed when restart health verification fails", async () => { @@ -5966,17 +5961,12 @@ describe("update-cli", () => { }, ); - const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8"); - const sentinel = JSON.parse(raw) as { - payload?: { - status?: string; - continuation?: unknown; - stats?: { reason?: string | null }; - }; - }; - expect(sentinel.payload?.status).toBe("error"); - expect(sentinel.payload?.stats?.reason).toBe("restart-unhealthy"); - expect(sentinel.payload?.continuation).toBeUndefined(); + const sentinel = await readRestartSentinel({ + OPENCLAW_STATE_DIR: stateDir, + } as NodeJS.ProcessEnv); + expect(sentinel?.payload.status).toBe("error"); + expect(sentinel?.payload.stats?.reason).toBe("restart-unhealthy"); + expect(sentinel?.payload.continuation).toBeUndefined(); expect(defaultRuntime.exit).toHaveBeenCalledWith(1); }); diff --git a/src/gateway/server-methods/config-write-flow.ts b/src/gateway/server-methods/config-write-flow.ts index a5461d4e223..1616a6569d1 100644 --- a/src/gateway/server-methods/config-write-flow.ts +++ b/src/gateway/server-methods/config-write-flow.ts @@ -203,13 +203,12 @@ function buildConfigRestartSentinelPayload(params: { }; } -async function tryWriteRestartSentinelPayload( - payload: RestartSentinelPayload, -): Promise { +async function tryWriteRestartSentinelPayload(payload: RestartSentinelPayload): Promise { try { - return await writeRestartSentinel(payload); + await writeRestartSentinel(payload); + return true; } catch { - return null; + return false; } } @@ -256,7 +255,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: { context?: GatewayRequestContext; }): Promise<{ payload: RestartSentinelPayload; - sentinelPath: string | null; + sentinelPersisted: boolean; restart: ReturnType | undefined; }> { const { sessionKey, note, restartDelayMs, deliveryContext, threadId } = @@ -270,7 +269,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: { threadId, note, }); - const sentinelPath = await tryWriteRestartSentinelPayload(payload); + const sentinelPersisted = await tryWriteRestartSentinelPayload(payload); const restart = shouldScheduleDirectConfigRestart({ changedPaths: params.changedPaths, nextConfig: params.nextConfig, @@ -291,5 +290,5 @@ export async function resolveGatewayConfigRestartWriteResult(params: { `${params.mode} restart coalesced ${formatControlPlaneActor(params.actor)} delayMs=${restart.delayMs}`, ); } - return { payload, sentinelPath, restart }; + return { payload, sentinelPersisted, restart }; } diff --git a/src/gateway/server-methods/config.shared-auth.test.ts b/src/gateway/server-methods/config.shared-auth.test.ts index 8adfc1145c3..51bc4b22400 100644 --- a/src/gateway/server-methods/config.shared-auth.test.ts +++ b/src/gateway/server-methods/config.shared-auth.test.ts @@ -21,9 +21,7 @@ const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ coalesced: false, })); const restartSentinelMocks = vi.hoisted(() => ({ - writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => { - return "/tmp/restart-sentinel.json"; - }), + writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined), })); vi.mock("../../config/config.js", async () => { diff --git a/src/gateway/server-methods/config.ts b/src/gateway/server-methods/config.ts index 90e957b34ea..2ab91843515 100644 --- a/src/gateway/server-methods/config.ts +++ b/src/gateway/server-methods/config.ts @@ -594,7 +594,7 @@ async function respondWithConfigRestartWrite(params: { uiHints: ConfigRedactionHints; }): Promise { clearConfigSchemaResponseCache(); - const { payload, sentinelPath, restart } = await resolveGatewayConfigRestartWriteResult({ + const { payload, sentinelPersisted, restart } = await resolveGatewayConfigRestartWriteResult({ requestParams: params.requestParams, kind: params.kind, mode: params.mode, @@ -612,7 +612,7 @@ async function respondWithConfigRestartWrite(params: { config: redactConfigObject(params.writeResult.config, params.uiHints), restart, sentinel: { - path: sentinelPath, + persisted: sentinelPersisted, payload, }, }, diff --git a/src/gateway/server-methods/update.test.ts b/src/gateway/server-methods/update.test.ts index 7ce693c2749..fdd121366db 100644 --- a/src/gateway/server-methods/update.test.ts +++ b/src/gateway/server-methods/update.test.ts @@ -49,7 +49,7 @@ type UpdateRunPayload = { ok: boolean; result?: { status?: string; reason?: string; mode?: string }; handoff?: { status?: string; command?: string; message?: string }; - sentinel?: { path?: string | null }; + sentinel?: { persisted?: boolean }; restart?: unknown; }; @@ -97,7 +97,6 @@ vi.mock("../../infra/restart-sentinel.js", async () => { ...(actual as Record), writeRestartSentinel: async (payload: RestartSentinelPayload) => { capturedPayload = payload; - return "/tmp/sentinel.json"; }, }; }); @@ -462,7 +461,7 @@ describe("update.run restart scheduling", () => { pid: 12345, command: "openclaw update --yes --timeout 1800", }); - expect(payload?.sentinel?.path).toBe("/tmp/sentinel.json"); + expect(payload?.sentinel?.persisted).toBe(true); const sentinel = readCapturedPayload(); expect(sentinel.kind).toBe("update"); expect(sentinel.status).toBe("skipped"); diff --git a/src/gateway/server-methods/update.ts b/src/gateway/server-methods/update.ts index d24b2c28aa4..1e15bf69c40 100644 --- a/src/gateway/server-methods/update.ts +++ b/src/gateway/server-methods/update.ts @@ -340,12 +340,13 @@ export const updateHandlers: GatewayRequestHandlers = { meta: sentinelMeta, }); - let sentinelPath: string | null; + let sentinelPersisted: boolean; try { - sentinelPath = await writeRestartSentinel(payload); + await writeRestartSentinel(payload); + sentinelPersisted = true; recordLatestUpdateRestartSentinel(payload); } catch { - sentinelPath = null; + sentinelPersisted = false; } // Only restart the gateway when the update actually succeeded. @@ -391,7 +392,7 @@ export const updateHandlers: GatewayRequestHandlers = { ...(handoff ? { handoff } : {}), restart, sentinel: { - path: sentinelPath, + persisted: sentinelPersisted, payload, }, }, diff --git a/src/gateway/server-restart-sentinel.test.ts b/src/gateway/server-restart-sentinel.test.ts index 6f00897bde9..464caca3122 100644 --- a/src/gateway/server-restart-sentinel.test.ts +++ b/src/gateway/server-restart-sentinel.test.ts @@ -42,8 +42,7 @@ const mocks = vi.hoisted(() => { }), ), finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null), - removeRestartSentinelFile: vi.fn(async () => undefined), - resolveRestartSentinelPath: vi.fn(() => "/tmp/restart-sentinel.json"), + clearRestartSentinel: vi.fn(async () => undefined), formatRestartSentinelMessage: vi.fn(() => "restart message"), summarizeRestartSentinel: vi.fn(() => "restart summary"), resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"), @@ -183,8 +182,7 @@ vi.mock("../agents/agent-scope.js", async () => { vi.mock("../infra/restart-sentinel.js", () => ({ finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion, readRestartSentinel: mocks.readRestartSentinel, - removeRestartSentinelFile: mocks.removeRestartSentinelFile, - resolveRestartSentinelPath: mocks.resolveRestartSentinelPath, + clearRestartSentinel: mocks.clearRestartSentinel, formatRestartSentinelMessage: mocks.formatRestartSentinelMessage, summarizeRestartSentinel: mocks.summarizeRestartSentinel, })); @@ -427,7 +425,7 @@ describe("scheduleRestartSentinelWake", () => { mocks.recoverPendingSessionDeliveries.mockClear(); mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset(); mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null); - mocks.removeRestartSentinelFile.mockClear(); + mocks.clearRestartSentinel.mockClear(); mocks.injectTimestamp.mockClear(); mocks.timestampOptsFromConfig.mockClear(); mocks.recordInboundSessionAndDispatchReply.mockReset(); @@ -1295,7 +1293,7 @@ describe("scheduleRestartSentinelWake", () => { await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.removeRestartSentinelFile).not.toHaveBeenCalled(); + expect(mocks.clearRestartSentinel).not.toHaveBeenCalled(); expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled(); expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", { source: "restart-sentinel", @@ -1359,7 +1357,7 @@ describe("scheduleRestartSentinelWake", () => { await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.removeRestartSentinelFile).toHaveBeenCalledWith("/tmp/restart-sentinel.json"); + expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce(); expect(getLatestUpdateRestartSentinel()).toEqual(payload); }); diff --git a/src/gateway/server-restart-sentinel.ts b/src/gateway/server-restart-sentinel.ts index 29dee193a71..c80ff64107b 100644 --- a/src/gateway/server-restart-sentinel.ts +++ b/src/gateway/server-restart-sentinel.ts @@ -18,13 +18,12 @@ import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/de import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; import { resolveOutboundTarget } from "../infra/outbound/targets.js"; import { + clearRestartSentinel, finalizeUpdateRestartSentinelRunningVersion, formatRestartSentinelMessage, readRestartSentinel, - removeRestartSentinelFile, type RestartSentinelContinuation, type RestartSentinelPayload, - resolveRestartSentinelPath, summarizeRestartSentinel, } from "../infra/restart-sentinel.js"; import { @@ -449,7 +448,6 @@ async function loadRestartSentinelStartupTask(params: { if (!sentinel) { return null; } - const sentinelPath = resolveRestartSentinelPath(); const payload = sentinel.payload; if (payload.kind === "update") { recordLatestUpdateRestartSentinel(payload); @@ -494,7 +492,7 @@ async function loadRestartSentinelStartupTask(params: { continuationKind: payload.continuation.kind, }); } - await removeRestartSentinelFile(sentinelPath); + await clearRestartSentinel(); return { status: "ran" as const }; } @@ -588,7 +586,7 @@ async function loadRestartSentinelStartupTask(params: { ); } - await removeRestartSentinelFile(sentinelPath); + await clearRestartSentinel(); const routedAgentTurnContinuation = payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined; if (!routedAgentTurnContinuation) { diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index e9bbab8985f..325364a52be 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -5,11 +5,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { writeRestartSentinel } from "../infra/restart-sentinel.js"; import type { PluginHookGatewayContext, PluginHookGatewayStartEvent, } from "../plugins/hook-types.js"; import type { PluginServicesHandle } from "../plugins/services.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { withEnvAsync } from "../test-utils/env.js"; const hoisted = vi.hoisted(() => { @@ -135,7 +137,9 @@ vi.mock("../config/paths.js", async () => { STATE_DIR: "/tmp/openclaw-state", resolveConfigPath: vi.fn(() => "/tmp/openclaw-state/openclaw.json"), resolveGatewayPort: vi.fn(() => 18789), - resolveStateDir: vi.fn(() => "/tmp/openclaw-state"), + resolveStateDir: vi.fn((env: NodeJS.ProcessEnv = process.env) => + env.OPENCLAW_STATE_DIR?.trim() ? actual.resolveStateDir(env) : "/tmp/openclaw-state", + ), }; }); @@ -295,6 +299,7 @@ function firstGatewayStartCall( describe("startGatewayPostAttachRuntime", () => { beforeEach(() => { + closeOpenClawStateDatabaseForTest(); vi.stubEnv("OPENCLAW_SKIP_CHANNELS", "0"); vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", "0"); hoisted.startPluginServices.mockClear(); @@ -325,6 +330,8 @@ describe("startGatewayPostAttachRuntime", () => { }); hoisted.scheduleRestartAbortedMainSessionRecovery.mockClear(); hoisted.scheduleRestartSentinelWake.mockClear(); + hoisted.refreshLatestUpdateRestartSentinel.mockReset(); + hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(null); hoisted.getAcpRuntimeBackend.mockReset(); hoisted.getAcpRuntimeBackend.mockReturnValue(null); hoisted.reconcilePendingSessionIdentities.mockClear(); @@ -355,6 +362,7 @@ describe("startGatewayPostAttachRuntime", () => { }); afterEach(() => { + closeOpenClawStateDatabaseForTest(); vi.useRealTimers(); vi.unstubAllEnvs(); }); @@ -556,62 +564,76 @@ describe("startGatewayPostAttachRuntime", () => { it("skips heavy restart sentinel refresh when no sentinel file exists", async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-no-sentinel-")); vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + hoisted.refreshLatestUpdateRestartSentinel.mockClear(); const result = await testing.refreshLatestUpdateRestartSentinelIfPresent(); expect(result).toBeNull(); expect(hoisted.refreshLatestUpdateRestartSentinel).not.toHaveBeenCalled(); + closeOpenClawStateDatabaseForTest(); fs.rmSync(stateDir, { recursive: true, force: true }); }); - it("refreshes the restart sentinel when the sentinel file exists", async () => { + it("refreshes the restart sentinel when the sentinel row exists", async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-")); - fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n"); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - const sentinel = { kind: "update", status: "ok", ts: 1 } as const; - hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel); + try { + await writeRestartSentinel( + { + kind: "update", + status: "ok", + ts: 1, + }, + { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv, + ); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + const sentinel = { kind: "update", status: "ok", ts: 1 } as const; + hoisted.refreshLatestUpdateRestartSentinel.mockClear(); + hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel); - const result = await testing.refreshLatestUpdateRestartSentinelIfPresent(); + const result = await testing.refreshLatestUpdateRestartSentinelIfPresent(); - expect(result).toBe(sentinel); - expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce(); - fs.rmSync(stateDir, { recursive: true, force: true }); + expect(result).toBe(sentinel); + expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce(); + } finally { + closeOpenClawStateDatabaseForTest(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } }); - it("expands tilde-based restart sentinel state paths", async () => { - const osHome = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-home-")); + it("detects restart sentinel rows in explicit state directories", async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-state-")); try { - const openclawHome = path.join(osHome, "openclaw-home"); - const stateDirFromHome = path.join(openclawHome, ".openclaw"); - fs.mkdirSync(stateDirFromHome, { recursive: true }); - fs.writeFileSync(path.join(stateDirFromHome, "restart-sentinel.json"), "{}\n"); + await writeRestartSentinel( + { + kind: "update", + status: "ok", + ts: 1, + }, + { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv, + ); expect( - await testing.hasRestartSentinelFileFast({ - HOME: osHome, - OPENCLAW_HOME: "~/openclaw-home", - } as NodeJS.ProcessEnv), - ).toBe(true); - - const backslashStateDir = path.resolve(`${osHome}\\openclaw-state`); - fs.mkdirSync(backslashStateDir, { recursive: true }); - fs.writeFileSync(path.join(backslashStateDir, "restart-sentinel.json"), "{}\n"); - - expect( - await testing.hasRestartSentinelFileFast({ - HOME: osHome, - OPENCLAW_STATE_DIR: "~\\openclaw-state", + await testing.hasRestartSentinelFast({ + OPENCLAW_STATE_DIR: stateDir, } as NodeJS.ProcessEnv), ).toBe(true); } finally { - fs.rmSync(osHome, { recursive: true, force: true }); + closeOpenClawStateDatabaseForTest(); + fs.rmSync(stateDir, { recursive: true, force: true }); } }); it("avoids sync filesystem probes while checking restart sentinel presence", async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-async-sentinel-")); try { - fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n"); + await writeRestartSentinel( + { + kind: "update", + status: "ok", + ts: 1, + }, + { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv, + ); const actualExistsSync = fs.existsSync; const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => { if (String(candidate).startsWith(stateDir)) { @@ -621,7 +643,7 @@ describe("startGatewayPostAttachRuntime", () => { }); try { await expect( - testing.hasRestartSentinelFileFast({ + testing.hasRestartSentinelFast({ OPENCLAW_STATE_DIR: stateDir, } as NodeJS.ProcessEnv), ).resolves.toBe(true); @@ -632,6 +654,7 @@ describe("startGatewayPostAttachRuntime", () => { existsSync.mockRestore(); } } finally { + closeOpenClawStateDatabaseForTest(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 6e03fe55526..6950b6594fb 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -1,8 +1,5 @@ // Gateway post-attach startup sidecars. // Schedules warmups, sentinels, update checks, memory backend, and plugin services. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import { monitorEventLoopDelay, performance } from "node:perf_hooks"; import { setTimeout as sleep } from "node:timers/promises"; import type { CliDeps } from "../cli/deps.types.js"; @@ -11,6 +8,7 @@ import type { GatewayTailscaleMode } from "../config/types.gateway.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { hasConfiguredInternalHooks } from "../hooks/configured.js"; import { isTruthyEnvValue } from "../infra/env.js"; +import { hasRestartSentinel } from "../infra/restart-sentinel.js"; import type { scheduleGatewayUpdateCheck } from "../infra/update-startup.js"; import type { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; import type { PluginHookGatewayCronService } from "../plugins/hook-types.js"; @@ -38,7 +36,6 @@ const DEFERRED_SIDECAR_START_DELAY_MS = 100; const SESSION_LOCK_CLEANUP_CONCURRENCY = 4; const SKIP_STARTUP_MODEL_PREWARM_ENV = "OPENCLAW_SKIP_STARTUP_MODEL_PREWARM"; const QMD_STARTUP_IDLE_DELAY_MS = 120_000; -const RESTART_SENTINEL_FILENAME = "restart-sentinel.json"; type Awaitable = T | Promise; @@ -506,67 +503,14 @@ function scheduleTranscriptsAutoStartSidecar(params: { }); } -async function pathExists(filePath: string): Promise { - try { - await fs.promises.access(filePath); - return true; - } catch { - return false; - } -} - -async function resolveRestartSentinelPathFast( - env: NodeJS.ProcessEnv = process.env, -): Promise { - const normalizePathEnv = (value: string | undefined) => { - const trimmed = value?.trim(); - return trimmed && trimmed !== "undefined" && trimmed !== "null" ? trimmed : undefined; - }; - const resolveRawOsHome = () => normalizePathEnv(env.HOME) ?? normalizePathEnv(env.USERPROFILE); - const expandHomePrefix = (input: string, home: string) => input.replace(/^~(?=$|[\\/])/, home); - const resolveHome = () => { - const explicitHome = normalizePathEnv(env.OPENCLAW_HOME); - if (explicitHome) { - const osHome = resolveRawOsHome() ?? os.homedir(); - return path.resolve(expandHomePrefix(explicitHome, osHome)); - } - return path.resolve(resolveRawOsHome() ?? os.homedir()); - }; - const resolveUserPath = (input: string) => { - const trimmed = input.trim(); - if (trimmed.startsWith("~")) { - return path.resolve(expandHomePrefix(trimmed, resolveHome())); - } - return path.resolve(trimmed); - }; - const override = normalizePathEnv(env.OPENCLAW_STATE_DIR); - if (override) { - return path.join(resolveUserPath(override), RESTART_SENTINEL_FILENAME); - } - const home = resolveHome(); - const newStateDir = path.join(home, ".openclaw"); - if (env.OPENCLAW_TEST_FAST === "1" || (await pathExists(newStateDir))) { - return path.join(newStateDir, RESTART_SENTINEL_FILENAME); - } - const legacyStateDir = path.join(home, ".clawdbot"); - if (await pathExists(legacyStateDir)) { - return path.join(legacyStateDir, RESTART_SENTINEL_FILENAME); - } - return path.join(newStateDir, RESTART_SENTINEL_FILENAME); -} - -async function hasRestartSentinelFileFast(env: NodeJS.ProcessEnv = process.env): Promise { - try { - return await pathExists(await resolveRestartSentinelPathFast(env)); - } catch { - return false; - } +async function hasRestartSentinelFast(env: NodeJS.ProcessEnv = process.env): Promise { + return await hasRestartSentinel(env); } async function refreshLatestUpdateRestartSentinelIfPresent(): Promise > | null> { - if (!(await hasRestartSentinelFileFast())) { + if (!(await hasRestartSentinelFast())) { return null; } return await (await loadGatewayRestartSentinelModule()).refreshLatestUpdateRestartSentinel(); @@ -930,7 +874,7 @@ export async function startGatewaySidecars(params: { if (!shouldCheckRestartSentinel()) { return; } - if (!(await hasRestartSentinelFileFast())) { + if (!(await hasRestartSentinelFast())) { return; } setTimeout(() => { @@ -1457,7 +1401,7 @@ export async function startGatewayPostAttachRuntime( export const testing = { providerAuthPrewarmStartDelayMs: PROVIDER_AUTH_PREWARM_START_DELAY_MS, - hasRestartSentinelFileFast, + hasRestartSentinelFast, prewarmConfiguredPrimaryModel, prewarmConfiguredPrimaryModelWithTimeout, refreshLatestUpdateRestartSentinelIfPresent, diff --git a/src/gateway/server.roles-allowlist-update.test.ts b/src/gateway/server.roles-allowlist-update.test.ts index 88fafbdcae0..cdde4679b30 100644 --- a/src/gateway/server.roles-allowlist-update.test.ts +++ b/src/gateway/server.roles-allowlist-update.test.ts @@ -10,7 +10,7 @@ import type { DeviceIdentity } from "../infra/device-identity.js"; import { loadOrCreateDeviceIdentity } from "../infra/device-identity.js"; import { approveDevicePairing, listDevicePairing } from "../infra/device-pairing.js"; import { approveNodePairing, requestNodePairing } from "../infra/node-pairing.js"; -import { resolveRestartSentinelPath } from "../infra/restart-sentinel.js"; +import { readRestartSentinel } from "../infra/restart-sentinel.js"; import { SUPERVISOR_HINT_ENV_VARS } from "../infra/supervisor-markers.js"; import { getActiveRuntimePluginRegistry } from "../plugins/active-runtime-registry.js"; import { @@ -417,13 +417,9 @@ describe("gateway update.run", () => { }, FAST_WAIT_OPTS); expect(sigusr1).toHaveBeenCalled(); - const sentinelPath = resolveRestartSentinelPath(); - const raw = await fs.readFile(sentinelPath, "utf-8"); - const parsed = JSON.parse(raw) as { - payload?: { kind?: string; stats?: { mode?: string } }; - }; - expect(parsed.payload?.kind).toBe("update"); - expect(parsed.payload?.stats?.mode).toBe("git"); + const sentinel = await readRestartSentinel(); + expect(sentinel?.payload.kind).toBe("update"); + expect(sentinel?.payload.stats?.mode).toBe("git"); } finally { process.off("SIGUSR1", sigusr1); } diff --git a/src/infra/restart-sentinel.test.ts b/src/infra/restart-sentinel.test.ts index 6c58af0273b..54f0fdc8a26 100644 --- a/src/infra/restart-sentinel.test.ts +++ b/src/infra/restart-sentinel.test.ts @@ -1,17 +1,28 @@ -// Covers restart sentinel persistence, summaries, and messages. import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; +// Covers restart sentinel persistence, summaries, and messages. +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { withEnvAsync } from "../test-utils/env.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; import { buildRestartSuccessContinuation, + clearRestartSentinel, finalizeUpdateRestartSentinelRunningVersion, formatDoctorNonInteractiveHint, formatRestartSentinelMessage, + hasRestartSentinel, markUpdateRestartSentinelFailure, readRestartSentinel, - resolveRestartSentinelPath, summarizeRestartSentinel, trimLogTail, writeRestartSentinel, @@ -25,28 +36,52 @@ import { buildUpdateRestartSentinelPayload } from "./update-restart-sentinel-pay async function withRestartSentinelStateDir(run: () => Promise): Promise { await withTempDir({ prefix: "openclaw-sentinel-" }, async (tempDir) => { - await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run); + try { + await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run); + } finally { + closeOpenClawStateDatabaseForTest(); + } }); } -async function expectPathMissing(targetPath: string): Promise { - try { - await fs.stat(targetPath); - } catch (error) { - expect(error).toBeInstanceOf(Error); - const statError = error as NodeJS.ErrnoException; - expect({ - code: statError.code, - path: statError.path, - syscall: statError.syscall, - }).toEqual({ - code: "ENOENT", - path: targetPath, - syscall: "stat", - }); - return; - } - throw new Error(`Expected path to be missing: ${targetPath}`); +type GatewayRestartSentinelDatabase = Pick; + +function readSentinelRow() { + const { db } = openOpenClawStateDatabase(); + const stateDb = getNodeSqliteKysely(db); + return executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select(["sentinel_key", "version", "kind", "status", "payload_json"]) + .where("sentinel_key", "=", "current"), + ); +} + +function insertSentinelRow(values: { version?: number; payloadJson: string }) { + const { db } = openOpenClawStateDatabase(); + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb.insertInto("gateway_restart_sentinel").values({ + sentinel_key: "current", + version: values.version ?? 1, + kind: "update", + status: "ok", + ts: Date.now(), + session_key: null, + thread_id: null, + delivery_channel: null, + delivery_to: null, + delivery_account_id: null, + message: null, + continuation_json: null, + doctor_hint: null, + stats_json: null, + payload_json: values.payloadJson, + updated_at_ms: Date.now(), + }), + ); } describe("restart sentinel", () => { @@ -63,8 +98,14 @@ describe("restart sentinel", () => { }, stats: { mode: "git" }, }; - const filePath = await writeRestartSentinel(payload); - expect(filePath).toBe(resolveRestartSentinelPath()); + await writeRestartSentinel(payload); + expect(readSentinelRow()).toMatchObject({ + sentinel_key: "current", + version: 1, + kind: "update", + status: "ok", + payload_json: JSON.stringify(payload), + }); const read = await readRestartSentinel(); expect(read?.payload.kind).toBe("update"); @@ -72,27 +113,84 @@ describe("restart sentinel", () => { }); }); + it("imports a legacy file sentinel into sqlite once", async () => { + await withRestartSentinelStateDir(async () => { + const payload = { + kind: "update" as const, + status: "skipped" as const, + ts: Date.now(), + sessionKey: "agent:main:webchat:dm:user-123", + message: "update restart pending", + stats: { + mode: "npm", + reason: "restart-health-pending", + }, + }; + const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); + await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8"); + + await expect(hasRestartSentinel()).resolves.toBe(true); + expect(readSentinelRow()).toMatchObject({ + sentinel_key: "current", + version: 1, + kind: "update", + status: "skipped", + payload_json: JSON.stringify(payload), + }); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload }); + }); + }); + + it("does not replay a legacy file superseded by a sqlite sentinel", async () => { + await withRestartSentinelStateDir(async () => { + const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json"); + await fs.writeFile( + legacyPath, + `${JSON.stringify({ + version: 1, + payload: { + kind: "update", + status: "ok", + ts: 1, + message: "stale legacy sentinel", + }, + })}\n`, + "utf-8", + ); + + await writeRestartSentinel({ + kind: "restart", + status: "ok", + ts: 2, + message: "current sqlite sentinel", + }); + await expect(fs.access(legacyPath)).rejects.toThrow(); + + await clearRestartSentinel(); + + await expect(hasRestartSentinel()).resolves.toBe(false); + await expect(readRestartSentinel()).resolves.toBeNull(); + }); + }); + it("drops invalid sentinel payloads", async () => { await withRestartSentinelStateDir(async () => { - const filePath = resolveRestartSentinelPath(); - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, "not-json", "utf-8"); + insertSentinelRow({ payloadJson: "not-json" }); const read = await readRestartSentinel(); expect(read).toBeNull(); - await expectPathMissing(filePath); + expect(readSentinelRow()).toBeUndefined(); }); }); it("drops structurally invalid sentinel payloads", async () => { await withRestartSentinelStateDir(async () => { - const filePath = resolveRestartSentinelPath(); - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, JSON.stringify({ version: 2, payload: null }), "utf-8"); + insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) }); await expect(readRestartSentinel()).resolves.toBeNull(); - await expectPathMissing(filePath); + expect(readSentinelRow()).toBeUndefined(); }); }); diff --git a/src/infra/restart-sentinel.ts b/src/infra/restart-sentinel.ts index 0547f1200d8..5974449c453 100644 --- a/src/infra/restart-sentinel.ts +++ b/src/infra/restart-sentinel.ts @@ -1,11 +1,20 @@ -// Persists restart sentinel files that coordinate deferred restarts. -import fs from "node:fs/promises"; +// Persists restart sentinel state that coordinates deferred restarts. +import { readFile, rm } from "node:fs/promises"; import path from "node:path"; import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce"; import { formatCliCommand } from "../cli/command-format.js"; import { resolveStateDir } from "../config/paths.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import { resolveRuntimeServiceVersion } from "../version.js"; -import { writeJson } from "./json-files.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; export type RestartSentinelLog = { stdoutTail?: string | null; @@ -66,7 +75,9 @@ export type RestartSentinel = { payload: RestartSentinelPayload; }; -const SENTINEL_FILENAME = "restart-sentinel.json"; +const RESTART_SENTINEL_KEY = "current"; +const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json"; +type GatewayRestartSentinelDatabase = Pick; export function formatDoctorNonInteractiveHint( env: Record = process.env as Record, @@ -77,18 +88,60 @@ export function formatDoctorNonInteractiveHint( )} in a terminal or approvals-capable OpenClaw surface.`; } -export function resolveRestartSentinelPath(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveStateDir(env), SENTINEL_FILENAME); -} - export async function writeRestartSentinel( payload: RestartSentinelPayload, env: NodeJS.ProcessEnv = process.env, -) { - const filePath = resolveRestartSentinelPath(env); - const data: RestartSentinel = { version: 1, payload }; - await writeJson(filePath, data, { trailingNewline: true, dirMode: 0o700 }); - return filePath; +): Promise { + const updatedAtMs = Date.now(); + runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .insertInto("gateway_restart_sentinel") + .values({ + sentinel_key: RESTART_SENTINEL_KEY, + version: 1, + kind: payload.kind, + status: payload.status, + ts: payload.ts, + session_key: payload.sessionKey ?? null, + thread_id: payload.threadId ?? null, + delivery_channel: payload.deliveryContext?.channel ?? null, + delivery_to: payload.deliveryContext?.to ?? null, + delivery_account_id: payload.deliveryContext?.accountId ?? null, + message: payload.message ?? null, + continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null, + doctor_hint: payload.doctorHint ?? null, + stats_json: payload.stats ? JSON.stringify(payload.stats) : null, + payload_json: JSON.stringify(payload), + updated_at_ms: updatedAtMs, + }) + .onConflict((conflict) => + conflict.column("sentinel_key").doUpdateSet({ + version: (eb) => eb.ref("excluded.version"), + kind: (eb) => eb.ref("excluded.kind"), + status: (eb) => eb.ref("excluded.status"), + ts: (eb) => eb.ref("excluded.ts"), + session_key: (eb) => eb.ref("excluded.session_key"), + thread_id: (eb) => eb.ref("excluded.thread_id"), + delivery_channel: (eb) => eb.ref("excluded.delivery_channel"), + delivery_to: (eb) => eb.ref("excluded.delivery_to"), + delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"), + message: (eb) => eb.ref("excluded.message"), + continuation_json: (eb) => eb.ref("excluded.continuation_json"), + doctor_hint: (eb) => eb.ref("excluded.doctor_hint"), + stats_json: (eb) => eb.ref("excluded.stats_json"), + payload_json: (eb) => eb.ref("excluded.payload_json"), + updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"), + }), + ), + ); + }, + { env }, + ); + await removeLegacyRestartSentinel(env); } function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload { @@ -156,11 +209,52 @@ export async function markUpdateRestartSentinelFailure( }, env); } -export async function removeRestartSentinelFile(filePath: string | null | undefined) { - if (!filePath) { - return; +export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { + try { + runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .deleteFrom("gateway_restart_sentinel") + .where("sentinel_key", "=", RESTART_SENTINEL_KEY), + ); + }, + { env }, + ); + } catch {} + await removeLegacyRestartSentinel(env); +} + +function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string { + return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME); +} + +async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise { + try { + await rm(resolveLegacyRestartSentinelPath(env), { force: true }); + } catch {} +} + +async function importLegacyRestartSentinel( + env: NodeJS.ProcessEnv = process.env, +): Promise { + const legacyPath = resolveLegacyRestartSentinelPath(env); + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown; + } catch { + return null; } - await fs.unlink(filePath).catch(() => {}); + if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) { + await removeLegacyRestartSentinel(env); + return null; + } + const payload = parsed.payload as RestartSentinelPayload; + await writeRestartSentinel(payload, env); + await removeLegacyRestartSentinel(env); + return { version: 1, payload }; } export function buildRestartSuccessContinuation(params: { @@ -177,26 +271,56 @@ export function buildRestartSuccessContinuation(params: { export async function readRestartSentinel( env: NodeJS.ProcessEnv = process.env, ): Promise { - const filePath = resolveRestartSentinelPath(env); try { - const raw = await fs.readFile(filePath, "utf-8"); - let parsed: RestartSentinel | undefined; + const database = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select(["version", "payload_json"]) + .where("sentinel_key", "=", RESTART_SENTINEL_KEY), + ); + if (!row) { + return await importLegacyRestartSentinel(env); + } + let payload: RestartSentinelPayload | undefined; try { - parsed = JSON.parse(raw) as RestartSentinel | undefined; + payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined; } catch { - await fs.unlink(filePath).catch(() => {}); + await clearRestartSentinel(env); return null; } - if (!parsed || parsed.version !== 1 || !parsed.payload) { - await fs.unlink(filePath).catch(() => {}); + if (row.version !== 1 || !payload) { + await clearRestartSentinel(env); return null; } - return parsed; + return { version: 1, payload }; } catch { return null; } } +export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { + try { + const database = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select("sentinel_key") + .where("sentinel_key", "=", RESTART_SENTINEL_KEY), + ); + if (row) { + return true; + } + return Boolean(await importLegacyRestartSentinel(env)); + } catch { + return false; + } +} + export function formatRestartSentinelMessage(payload: RestartSentinelPayload): string { const message = payload.message?.trim(); if (message && (!payload.stats || payload.kind === "config-auto-recovery")) { diff --git a/src/infra/update-control-plane-sentinel.ts b/src/infra/update-control-plane-sentinel.ts index b5dafcb4db9..45073791240 100644 --- a/src/infra/update-control-plane-sentinel.ts +++ b/src/infra/update-control-plane-sentinel.ts @@ -118,8 +118,8 @@ export async function readControlPlaneUpdateSentinelMeta( export async function writeControlPlaneUpdateRestartSentinel(params: { result: UpdateRunResult; meta: UpdateRestartSentinelMeta; -}): Promise { - return await writeRestartSentinel( +}): Promise { + await writeRestartSentinel( buildUpdateRestartSentinelPayload({ result: params.result, meta: params.meta, diff --git a/src/infra/update-managed-service-handoff.test.ts b/src/infra/update-managed-service-handoff.test.ts index 8777e806744..179db87f26d 100644 --- a/src/infra/update-managed-service-handoff.test.ts +++ b/src/infra/update-managed-service-handoff.test.ts @@ -5,6 +5,17 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; import { SUPERVISOR_HINT_ENV_VARS } from "./supervisor-markers.js"; import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "./update-control-plane-sentinel.js"; import { @@ -28,8 +39,10 @@ vi.mock("node:child_process", async () => { }); const tempDirs = new Set(); +type GatewayRestartSentinelDatabase = Pick; afterEach(async () => { + closeOpenClawStateDatabaseForTest(); spawnMock.mockClear(); await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); @@ -44,10 +57,74 @@ async function pathExists(filePath: string): Promise { } } +function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { + const { db } = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(db); + const payload = + sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1 + ? (sentinel as { payload?: unknown }).payload + : null; + if (!payload || typeof payload !== "object") { + throw new Error("expected versioned restart sentinel payload"); + } + const record = payload as { + kind?: unknown; + status?: unknown; + ts?: unknown; + sessionKey?: unknown; + threadId?: unknown; + deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown }; + message?: unknown; + continuation?: unknown; + doctorHint?: unknown; + stats?: unknown; + }; + executeSqliteQuerySync( + db, + stateDb.insertInto("gateway_restart_sentinel").values({ + sentinel_key: "current", + version: 1, + kind: typeof record.kind === "string" ? record.kind : "update", + status: typeof record.status === "string" ? record.status : "skipped", + ts: typeof record.ts === "number" ? record.ts : Date.now(), + session_key: typeof record.sessionKey === "string" ? record.sessionKey : null, + thread_id: typeof record.threadId === "string" ? record.threadId : null, + delivery_channel: + typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null, + delivery_to: + typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null, + delivery_account_id: + typeof record.deliveryContext?.accountId === "string" + ? record.deliveryContext.accountId + : null, + message: typeof record.message === "string" ? record.message : null, + continuation_json: record.continuation ? JSON.stringify(record.continuation) : null, + doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null, + stats_json: record.stats ? JSON.stringify(record.stats) : null, + payload_json: JSON.stringify(payload), + updated_at_ms: Date.now(), + }), + ); +} + +function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown { + const { db } = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select(["version", "payload_json"]) + .where("sentinel_key", "=", "current"), + ); + return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null; +} + async function runHelperWithExistingSentinel(params: { handoffId?: string; metaHandoffId?: string; - sentinel: unknown; + prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise | void; + sentinel?: unknown; }) { const { execFile } = await vi.importActual("node:child_process"); @@ -82,8 +159,11 @@ async function runHelperWithExistingSentinel(params: { string, unknown >; - const sentinelPath = path.join(tmpDir, "restart-sentinel.json"); - await fs.writeFile(sentinelPath, `${JSON.stringify(params.sentinel, null, 2)}\n`); + const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv; + await params.prepareStateDatabase?.(env); + if (params.sentinel !== undefined) { + writeRestartSentinelRow(env, params.sentinel); + } const helperParamsPath = path.join(tmpDir, "helper-params.json"); await fs.writeFile( helperParamsPath, @@ -92,7 +172,7 @@ async function runHelperWithExistingSentinel(params: { ...helperParams, parentPid: process.pid, parentExitTimeoutMs: 1, - sentinelPath, + stateDatabasePath: resolveOpenClawStateSqlitePath(env), logPath: path.join(tmpDir, "handoff.log"), sensitivePaths: [], }, @@ -113,7 +193,31 @@ async function runHelperWithExistingSentinel(params: { }, ); - return { result, sentinelPath }; + return { result, env }; +} + +async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise { + const sqlite = await import("node:sqlite"); + const stateDatabasePath = resolveOpenClawStateSqlitePath(env); + await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true }); + const db = new sqlite.DatabaseSync(stateDatabasePath); + try { + db.exec(` + CREATE TABLE gateway_restart_sentinel ( + sentinel_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + kind TEXT NOT NULL, + status TEXT NOT NULL, + ts INTEGER NOT NULL, + session_key TEXT, + thread_id TEXT, + payload_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + `); + } finally { + db.close(); + } } async function spawnExitedPid(): Promise { @@ -166,7 +270,7 @@ async function runHelperWithCommand(params: { parentExitTimeoutMs: 5000, cwd: tmpDir, commandArgv: params.commandArgv, - sentinelPath: path.join(tmpDir, "restart-sentinel.json"), + stateDatabasePath: resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: tmpDir }), logPath: path.join(tmpDir, "handoff.log"), sensitivePaths: [], ...(params.serviceRecovery ? { serviceRecovery: params.serviceRecovery } : {}), @@ -284,10 +388,10 @@ describe("managed service update handoff", () => { const helperParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as { cwd?: string; metaPath?: string; - sentinelPath?: string; + stateDatabasePath?: string; }; expect(helperParams.metaPath).toMatch(/sentinel-meta\.json$/u); - expect(helperParams.sentinelPath).toMatch(/restart-sentinel\.json$/u); + expect(helperParams.stateDatabasePath).toMatch(/openclaw\.sqlite$/u); expect(options.cwd).toBe(os.homedir()); expect(helperParams.cwd).toBe(os.homedir()); expect(options.detached).toBe(true); @@ -477,6 +581,51 @@ describe("managed service update handoff", () => { } }); + it("writes a fallback update failure when no restart sentinel row exists", async () => { + const { result, env } = await runHelperWithExistingSentinel({ + handoffId: "handoff-123", + metaHandoffId: "handoff-123", + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + kind: "update", + status: "error", + sessionKey: "agent:test:webchat:dm:user-123", + stats: { + handoffId: "handoff-123", + reason: "managed-service-handoff-parent-timeout", + }, + }, + }); + if (process.platform !== "win32") { + const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777; + expect(mode).toBe(0o600); + } + }); + + it("repairs legacy restart sentinel columns before writing fallback failures", async () => { + const { result, env } = await runHelperWithExistingSentinel({ + handoffId: "handoff-123", + metaHandoffId: "handoff-123", + prepareStateDatabase: createLegacyRestartSentinelTable, + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + kind: "update", + status: "error", + stats: { + reason: "managed-service-handoff-parent-timeout", + }, + }, + }); + }); + it("does not overwrite a restart sentinel owned by another startup task", async () => { const unrelatedSentinel = { version: 1, @@ -487,14 +636,12 @@ describe("managed service update handoff", () => { stats: { reason: "config-restart-pending" }, }, }; - const { result, sentinelPath } = await runHelperWithExistingSentinel({ + const { result, env } = await runHelperWithExistingSentinel({ sentinel: unrelatedSentinel, }); expect(result).toEqual({ code: 1, signal: null }); - await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual( - unrelatedSentinel, - ); + expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel); }); it("does not overwrite a newer pending update handoff sentinel", async () => { @@ -513,16 +660,14 @@ describe("managed service update handoff", () => { }, }, }; - const { result, sentinelPath } = await runHelperWithExistingSentinel({ + const { result, env } = await runHelperWithExistingSentinel({ handoffId: "old-handoff", metaHandoffId: "old-handoff", sentinel: newerSentinel, }); expect(result).toEqual({ code: 1, signal: null }); - await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual( - newerSentinel, - ); + expect(readRestartSentinelPayload(env)).toEqual(newerSentinel); }); it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => { diff --git a/src/infra/update-managed-service-handoff.ts b/src/infra/update-managed-service-handoff.ts index f8451f3f936..b4f0f8bc8e0 100644 --- a/src/infra/update-managed-service-handoff.ts +++ b/src/infra/update-managed-service-handoff.ts @@ -9,7 +9,7 @@ import { resolveGatewaySystemdServiceName, resolveGatewayWindowsTaskName, } from "../daemon/constants.js"; -import { resolveRestartSentinelPath } from "./restart-sentinel.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { SUPERVISOR_HINT_ENV_VARS, type RespawnSupervisor } from "./supervisor-markers.js"; import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV, @@ -96,26 +96,6 @@ function readJsonFile(filePath) { } } -function writeJsonFile(filePath, value) { - const dir = path.dirname(filePath); - const tempPath = path.join( - dir, - "." + path.basename(filePath) + "." + process.pid + "." + Date.now() + ".tmp", - ); - try { - fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); - fs.writeFileSync(tempPath, JSON.stringify(value, null, 2) + "\n", { mode: 0o600 }); - fs.renameSync(tempPath, filePath); - } catch (err) { - appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err))); - try { - fs.rmSync(tempPath, { force: true }); - } catch { - // Best effort only. - } - } -} - function isPendingUpdatePayload(payload) { const reason = payload && payload.stats && payload.stats.reason; return ( @@ -126,6 +106,170 @@ function isPendingUpdatePayload(payload) { ); } +function openStateDatabase() { + if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") { + return null; + } + try { + const sqlite = require("node:sqlite"); + fs.mkdirSync(path.dirname(params.stateDatabasePath), { recursive: true, mode: 0o700 }); + const db = new sqlite.DatabaseSync(params.stateDatabasePath); + db.exec([ + "CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (", + "sentinel_key TEXT NOT NULL PRIMARY KEY,", + "version INTEGER NOT NULL,", + "kind TEXT NOT NULL,", + "status TEXT NOT NULL,", + "ts INTEGER NOT NULL,", + "session_key TEXT,", + "thread_id TEXT,", + "delivery_channel TEXT,", + "delivery_to TEXT,", + "delivery_account_id TEXT,", + "message TEXT,", + "continuation_json TEXT,", + "doctor_hint TEXT,", + "stats_json TEXT,", + "payload_json TEXT NOT NULL,", + "updated_at_ms INTEGER NOT NULL", + ");", + "CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts", + "ON gateway_restart_sentinel(ts DESC, sentinel_key);", + ].join(" ")); + ensureGatewayRestartSentinelColumns(db); + hardenStateDatabaseFiles(); + return db; + } catch (err) { + appendLog("failed to open restart sentinel database: " + (err && err.stack ? err.stack : String(err))); + return null; + } +} + +function tableHasColumn(db, tableName, columnName) { + try { + return db.prepare("PRAGMA table_info(" + tableName + ")").all().some((row) => row && row.name === columnName); + } catch { + return false; + } +} + +function ensureColumn(db, tableName, columnSql) { + const columnName = columnSql.trim().split(/\s+/, 1)[0]; + if (!columnName || tableHasColumn(db, tableName, columnName)) { + return; + } + db.exec("ALTER TABLE " + tableName + " ADD COLUMN " + columnSql + ";"); +} + +function ensureGatewayRestartSentinelColumns(db) { + ensureColumn(db, "gateway_restart_sentinel", "delivery_channel TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "delivery_to TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "delivery_account_id TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "message TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "continuation_json TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "doctor_hint TEXT"); + ensureColumn(db, "gateway_restart_sentinel", "stats_json TEXT"); +} + +function hardenStateDatabaseFiles() { + if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") { + return; + } + for (const filePath of [ + params.stateDatabasePath, + params.stateDatabasePath + "-wal", + params.stateDatabasePath + "-shm", + ]) { + try { + if (fs.existsSync(filePath)) { + fs.chmodSync(filePath, 0o600); + } + } catch { + // Best effort only. + } + } +} + +function readRestartSentinelPayload() { + const db = openStateDatabase(); + if (!db) { + return null; + } + try { + const row = db + .prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?") + .get("current"); + if (!row || row.version !== 1 || typeof row.payload_json !== "string") { + return null; + } + return JSON.parse(row.payload_json); + } catch { + return null; + } finally { + hardenStateDatabaseFiles(); + try { + db.close(); + } catch {} + } +} + +function writeRestartSentinelPayload(payload) { + const db = openStateDatabase(); + if (!db) { + return; + } + try { + const updatedAtMs = Date.now(); + db.prepare( + [ + "INSERT INTO gateway_restart_sentinel (", + "sentinel_key, version, kind, status, ts, session_key, thread_id,", + "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", + "doctor_hint, stats_json, payload_json, updated_at_ms", + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "ON CONFLICT(sentinel_key) DO UPDATE SET", + "version = excluded.version, kind = excluded.kind, status = excluded.status,", + "ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,", + "delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,", + "delivery_account_id = excluded.delivery_account_id, message = excluded.message,", + "continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,", + "stats_json = excluded.stats_json, payload_json = excluded.payload_json,", + "updated_at_ms = excluded.updated_at_ms", + ].join(" "), + ).run( + "current", + 1, + payload.kind, + payload.status, + payload.ts, + payload.sessionKey || null, + payload.threadId || null, + payload.deliveryContext && typeof payload.deliveryContext.channel === "string" + ? payload.deliveryContext.channel + : null, + payload.deliveryContext && typeof payload.deliveryContext.to === "string" + ? payload.deliveryContext.to + : null, + payload.deliveryContext && typeof payload.deliveryContext.accountId === "string" + ? payload.deliveryContext.accountId + : null, + payload.message || null, + payload.continuation ? JSON.stringify(payload.continuation) : null, + payload.doctorHint || null, + payload.stats ? JSON.stringify(payload.stats) : null, + JSON.stringify(payload), + updatedAtMs, + ); + } catch (err) { + appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err))); + } finally { + hardenStateDatabaseFiles(); + try { + db.close(); + } catch {} + } +} + function buildFallbackFailurePayload(reason) { const metaFile = params.metaPath ? readJsonFile(params.metaPath) : null; const meta = metaFile && metaFile.version === 1 && metaFile.meta ? metaFile.meta : {}; @@ -157,11 +301,7 @@ function buildFallbackFailurePayload(reason) { } function markUpdateSentinelFailureIfPending(reason) { - if (!params.sentinelPath) { - return; - } - const current = readJsonFile(params.sentinelPath); - let payload = current && current.version === 1 ? current.payload : null; + let payload = readRestartSentinelPayload(); if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) { return; } @@ -176,7 +316,7 @@ function markUpdateSentinelFailureIfPending(reason) { } else { payload = buildFallbackFailurePayload(reason); } - writeJsonFile(params.sentinelPath, { version: 1, payload }); + writeRestartSentinelPayload(payload); } function runServiceCommand(command, args) { @@ -553,7 +693,7 @@ export async function startManagedServiceUpdateHandoff(params: { handoffId: params.handoffId, logPath, metaPath, - sentinelPath: resolveRestartSentinelPath(), + stateDatabasePath: resolveOpenClawStateSqlitePath(params.env ?? process.env), sensitivePaths: [scriptPath, paramsPath, metaPath], serviceRecovery: resolveGatewayServiceRecovery(params.supervisor, params.env ?? process.env), };