fix(deadcode): move restart sentinels to sqlite

This commit is contained in:
Vincent Koc
2026-06-21 23:28:49 +08:00
parent 2804c24dc6
commit 514b3365b5
21 changed files with 757 additions and 306 deletions

View File

@@ -3,10 +3,11 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js";
import { GatewayClientRequestError } from "../gateway/client.js";
import { readRestartSentinel } from "../infra/restart-sentinel.js";
import { testing as restartTesting } from "../infra/restart.js";
import { withEnvAsync } from "../test-utils/env.js";
import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js";
import { createGatewayTool } from "./tools/gateway-tool.js";
import { callGatewayTool } from "./tools/gateway.js";
@@ -332,13 +333,9 @@ describe("gateway tool", () => {
});
expect(restartSignalKillCalls()).toHaveLength(0);
const sentinelPath = path.join(stateDir, "restart-sentinel.json");
const raw = await fs.readFile(sentinelPath, "utf-8");
const parsed = JSON.parse(raw) as {
payload?: { kind?: string; doctorHint?: string | null };
};
expect(parsed.payload?.kind).toBe("restart");
expect(parsed.payload?.doctorHint).toBe(
const sentinel = await readRestartSentinel();
expect(sentinel?.payload.kind).toBe("restart");
expect(sentinel?.payload.doctorHint).toBe(
"Recommended follow-up: run openclaw --profile isolated doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
);
},
@@ -507,15 +504,13 @@ describe("gateway tool", () => {
it("distinguishes explicit terminal array consent from indexed consent", () => {
expect(normalizeConfigPatchReplacePath("bindings[]")).toBe("bindings");
expect(normalizeConfigPatchReplacePath("bindings[0]")).toBe("bindings[0]");
expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe(
"agents.list[].skills",
);
expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe("agents.list[].skills");
expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[]"))).toBe(
"bindings",
);
expect(
normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]")),
).toBe("bindings[0]");
expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]"))).toBe(
"bindings[0]",
);
});
it("rejects config.patch when it changes safe bin approval paths", async () => {

View File

@@ -12,7 +12,7 @@ const {
formatDoctorNonInteractiveHintMock,
isRestartEnabledMock,
callGatewayToolMock,
removeRestartSentinelFileMock,
clearRestartSentinelMock,
scheduleGatewaySigusr1RestartMock,
writeRestartSentinelMock,
} = vi.hoisted(() => ({
@@ -30,8 +30,8 @@ const {
() =>
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
),
writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/restart"),
removeRestartSentinelFileMock: vi.fn(async (_path: string | null | undefined) => undefined),
writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
clearRestartSentinelMock: vi.fn(async () => undefined),
scheduleGatewaySigusr1RestartMock: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({
ok: true,
pid: 123,
@@ -59,7 +59,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
return {
...actual,
formatDoctorNonInteractiveHint: formatDoctorNonInteractiveHintMock,
removeRestartSentinelFile: removeRestartSentinelFileMock,
clearRestartSentinel: clearRestartSentinelMock,
writeRestartSentinel: writeRestartSentinelMock,
};
});
@@ -115,8 +115,8 @@ describe("gateway tool restart continuation", () => {
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
);
writeRestartSentinelMock.mockReset();
writeRestartSentinelMock.mockResolvedValue("/tmp/restart");
removeRestartSentinelFileMock.mockClear();
writeRestartSentinelMock.mockResolvedValue(undefined);
clearRestartSentinelMock.mockClear();
scheduleGatewaySigusr1RestartMock.mockReset();
scheduleGatewaySigusr1RestartMock.mockReturnValue({
ok: true,
@@ -354,7 +354,7 @@ describe("gateway tool restart continuation", () => {
await scheduledArgs.emitHooks?.beforeEmit?.();
await scheduledArgs.emitHooks?.afterEmitRejected?.();
expect(removeRestartSentinelFileMock).toHaveBeenCalledWith("/tmp/restart");
expect(clearRestartSentinelMock).toHaveBeenCalledOnce();
});
it("uses the runtime session for update.run continuation routing (#86742)", async () => {

View File

@@ -19,8 +19,8 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { GatewayClientRequestError } from "../../gateway/client.js";
import {
buildRestartSuccessContinuation,
clearRestartSentinel,
formatDoctorNonInteractiveHint,
removeRestartSentinelFile,
type RestartSentinelPayload,
writeRestartSentinel,
} from "../../infra/restart-sentinel.js";
@@ -493,7 +493,7 @@ export function createGatewayTool(opts?: {
log.info(
`gateway tool: restart requested (delayMs=${delayMs ?? "default"}, reason=${reason ?? "none"})`,
);
let sentinelPath: string | null = null;
let sentinelWritten = false;
const scheduled = scheduleGatewaySigusr1Restart({
delayMs,
reason,
@@ -502,10 +502,13 @@ export function createGatewayTool(opts?: {
sessionKey,
emitHooks: {
beforeEmit: async () => {
sentinelPath = await writeRestartSentinel(payload);
await writeRestartSentinel(payload);
sentinelWritten = true;
},
afterEmitRejected: async () => {
await removeRestartSentinelFile(sentinelPath);
if (sentinelWritten) {
await clearRestartSentinel();
}
},
},
});

View File

@@ -7,7 +7,7 @@ import type { HandleCommandsParams } from "./commands-types.js";
type ScheduleGatewayRestartArgs = Parameters<typeof scheduleGatewaySigusr1Restart>[0];
const mocks = vi.hoisted(() => ({
unlink: vi.fn(async (_path: string) => undefined),
clearRestartSentinel: vi.fn(async () => undefined),
isRestartEnabled: vi.fn(() => true),
extractDeliveryInfo: vi.fn(() => ({
deliveryContext: {
@@ -21,20 +21,13 @@ const mocks = vi.hoisted(() => ({
() =>
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
),
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/sentinel.json"),
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
scheduleGatewaySigusr1Restart: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({
scheduled: true,
})),
triggerOpenClawRestart: vi.fn(() => ({ ok: true, method: "launchctl" })),
}));
vi.mock("node:fs/promises", () => ({
default: {
unlink: mocks.unlink,
},
unlink: mocks.unlink,
}));
vi.mock("../../config/commands.flags.js", () => ({
isRestartEnabled: mocks.isRestartEnabled,
}));
@@ -67,6 +60,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
);
return {
...actual,
clearRestartSentinel: mocks.clearRestartSentinel,
formatDoctorNonInteractiveHint: mocks.formatDoctorNonInteractiveHint,
writeRestartSentinel: mocks.writeRestartSentinel,
};
@@ -119,7 +113,7 @@ describe("handleRestartCommand", () => {
beforeEach(() => {
mocks.isRestartEnabled.mockReset();
mocks.isRestartEnabled.mockReturnValue(true);
mocks.unlink.mockClear();
mocks.clearRestartSentinel.mockClear();
mocks.extractDeliveryInfo.mockClear();
mocks.formatDoctorNonInteractiveHint.mockClear();
mocks.writeRestartSentinel.mockClear();
@@ -218,7 +212,7 @@ describe("handleRestartCommand", () => {
expect(mocks.triggerOpenClawRestart).not.toHaveBeenCalled();
});
it("removes the success sentinel when fallback restart fails", async () => {
it("clears the success sentinel when fallback restart fails", async () => {
mocks.triggerOpenClawRestart.mockReturnValueOnce({
ok: false,
method: "launchctl",
@@ -227,6 +221,6 @@ describe("handleRestartCommand", () => {
const result = await handleRestartCommand(restartCommandParams(), true);
expect(result?.reply?.text).toContain("Restart failed");
expect(mocks.unlink).toHaveBeenCalledWith("/tmp/sentinel.json");
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
});
});

View File

@@ -21,8 +21,8 @@ import { getSessionBindingService } from "../../infra/outbound/session-binding-s
import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js";
import {
buildRestartSuccessContinuation,
clearRestartSentinel,
formatDoctorNonInteractiveHint,
removeRestartSentinelFile,
type RestartSentinelPayload,
writeRestartSentinel,
} from "../../infra/restart-sentinel.js";
@@ -703,7 +703,7 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0;
const sentinelPayload = buildRestartCommandSentinel(params);
if (hasSigusr1Listener) {
let sentinelPath: string | null = null;
let sentinelWritten = false;
scheduleGatewaySigusr1Restart({
reason: "/restart",
// Sibling session-routing guard: /restart writes a session-scoped sentinel
@@ -713,10 +713,13 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
emitHooks: sentinelPayload
? {
beforeEmit: async () => {
sentinelPath = await writeRestartSentinel(sentinelPayload);
await writeRestartSentinel(sentinelPayload);
sentinelWritten = true;
},
afterEmitRejected: async () => {
await removeRestartSentinelFile(sentinelPath);
if (sentinelWritten) {
await clearRestartSentinel();
}
},
}
: undefined,
@@ -728,10 +731,11 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
},
};
}
let sentinelPath: string | null = null;
let sentinelWritten = false;
try {
if (sentinelPayload) {
sentinelPath = await writeRestartSentinel(sentinelPayload);
await writeRestartSentinel(sentinelPayload);
sentinelWritten = true;
}
} catch (err) {
logVerbose(`failed to write /restart sentinel: ${String(err)}`);
@@ -744,7 +748,9 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
}
const restartMethod = triggerOpenClawRestart();
if (!restartMethod.ok) {
await removeRestartSentinelFile(sentinelPath);
if (sentinelWritten) {
await clearRestartSentinel();
}
const detail = restartMethod.detail ? ` Details: ${restartMethod.detail}` : "";
return {
shouldContinue: false,

View File

@@ -366,6 +366,7 @@ const { updateCommand, updateFinalizeCommand, updateStatusCommand, updateWizardC
const updateCliShared = await import("./update-cli/shared.js");
const { ensureGitCheckout, resolveGitInstallDir } = updateCliShared;
const { spawnSync } = await import("node:child_process");
const { readRestartSentinel } = await import("../infra/restart-sentinel.js");
function requireValue<T>(value: T | undefined, label: string): T {
if (value === undefined) {
@@ -5891,23 +5892,17 @@ describe("update-cli", () => {
},
);
const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8");
const sentinel = JSON.parse(raw) as {
payload?: {
status?: string;
message?: string | null;
continuation?: { kind?: string; message?: string };
stats?: { mode?: string; after?: { version?: string | null } };
};
};
expect(sentinel.payload?.status).toBe("ok");
expect(sentinel.payload?.message).toBe("Update requested from the agent.");
expect(sentinel.payload?.continuation).toEqual({
const sentinel = await readRestartSentinel({
OPENCLAW_STATE_DIR: stateDir,
} as NodeJS.ProcessEnv);
expect(sentinel?.payload.status).toBe("ok");
expect(sentinel?.payload.message).toBe("Update requested from the agent.");
expect(sentinel?.payload.continuation).toEqual({
kind: "agentTurn",
message: "Check the running version and finish the update report.",
});
expect(sentinel.payload?.stats?.mode).toBe("npm");
expect(sentinel.payload?.stats?.after?.version).toBe("2026.4.24");
expect(sentinel?.payload.stats?.mode).toBe("npm");
expect(sentinel?.payload.stats?.after?.version).toBe("2026.4.24");
});
it("marks the control-plane update sentinel failed when restart health verification fails", async () => {
@@ -5966,17 +5961,12 @@ describe("update-cli", () => {
},
);
const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8");
const sentinel = JSON.parse(raw) as {
payload?: {
status?: string;
continuation?: unknown;
stats?: { reason?: string | null };
};
};
expect(sentinel.payload?.status).toBe("error");
expect(sentinel.payload?.stats?.reason).toBe("restart-unhealthy");
expect(sentinel.payload?.continuation).toBeUndefined();
const sentinel = await readRestartSentinel({
OPENCLAW_STATE_DIR: stateDir,
} as NodeJS.ProcessEnv);
expect(sentinel?.payload.status).toBe("error");
expect(sentinel?.payload.stats?.reason).toBe("restart-unhealthy");
expect(sentinel?.payload.continuation).toBeUndefined();
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});

View File

@@ -203,13 +203,12 @@ function buildConfigRestartSentinelPayload(params: {
};
}
async function tryWriteRestartSentinelPayload(
payload: RestartSentinelPayload,
): Promise<string | null> {
async function tryWriteRestartSentinelPayload(payload: RestartSentinelPayload): Promise<boolean> {
try {
return await writeRestartSentinel(payload);
await writeRestartSentinel(payload);
return true;
} catch {
return null;
return false;
}
}
@@ -256,7 +255,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
context?: GatewayRequestContext;
}): Promise<{
payload: RestartSentinelPayload;
sentinelPath: string | null;
sentinelPersisted: boolean;
restart: ReturnType<typeof scheduleGatewaySigusr1Restart> | undefined;
}> {
const { sessionKey, note, restartDelayMs, deliveryContext, threadId } =
@@ -270,7 +269,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
threadId,
note,
});
const sentinelPath = await tryWriteRestartSentinelPayload(payload);
const sentinelPersisted = await tryWriteRestartSentinelPayload(payload);
const restart = shouldScheduleDirectConfigRestart({
changedPaths: params.changedPaths,
nextConfig: params.nextConfig,
@@ -291,5 +290,5 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
`${params.mode} restart coalesced ${formatControlPlaneActor(params.actor)} delayMs=${restart.delayMs}`,
);
}
return { payload, sentinelPath, restart };
return { payload, sentinelPersisted, restart };
}

View File

@@ -21,9 +21,7 @@ const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({
coalesced: false,
}));
const restartSentinelMocks = vi.hoisted(() => ({
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => {
return "/tmp/restart-sentinel.json";
}),
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
}));
vi.mock("../../config/config.js", async () => {

View File

@@ -594,7 +594,7 @@ async function respondWithConfigRestartWrite(params: {
uiHints: ConfigRedactionHints;
}): Promise<void> {
clearConfigSchemaResponseCache();
const { payload, sentinelPath, restart } = await resolveGatewayConfigRestartWriteResult({
const { payload, sentinelPersisted, restart } = await resolveGatewayConfigRestartWriteResult({
requestParams: params.requestParams,
kind: params.kind,
mode: params.mode,
@@ -612,7 +612,7 @@ async function respondWithConfigRestartWrite(params: {
config: redactConfigObject(params.writeResult.config, params.uiHints),
restart,
sentinel: {
path: sentinelPath,
persisted: sentinelPersisted,
payload,
},
},

View File

@@ -49,7 +49,7 @@ type UpdateRunPayload = {
ok: boolean;
result?: { status?: string; reason?: string; mode?: string };
handoff?: { status?: string; command?: string; message?: string };
sentinel?: { path?: string | null };
sentinel?: { persisted?: boolean };
restart?: unknown;
};
@@ -97,7 +97,6 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
...(actual as Record<string, unknown>),
writeRestartSentinel: async (payload: RestartSentinelPayload) => {
capturedPayload = payload;
return "/tmp/sentinel.json";
},
};
});
@@ -462,7 +461,7 @@ describe("update.run restart scheduling", () => {
pid: 12345,
command: "openclaw update --yes --timeout 1800",
});
expect(payload?.sentinel?.path).toBe("/tmp/sentinel.json");
expect(payload?.sentinel?.persisted).toBe(true);
const sentinel = readCapturedPayload();
expect(sentinel.kind).toBe("update");
expect(sentinel.status).toBe("skipped");

View File

@@ -340,12 +340,13 @@ export const updateHandlers: GatewayRequestHandlers = {
meta: sentinelMeta,
});
let sentinelPath: string | null;
let sentinelPersisted: boolean;
try {
sentinelPath = await writeRestartSentinel(payload);
await writeRestartSentinel(payload);
sentinelPersisted = true;
recordLatestUpdateRestartSentinel(payload);
} catch {
sentinelPath = null;
sentinelPersisted = false;
}
// Only restart the gateway when the update actually succeeded.
@@ -391,7 +392,7 @@ export const updateHandlers: GatewayRequestHandlers = {
...(handoff ? { handoff } : {}),
restart,
sentinel: {
path: sentinelPath,
persisted: sentinelPersisted,
payload,
},
},

View File

@@ -42,8 +42,7 @@ const mocks = vi.hoisted(() => {
}),
),
finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null),
removeRestartSentinelFile: vi.fn(async () => undefined),
resolveRestartSentinelPath: vi.fn(() => "/tmp/restart-sentinel.json"),
clearRestartSentinel: vi.fn(async () => undefined),
formatRestartSentinelMessage: vi.fn(() => "restart message"),
summarizeRestartSentinel: vi.fn(() => "restart summary"),
resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"),
@@ -183,8 +182,7 @@ vi.mock("../agents/agent-scope.js", async () => {
vi.mock("../infra/restart-sentinel.js", () => ({
finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion,
readRestartSentinel: mocks.readRestartSentinel,
removeRestartSentinelFile: mocks.removeRestartSentinelFile,
resolveRestartSentinelPath: mocks.resolveRestartSentinelPath,
clearRestartSentinel: mocks.clearRestartSentinel,
formatRestartSentinelMessage: mocks.formatRestartSentinelMessage,
summarizeRestartSentinel: mocks.summarizeRestartSentinel,
}));
@@ -427,7 +425,7 @@ describe("scheduleRestartSentinelWake", () => {
mocks.recoverPendingSessionDeliveries.mockClear();
mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset();
mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null);
mocks.removeRestartSentinelFile.mockClear();
mocks.clearRestartSentinel.mockClear();
mocks.injectTimestamp.mockClear();
mocks.timestampOptsFromConfig.mockClear();
mocks.recordInboundSessionAndDispatchReply.mockReset();
@@ -1295,7 +1293,7 @@ describe("scheduleRestartSentinelWake", () => {
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.removeRestartSentinelFile).not.toHaveBeenCalled();
expect(mocks.clearRestartSentinel).not.toHaveBeenCalled();
expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled();
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
source: "restart-sentinel",
@@ -1359,7 +1357,7 @@ describe("scheduleRestartSentinelWake", () => {
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.removeRestartSentinelFile).toHaveBeenCalledWith("/tmp/restart-sentinel.json");
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
});

View File

@@ -18,13 +18,12 @@ import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/de
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
import { resolveOutboundTarget } from "../infra/outbound/targets.js";
import {
clearRestartSentinel,
finalizeUpdateRestartSentinelRunningVersion,
formatRestartSentinelMessage,
readRestartSentinel,
removeRestartSentinelFile,
type RestartSentinelContinuation,
type RestartSentinelPayload,
resolveRestartSentinelPath,
summarizeRestartSentinel,
} from "../infra/restart-sentinel.js";
import {
@@ -449,7 +448,6 @@ async function loadRestartSentinelStartupTask(params: {
if (!sentinel) {
return null;
}
const sentinelPath = resolveRestartSentinelPath();
const payload = sentinel.payload;
if (payload.kind === "update") {
recordLatestUpdateRestartSentinel(payload);
@@ -494,7 +492,7 @@ async function loadRestartSentinelStartupTask(params: {
continuationKind: payload.continuation.kind,
});
}
await removeRestartSentinelFile(sentinelPath);
await clearRestartSentinel();
return { status: "ran" as const };
}
@@ -588,7 +586,7 @@ async function loadRestartSentinelStartupTask(params: {
);
}
await removeRestartSentinelFile(sentinelPath);
await clearRestartSentinel();
const routedAgentTurnContinuation =
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
if (!routedAgentTurnContinuation) {

View File

@@ -5,11 +5,13 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { writeRestartSentinel } from "../infra/restart-sentinel.js";
import type {
PluginHookGatewayContext,
PluginHookGatewayStartEvent,
} from "../plugins/hook-types.js";
import type { PluginServicesHandle } from "../plugins/services.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { withEnvAsync } from "../test-utils/env.js";
const hoisted = vi.hoisted(() => {
@@ -135,7 +137,9 @@ vi.mock("../config/paths.js", async () => {
STATE_DIR: "/tmp/openclaw-state",
resolveConfigPath: vi.fn(() => "/tmp/openclaw-state/openclaw.json"),
resolveGatewayPort: vi.fn(() => 18789),
resolveStateDir: vi.fn(() => "/tmp/openclaw-state"),
resolveStateDir: vi.fn((env: NodeJS.ProcessEnv = process.env) =>
env.OPENCLAW_STATE_DIR?.trim() ? actual.resolveStateDir(env) : "/tmp/openclaw-state",
),
};
});
@@ -295,6 +299,7 @@ function firstGatewayStartCall(
describe("startGatewayPostAttachRuntime", () => {
beforeEach(() => {
closeOpenClawStateDatabaseForTest();
vi.stubEnv("OPENCLAW_SKIP_CHANNELS", "0");
vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", "0");
hoisted.startPluginServices.mockClear();
@@ -325,6 +330,8 @@ describe("startGatewayPostAttachRuntime", () => {
});
hoisted.scheduleRestartAbortedMainSessionRecovery.mockClear();
hoisted.scheduleRestartSentinelWake.mockClear();
hoisted.refreshLatestUpdateRestartSentinel.mockReset();
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(null);
hoisted.getAcpRuntimeBackend.mockReset();
hoisted.getAcpRuntimeBackend.mockReturnValue(null);
hoisted.reconcilePendingSessionIdentities.mockClear();
@@ -355,6 +362,7 @@ describe("startGatewayPostAttachRuntime", () => {
});
afterEach(() => {
closeOpenClawStateDatabaseForTest();
vi.useRealTimers();
vi.unstubAllEnvs();
});
@@ -556,62 +564,76 @@ describe("startGatewayPostAttachRuntime", () => {
it("skips heavy restart sentinel refresh when no sentinel file exists", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-no-sentinel-"));
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
hoisted.refreshLatestUpdateRestartSentinel.mockClear();
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
expect(result).toBeNull();
expect(hoisted.refreshLatestUpdateRestartSentinel).not.toHaveBeenCalled();
closeOpenClawStateDatabaseForTest();
fs.rmSync(stateDir, { recursive: true, force: true });
});
it("refreshes the restart sentinel when the sentinel file exists", async () => {
it("refreshes the restart sentinel when the sentinel row exists", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-"));
fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n");
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
const sentinel = { kind: "update", status: "ok", ts: 1 } as const;
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel);
try {
await writeRestartSentinel(
{
kind: "update",
status: "ok",
ts: 1,
},
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
);
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
const sentinel = { kind: "update", status: "ok", ts: 1 } as const;
hoisted.refreshLatestUpdateRestartSentinel.mockClear();
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel);
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
expect(result).toBe(sentinel);
expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce();
fs.rmSync(stateDir, { recursive: true, force: true });
expect(result).toBe(sentinel);
expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce();
} finally {
closeOpenClawStateDatabaseForTest();
fs.rmSync(stateDir, { recursive: true, force: true });
}
});
it("expands tilde-based restart sentinel state paths", async () => {
const osHome = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-home-"));
it("detects restart sentinel rows in explicit state directories", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-state-"));
try {
const openclawHome = path.join(osHome, "openclaw-home");
const stateDirFromHome = path.join(openclawHome, ".openclaw");
fs.mkdirSync(stateDirFromHome, { recursive: true });
fs.writeFileSync(path.join(stateDirFromHome, "restart-sentinel.json"), "{}\n");
await writeRestartSentinel(
{
kind: "update",
status: "ok",
ts: 1,
},
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
);
expect(
await testing.hasRestartSentinelFileFast({
HOME: osHome,
OPENCLAW_HOME: "~/openclaw-home",
} as NodeJS.ProcessEnv),
).toBe(true);
const backslashStateDir = path.resolve(`${osHome}\\openclaw-state`);
fs.mkdirSync(backslashStateDir, { recursive: true });
fs.writeFileSync(path.join(backslashStateDir, "restart-sentinel.json"), "{}\n");
expect(
await testing.hasRestartSentinelFileFast({
HOME: osHome,
OPENCLAW_STATE_DIR: "~\\openclaw-state",
await testing.hasRestartSentinelFast({
OPENCLAW_STATE_DIR: stateDir,
} as NodeJS.ProcessEnv),
).toBe(true);
} finally {
fs.rmSync(osHome, { recursive: true, force: true });
closeOpenClawStateDatabaseForTest();
fs.rmSync(stateDir, { recursive: true, force: true });
}
});
it("avoids sync filesystem probes while checking restart sentinel presence", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-async-sentinel-"));
try {
fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n");
await writeRestartSentinel(
{
kind: "update",
status: "ok",
ts: 1,
},
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
);
const actualExistsSync = fs.existsSync;
const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => {
if (String(candidate).startsWith(stateDir)) {
@@ -621,7 +643,7 @@ describe("startGatewayPostAttachRuntime", () => {
});
try {
await expect(
testing.hasRestartSentinelFileFast({
testing.hasRestartSentinelFast({
OPENCLAW_STATE_DIR: stateDir,
} as NodeJS.ProcessEnv),
).resolves.toBe(true);
@@ -632,6 +654,7 @@ describe("startGatewayPostAttachRuntime", () => {
existsSync.mockRestore();
}
} finally {
closeOpenClawStateDatabaseForTest();
fs.rmSync(stateDir, { recursive: true, force: true });
}
});

View File

@@ -1,8 +1,5 @@
// Gateway post-attach startup sidecars.
// Schedules warmups, sentinels, update checks, memory backend, and plugin services.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
import { setTimeout as sleep } from "node:timers/promises";
import type { CliDeps } from "../cli/deps.types.js";
@@ -11,6 +8,7 @@ import type { GatewayTailscaleMode } from "../config/types.gateway.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { hasConfiguredInternalHooks } from "../hooks/configured.js";
import { isTruthyEnvValue } from "../infra/env.js";
import { hasRestartSentinel } from "../infra/restart-sentinel.js";
import type { scheduleGatewayUpdateCheck } from "../infra/update-startup.js";
import type { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
import type { PluginHookGatewayCronService } from "../plugins/hook-types.js";
@@ -38,7 +36,6 @@ const DEFERRED_SIDECAR_START_DELAY_MS = 100;
const SESSION_LOCK_CLEANUP_CONCURRENCY = 4;
const SKIP_STARTUP_MODEL_PREWARM_ENV = "OPENCLAW_SKIP_STARTUP_MODEL_PREWARM";
const QMD_STARTUP_IDLE_DELAY_MS = 120_000;
const RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
type Awaitable<T> = T | Promise<T>;
@@ -506,67 +503,14 @@ function scheduleTranscriptsAutoStartSidecar(params: {
});
}
async function pathExists(filePath: string): Promise<boolean> {
try {
await fs.promises.access(filePath);
return true;
} catch {
return false;
}
}
async function resolveRestartSentinelPathFast(
env: NodeJS.ProcessEnv = process.env,
): Promise<string> {
const normalizePathEnv = (value: string | undefined) => {
const trimmed = value?.trim();
return trimmed && trimmed !== "undefined" && trimmed !== "null" ? trimmed : undefined;
};
const resolveRawOsHome = () => normalizePathEnv(env.HOME) ?? normalizePathEnv(env.USERPROFILE);
const expandHomePrefix = (input: string, home: string) => input.replace(/^~(?=$|[\\/])/, home);
const resolveHome = () => {
const explicitHome = normalizePathEnv(env.OPENCLAW_HOME);
if (explicitHome) {
const osHome = resolveRawOsHome() ?? os.homedir();
return path.resolve(expandHomePrefix(explicitHome, osHome));
}
return path.resolve(resolveRawOsHome() ?? os.homedir());
};
const resolveUserPath = (input: string) => {
const trimmed = input.trim();
if (trimmed.startsWith("~")) {
return path.resolve(expandHomePrefix(trimmed, resolveHome()));
}
return path.resolve(trimmed);
};
const override = normalizePathEnv(env.OPENCLAW_STATE_DIR);
if (override) {
return path.join(resolveUserPath(override), RESTART_SENTINEL_FILENAME);
}
const home = resolveHome();
const newStateDir = path.join(home, ".openclaw");
if (env.OPENCLAW_TEST_FAST === "1" || (await pathExists(newStateDir))) {
return path.join(newStateDir, RESTART_SENTINEL_FILENAME);
}
const legacyStateDir = path.join(home, ".clawdbot");
if (await pathExists(legacyStateDir)) {
return path.join(legacyStateDir, RESTART_SENTINEL_FILENAME);
}
return path.join(newStateDir, RESTART_SENTINEL_FILENAME);
}
async function hasRestartSentinelFileFast(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
try {
return await pathExists(await resolveRestartSentinelPathFast(env));
} catch {
return false;
}
async function hasRestartSentinelFast(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
return await hasRestartSentinel(env);
}
async function refreshLatestUpdateRestartSentinelIfPresent(): Promise<Awaited<
ReturnType<typeof refreshLatestUpdateRestartSentinel>
> | null> {
if (!(await hasRestartSentinelFileFast())) {
if (!(await hasRestartSentinelFast())) {
return null;
}
return await (await loadGatewayRestartSentinelModule()).refreshLatestUpdateRestartSentinel();
@@ -930,7 +874,7 @@ export async function startGatewaySidecars(params: {
if (!shouldCheckRestartSentinel()) {
return;
}
if (!(await hasRestartSentinelFileFast())) {
if (!(await hasRestartSentinelFast())) {
return;
}
setTimeout(() => {
@@ -1457,7 +1401,7 @@ export async function startGatewayPostAttachRuntime(
export const testing = {
providerAuthPrewarmStartDelayMs: PROVIDER_AUTH_PREWARM_START_DELAY_MS,
hasRestartSentinelFileFast,
hasRestartSentinelFast,
prewarmConfiguredPrimaryModel,
prewarmConfiguredPrimaryModelWithTimeout,
refreshLatestUpdateRestartSentinelIfPresent,

View File

@@ -10,7 +10,7 @@ import type { DeviceIdentity } from "../infra/device-identity.js";
import { loadOrCreateDeviceIdentity } from "../infra/device-identity.js";
import { approveDevicePairing, listDevicePairing } from "../infra/device-pairing.js";
import { approveNodePairing, requestNodePairing } from "../infra/node-pairing.js";
import { resolveRestartSentinelPath } from "../infra/restart-sentinel.js";
import { readRestartSentinel } from "../infra/restart-sentinel.js";
import { SUPERVISOR_HINT_ENV_VARS } from "../infra/supervisor-markers.js";
import { getActiveRuntimePluginRegistry } from "../plugins/active-runtime-registry.js";
import {
@@ -417,13 +417,9 @@ describe("gateway update.run", () => {
}, FAST_WAIT_OPTS);
expect(sigusr1).toHaveBeenCalled();
const sentinelPath = resolveRestartSentinelPath();
const raw = await fs.readFile(sentinelPath, "utf-8");
const parsed = JSON.parse(raw) as {
payload?: { kind?: string; stats?: { mode?: string } };
};
expect(parsed.payload?.kind).toBe("update");
expect(parsed.payload?.stats?.mode).toBe("git");
const sentinel = await readRestartSentinel();
expect(sentinel?.payload.kind).toBe("update");
expect(sentinel?.payload.stats?.mode).toBe("git");
} finally {
process.off("SIGUSR1", sigusr1);
}

View File

@@ -1,17 +1,28 @@
// Covers restart sentinel persistence, summaries, and messages.
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
// Covers restart sentinel persistence, summaries, and messages.
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { withEnvAsync } from "../test-utils/env.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import {
buildRestartSuccessContinuation,
clearRestartSentinel,
finalizeUpdateRestartSentinelRunningVersion,
formatDoctorNonInteractiveHint,
formatRestartSentinelMessage,
hasRestartSentinel,
markUpdateRestartSentinelFailure,
readRestartSentinel,
resolveRestartSentinelPath,
summarizeRestartSentinel,
trimLogTail,
writeRestartSentinel,
@@ -25,28 +36,52 @@ import { buildUpdateRestartSentinelPayload } from "./update-restart-sentinel-pay
async function withRestartSentinelStateDir(run: () => Promise<void>): Promise<void> {
await withTempDir({ prefix: "openclaw-sentinel-" }, async (tempDir) => {
await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run);
try {
await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run);
} finally {
closeOpenClawStateDatabaseForTest();
}
});
}
async function expectPathMissing(targetPath: string): Promise<void> {
try {
await fs.stat(targetPath);
} catch (error) {
expect(error).toBeInstanceOf(Error);
const statError = error as NodeJS.ErrnoException;
expect({
code: statError.code,
path: statError.path,
syscall: statError.syscall,
}).toEqual({
code: "ENOENT",
path: targetPath,
syscall: "stat",
});
return;
}
throw new Error(`Expected path to be missing: ${targetPath}`);
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
function readSentinelRow() {
const { db } = openOpenClawStateDatabase();
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
return executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["sentinel_key", "version", "kind", "status", "payload_json"])
.where("sentinel_key", "=", "current"),
);
}
function insertSentinelRow(values: { version?: number; payloadJson: string }) {
const { db } = openOpenClawStateDatabase();
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
version: values.version ?? 1,
kind: "update",
status: "ok",
ts: Date.now(),
session_key: null,
thread_id: null,
delivery_channel: null,
delivery_to: null,
delivery_account_id: null,
message: null,
continuation_json: null,
doctor_hint: null,
stats_json: null,
payload_json: values.payloadJson,
updated_at_ms: Date.now(),
}),
);
}
describe("restart sentinel", () => {
@@ -63,8 +98,14 @@ describe("restart sentinel", () => {
},
stats: { mode: "git" },
};
const filePath = await writeRestartSentinel(payload);
expect(filePath).toBe(resolveRestartSentinelPath());
await writeRestartSentinel(payload);
expect(readSentinelRow()).toMatchObject({
sentinel_key: "current",
version: 1,
kind: "update",
status: "ok",
payload_json: JSON.stringify(payload),
});
const read = await readRestartSentinel();
expect(read?.payload.kind).toBe("update");
@@ -72,27 +113,84 @@ describe("restart sentinel", () => {
});
});
it("imports a legacy file sentinel into sqlite once", async () => {
await withRestartSentinelStateDir(async () => {
const payload = {
kind: "update" as const,
status: "skipped" as const,
ts: Date.now(),
sessionKey: "agent:main:webchat:dm:user-123",
message: "update restart pending",
stats: {
mode: "npm",
reason: "restart-health-pending",
},
};
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8");
await expect(hasRestartSentinel()).resolves.toBe(true);
expect(readSentinelRow()).toMatchObject({
sentinel_key: "current",
version: 1,
kind: "update",
status: "skipped",
payload_json: JSON.stringify(payload),
});
await expect(fs.access(legacyPath)).rejects.toThrow();
await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload });
});
});
it("does not replay a legacy file superseded by a sqlite sentinel", async () => {
await withRestartSentinelStateDir(async () => {
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
await fs.writeFile(
legacyPath,
`${JSON.stringify({
version: 1,
payload: {
kind: "update",
status: "ok",
ts: 1,
message: "stale legacy sentinel",
},
})}\n`,
"utf-8",
);
await writeRestartSentinel({
kind: "restart",
status: "ok",
ts: 2,
message: "current sqlite sentinel",
});
await expect(fs.access(legacyPath)).rejects.toThrow();
await clearRestartSentinel();
await expect(hasRestartSentinel()).resolves.toBe(false);
await expect(readRestartSentinel()).resolves.toBeNull();
});
});
it("drops invalid sentinel payloads", async () => {
await withRestartSentinelStateDir(async () => {
const filePath = resolveRestartSentinelPath();
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, "not-json", "utf-8");
insertSentinelRow({ payloadJson: "not-json" });
const read = await readRestartSentinel();
expect(read).toBeNull();
await expectPathMissing(filePath);
expect(readSentinelRow()).toBeUndefined();
});
});
it("drops structurally invalid sentinel payloads", async () => {
await withRestartSentinelStateDir(async () => {
const filePath = resolveRestartSentinelPath();
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, JSON.stringify({ version: 2, payload: null }), "utf-8");
insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) });
await expect(readRestartSentinel()).resolves.toBeNull();
await expectPathMissing(filePath);
expect(readSentinelRow()).toBeUndefined();
});
});

View File

@@ -1,11 +1,20 @@
// Persists restart sentinel files that coordinate deferred restarts.
import fs from "node:fs/promises";
// Persists restart sentinel state that coordinates deferred restarts.
import { readFile, rm } from "node:fs/promises";
import path from "node:path";
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
import { formatCliCommand } from "../cli/command-format.js";
import { resolveStateDir } from "../config/paths.js";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
} from "../state/openclaw-state-db.js";
import { resolveRuntimeServiceVersion } from "../version.js";
import { writeJson } from "./json-files.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
export type RestartSentinelLog = {
stdoutTail?: string | null;
@@ -66,7 +75,9 @@ export type RestartSentinel = {
payload: RestartSentinelPayload;
};
const SENTINEL_FILENAME = "restart-sentinel.json";
const RESTART_SENTINEL_KEY = "current";
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
export function formatDoctorNonInteractiveHint(
env: Record<string, string | undefined> = process.env as Record<string, string | undefined>,
@@ -77,18 +88,60 @@ export function formatDoctorNonInteractiveHint(
)} in a terminal or approvals-capable OpenClaw surface.`;
}
export function resolveRestartSentinelPath(env: NodeJS.ProcessEnv = process.env): string {
return path.join(resolveStateDir(env), SENTINEL_FILENAME);
}
export async function writeRestartSentinel(
payload: RestartSentinelPayload,
env: NodeJS.ProcessEnv = process.env,
) {
const filePath = resolveRestartSentinelPath(env);
const data: RestartSentinel = { version: 1, payload };
await writeJson(filePath, data, { trailingNewline: true, dirMode: 0o700 });
return filePath;
): Promise<void> {
const updatedAtMs = Date.now();
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.insertInto("gateway_restart_sentinel")
.values({
sentinel_key: RESTART_SENTINEL_KEY,
version: 1,
kind: payload.kind,
status: payload.status,
ts: payload.ts,
session_key: payload.sessionKey ?? null,
thread_id: payload.threadId ?? null,
delivery_channel: payload.deliveryContext?.channel ?? null,
delivery_to: payload.deliveryContext?.to ?? null,
delivery_account_id: payload.deliveryContext?.accountId ?? null,
message: payload.message ?? null,
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
doctor_hint: payload.doctorHint ?? null,
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
payload_json: JSON.stringify(payload),
updated_at_ms: updatedAtMs,
})
.onConflict((conflict) =>
conflict.column("sentinel_key").doUpdateSet({
version: (eb) => eb.ref("excluded.version"),
kind: (eb) => eb.ref("excluded.kind"),
status: (eb) => eb.ref("excluded.status"),
ts: (eb) => eb.ref("excluded.ts"),
session_key: (eb) => eb.ref("excluded.session_key"),
thread_id: (eb) => eb.ref("excluded.thread_id"),
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
message: (eb) => eb.ref("excluded.message"),
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
stats_json: (eb) => eb.ref("excluded.stats_json"),
payload_json: (eb) => eb.ref("excluded.payload_json"),
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
}),
),
);
},
{ env },
);
await removeLegacyRestartSentinel(env);
}
function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
@@ -156,11 +209,52 @@ export async function markUpdateRestartSentinelFailure(
}, env);
}
export async function removeRestartSentinelFile(filePath: string | null | undefined) {
if (!filePath) {
return;
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<void> {
try {
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.deleteFrom("gateway_restart_sentinel")
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
},
{ env },
);
} catch {}
await removeLegacyRestartSentinel(env);
}
function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string {
return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME);
}
async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise<void> {
try {
await rm(resolveLegacyRestartSentinelPath(env), { force: true });
} catch {}
}
async function importLegacyRestartSentinel(
env: NodeJS.ProcessEnv = process.env,
): Promise<RestartSentinel | null> {
const legacyPath = resolveLegacyRestartSentinelPath(env);
let parsed: unknown;
try {
parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown;
} catch {
return null;
}
await fs.unlink(filePath).catch(() => {});
if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) {
await removeLegacyRestartSentinel(env);
return null;
}
const payload = parsed.payload as RestartSentinelPayload;
await writeRestartSentinel(payload, env);
await removeLegacyRestartSentinel(env);
return { version: 1, payload };
}
export function buildRestartSuccessContinuation(params: {
@@ -177,26 +271,56 @@ export function buildRestartSuccessContinuation(params: {
export async function readRestartSentinel(
env: NodeJS.ProcessEnv = process.env,
): Promise<RestartSentinel | null> {
const filePath = resolveRestartSentinelPath(env);
try {
const raw = await fs.readFile(filePath, "utf-8");
let parsed: RestartSentinel | undefined;
const database = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["version", "payload_json"])
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
if (!row) {
return await importLegacyRestartSentinel(env);
}
let payload: RestartSentinelPayload | undefined;
try {
parsed = JSON.parse(raw) as RestartSentinel | undefined;
payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined;
} catch {
await fs.unlink(filePath).catch(() => {});
await clearRestartSentinel(env);
return null;
}
if (!parsed || parsed.version !== 1 || !parsed.payload) {
await fs.unlink(filePath).catch(() => {});
if (row.version !== 1 || !payload) {
await clearRestartSentinel(env);
return null;
}
return parsed;
return { version: 1, payload };
} catch {
return null;
}
}
export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
try {
const database = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select("sentinel_key")
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
);
if (row) {
return true;
}
return Boolean(await importLegacyRestartSentinel(env));
} catch {
return false;
}
}
export function formatRestartSentinelMessage(payload: RestartSentinelPayload): string {
const message = payload.message?.trim();
if (message && (!payload.stats || payload.kind === "config-auto-recovery")) {

View File

@@ -118,8 +118,8 @@ export async function readControlPlaneUpdateSentinelMeta(
export async function writeControlPlaneUpdateRestartSentinel(params: {
result: UpdateRunResult;
meta: UpdateRestartSentinelMeta;
}): Promise<string> {
return await writeRestartSentinel(
}): Promise<void> {
await writeRestartSentinel(
buildUpdateRestartSentinelPayload({
result: params.result,
meta: params.meta,

View File

@@ -5,6 +5,17 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import { SUPERVISOR_HINT_ENV_VARS } from "./supervisor-markers.js";
import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "./update-control-plane-sentinel.js";
import {
@@ -28,8 +39,10 @@ vi.mock("node:child_process", async () => {
});
const tempDirs = new Set<string>();
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
afterEach(async () => {
closeOpenClawStateDatabaseForTest();
spawnMock.mockClear();
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
tempDirs.clear();
@@ -44,10 +57,74 @@ async function pathExists(filePath: string): Promise<boolean> {
}
}
function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const payload =
sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1
? (sentinel as { payload?: unknown }).payload
: null;
if (!payload || typeof payload !== "object") {
throw new Error("expected versioned restart sentinel payload");
}
const record = payload as {
kind?: unknown;
status?: unknown;
ts?: unknown;
sessionKey?: unknown;
threadId?: unknown;
deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown };
message?: unknown;
continuation?: unknown;
doctorHint?: unknown;
stats?: unknown;
};
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
version: 1,
kind: typeof record.kind === "string" ? record.kind : "update",
status: typeof record.status === "string" ? record.status : "skipped",
ts: typeof record.ts === "number" ? record.ts : Date.now(),
session_key: typeof record.sessionKey === "string" ? record.sessionKey : null,
thread_id: typeof record.threadId === "string" ? record.threadId : null,
delivery_channel:
typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null,
delivery_to:
typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null,
delivery_account_id:
typeof record.deliveryContext?.accountId === "string"
? record.deliveryContext.accountId
: null,
message: typeof record.message === "string" ? record.message : null,
continuation_json: record.continuation ? JSON.stringify(record.continuation) : null,
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
stats_json: record.stats ? JSON.stringify(record.stats) : null,
payload_json: JSON.stringify(payload),
updated_at_ms: Date.now(),
}),
);
}
function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["version", "payload_json"])
.where("sentinel_key", "=", "current"),
);
return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null;
}
async function runHelperWithExistingSentinel(params: {
handoffId?: string;
metaHandoffId?: string;
sentinel: unknown;
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
sentinel?: unknown;
}) {
const { execFile } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
@@ -82,8 +159,11 @@ async function runHelperWithExistingSentinel(params: {
string,
unknown
>;
const sentinelPath = path.join(tmpDir, "restart-sentinel.json");
await fs.writeFile(sentinelPath, `${JSON.stringify(params.sentinel, null, 2)}\n`);
const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv;
await params.prepareStateDatabase?.(env);
if (params.sentinel !== undefined) {
writeRestartSentinelRow(env, params.sentinel);
}
const helperParamsPath = path.join(tmpDir, "helper-params.json");
await fs.writeFile(
helperParamsPath,
@@ -92,7 +172,7 @@ async function runHelperWithExistingSentinel(params: {
...helperParams,
parentPid: process.pid,
parentExitTimeoutMs: 1,
sentinelPath,
stateDatabasePath: resolveOpenClawStateSqlitePath(env),
logPath: path.join(tmpDir, "handoff.log"),
sensitivePaths: [],
},
@@ -113,7 +193,31 @@ async function runHelperWithExistingSentinel(params: {
},
);
return { result, sentinelPath };
return { result, env };
}
async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise<void> {
const sqlite = await import("node:sqlite");
const stateDatabasePath = resolveOpenClawStateSqlitePath(env);
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
const db = new sqlite.DatabaseSync(stateDatabasePath);
try {
db.exec(`
CREATE TABLE gateway_restart_sentinel (
sentinel_key TEXT NOT NULL PRIMARY KEY,
version INTEGER NOT NULL,
kind TEXT NOT NULL,
status TEXT NOT NULL,
ts INTEGER NOT NULL,
session_key TEXT,
thread_id TEXT,
payload_json TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL
);
`);
} finally {
db.close();
}
}
async function spawnExitedPid(): Promise<number> {
@@ -166,7 +270,7 @@ async function runHelperWithCommand(params: {
parentExitTimeoutMs: 5000,
cwd: tmpDir,
commandArgv: params.commandArgv,
sentinelPath: path.join(tmpDir, "restart-sentinel.json"),
stateDatabasePath: resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: tmpDir }),
logPath: path.join(tmpDir, "handoff.log"),
sensitivePaths: [],
...(params.serviceRecovery ? { serviceRecovery: params.serviceRecovery } : {}),
@@ -284,10 +388,10 @@ describe("managed service update handoff", () => {
const helperParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as {
cwd?: string;
metaPath?: string;
sentinelPath?: string;
stateDatabasePath?: string;
};
expect(helperParams.metaPath).toMatch(/sentinel-meta\.json$/u);
expect(helperParams.sentinelPath).toMatch(/restart-sentinel\.json$/u);
expect(helperParams.stateDatabasePath).toMatch(/openclaw\.sqlite$/u);
expect(options.cwd).toBe(os.homedir());
expect(helperParams.cwd).toBe(os.homedir());
expect(options.detached).toBe(true);
@@ -477,6 +581,51 @@ describe("managed service update handoff", () => {
}
});
it("writes a fallback update failure when no restart sentinel row exists", async () => {
const { result, env } = await runHelperWithExistingSentinel({
handoffId: "handoff-123",
metaHandoffId: "handoff-123",
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
version: 1,
payload: {
kind: "update",
status: "error",
sessionKey: "agent:test:webchat:dm:user-123",
stats: {
handoffId: "handoff-123",
reason: "managed-service-handoff-parent-timeout",
},
},
});
if (process.platform !== "win32") {
const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777;
expect(mode).toBe(0o600);
}
});
it("repairs legacy restart sentinel columns before writing fallback failures", async () => {
const { result, env } = await runHelperWithExistingSentinel({
handoffId: "handoff-123",
metaHandoffId: "handoff-123",
prepareStateDatabase: createLegacyRestartSentinelTable,
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
version: 1,
payload: {
kind: "update",
status: "error",
stats: {
reason: "managed-service-handoff-parent-timeout",
},
},
});
});
it("does not overwrite a restart sentinel owned by another startup task", async () => {
const unrelatedSentinel = {
version: 1,
@@ -487,14 +636,12 @@ describe("managed service update handoff", () => {
stats: { reason: "config-restart-pending" },
},
};
const { result, sentinelPath } = await runHelperWithExistingSentinel({
const { result, env } = await runHelperWithExistingSentinel({
sentinel: unrelatedSentinel,
});
expect(result).toEqual({ code: 1, signal: null });
await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual(
unrelatedSentinel,
);
expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel);
});
it("does not overwrite a newer pending update handoff sentinel", async () => {
@@ -513,16 +660,14 @@ describe("managed service update handoff", () => {
},
},
};
const { result, sentinelPath } = await runHelperWithExistingSentinel({
const { result, env } = await runHelperWithExistingSentinel({
handoffId: "old-handoff",
metaHandoffId: "old-handoff",
sentinel: newerSentinel,
});
expect(result).toEqual({ code: 1, signal: null });
await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual(
newerSentinel,
);
expect(readRestartSentinelPayload(env)).toEqual(newerSentinel);
});
it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => {

View File

@@ -9,7 +9,7 @@ import {
resolveGatewaySystemdServiceName,
resolveGatewayWindowsTaskName,
} from "../daemon/constants.js";
import { resolveRestartSentinelPath } from "./restart-sentinel.js";
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
import { SUPERVISOR_HINT_ENV_VARS, type RespawnSupervisor } from "./supervisor-markers.js";
import {
CONTROL_PLANE_UPDATE_SENTINEL_META_ENV,
@@ -96,26 +96,6 @@ function readJsonFile(filePath) {
}
}
function writeJsonFile(filePath, value) {
const dir = path.dirname(filePath);
const tempPath = path.join(
dir,
"." + path.basename(filePath) + "." + process.pid + "." + Date.now() + ".tmp",
);
try {
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
fs.writeFileSync(tempPath, JSON.stringify(value, null, 2) + "\n", { mode: 0o600 });
fs.renameSync(tempPath, filePath);
} catch (err) {
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
try {
fs.rmSync(tempPath, { force: true });
} catch {
// Best effort only.
}
}
}
function isPendingUpdatePayload(payload) {
const reason = payload && payload.stats && payload.stats.reason;
return (
@@ -126,6 +106,170 @@ function isPendingUpdatePayload(payload) {
);
}
function openStateDatabase() {
if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") {
return null;
}
try {
const sqlite = require("node:sqlite");
fs.mkdirSync(path.dirname(params.stateDatabasePath), { recursive: true, mode: 0o700 });
const db = new sqlite.DatabaseSync(params.stateDatabasePath);
db.exec([
"CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (",
"sentinel_key TEXT NOT NULL PRIMARY KEY,",
"version INTEGER NOT NULL,",
"kind TEXT NOT NULL,",
"status TEXT NOT NULL,",
"ts INTEGER NOT NULL,",
"session_key TEXT,",
"thread_id TEXT,",
"delivery_channel TEXT,",
"delivery_to TEXT,",
"delivery_account_id TEXT,",
"message TEXT,",
"continuation_json TEXT,",
"doctor_hint TEXT,",
"stats_json TEXT,",
"payload_json TEXT NOT NULL,",
"updated_at_ms INTEGER NOT NULL",
");",
"CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts",
"ON gateway_restart_sentinel(ts DESC, sentinel_key);",
].join(" "));
ensureGatewayRestartSentinelColumns(db);
hardenStateDatabaseFiles();
return db;
} catch (err) {
appendLog("failed to open restart sentinel database: " + (err && err.stack ? err.stack : String(err)));
return null;
}
}
function tableHasColumn(db, tableName, columnName) {
try {
return db.prepare("PRAGMA table_info(" + tableName + ")").all().some((row) => row && row.name === columnName);
} catch {
return false;
}
}
function ensureColumn(db, tableName, columnSql) {
const columnName = columnSql.trim().split(/\s+/, 1)[0];
if (!columnName || tableHasColumn(db, tableName, columnName)) {
return;
}
db.exec("ALTER TABLE " + tableName + " ADD COLUMN " + columnSql + ";");
}
function ensureGatewayRestartSentinelColumns(db) {
ensureColumn(db, "gateway_restart_sentinel", "delivery_channel TEXT");
ensureColumn(db, "gateway_restart_sentinel", "delivery_to TEXT");
ensureColumn(db, "gateway_restart_sentinel", "delivery_account_id TEXT");
ensureColumn(db, "gateway_restart_sentinel", "message TEXT");
ensureColumn(db, "gateway_restart_sentinel", "continuation_json TEXT");
ensureColumn(db, "gateway_restart_sentinel", "doctor_hint TEXT");
ensureColumn(db, "gateway_restart_sentinel", "stats_json TEXT");
}
function hardenStateDatabaseFiles() {
if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") {
return;
}
for (const filePath of [
params.stateDatabasePath,
params.stateDatabasePath + "-wal",
params.stateDatabasePath + "-shm",
]) {
try {
if (fs.existsSync(filePath)) {
fs.chmodSync(filePath, 0o600);
}
} catch {
// Best effort only.
}
}
}
function readRestartSentinelPayload() {
const db = openStateDatabase();
if (!db) {
return null;
}
try {
const row = db
.prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?")
.get("current");
if (!row || row.version !== 1 || typeof row.payload_json !== "string") {
return null;
}
return JSON.parse(row.payload_json);
} catch {
return null;
} finally {
hardenStateDatabaseFiles();
try {
db.close();
} catch {}
}
}
function writeRestartSentinelPayload(payload) {
const db = openStateDatabase();
if (!db) {
return;
}
try {
const updatedAtMs = Date.now();
db.prepare(
[
"INSERT INTO gateway_restart_sentinel (",
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
"doctor_hint, stats_json, payload_json, updated_at_ms",
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"ON CONFLICT(sentinel_key) DO UPDATE SET",
"version = excluded.version, kind = excluded.kind, status = excluded.status,",
"ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,",
"delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,",
"delivery_account_id = excluded.delivery_account_id, message = excluded.message,",
"continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,",
"stats_json = excluded.stats_json, payload_json = excluded.payload_json,",
"updated_at_ms = excluded.updated_at_ms",
].join(" "),
).run(
"current",
1,
payload.kind,
payload.status,
payload.ts,
payload.sessionKey || null,
payload.threadId || null,
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
? payload.deliveryContext.channel
: null,
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
? payload.deliveryContext.to
: null,
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
? payload.deliveryContext.accountId
: null,
payload.message || null,
payload.continuation ? JSON.stringify(payload.continuation) : null,
payload.doctorHint || null,
payload.stats ? JSON.stringify(payload.stats) : null,
JSON.stringify(payload),
updatedAtMs,
);
} catch (err) {
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
} finally {
hardenStateDatabaseFiles();
try {
db.close();
} catch {}
}
}
function buildFallbackFailurePayload(reason) {
const metaFile = params.metaPath ? readJsonFile(params.metaPath) : null;
const meta = metaFile && metaFile.version === 1 && metaFile.meta ? metaFile.meta : {};
@@ -157,11 +301,7 @@ function buildFallbackFailurePayload(reason) {
}
function markUpdateSentinelFailureIfPending(reason) {
if (!params.sentinelPath) {
return;
}
const current = readJsonFile(params.sentinelPath);
let payload = current && current.version === 1 ? current.payload : null;
let payload = readRestartSentinelPayload();
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
return;
}
@@ -176,7 +316,7 @@ function markUpdateSentinelFailureIfPending(reason) {
} else {
payload = buildFallbackFailurePayload(reason);
}
writeJsonFile(params.sentinelPath, { version: 1, payload });
writeRestartSentinelPayload(payload);
}
function runServiceCommand(command, args) {
@@ -553,7 +693,7 @@ export async function startManagedServiceUpdateHandoff(params: {
handoffId: params.handoffId,
logPath,
metaPath,
sentinelPath: resolveRestartSentinelPath(),
stateDatabasePath: resolveOpenClawStateSqlitePath(params.env ?? process.env),
sensitivePaths: [scriptPath, paramsPath, metaPath],
serviceRecovery: resolveGatewayServiceRecovery(params.supervisor, params.env ?? process.env),
};