mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-24 04:29:29 +00:00
fix(deadcode): move restart sentinels to sqlite
This commit is contained in:
@@ -3,10 +3,11 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js";
|
||||
import { GatewayClientRequestError } from "../gateway/client.js";
|
||||
import { readRestartSentinel } from "../infra/restart-sentinel.js";
|
||||
import { testing as restartTesting } from "../infra/restart.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import { normalizeConfigPatchReplacePath } from "../config/patch-replace-paths.js";
|
||||
import { createGatewayTool } from "./tools/gateway-tool.js";
|
||||
import { callGatewayTool } from "./tools/gateway.js";
|
||||
|
||||
@@ -332,13 +333,9 @@ describe("gateway tool", () => {
|
||||
});
|
||||
expect(restartSignalKillCalls()).toHaveLength(0);
|
||||
|
||||
const sentinelPath = path.join(stateDir, "restart-sentinel.json");
|
||||
const raw = await fs.readFile(sentinelPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as {
|
||||
payload?: { kind?: string; doctorHint?: string | null };
|
||||
};
|
||||
expect(parsed.payload?.kind).toBe("restart");
|
||||
expect(parsed.payload?.doctorHint).toBe(
|
||||
const sentinel = await readRestartSentinel();
|
||||
expect(sentinel?.payload.kind).toBe("restart");
|
||||
expect(sentinel?.payload.doctorHint).toBe(
|
||||
"Recommended follow-up: run openclaw --profile isolated doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
|
||||
);
|
||||
},
|
||||
@@ -507,15 +504,13 @@ describe("gateway tool", () => {
|
||||
it("distinguishes explicit terminal array consent from indexed consent", () => {
|
||||
expect(normalizeConfigPatchReplacePath("bindings[]")).toBe("bindings");
|
||||
expect(normalizeConfigPatchReplacePath("bindings[0]")).toBe("bindings[0]");
|
||||
expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe(
|
||||
"agents.list[].skills",
|
||||
);
|
||||
expect(normalizeConfigPatchReplacePath("agents.list[0].skills")).toBe("agents.list[].skills");
|
||||
expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[]"))).toBe(
|
||||
"bindings",
|
||||
);
|
||||
expect(
|
||||
normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]")),
|
||||
).toBe("bindings[0]");
|
||||
expect(normalizeConfigPatchReplacePath(normalizeConfigPatchReplacePath("bindings[0]"))).toBe(
|
||||
"bindings[0]",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects config.patch when it changes safe bin approval paths", async () => {
|
||||
|
||||
@@ -12,7 +12,7 @@ const {
|
||||
formatDoctorNonInteractiveHintMock,
|
||||
isRestartEnabledMock,
|
||||
callGatewayToolMock,
|
||||
removeRestartSentinelFileMock,
|
||||
clearRestartSentinelMock,
|
||||
scheduleGatewaySigusr1RestartMock,
|
||||
writeRestartSentinelMock,
|
||||
} = vi.hoisted(() => ({
|
||||
@@ -30,8 +30,8 @@ const {
|
||||
() =>
|
||||
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
|
||||
),
|
||||
writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/restart"),
|
||||
removeRestartSentinelFileMock: vi.fn(async (_path: string | null | undefined) => undefined),
|
||||
writeRestartSentinelMock: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
|
||||
clearRestartSentinelMock: vi.fn(async () => undefined),
|
||||
scheduleGatewaySigusr1RestartMock: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({
|
||||
ok: true,
|
||||
pid: 123,
|
||||
@@ -59,7 +59,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
|
||||
return {
|
||||
...actual,
|
||||
formatDoctorNonInteractiveHint: formatDoctorNonInteractiveHintMock,
|
||||
removeRestartSentinelFile: removeRestartSentinelFileMock,
|
||||
clearRestartSentinel: clearRestartSentinelMock,
|
||||
writeRestartSentinel: writeRestartSentinelMock,
|
||||
};
|
||||
});
|
||||
@@ -115,8 +115,8 @@ describe("gateway tool restart continuation", () => {
|
||||
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
|
||||
);
|
||||
writeRestartSentinelMock.mockReset();
|
||||
writeRestartSentinelMock.mockResolvedValue("/tmp/restart");
|
||||
removeRestartSentinelFileMock.mockClear();
|
||||
writeRestartSentinelMock.mockResolvedValue(undefined);
|
||||
clearRestartSentinelMock.mockClear();
|
||||
scheduleGatewaySigusr1RestartMock.mockReset();
|
||||
scheduleGatewaySigusr1RestartMock.mockReturnValue({
|
||||
ok: true,
|
||||
@@ -354,7 +354,7 @@ describe("gateway tool restart continuation", () => {
|
||||
await scheduledArgs.emitHooks?.beforeEmit?.();
|
||||
await scheduledArgs.emitHooks?.afterEmitRejected?.();
|
||||
|
||||
expect(removeRestartSentinelFileMock).toHaveBeenCalledWith("/tmp/restart");
|
||||
expect(clearRestartSentinelMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("uses the runtime session for update.run continuation routing (#86742)", async () => {
|
||||
|
||||
@@ -19,8 +19,8 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { GatewayClientRequestError } from "../../gateway/client.js";
|
||||
import {
|
||||
buildRestartSuccessContinuation,
|
||||
clearRestartSentinel,
|
||||
formatDoctorNonInteractiveHint,
|
||||
removeRestartSentinelFile,
|
||||
type RestartSentinelPayload,
|
||||
writeRestartSentinel,
|
||||
} from "../../infra/restart-sentinel.js";
|
||||
@@ -493,7 +493,7 @@ export function createGatewayTool(opts?: {
|
||||
log.info(
|
||||
`gateway tool: restart requested (delayMs=${delayMs ?? "default"}, reason=${reason ?? "none"})`,
|
||||
);
|
||||
let sentinelPath: string | null = null;
|
||||
let sentinelWritten = false;
|
||||
const scheduled = scheduleGatewaySigusr1Restart({
|
||||
delayMs,
|
||||
reason,
|
||||
@@ -502,10 +502,13 @@ export function createGatewayTool(opts?: {
|
||||
sessionKey,
|
||||
emitHooks: {
|
||||
beforeEmit: async () => {
|
||||
sentinelPath = await writeRestartSentinel(payload);
|
||||
await writeRestartSentinel(payload);
|
||||
sentinelWritten = true;
|
||||
},
|
||||
afterEmitRejected: async () => {
|
||||
await removeRestartSentinelFile(sentinelPath);
|
||||
if (sentinelWritten) {
|
||||
await clearRestartSentinel();
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { HandleCommandsParams } from "./commands-types.js";
|
||||
type ScheduleGatewayRestartArgs = Parameters<typeof scheduleGatewaySigusr1Restart>[0];
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
unlink: vi.fn(async (_path: string) => undefined),
|
||||
clearRestartSentinel: vi.fn(async () => undefined),
|
||||
isRestartEnabled: vi.fn(() => true),
|
||||
extractDeliveryInfo: vi.fn(() => ({
|
||||
deliveryContext: {
|
||||
@@ -21,20 +21,13 @@ const mocks = vi.hoisted(() => ({
|
||||
() =>
|
||||
"Recommended follow-up: run openclaw doctor --non-interactive in a terminal or approvals-capable OpenClaw surface.",
|
||||
),
|
||||
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => "/tmp/sentinel.json"),
|
||||
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
|
||||
scheduleGatewaySigusr1Restart: vi.fn((_opts?: ScheduleGatewayRestartArgs) => ({
|
||||
scheduled: true,
|
||||
})),
|
||||
triggerOpenClawRestart: vi.fn(() => ({ ok: true, method: "launchctl" })),
|
||||
}));
|
||||
|
||||
vi.mock("node:fs/promises", () => ({
|
||||
default: {
|
||||
unlink: mocks.unlink,
|
||||
},
|
||||
unlink: mocks.unlink,
|
||||
}));
|
||||
|
||||
vi.mock("../../config/commands.flags.js", () => ({
|
||||
isRestartEnabled: mocks.isRestartEnabled,
|
||||
}));
|
||||
@@ -67,6 +60,7 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
clearRestartSentinel: mocks.clearRestartSentinel,
|
||||
formatDoctorNonInteractiveHint: mocks.formatDoctorNonInteractiveHint,
|
||||
writeRestartSentinel: mocks.writeRestartSentinel,
|
||||
};
|
||||
@@ -119,7 +113,7 @@ describe("handleRestartCommand", () => {
|
||||
beforeEach(() => {
|
||||
mocks.isRestartEnabled.mockReset();
|
||||
mocks.isRestartEnabled.mockReturnValue(true);
|
||||
mocks.unlink.mockClear();
|
||||
mocks.clearRestartSentinel.mockClear();
|
||||
mocks.extractDeliveryInfo.mockClear();
|
||||
mocks.formatDoctorNonInteractiveHint.mockClear();
|
||||
mocks.writeRestartSentinel.mockClear();
|
||||
@@ -218,7 +212,7 @@ describe("handleRestartCommand", () => {
|
||||
expect(mocks.triggerOpenClawRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("removes the success sentinel when fallback restart fails", async () => {
|
||||
it("clears the success sentinel when fallback restart fails", async () => {
|
||||
mocks.triggerOpenClawRestart.mockReturnValueOnce({
|
||||
ok: false,
|
||||
method: "launchctl",
|
||||
@@ -227,6 +221,6 @@ describe("handleRestartCommand", () => {
|
||||
const result = await handleRestartCommand(restartCommandParams(), true);
|
||||
|
||||
expect(result?.reply?.text).toContain("Restart failed");
|
||||
expect(mocks.unlink).toHaveBeenCalledWith("/tmp/sentinel.json");
|
||||
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,8 +21,8 @@ import { getSessionBindingService } from "../../infra/outbound/session-binding-s
|
||||
import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js";
|
||||
import {
|
||||
buildRestartSuccessContinuation,
|
||||
clearRestartSentinel,
|
||||
formatDoctorNonInteractiveHint,
|
||||
removeRestartSentinelFile,
|
||||
type RestartSentinelPayload,
|
||||
writeRestartSentinel,
|
||||
} from "../../infra/restart-sentinel.js";
|
||||
@@ -703,7 +703,7 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
|
||||
const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0;
|
||||
const sentinelPayload = buildRestartCommandSentinel(params);
|
||||
if (hasSigusr1Listener) {
|
||||
let sentinelPath: string | null = null;
|
||||
let sentinelWritten = false;
|
||||
scheduleGatewaySigusr1Restart({
|
||||
reason: "/restart",
|
||||
// Sibling session-routing guard: /restart writes a session-scoped sentinel
|
||||
@@ -713,10 +713,13 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
|
||||
emitHooks: sentinelPayload
|
||||
? {
|
||||
beforeEmit: async () => {
|
||||
sentinelPath = await writeRestartSentinel(sentinelPayload);
|
||||
await writeRestartSentinel(sentinelPayload);
|
||||
sentinelWritten = true;
|
||||
},
|
||||
afterEmitRejected: async () => {
|
||||
await removeRestartSentinelFile(sentinelPath);
|
||||
if (sentinelWritten) {
|
||||
await clearRestartSentinel();
|
||||
}
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
@@ -728,10 +731,11 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
|
||||
},
|
||||
};
|
||||
}
|
||||
let sentinelPath: string | null = null;
|
||||
let sentinelWritten = false;
|
||||
try {
|
||||
if (sentinelPayload) {
|
||||
sentinelPath = await writeRestartSentinel(sentinelPayload);
|
||||
await writeRestartSentinel(sentinelPayload);
|
||||
sentinelWritten = true;
|
||||
}
|
||||
} catch (err) {
|
||||
logVerbose(`failed to write /restart sentinel: ${String(err)}`);
|
||||
@@ -744,7 +748,9 @@ export const handleRestartCommand: CommandHandler = async (params, allowTextComm
|
||||
}
|
||||
const restartMethod = triggerOpenClawRestart();
|
||||
if (!restartMethod.ok) {
|
||||
await removeRestartSentinelFile(sentinelPath);
|
||||
if (sentinelWritten) {
|
||||
await clearRestartSentinel();
|
||||
}
|
||||
const detail = restartMethod.detail ? ` Details: ${restartMethod.detail}` : "";
|
||||
return {
|
||||
shouldContinue: false,
|
||||
|
||||
@@ -366,6 +366,7 @@ const { updateCommand, updateFinalizeCommand, updateStatusCommand, updateWizardC
|
||||
const updateCliShared = await import("./update-cli/shared.js");
|
||||
const { ensureGitCheckout, resolveGitInstallDir } = updateCliShared;
|
||||
const { spawnSync } = await import("node:child_process");
|
||||
const { readRestartSentinel } = await import("../infra/restart-sentinel.js");
|
||||
|
||||
function requireValue<T>(value: T | undefined, label: string): T {
|
||||
if (value === undefined) {
|
||||
@@ -5891,23 +5892,17 @@ describe("update-cli", () => {
|
||||
},
|
||||
);
|
||||
|
||||
const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8");
|
||||
const sentinel = JSON.parse(raw) as {
|
||||
payload?: {
|
||||
status?: string;
|
||||
message?: string | null;
|
||||
continuation?: { kind?: string; message?: string };
|
||||
stats?: { mode?: string; after?: { version?: string | null } };
|
||||
};
|
||||
};
|
||||
expect(sentinel.payload?.status).toBe("ok");
|
||||
expect(sentinel.payload?.message).toBe("Update requested from the agent.");
|
||||
expect(sentinel.payload?.continuation).toEqual({
|
||||
const sentinel = await readRestartSentinel({
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
} as NodeJS.ProcessEnv);
|
||||
expect(sentinel?.payload.status).toBe("ok");
|
||||
expect(sentinel?.payload.message).toBe("Update requested from the agent.");
|
||||
expect(sentinel?.payload.continuation).toEqual({
|
||||
kind: "agentTurn",
|
||||
message: "Check the running version and finish the update report.",
|
||||
});
|
||||
expect(sentinel.payload?.stats?.mode).toBe("npm");
|
||||
expect(sentinel.payload?.stats?.after?.version).toBe("2026.4.24");
|
||||
expect(sentinel?.payload.stats?.mode).toBe("npm");
|
||||
expect(sentinel?.payload.stats?.after?.version).toBe("2026.4.24");
|
||||
});
|
||||
|
||||
it("marks the control-plane update sentinel failed when restart health verification fails", async () => {
|
||||
@@ -5966,17 +5961,12 @@ describe("update-cli", () => {
|
||||
},
|
||||
);
|
||||
|
||||
const raw = await fs.readFile(path.join(stateDir, "restart-sentinel.json"), "utf-8");
|
||||
const sentinel = JSON.parse(raw) as {
|
||||
payload?: {
|
||||
status?: string;
|
||||
continuation?: unknown;
|
||||
stats?: { reason?: string | null };
|
||||
};
|
||||
};
|
||||
expect(sentinel.payload?.status).toBe("error");
|
||||
expect(sentinel.payload?.stats?.reason).toBe("restart-unhealthy");
|
||||
expect(sentinel.payload?.continuation).toBeUndefined();
|
||||
const sentinel = await readRestartSentinel({
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
} as NodeJS.ProcessEnv);
|
||||
expect(sentinel?.payload.status).toBe("error");
|
||||
expect(sentinel?.payload.stats?.reason).toBe("restart-unhealthy");
|
||||
expect(sentinel?.payload.continuation).toBeUndefined();
|
||||
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
|
||||
});
|
||||
|
||||
|
||||
@@ -203,13 +203,12 @@ function buildConfigRestartSentinelPayload(params: {
|
||||
};
|
||||
}
|
||||
|
||||
async function tryWriteRestartSentinelPayload(
|
||||
payload: RestartSentinelPayload,
|
||||
): Promise<string | null> {
|
||||
async function tryWriteRestartSentinelPayload(payload: RestartSentinelPayload): Promise<boolean> {
|
||||
try {
|
||||
return await writeRestartSentinel(payload);
|
||||
await writeRestartSentinel(payload);
|
||||
return true;
|
||||
} catch {
|
||||
return null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,7 +255,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
|
||||
context?: GatewayRequestContext;
|
||||
}): Promise<{
|
||||
payload: RestartSentinelPayload;
|
||||
sentinelPath: string | null;
|
||||
sentinelPersisted: boolean;
|
||||
restart: ReturnType<typeof scheduleGatewaySigusr1Restart> | undefined;
|
||||
}> {
|
||||
const { sessionKey, note, restartDelayMs, deliveryContext, threadId } =
|
||||
@@ -270,7 +269,7 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
|
||||
threadId,
|
||||
note,
|
||||
});
|
||||
const sentinelPath = await tryWriteRestartSentinelPayload(payload);
|
||||
const sentinelPersisted = await tryWriteRestartSentinelPayload(payload);
|
||||
const restart = shouldScheduleDirectConfigRestart({
|
||||
changedPaths: params.changedPaths,
|
||||
nextConfig: params.nextConfig,
|
||||
@@ -291,5 +290,5 @@ export async function resolveGatewayConfigRestartWriteResult(params: {
|
||||
`${params.mode} restart coalesced ${formatControlPlaneActor(params.actor)} delayMs=${restart.delayMs}`,
|
||||
);
|
||||
}
|
||||
return { payload, sentinelPath, restart };
|
||||
return { payload, sentinelPersisted, restart };
|
||||
}
|
||||
|
||||
@@ -21,9 +21,7 @@ const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({
|
||||
coalesced: false,
|
||||
}));
|
||||
const restartSentinelMocks = vi.hoisted(() => ({
|
||||
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => {
|
||||
return "/tmp/restart-sentinel.json";
|
||||
}),
|
||||
writeRestartSentinel: vi.fn(async (_payload: RestartSentinelPayload) => undefined),
|
||||
}));
|
||||
|
||||
vi.mock("../../config/config.js", async () => {
|
||||
|
||||
@@ -594,7 +594,7 @@ async function respondWithConfigRestartWrite(params: {
|
||||
uiHints: ConfigRedactionHints;
|
||||
}): Promise<void> {
|
||||
clearConfigSchemaResponseCache();
|
||||
const { payload, sentinelPath, restart } = await resolveGatewayConfigRestartWriteResult({
|
||||
const { payload, sentinelPersisted, restart } = await resolveGatewayConfigRestartWriteResult({
|
||||
requestParams: params.requestParams,
|
||||
kind: params.kind,
|
||||
mode: params.mode,
|
||||
@@ -612,7 +612,7 @@ async function respondWithConfigRestartWrite(params: {
|
||||
config: redactConfigObject(params.writeResult.config, params.uiHints),
|
||||
restart,
|
||||
sentinel: {
|
||||
path: sentinelPath,
|
||||
persisted: sentinelPersisted,
|
||||
payload,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -49,7 +49,7 @@ type UpdateRunPayload = {
|
||||
ok: boolean;
|
||||
result?: { status?: string; reason?: string; mode?: string };
|
||||
handoff?: { status?: string; command?: string; message?: string };
|
||||
sentinel?: { path?: string | null };
|
||||
sentinel?: { persisted?: boolean };
|
||||
restart?: unknown;
|
||||
};
|
||||
|
||||
@@ -97,7 +97,6 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
|
||||
...(actual as Record<string, unknown>),
|
||||
writeRestartSentinel: async (payload: RestartSentinelPayload) => {
|
||||
capturedPayload = payload;
|
||||
return "/tmp/sentinel.json";
|
||||
},
|
||||
};
|
||||
});
|
||||
@@ -462,7 +461,7 @@ describe("update.run restart scheduling", () => {
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --timeout 1800",
|
||||
});
|
||||
expect(payload?.sentinel?.path).toBe("/tmp/sentinel.json");
|
||||
expect(payload?.sentinel?.persisted).toBe(true);
|
||||
const sentinel = readCapturedPayload();
|
||||
expect(sentinel.kind).toBe("update");
|
||||
expect(sentinel.status).toBe("skipped");
|
||||
|
||||
@@ -340,12 +340,13 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
meta: sentinelMeta,
|
||||
});
|
||||
|
||||
let sentinelPath: string | null;
|
||||
let sentinelPersisted: boolean;
|
||||
try {
|
||||
sentinelPath = await writeRestartSentinel(payload);
|
||||
await writeRestartSentinel(payload);
|
||||
sentinelPersisted = true;
|
||||
recordLatestUpdateRestartSentinel(payload);
|
||||
} catch {
|
||||
sentinelPath = null;
|
||||
sentinelPersisted = false;
|
||||
}
|
||||
|
||||
// Only restart the gateway when the update actually succeeded.
|
||||
@@ -391,7 +392,7 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
...(handoff ? { handoff } : {}),
|
||||
restart,
|
||||
sentinel: {
|
||||
path: sentinelPath,
|
||||
persisted: sentinelPersisted,
|
||||
payload,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -42,8 +42,7 @@ const mocks = vi.hoisted(() => {
|
||||
}),
|
||||
),
|
||||
finalizeUpdateRestartSentinelRunningVersion: vi.fn(async () => null),
|
||||
removeRestartSentinelFile: vi.fn(async () => undefined),
|
||||
resolveRestartSentinelPath: vi.fn(() => "/tmp/restart-sentinel.json"),
|
||||
clearRestartSentinel: vi.fn(async () => undefined),
|
||||
formatRestartSentinelMessage: vi.fn(() => "restart message"),
|
||||
summarizeRestartSentinel: vi.fn(() => "restart summary"),
|
||||
resolveMainSessionKeyFromConfig: vi.fn(() => "agent:main:main"),
|
||||
@@ -183,8 +182,7 @@ vi.mock("../agents/agent-scope.js", async () => {
|
||||
vi.mock("../infra/restart-sentinel.js", () => ({
|
||||
finalizeUpdateRestartSentinelRunningVersion: mocks.finalizeUpdateRestartSentinelRunningVersion,
|
||||
readRestartSentinel: mocks.readRestartSentinel,
|
||||
removeRestartSentinelFile: mocks.removeRestartSentinelFile,
|
||||
resolveRestartSentinelPath: mocks.resolveRestartSentinelPath,
|
||||
clearRestartSentinel: mocks.clearRestartSentinel,
|
||||
formatRestartSentinelMessage: mocks.formatRestartSentinelMessage,
|
||||
summarizeRestartSentinel: mocks.summarizeRestartSentinel,
|
||||
}));
|
||||
@@ -427,7 +425,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
mocks.recoverPendingSessionDeliveries.mockClear();
|
||||
mocks.finalizeUpdateRestartSentinelRunningVersion.mockReset();
|
||||
mocks.finalizeUpdateRestartSentinelRunningVersion.mockResolvedValue(null);
|
||||
mocks.removeRestartSentinelFile.mockClear();
|
||||
mocks.clearRestartSentinel.mockClear();
|
||||
mocks.injectTimestamp.mockClear();
|
||||
mocks.timestampOptsFromConfig.mockClear();
|
||||
mocks.recordInboundSessionAndDispatchReply.mockReset();
|
||||
@@ -1295,7 +1293,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.removeRestartSentinelFile).not.toHaveBeenCalled();
|
||||
expect(mocks.clearRestartSentinel).not.toHaveBeenCalled();
|
||||
expect(mocks.drainPendingSessionDeliveries).not.toHaveBeenCalled();
|
||||
expect(mocks.logWarn).toHaveBeenCalledWith("startup task failed", {
|
||||
source: "restart-sentinel",
|
||||
@@ -1359,7 +1357,7 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
|
||||
expect(mocks.removeRestartSentinelFile).toHaveBeenCalledWith("/tmp/restart-sentinel.json");
|
||||
expect(mocks.clearRestartSentinel).toHaveBeenCalledOnce();
|
||||
expect(getLatestUpdateRestartSentinel()).toEqual(payload);
|
||||
});
|
||||
|
||||
|
||||
@@ -18,13 +18,12 @@ import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/de
|
||||
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
|
||||
import { resolveOutboundTarget } from "../infra/outbound/targets.js";
|
||||
import {
|
||||
clearRestartSentinel,
|
||||
finalizeUpdateRestartSentinelRunningVersion,
|
||||
formatRestartSentinelMessage,
|
||||
readRestartSentinel,
|
||||
removeRestartSentinelFile,
|
||||
type RestartSentinelContinuation,
|
||||
type RestartSentinelPayload,
|
||||
resolveRestartSentinelPath,
|
||||
summarizeRestartSentinel,
|
||||
} from "../infra/restart-sentinel.js";
|
||||
import {
|
||||
@@ -449,7 +448,6 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
if (!sentinel) {
|
||||
return null;
|
||||
}
|
||||
const sentinelPath = resolveRestartSentinelPath();
|
||||
const payload = sentinel.payload;
|
||||
if (payload.kind === "update") {
|
||||
recordLatestUpdateRestartSentinel(payload);
|
||||
@@ -494,7 +492,7 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
continuationKind: payload.continuation.kind,
|
||||
});
|
||||
}
|
||||
await removeRestartSentinelFile(sentinelPath);
|
||||
await clearRestartSentinel();
|
||||
return { status: "ran" as const };
|
||||
}
|
||||
|
||||
@@ -588,7 +586,7 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
);
|
||||
}
|
||||
|
||||
await removeRestartSentinelFile(sentinelPath);
|
||||
await clearRestartSentinel();
|
||||
const routedAgentTurnContinuation =
|
||||
payload.continuation?.kind === "agentTurn" && continuationRoute !== undefined;
|
||||
if (!routedAgentTurnContinuation) {
|
||||
|
||||
@@ -5,11 +5,13 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { writeRestartSentinel } from "../infra/restart-sentinel.js";
|
||||
import type {
|
||||
PluginHookGatewayContext,
|
||||
PluginHookGatewayStartEvent,
|
||||
} from "../plugins/hook-types.js";
|
||||
import type { PluginServicesHandle } from "../plugins/services.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
|
||||
const hoisted = vi.hoisted(() => {
|
||||
@@ -135,7 +137,9 @@ vi.mock("../config/paths.js", async () => {
|
||||
STATE_DIR: "/tmp/openclaw-state",
|
||||
resolveConfigPath: vi.fn(() => "/tmp/openclaw-state/openclaw.json"),
|
||||
resolveGatewayPort: vi.fn(() => 18789),
|
||||
resolveStateDir: vi.fn(() => "/tmp/openclaw-state"),
|
||||
resolveStateDir: vi.fn((env: NodeJS.ProcessEnv = process.env) =>
|
||||
env.OPENCLAW_STATE_DIR?.trim() ? actual.resolveStateDir(env) : "/tmp/openclaw-state",
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -295,6 +299,7 @@ function firstGatewayStartCall(
|
||||
|
||||
describe("startGatewayPostAttachRuntime", () => {
|
||||
beforeEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
vi.stubEnv("OPENCLAW_SKIP_CHANNELS", "0");
|
||||
vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", "0");
|
||||
hoisted.startPluginServices.mockClear();
|
||||
@@ -325,6 +330,8 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
});
|
||||
hoisted.scheduleRestartAbortedMainSessionRecovery.mockClear();
|
||||
hoisted.scheduleRestartSentinelWake.mockClear();
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockReset();
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(null);
|
||||
hoisted.getAcpRuntimeBackend.mockReset();
|
||||
hoisted.getAcpRuntimeBackend.mockReturnValue(null);
|
||||
hoisted.reconcilePendingSessionIdentities.mockClear();
|
||||
@@ -355,6 +362,7 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
@@ -556,62 +564,76 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
it("skips heavy restart sentinel refresh when no sentinel file exists", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-no-sentinel-"));
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockClear();
|
||||
|
||||
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(hoisted.refreshLatestUpdateRestartSentinel).not.toHaveBeenCalled();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("refreshes the restart sentinel when the sentinel file exists", async () => {
|
||||
it("refreshes the restart sentinel when the sentinel row exists", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-"));
|
||||
fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n");
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
const sentinel = { kind: "update", status: "ok", ts: 1 } as const;
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel);
|
||||
try {
|
||||
await writeRestartSentinel(
|
||||
{
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
},
|
||||
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
|
||||
);
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
const sentinel = { kind: "update", status: "ok", ts: 1 } as const;
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockClear();
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockResolvedValue(sentinel);
|
||||
|
||||
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
|
||||
const result = await testing.refreshLatestUpdateRestartSentinelIfPresent();
|
||||
|
||||
expect(result).toBe(sentinel);
|
||||
expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce();
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
expect(result).toBe(sentinel);
|
||||
expect(hoisted.refreshLatestUpdateRestartSentinel).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("expands tilde-based restart sentinel state paths", async () => {
|
||||
const osHome = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-home-"));
|
||||
it("detects restart sentinel rows in explicit state directories", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sentinel-state-"));
|
||||
try {
|
||||
const openclawHome = path.join(osHome, "openclaw-home");
|
||||
const stateDirFromHome = path.join(openclawHome, ".openclaw");
|
||||
fs.mkdirSync(stateDirFromHome, { recursive: true });
|
||||
fs.writeFileSync(path.join(stateDirFromHome, "restart-sentinel.json"), "{}\n");
|
||||
await writeRestartSentinel(
|
||||
{
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
},
|
||||
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
|
||||
);
|
||||
|
||||
expect(
|
||||
await testing.hasRestartSentinelFileFast({
|
||||
HOME: osHome,
|
||||
OPENCLAW_HOME: "~/openclaw-home",
|
||||
} as NodeJS.ProcessEnv),
|
||||
).toBe(true);
|
||||
|
||||
const backslashStateDir = path.resolve(`${osHome}\\openclaw-state`);
|
||||
fs.mkdirSync(backslashStateDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(backslashStateDir, "restart-sentinel.json"), "{}\n");
|
||||
|
||||
expect(
|
||||
await testing.hasRestartSentinelFileFast({
|
||||
HOME: osHome,
|
||||
OPENCLAW_STATE_DIR: "~\\openclaw-state",
|
||||
await testing.hasRestartSentinelFast({
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
} as NodeJS.ProcessEnv),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
fs.rmSync(osHome, { recursive: true, force: true });
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("avoids sync filesystem probes while checking restart sentinel presence", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-async-sentinel-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(stateDir, "restart-sentinel.json"), "{}\n");
|
||||
await writeRestartSentinel(
|
||||
{
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
},
|
||||
{ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv,
|
||||
);
|
||||
const actualExistsSync = fs.existsSync;
|
||||
const existsSync = vi.spyOn(fs, "existsSync").mockImplementation((candidate) => {
|
||||
if (String(candidate).startsWith(stateDir)) {
|
||||
@@ -621,7 +643,7 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
});
|
||||
try {
|
||||
await expect(
|
||||
testing.hasRestartSentinelFileFast({
|
||||
testing.hasRestartSentinelFast({
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
} as NodeJS.ProcessEnv),
|
||||
).resolves.toBe(true);
|
||||
@@ -632,6 +654,7 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
existsSync.mockRestore();
|
||||
}
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
// Gateway post-attach startup sidecars.
|
||||
// Schedules warmups, sentinels, update checks, memory backend, and plugin services.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
@@ -11,6 +8,7 @@ import type { GatewayTailscaleMode } from "../config/types.gateway.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { hasConfiguredInternalHooks } from "../hooks/configured.js";
|
||||
import { isTruthyEnvValue } from "../infra/env.js";
|
||||
import { hasRestartSentinel } from "../infra/restart-sentinel.js";
|
||||
import type { scheduleGatewayUpdateCheck } from "../infra/update-startup.js";
|
||||
import type { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import type { PluginHookGatewayCronService } from "../plugins/hook-types.js";
|
||||
@@ -38,7 +36,6 @@ const DEFERRED_SIDECAR_START_DELAY_MS = 100;
|
||||
const SESSION_LOCK_CLEANUP_CONCURRENCY = 4;
|
||||
const SKIP_STARTUP_MODEL_PREWARM_ENV = "OPENCLAW_SKIP_STARTUP_MODEL_PREWARM";
|
||||
const QMD_STARTUP_IDLE_DELAY_MS = 120_000;
|
||||
const RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
@@ -506,67 +503,14 @@ function scheduleTranscriptsAutoStartSidecar(params: {
|
||||
});
|
||||
}
|
||||
|
||||
async function pathExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.promises.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveRestartSentinelPathFast(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<string> {
|
||||
const normalizePathEnv = (value: string | undefined) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed && trimmed !== "undefined" && trimmed !== "null" ? trimmed : undefined;
|
||||
};
|
||||
const resolveRawOsHome = () => normalizePathEnv(env.HOME) ?? normalizePathEnv(env.USERPROFILE);
|
||||
const expandHomePrefix = (input: string, home: string) => input.replace(/^~(?=$|[\\/])/, home);
|
||||
const resolveHome = () => {
|
||||
const explicitHome = normalizePathEnv(env.OPENCLAW_HOME);
|
||||
if (explicitHome) {
|
||||
const osHome = resolveRawOsHome() ?? os.homedir();
|
||||
return path.resolve(expandHomePrefix(explicitHome, osHome));
|
||||
}
|
||||
return path.resolve(resolveRawOsHome() ?? os.homedir());
|
||||
};
|
||||
const resolveUserPath = (input: string) => {
|
||||
const trimmed = input.trim();
|
||||
if (trimmed.startsWith("~")) {
|
||||
return path.resolve(expandHomePrefix(trimmed, resolveHome()));
|
||||
}
|
||||
return path.resolve(trimmed);
|
||||
};
|
||||
const override = normalizePathEnv(env.OPENCLAW_STATE_DIR);
|
||||
if (override) {
|
||||
return path.join(resolveUserPath(override), RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
const home = resolveHome();
|
||||
const newStateDir = path.join(home, ".openclaw");
|
||||
if (env.OPENCLAW_TEST_FAST === "1" || (await pathExists(newStateDir))) {
|
||||
return path.join(newStateDir, RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
const legacyStateDir = path.join(home, ".clawdbot");
|
||||
if (await pathExists(legacyStateDir)) {
|
||||
return path.join(legacyStateDir, RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
return path.join(newStateDir, RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
|
||||
async function hasRestartSentinelFileFast(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
try {
|
||||
return await pathExists(await resolveRestartSentinelPathFast(env));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
async function hasRestartSentinelFast(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
return await hasRestartSentinel(env);
|
||||
}
|
||||
|
||||
async function refreshLatestUpdateRestartSentinelIfPresent(): Promise<Awaited<
|
||||
ReturnType<typeof refreshLatestUpdateRestartSentinel>
|
||||
> | null> {
|
||||
if (!(await hasRestartSentinelFileFast())) {
|
||||
if (!(await hasRestartSentinelFast())) {
|
||||
return null;
|
||||
}
|
||||
return await (await loadGatewayRestartSentinelModule()).refreshLatestUpdateRestartSentinel();
|
||||
@@ -930,7 +874,7 @@ export async function startGatewaySidecars(params: {
|
||||
if (!shouldCheckRestartSentinel()) {
|
||||
return;
|
||||
}
|
||||
if (!(await hasRestartSentinelFileFast())) {
|
||||
if (!(await hasRestartSentinelFast())) {
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
@@ -1457,7 +1401,7 @@ export async function startGatewayPostAttachRuntime(
|
||||
|
||||
export const testing = {
|
||||
providerAuthPrewarmStartDelayMs: PROVIDER_AUTH_PREWARM_START_DELAY_MS,
|
||||
hasRestartSentinelFileFast,
|
||||
hasRestartSentinelFast,
|
||||
prewarmConfiguredPrimaryModel,
|
||||
prewarmConfiguredPrimaryModelWithTimeout,
|
||||
refreshLatestUpdateRestartSentinelIfPresent,
|
||||
|
||||
@@ -10,7 +10,7 @@ import type { DeviceIdentity } from "../infra/device-identity.js";
|
||||
import { loadOrCreateDeviceIdentity } from "../infra/device-identity.js";
|
||||
import { approveDevicePairing, listDevicePairing } from "../infra/device-pairing.js";
|
||||
import { approveNodePairing, requestNodePairing } from "../infra/node-pairing.js";
|
||||
import { resolveRestartSentinelPath } from "../infra/restart-sentinel.js";
|
||||
import { readRestartSentinel } from "../infra/restart-sentinel.js";
|
||||
import { SUPERVISOR_HINT_ENV_VARS } from "../infra/supervisor-markers.js";
|
||||
import { getActiveRuntimePluginRegistry } from "../plugins/active-runtime-registry.js";
|
||||
import {
|
||||
@@ -417,13 +417,9 @@ describe("gateway update.run", () => {
|
||||
}, FAST_WAIT_OPTS);
|
||||
expect(sigusr1).toHaveBeenCalled();
|
||||
|
||||
const sentinelPath = resolveRestartSentinelPath();
|
||||
const raw = await fs.readFile(sentinelPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as {
|
||||
payload?: { kind?: string; stats?: { mode?: string } };
|
||||
};
|
||||
expect(parsed.payload?.kind).toBe("update");
|
||||
expect(parsed.payload?.stats?.mode).toBe("git");
|
||||
const sentinel = await readRestartSentinel();
|
||||
expect(sentinel?.payload.kind).toBe("update");
|
||||
expect(sentinel?.payload.stats?.mode).toBe("git");
|
||||
} finally {
|
||||
process.off("SIGUSR1", sigusr1);
|
||||
}
|
||||
|
||||
@@ -1,17 +1,28 @@
|
||||
// Covers restart sentinel persistence, summaries, and messages.
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
// Covers restart sentinel persistence, summaries, and messages.
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { withTempDir } from "../test-helpers/temp-dir.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
import {
|
||||
buildRestartSuccessContinuation,
|
||||
clearRestartSentinel,
|
||||
finalizeUpdateRestartSentinelRunningVersion,
|
||||
formatDoctorNonInteractiveHint,
|
||||
formatRestartSentinelMessage,
|
||||
hasRestartSentinel,
|
||||
markUpdateRestartSentinelFailure,
|
||||
readRestartSentinel,
|
||||
resolveRestartSentinelPath,
|
||||
summarizeRestartSentinel,
|
||||
trimLogTail,
|
||||
writeRestartSentinel,
|
||||
@@ -25,28 +36,52 @@ import { buildUpdateRestartSentinelPayload } from "./update-restart-sentinel-pay
|
||||
|
||||
async function withRestartSentinelStateDir(run: () => Promise<void>): Promise<void> {
|
||||
await withTempDir({ prefix: "openclaw-sentinel-" }, async (tempDir) => {
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run);
|
||||
try {
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: tempDir }, run);
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function expectPathMissing(targetPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.stat(targetPath);
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const statError = error as NodeJS.ErrnoException;
|
||||
expect({
|
||||
code: statError.code,
|
||||
path: statError.path,
|
||||
syscall: statError.syscall,
|
||||
}).toEqual({
|
||||
code: "ENOENT",
|
||||
path: targetPath,
|
||||
syscall: "stat",
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw new Error(`Expected path to be missing: ${targetPath}`);
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
function readSentinelRow() {
|
||||
const { db } = openOpenClawStateDatabase();
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
return executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["sentinel_key", "version", "kind", "status", "payload_json"])
|
||||
.where("sentinel_key", "=", "current"),
|
||||
);
|
||||
}
|
||||
|
||||
function insertSentinelRow(values: { version?: number; payloadJson: string }) {
|
||||
const { db } = openOpenClawStateDatabase();
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
version: values.version ?? 1,
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: Date.now(),
|
||||
session_key: null,
|
||||
thread_id: null,
|
||||
delivery_channel: null,
|
||||
delivery_to: null,
|
||||
delivery_account_id: null,
|
||||
message: null,
|
||||
continuation_json: null,
|
||||
doctor_hint: null,
|
||||
stats_json: null,
|
||||
payload_json: values.payloadJson,
|
||||
updated_at_ms: Date.now(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("restart sentinel", () => {
|
||||
@@ -63,8 +98,14 @@ describe("restart sentinel", () => {
|
||||
},
|
||||
stats: { mode: "git" },
|
||||
};
|
||||
const filePath = await writeRestartSentinel(payload);
|
||||
expect(filePath).toBe(resolveRestartSentinelPath());
|
||||
await writeRestartSentinel(payload);
|
||||
expect(readSentinelRow()).toMatchObject({
|
||||
sentinel_key: "current",
|
||||
version: 1,
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
payload_json: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
const read = await readRestartSentinel();
|
||||
expect(read?.payload.kind).toBe("update");
|
||||
@@ -72,27 +113,84 @@ describe("restart sentinel", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("imports a legacy file sentinel into sqlite once", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const payload = {
|
||||
kind: "update" as const,
|
||||
status: "skipped" as const,
|
||||
ts: Date.now(),
|
||||
sessionKey: "agent:main:webchat:dm:user-123",
|
||||
message: "update restart pending",
|
||||
stats: {
|
||||
mode: "npm",
|
||||
reason: "restart-health-pending",
|
||||
},
|
||||
};
|
||||
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
await fs.writeFile(legacyPath, `${JSON.stringify({ version: 1, payload })}\n`, "utf-8");
|
||||
|
||||
await expect(hasRestartSentinel()).resolves.toBe(true);
|
||||
expect(readSentinelRow()).toMatchObject({
|
||||
sentinel_key: "current",
|
||||
version: 1,
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
payload_json: JSON.stringify(payload),
|
||||
});
|
||||
await expect(fs.access(legacyPath)).rejects.toThrow();
|
||||
await expect(readRestartSentinel()).resolves.toEqual({ version: 1, payload });
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replay a legacy file superseded by a sqlite sentinel", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const legacyPath = path.join(process.env.OPENCLAW_STATE_DIR ?? "", "restart-sentinel.json");
|
||||
await fs.writeFile(
|
||||
legacyPath,
|
||||
`${JSON.stringify({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
ts: 1,
|
||||
message: "stale legacy sentinel",
|
||||
},
|
||||
})}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
await writeRestartSentinel({
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 2,
|
||||
message: "current sqlite sentinel",
|
||||
});
|
||||
await expect(fs.access(legacyPath)).rejects.toThrow();
|
||||
|
||||
await clearRestartSentinel();
|
||||
|
||||
await expect(hasRestartSentinel()).resolves.toBe(false);
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it("drops invalid sentinel payloads", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const filePath = resolveRestartSentinelPath();
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, "not-json", "utf-8");
|
||||
insertSentinelRow({ payloadJson: "not-json" });
|
||||
|
||||
const read = await readRestartSentinel();
|
||||
expect(read).toBeNull();
|
||||
|
||||
await expectPathMissing(filePath);
|
||||
expect(readSentinelRow()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("drops structurally invalid sentinel payloads", async () => {
|
||||
await withRestartSentinelStateDir(async () => {
|
||||
const filePath = resolveRestartSentinelPath();
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, JSON.stringify({ version: 2, payload: null }), "utf-8");
|
||||
insertSentinelRow({ version: 2, payloadJson: JSON.stringify(null) });
|
||||
|
||||
await expect(readRestartSentinel()).resolves.toBeNull();
|
||||
await expectPathMissing(filePath);
|
||||
expect(readSentinelRow()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,11 +1,20 @@
|
||||
// Persists restart sentinel files that coordinate deferred restarts.
|
||||
import fs from "node:fs/promises";
|
||||
// Persists restart sentinel state that coordinates deferred restarts.
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isRecord as isPlainRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { resolveRuntimeServiceVersion } from "../version.js";
|
||||
import { writeJson } from "./json-files.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
|
||||
export type RestartSentinelLog = {
|
||||
stdoutTail?: string | null;
|
||||
@@ -66,7 +75,9 @@ export type RestartSentinel = {
|
||||
payload: RestartSentinelPayload;
|
||||
};
|
||||
|
||||
const SENTINEL_FILENAME = "restart-sentinel.json";
|
||||
const RESTART_SENTINEL_KEY = "current";
|
||||
const LEGACY_RESTART_SENTINEL_FILENAME = "restart-sentinel.json";
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
export function formatDoctorNonInteractiveHint(
|
||||
env: Record<string, string | undefined> = process.env as Record<string, string | undefined>,
|
||||
@@ -77,18 +88,60 @@ export function formatDoctorNonInteractiveHint(
|
||||
)} in a terminal or approvals-capable OpenClaw surface.`;
|
||||
}
|
||||
|
||||
export function resolveRestartSentinelPath(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return path.join(resolveStateDir(env), SENTINEL_FILENAME);
|
||||
}
|
||||
|
||||
export async function writeRestartSentinel(
|
||||
payload: RestartSentinelPayload,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
) {
|
||||
const filePath = resolveRestartSentinelPath(env);
|
||||
const data: RestartSentinel = { version: 1, payload };
|
||||
await writeJson(filePath, data, { trailingNewline: true, dirMode: 0o700 });
|
||||
return filePath;
|
||||
): Promise<void> {
|
||||
const updatedAtMs = Date.now();
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.insertInto("gateway_restart_sentinel")
|
||||
.values({
|
||||
sentinel_key: RESTART_SENTINEL_KEY,
|
||||
version: 1,
|
||||
kind: payload.kind,
|
||||
status: payload.status,
|
||||
ts: payload.ts,
|
||||
session_key: payload.sessionKey ?? null,
|
||||
thread_id: payload.threadId ?? null,
|
||||
delivery_channel: payload.deliveryContext?.channel ?? null,
|
||||
delivery_to: payload.deliveryContext?.to ?? null,
|
||||
delivery_account_id: payload.deliveryContext?.accountId ?? null,
|
||||
message: payload.message ?? null,
|
||||
continuation_json: payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
doctor_hint: payload.doctorHint ?? null,
|
||||
stats_json: payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: updatedAtMs,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("sentinel_key").doUpdateSet({
|
||||
version: (eb) => eb.ref("excluded.version"),
|
||||
kind: (eb) => eb.ref("excluded.kind"),
|
||||
status: (eb) => eb.ref("excluded.status"),
|
||||
ts: (eb) => eb.ref("excluded.ts"),
|
||||
session_key: (eb) => eb.ref("excluded.session_key"),
|
||||
thread_id: (eb) => eb.ref("excluded.thread_id"),
|
||||
delivery_channel: (eb) => eb.ref("excluded.delivery_channel"),
|
||||
delivery_to: (eb) => eb.ref("excluded.delivery_to"),
|
||||
delivery_account_id: (eb) => eb.ref("excluded.delivery_account_id"),
|
||||
message: (eb) => eb.ref("excluded.message"),
|
||||
continuation_json: (eb) => eb.ref("excluded.continuation_json"),
|
||||
doctor_hint: (eb) => eb.ref("excluded.doctor_hint"),
|
||||
stats_json: (eb) => eb.ref("excluded.stats_json"),
|
||||
payload_json: (eb) => eb.ref("excluded.payload_json"),
|
||||
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
await removeLegacyRestartSentinel(env);
|
||||
}
|
||||
|
||||
function cloneRestartSentinelPayload(payload: RestartSentinelPayload): RestartSentinelPayload {
|
||||
@@ -156,11 +209,52 @@ export async function markUpdateRestartSentinelFailure(
|
||||
}, env);
|
||||
}
|
||||
|
||||
export async function removeRestartSentinelFile(filePath: string | null | undefined) {
|
||||
if (!filePath) {
|
||||
return;
|
||||
export async function clearRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<void> {
|
||||
try {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb
|
||||
.deleteFrom("gateway_restart_sentinel")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
},
|
||||
{ env },
|
||||
);
|
||||
} catch {}
|
||||
await removeLegacyRestartSentinel(env);
|
||||
}
|
||||
|
||||
function resolveLegacyRestartSentinelPath(env: NodeJS.ProcessEnv): string {
|
||||
return path.join(resolveStateDir(env), LEGACY_RESTART_SENTINEL_FILENAME);
|
||||
}
|
||||
|
||||
async function removeLegacyRestartSentinel(env: NodeJS.ProcessEnv): Promise<void> {
|
||||
try {
|
||||
await rm(resolveLegacyRestartSentinelPath(env), { force: true });
|
||||
} catch {}
|
||||
}
|
||||
|
||||
async function importLegacyRestartSentinel(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<RestartSentinel | null> {
|
||||
const legacyPath = resolveLegacyRestartSentinelPath(env);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(await readFile(legacyPath, "utf-8")) as unknown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
await fs.unlink(filePath).catch(() => {});
|
||||
if (!isPlainRecord(parsed) || parsed.version !== 1 || !isPlainRecord(parsed.payload)) {
|
||||
await removeLegacyRestartSentinel(env);
|
||||
return null;
|
||||
}
|
||||
const payload = parsed.payload as RestartSentinelPayload;
|
||||
await writeRestartSentinel(payload, env);
|
||||
await removeLegacyRestartSentinel(env);
|
||||
return { version: 1, payload };
|
||||
}
|
||||
|
||||
export function buildRestartSuccessContinuation(params: {
|
||||
@@ -177,26 +271,56 @@ export function buildRestartSuccessContinuation(params: {
|
||||
export async function readRestartSentinel(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<RestartSentinel | null> {
|
||||
const filePath = resolveRestartSentinelPath(env);
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
let parsed: RestartSentinel | undefined;
|
||||
const database = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["version", "payload_json"])
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
if (!row) {
|
||||
return await importLegacyRestartSentinel(env);
|
||||
}
|
||||
let payload: RestartSentinelPayload | undefined;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as RestartSentinel | undefined;
|
||||
payload = JSON.parse(row.payload_json) as RestartSentinelPayload | undefined;
|
||||
} catch {
|
||||
await fs.unlink(filePath).catch(() => {});
|
||||
await clearRestartSentinel(env);
|
||||
return null;
|
||||
}
|
||||
if (!parsed || parsed.version !== 1 || !parsed.payload) {
|
||||
await fs.unlink(filePath).catch(() => {});
|
||||
if (row.version !== 1 || !payload) {
|
||||
await clearRestartSentinel(env);
|
||||
return null;
|
||||
}
|
||||
return parsed;
|
||||
return { version: 1, payload };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise<boolean> {
|
||||
try {
|
||||
const database = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(database.db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select("sentinel_key")
|
||||
.where("sentinel_key", "=", RESTART_SENTINEL_KEY),
|
||||
);
|
||||
if (row) {
|
||||
return true;
|
||||
}
|
||||
return Boolean(await importLegacyRestartSentinel(env));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function formatRestartSentinelMessage(payload: RestartSentinelPayload): string {
|
||||
const message = payload.message?.trim();
|
||||
if (message && (!payload.stats || payload.kind === "config-auto-recovery")) {
|
||||
|
||||
@@ -118,8 +118,8 @@ export async function readControlPlaneUpdateSentinelMeta(
|
||||
export async function writeControlPlaneUpdateRestartSentinel(params: {
|
||||
result: UpdateRunResult;
|
||||
meta: UpdateRestartSentinelMeta;
|
||||
}): Promise<string> {
|
||||
return await writeRestartSentinel(
|
||||
}): Promise<void> {
|
||||
await writeRestartSentinel(
|
||||
buildUpdateRestartSentinelPayload({
|
||||
result: params.result,
|
||||
meta: params.meta,
|
||||
|
||||
@@ -5,6 +5,17 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
import { SUPERVISOR_HINT_ENV_VARS } from "./supervisor-markers.js";
|
||||
import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "./update-control-plane-sentinel.js";
|
||||
import {
|
||||
@@ -28,8 +39,10 @@ vi.mock("node:child_process", async () => {
|
||||
});
|
||||
|
||||
const tempDirs = new Set<string>();
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
afterEach(async () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
spawnMock.mockClear();
|
||||
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
tempDirs.clear();
|
||||
@@ -44,10 +57,74 @@ async function pathExists(filePath: string): Promise<boolean> {
|
||||
}
|
||||
}
|
||||
|
||||
function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const payload =
|
||||
sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1
|
||||
? (sentinel as { payload?: unknown }).payload
|
||||
: null;
|
||||
if (!payload || typeof payload !== "object") {
|
||||
throw new Error("expected versioned restart sentinel payload");
|
||||
}
|
||||
const record = payload as {
|
||||
kind?: unknown;
|
||||
status?: unknown;
|
||||
ts?: unknown;
|
||||
sessionKey?: unknown;
|
||||
threadId?: unknown;
|
||||
deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown };
|
||||
message?: unknown;
|
||||
continuation?: unknown;
|
||||
doctorHint?: unknown;
|
||||
stats?: unknown;
|
||||
};
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
version: 1,
|
||||
kind: typeof record.kind === "string" ? record.kind : "update",
|
||||
status: typeof record.status === "string" ? record.status : "skipped",
|
||||
ts: typeof record.ts === "number" ? record.ts : Date.now(),
|
||||
session_key: typeof record.sessionKey === "string" ? record.sessionKey : null,
|
||||
thread_id: typeof record.threadId === "string" ? record.threadId : null,
|
||||
delivery_channel:
|
||||
typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null,
|
||||
delivery_to:
|
||||
typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null,
|
||||
delivery_account_id:
|
||||
typeof record.deliveryContext?.accountId === "string"
|
||||
? record.deliveryContext.accountId
|
||||
: null,
|
||||
message: typeof record.message === "string" ? record.message : null,
|
||||
continuation_json: record.continuation ? JSON.stringify(record.continuation) : null,
|
||||
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
|
||||
stats_json: record.stats ? JSON.stringify(record.stats) : null,
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: Date.now(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload(env: NodeJS.ProcessEnv): unknown {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["version", "payload_json"])
|
||||
.where("sentinel_key", "=", "current"),
|
||||
);
|
||||
return row ? { version: row.version, payload: JSON.parse(row.payload_json) } : null;
|
||||
}
|
||||
|
||||
async function runHelperWithExistingSentinel(params: {
|
||||
handoffId?: string;
|
||||
metaHandoffId?: string;
|
||||
sentinel: unknown;
|
||||
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
|
||||
sentinel?: unknown;
|
||||
}) {
|
||||
const { execFile } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
@@ -82,8 +159,11 @@ async function runHelperWithExistingSentinel(params: {
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const sentinelPath = path.join(tmpDir, "restart-sentinel.json");
|
||||
await fs.writeFile(sentinelPath, `${JSON.stringify(params.sentinel, null, 2)}\n`);
|
||||
const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv;
|
||||
await params.prepareStateDatabase?.(env);
|
||||
if (params.sentinel !== undefined) {
|
||||
writeRestartSentinelRow(env, params.sentinel);
|
||||
}
|
||||
const helperParamsPath = path.join(tmpDir, "helper-params.json");
|
||||
await fs.writeFile(
|
||||
helperParamsPath,
|
||||
@@ -92,7 +172,7 @@ async function runHelperWithExistingSentinel(params: {
|
||||
...helperParams,
|
||||
parentPid: process.pid,
|
||||
parentExitTimeoutMs: 1,
|
||||
sentinelPath,
|
||||
stateDatabasePath: resolveOpenClawStateSqlitePath(env),
|
||||
logPath: path.join(tmpDir, "handoff.log"),
|
||||
sensitivePaths: [],
|
||||
},
|
||||
@@ -113,7 +193,31 @@ async function runHelperWithExistingSentinel(params: {
|
||||
},
|
||||
);
|
||||
|
||||
return { result, sentinelPath };
|
||||
return { result, env };
|
||||
}
|
||||
|
||||
async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise<void> {
|
||||
const sqlite = await import("node:sqlite");
|
||||
const stateDatabasePath = resolveOpenClawStateSqlitePath(env);
|
||||
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
|
||||
const db = new sqlite.DatabaseSync(stateDatabasePath);
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE gateway_restart_sentinel (
|
||||
sentinel_key TEXT NOT NULL PRIMARY KEY,
|
||||
version INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
session_key TEXT,
|
||||
thread_id TEXT,
|
||||
payload_json TEXT NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
`);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function spawnExitedPid(): Promise<number> {
|
||||
@@ -166,7 +270,7 @@ async function runHelperWithCommand(params: {
|
||||
parentExitTimeoutMs: 5000,
|
||||
cwd: tmpDir,
|
||||
commandArgv: params.commandArgv,
|
||||
sentinelPath: path.join(tmpDir, "restart-sentinel.json"),
|
||||
stateDatabasePath: resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: tmpDir }),
|
||||
logPath: path.join(tmpDir, "handoff.log"),
|
||||
sensitivePaths: [],
|
||||
...(params.serviceRecovery ? { serviceRecovery: params.serviceRecovery } : {}),
|
||||
@@ -284,10 +388,10 @@ describe("managed service update handoff", () => {
|
||||
const helperParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as {
|
||||
cwd?: string;
|
||||
metaPath?: string;
|
||||
sentinelPath?: string;
|
||||
stateDatabasePath?: string;
|
||||
};
|
||||
expect(helperParams.metaPath).toMatch(/sentinel-meta\.json$/u);
|
||||
expect(helperParams.sentinelPath).toMatch(/restart-sentinel\.json$/u);
|
||||
expect(helperParams.stateDatabasePath).toMatch(/openclaw\.sqlite$/u);
|
||||
expect(options.cwd).toBe(os.homedir());
|
||||
expect(helperParams.cwd).toBe(os.homedir());
|
||||
expect(options.detached).toBe(true);
|
||||
@@ -477,6 +581,51 @@ describe("managed service update handoff", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("writes a fallback update failure when no restart sentinel row exists", async () => {
|
||||
const { result, env } = await runHelperWithExistingSentinel({
|
||||
handoffId: "handoff-123",
|
||||
metaHandoffId: "handoff-123",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
sessionKey: "agent:test:webchat:dm:user-123",
|
||||
stats: {
|
||||
handoffId: "handoff-123",
|
||||
reason: "managed-service-handoff-parent-timeout",
|
||||
},
|
||||
},
|
||||
});
|
||||
if (process.platform !== "win32") {
|
||||
const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777;
|
||||
expect(mode).toBe(0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("repairs legacy restart sentinel columns before writing fallback failures", async () => {
|
||||
const { result, env } = await runHelperWithExistingSentinel({
|
||||
handoffId: "handoff-123",
|
||||
metaHandoffId: "handoff-123",
|
||||
prepareStateDatabase: createLegacyRestartSentinelTable,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
stats: {
|
||||
reason: "managed-service-handoff-parent-timeout",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not overwrite a restart sentinel owned by another startup task", async () => {
|
||||
const unrelatedSentinel = {
|
||||
version: 1,
|
||||
@@ -487,14 +636,12 @@ describe("managed service update handoff", () => {
|
||||
stats: { reason: "config-restart-pending" },
|
||||
},
|
||||
};
|
||||
const { result, sentinelPath } = await runHelperWithExistingSentinel({
|
||||
const { result, env } = await runHelperWithExistingSentinel({
|
||||
sentinel: unrelatedSentinel,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual(
|
||||
unrelatedSentinel,
|
||||
);
|
||||
expect(readRestartSentinelPayload(env)).toEqual(unrelatedSentinel);
|
||||
});
|
||||
|
||||
it("does not overwrite a newer pending update handoff sentinel", async () => {
|
||||
@@ -513,16 +660,14 @@ describe("managed service update handoff", () => {
|
||||
},
|
||||
},
|
||||
};
|
||||
const { result, sentinelPath } = await runHelperWithExistingSentinel({
|
||||
const { result, env } = await runHelperWithExistingSentinel({
|
||||
handoffId: "old-handoff",
|
||||
metaHandoffId: "old-handoff",
|
||||
sentinel: newerSentinel,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
await expect(fs.readFile(sentinelPath, "utf-8").then(JSON.parse)).resolves.toEqual(
|
||||
newerSentinel,
|
||||
);
|
||||
expect(readRestartSentinelPayload(env)).toEqual(newerSentinel);
|
||||
});
|
||||
|
||||
it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => {
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
resolveGatewaySystemdServiceName,
|
||||
resolveGatewayWindowsTaskName,
|
||||
} from "../daemon/constants.js";
|
||||
import { resolveRestartSentinelPath } from "./restart-sentinel.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
|
||||
import { SUPERVISOR_HINT_ENV_VARS, type RespawnSupervisor } from "./supervisor-markers.js";
|
||||
import {
|
||||
CONTROL_PLANE_UPDATE_SENTINEL_META_ENV,
|
||||
@@ -96,26 +96,6 @@ function readJsonFile(filePath) {
|
||||
}
|
||||
}
|
||||
|
||||
function writeJsonFile(filePath, value) {
|
||||
const dir = path.dirname(filePath);
|
||||
const tempPath = path.join(
|
||||
dir,
|
||||
"." + path.basename(filePath) + "." + process.pid + "." + Date.now() + ".tmp",
|
||||
);
|
||||
try {
|
||||
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(tempPath, JSON.stringify(value, null, 2) + "\n", { mode: 0o600 });
|
||||
fs.renameSync(tempPath, filePath);
|
||||
} catch (err) {
|
||||
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
|
||||
try {
|
||||
fs.rmSync(tempPath, { force: true });
|
||||
} catch {
|
||||
// Best effort only.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isPendingUpdatePayload(payload) {
|
||||
const reason = payload && payload.stats && payload.stats.reason;
|
||||
return (
|
||||
@@ -126,6 +106,170 @@ function isPendingUpdatePayload(payload) {
|
||||
);
|
||||
}
|
||||
|
||||
function openStateDatabase() {
|
||||
if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const sqlite = require("node:sqlite");
|
||||
fs.mkdirSync(path.dirname(params.stateDatabasePath), { recursive: true, mode: 0o700 });
|
||||
const db = new sqlite.DatabaseSync(params.stateDatabasePath);
|
||||
db.exec([
|
||||
"CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (",
|
||||
"sentinel_key TEXT NOT NULL PRIMARY KEY,",
|
||||
"version INTEGER NOT NULL,",
|
||||
"kind TEXT NOT NULL,",
|
||||
"status TEXT NOT NULL,",
|
||||
"ts INTEGER NOT NULL,",
|
||||
"session_key TEXT,",
|
||||
"thread_id TEXT,",
|
||||
"delivery_channel TEXT,",
|
||||
"delivery_to TEXT,",
|
||||
"delivery_account_id TEXT,",
|
||||
"message TEXT,",
|
||||
"continuation_json TEXT,",
|
||||
"doctor_hint TEXT,",
|
||||
"stats_json TEXT,",
|
||||
"payload_json TEXT NOT NULL,",
|
||||
"updated_at_ms INTEGER NOT NULL",
|
||||
");",
|
||||
"CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts",
|
||||
"ON gateway_restart_sentinel(ts DESC, sentinel_key);",
|
||||
].join(" "));
|
||||
ensureGatewayRestartSentinelColumns(db);
|
||||
hardenStateDatabaseFiles();
|
||||
return db;
|
||||
} catch (err) {
|
||||
appendLog("failed to open restart sentinel database: " + (err && err.stack ? err.stack : String(err)));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function tableHasColumn(db, tableName, columnName) {
|
||||
try {
|
||||
return db.prepare("PRAGMA table_info(" + tableName + ")").all().some((row) => row && row.name === columnName);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function ensureColumn(db, tableName, columnSql) {
|
||||
const columnName = columnSql.trim().split(/\s+/, 1)[0];
|
||||
if (!columnName || tableHasColumn(db, tableName, columnName)) {
|
||||
return;
|
||||
}
|
||||
db.exec("ALTER TABLE " + tableName + " ADD COLUMN " + columnSql + ";");
|
||||
}
|
||||
|
||||
function ensureGatewayRestartSentinelColumns(db) {
|
||||
ensureColumn(db, "gateway_restart_sentinel", "delivery_channel TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "delivery_to TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "delivery_account_id TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "message TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "continuation_json TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "doctor_hint TEXT");
|
||||
ensureColumn(db, "gateway_restart_sentinel", "stats_json TEXT");
|
||||
}
|
||||
|
||||
function hardenStateDatabaseFiles() {
|
||||
if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") {
|
||||
return;
|
||||
}
|
||||
for (const filePath of [
|
||||
params.stateDatabasePath,
|
||||
params.stateDatabasePath + "-wal",
|
||||
params.stateDatabasePath + "-shm",
|
||||
]) {
|
||||
try {
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.chmodSync(filePath, 0o600);
|
||||
}
|
||||
} catch {
|
||||
// Best effort only.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload() {
|
||||
const db = openStateDatabase();
|
||||
if (!db) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const row = db
|
||||
.prepare("SELECT version, payload_json FROM gateway_restart_sentinel WHERE sentinel_key = ?")
|
||||
.get("current");
|
||||
if (!row || row.version !== 1 || typeof row.payload_json !== "string") {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(row.payload_json);
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
hardenStateDatabaseFiles();
|
||||
try {
|
||||
db.close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
function writeRestartSentinelPayload(payload) {
|
||||
const db = openStateDatabase();
|
||||
if (!db) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const updatedAtMs = Date.now();
|
||||
db.prepare(
|
||||
[
|
||||
"INSERT INTO gateway_restart_sentinel (",
|
||||
"sentinel_key, version, kind, status, ts, session_key, thread_id,",
|
||||
"delivery_channel, delivery_to, delivery_account_id, message, continuation_json,",
|
||||
"doctor_hint, stats_json, payload_json, updated_at_ms",
|
||||
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"ON CONFLICT(sentinel_key) DO UPDATE SET",
|
||||
"version = excluded.version, kind = excluded.kind, status = excluded.status,",
|
||||
"ts = excluded.ts, session_key = excluded.session_key, thread_id = excluded.thread_id,",
|
||||
"delivery_channel = excluded.delivery_channel, delivery_to = excluded.delivery_to,",
|
||||
"delivery_account_id = excluded.delivery_account_id, message = excluded.message,",
|
||||
"continuation_json = excluded.continuation_json, doctor_hint = excluded.doctor_hint,",
|
||||
"stats_json = excluded.stats_json, payload_json = excluded.payload_json,",
|
||||
"updated_at_ms = excluded.updated_at_ms",
|
||||
].join(" "),
|
||||
).run(
|
||||
"current",
|
||||
1,
|
||||
payload.kind,
|
||||
payload.status,
|
||||
payload.ts,
|
||||
payload.sessionKey || null,
|
||||
payload.threadId || null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.channel === "string"
|
||||
? payload.deliveryContext.channel
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.to === "string"
|
||||
? payload.deliveryContext.to
|
||||
: null,
|
||||
payload.deliveryContext && typeof payload.deliveryContext.accountId === "string"
|
||||
? payload.deliveryContext.accountId
|
||||
: null,
|
||||
payload.message || null,
|
||||
payload.continuation ? JSON.stringify(payload.continuation) : null,
|
||||
payload.doctorHint || null,
|
||||
payload.stats ? JSON.stringify(payload.stats) : null,
|
||||
JSON.stringify(payload),
|
||||
updatedAtMs,
|
||||
);
|
||||
} catch (err) {
|
||||
appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err)));
|
||||
} finally {
|
||||
hardenStateDatabaseFiles();
|
||||
try {
|
||||
db.close();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
function buildFallbackFailurePayload(reason) {
|
||||
const metaFile = params.metaPath ? readJsonFile(params.metaPath) : null;
|
||||
const meta = metaFile && metaFile.version === 1 && metaFile.meta ? metaFile.meta : {};
|
||||
@@ -157,11 +301,7 @@ function buildFallbackFailurePayload(reason) {
|
||||
}
|
||||
|
||||
function markUpdateSentinelFailureIfPending(reason) {
|
||||
if (!params.sentinelPath) {
|
||||
return;
|
||||
}
|
||||
const current = readJsonFile(params.sentinelPath);
|
||||
let payload = current && current.version === 1 ? current.payload : null;
|
||||
let payload = readRestartSentinelPayload();
|
||||
if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) {
|
||||
return;
|
||||
}
|
||||
@@ -176,7 +316,7 @@ function markUpdateSentinelFailureIfPending(reason) {
|
||||
} else {
|
||||
payload = buildFallbackFailurePayload(reason);
|
||||
}
|
||||
writeJsonFile(params.sentinelPath, { version: 1, payload });
|
||||
writeRestartSentinelPayload(payload);
|
||||
}
|
||||
|
||||
function runServiceCommand(command, args) {
|
||||
@@ -553,7 +693,7 @@ export async function startManagedServiceUpdateHandoff(params: {
|
||||
handoffId: params.handoffId,
|
||||
logPath,
|
||||
metaPath,
|
||||
sentinelPath: resolveRestartSentinelPath(),
|
||||
stateDatabasePath: resolveOpenClawStateSqlitePath(params.env ?? process.env),
|
||||
sensitivePaths: [scriptPath, paramsPath, metaPath],
|
||||
serviceRecovery: resolveGatewayServiceRecovery(params.supervisor, params.env ?? process.env),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user