fix(update): require applied gateway restarts

Require Control UI updates to observe a real gateway process replacement, surface skipped/error update outcomes, and verify the running gateway version after restart.\n\nAdds update.status restart-sentinel plumbing, docs, generated protocol model updates, and changelog attribution.\n\nLocal verification:\n- pnpm test src/gateway/server-methods/update.test.ts src/cli/gateway-cli/run-loop.test.ts src/infra/restart-sentinel.test.ts src/infra/process-respawn.test.ts src/infra/update-runner.test.ts ui/src/ui/app-gateway.node.test.ts ui/src/ui/controllers/config.test.ts\n- git diff --check\n- pnpm exec oxfmt --check --threads=1 CHANGELOG.md docs/gateway/protocol.md docs/gateway/configuration.md docs/web/control-ui.md\n- pnpm docs:check-mdx
This commit is contained in:
Samuel Rodda
2026-04-27 18:37:43 +09:30
committed by GitHub
parent b74f35ee6f
commit 6c252cc54c
41 changed files with 1265 additions and 54 deletions

View File

@@ -43,6 +43,9 @@ vi.mock("./gateway.ts", async (importOriginal) => {
readonly start = vi.fn();
readonly stop = vi.fn();
readonly request = vi.fn(async (method: string) => {
if (method === "update.status") {
return { sentinel: null };
}
if (method === "models.authStatus") {
return { ts: 0, providers: [] };
}
@@ -154,6 +157,8 @@ function createHost(): TestGatewayHost {
assistantAgentId: null,
localMediaPreviewRoots: [],
serverVersion: null,
pendingUpdateExpectedVersion: null,
updateStatusBanner: null,
sessionKey: "main",
chatMessages: [],
chatQueue: [],
@@ -283,6 +288,117 @@ describe("connectGateway", () => {
});
});
it("clears pending update verification when the restarted version matches", async () => {
const host = createHost();
host.pendingUpdateExpectedVersion = "2.0.0";
connectGateway(host);
const client = gatewayClientInstances[0];
expect(client).toBeDefined();
client.request.mockImplementation(async (method: string) => {
if (method === "update.status") {
return {
sentinel: {
kind: "update",
status: "ok",
stats: {
after: { version: "2.0.0" },
},
},
};
}
return {};
});
client.emitHello({
type: "hello-ok",
protocol: 3,
server: { version: "2.0.0" },
snapshot: {},
});
await vi.waitFor(() => {
expect(host.pendingUpdateExpectedVersion).toBeNull();
});
expect(host.updateStatusBanner).toBeNull();
});
it("shows a hard error when the restarted version does not match the expected update", async () => {
const host = createHost();
host.pendingUpdateExpectedVersion = "2.0.0";
connectGateway(host);
const client = gatewayClientInstances[0];
expect(client).toBeDefined();
client.request.mockImplementation(async (method: string) => {
if (method === "update.status") {
return {
sentinel: {
kind: "update",
status: "ok",
stats: {
after: { version: "1.0.0" },
},
},
};
}
return {};
});
client.emitHello({
type: "hello-ok",
protocol: 3,
server: { version: "1.0.0" },
snapshot: {},
});
await vi.waitFor(() => {
expect(host.pendingUpdateExpectedVersion).toBeNull();
expect(host.updateStatusBanner?.text).toContain(
"Update installed but running version did not change",
);
});
});
it("surfaces post-restart sentinel failures after reconnect", async () => {
const host = createHost();
host.pendingUpdateExpectedVersion = "2.0.0";
connectGateway(host);
const client = gatewayClientInstances[0];
expect(client).toBeDefined();
client.request.mockImplementation(async (method: string) => {
if (method === "update.status") {
return {
sentinel: {
kind: "update",
status: "error",
stats: {
reason: "restart-unhealthy",
after: { version: "1.0.0" },
},
},
};
}
return {};
});
client.emitHello({
type: "hello-ok",
protocol: 3,
server: { version: "1.0.0" },
snapshot: {},
});
await vi.waitFor(() => {
expect(host.pendingUpdateExpectedVersion).toBeNull();
expect(host.updateStatusBanner).toEqual({
tone: "danger",
text: "Update error: restart-unhealthy. The replacement process never became healthy and the previous process stayed up.",
});
});
});
it("ignores stale client onClose callbacks after reconnect", () => {
const host = createHost();

View File

@@ -97,6 +97,8 @@ type GatewayHost = {
assistantAvatar: string | null;
assistantAgentId: string | null;
serverVersion: string | null;
pendingUpdateExpectedVersion: string | null;
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
sessionKey: string;
chatRunId: string | null;
pendingAbort?: { runId: string; sessionKey: string } | null;
@@ -157,6 +159,94 @@ type ConnectGatewayOptions = {
reason?: "initial" | "seq-gap";
};
type UpdateRestartStatusResponse = {
sentinel?: {
kind?: string;
status?: string;
stats?: {
reason?: string | null;
after?: { version?: string | null } | null;
} | null;
} | null;
};
function resolveUpdateVerificationBanner(params: {
expectedVersion: string;
actualVersion: string | null;
}): { tone: "danger"; text: string } {
const actualSuffix = params.actualVersion
? ` Expected v${params.expectedVersion}, running v${params.actualVersion}.`
: "";
return {
tone: "danger",
text: `Update installed but running version did not change — restart may have been blocked.${actualSuffix}`,
};
}
function resolvePostRestartUpdateBanner(reason: string | null | undefined): {
tone: "danger";
text: string;
} {
const normalizedReason = reason?.trim() || "restart-unhealthy";
const guidance =
normalizedReason === "restart-unhealthy"
? "The replacement process never became healthy and the previous process stayed up."
: "Check the gateway logs for the replacement failure.";
return {
tone: "danger",
text: `Update error: ${normalizedReason}. ${guidance}`,
};
}
async function verifyPendingUpdateVersion(
host: GatewayHost,
client: GatewayBrowserClient,
): Promise<void> {
const expectedVersion = host.pendingUpdateExpectedVersion?.trim();
if (!expectedVersion) {
return;
}
const deadline = Date.now() + 10_000;
while (host.client === client && host.connected && Date.now() < deadline) {
let response: UpdateRestartStatusResponse | null = null;
try {
response = await client.request<UpdateRestartStatusResponse>("update.status", {});
} catch {
response = null;
}
const sentinel = response?.sentinel;
const actualVersion = sentinel?.stats?.after?.version?.trim() || null;
if (sentinel?.kind === "update" && actualVersion) {
host.pendingUpdateExpectedVersion = null;
if (sentinel.status && sentinel.status !== "ok") {
host.updateStatusBanner = resolvePostRestartUpdateBanner(sentinel.stats?.reason ?? null);
return;
}
if (actualVersion !== expectedVersion) {
host.updateStatusBanner = resolveUpdateVerificationBanner({
expectedVersion,
actualVersion,
});
}
return;
}
await new Promise<void>((resolve) => {
setTimeout(resolve, 250);
});
}
if (host.client !== client || !host.connected) {
return;
}
const currentVersion = host.hello?.server?.version?.trim() || null;
host.pendingUpdateExpectedVersion = null;
if (currentVersion !== expectedVersion) {
host.updateStatusBanner = resolveUpdateVerificationBanner({
expectedVersion,
actualVersion: currentVersion,
});
}
}
export function resolveControlUiClientVersion(params: {
gatewayUrl: string;
serverVersion: string | null;
@@ -344,6 +434,7 @@ export function connectGateway(host: GatewayHost, options?: ConnectGatewayOption
void refreshActiveTab(host as unknown as Parameters<typeof refreshActiveTab>[0]);
// Re-run push reconciliation now that the gateway client is available.
void host.reconcileWebPushState?.();
void verifyPendingUpdateVersion(host, client);
},
onClose: ({ code, reason, error }) => {
if (host.client !== client) {

View File

@@ -1482,6 +1482,11 @@ export function renderApp(state: AppViewState) {
</aside>
</div>
<main class="content ${isChat ? "content--chat" : ""}">
${state.updateStatusBanner
? html`<div class="callout ${state.updateStatusBanner.tone}" role="alert">
${state.updateStatusBanner.text}
</div>`
: nothing}
${state.updateAvailable &&
state.updateAvailable.latestVersion !== state.updateAvailable.currentVersion &&
!isUpdateBannerDismissed(state.updateAvailable)

View File

@@ -744,7 +744,7 @@ function buildAttentionItems(host: SettingsAppHost) {
// Use the same predicate as the Overview card so the two stay in sync.
// Without this, a `missing` provider shows up on the card but never
// produces the re-auth attention callout.
const monitored = modelAuth.providers.filter(isMonitoredAuthProvider);
const monitored = (modelAuth.providers ?? []).filter(isMonitoredAuthProvider);
const expiredProviders = monitored.filter(
(p) => p.status === "expired" || p.status === "missing",
);

View File

@@ -176,6 +176,8 @@ export type AppViewState = {
configSearchQuery: string;
configActiveSection: string | null;
configActiveSubsection: string | null;
pendingUpdateExpectedVersion: string | null;
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
communicationsFormMode: "form" | "raw";
communicationsSearchQuery: string;
communicationsActiveSection: string | null;

View File

@@ -284,6 +284,8 @@ export class OpenClawApp extends LitElement {
@state() configSearchQuery = "";
@state() configActiveSection: string | null = null;
@state() configActiveSubsection: string | null = null;
@state() pendingUpdateExpectedVersion: string | null = null;
@state() updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null = null;
@state() communicationsFormMode: "form" | "raw" = "form";
@state() communicationsSearchQuery = "";
@state() communicationsActiveSection: string | null = null;

View File

@@ -73,6 +73,8 @@ function createSaveState(): {
configSearchQuery: "",
configActiveSection: null,
configActiveSubsection: null,
pendingUpdateExpectedVersion: null,
updateStatusBanner: null,
lastError: null,
},
request,

View File

@@ -37,6 +37,8 @@ function createState(): ConfigState {
configValid: null,
connected: false,
lastError: null,
pendingUpdateExpectedVersion: null,
updateStatusBanner: null,
updateRunning: false,
};
}
@@ -554,6 +556,44 @@ describe("runUpdate", () => {
await runUpdate(state);
expect(state.lastError).toBe("Update error: network unavailable");
expect(state.updateStatusBanner).toEqual({
tone: "danger",
text: "Update error: network unavailable. See the gateway logs for the exact failure and retry once the cause is fixed.",
});
});
it("surfaces skipped updates with actionable guidance", async () => {
const request = vi.fn().mockResolvedValue({
ok: false,
result: { status: "skipped", reason: "dirty" },
});
const state = createState();
state.connected = true;
state.client = { request } as unknown as ConfigState["client"];
await runUpdate(state);
expect(state.updateStatusBanner).toEqual({
tone: "warn",
text: "Update skipped: dirty. Commit or stash changes, then retry.",
});
});
it("stores the expected post-update version when update.run succeeds", async () => {
const request = vi.fn().mockResolvedValue({
ok: true,
result: {
status: "ok",
after: { version: "2.0.0" },
},
});
const state = createState();
state.connected = true;
state.client = { request } as unknown as ConfigState["client"];
await runUpdate(state);
expect(state.pendingUpdateExpectedVersion).toBe("2.0.0");
expect(state.updateStatusBanner).toBeNull();
});
});

View File

@@ -34,6 +34,8 @@ export type ConfigState = {
configSearchQuery: string;
configActiveSection: string | null;
configActiveSubsection: string | null;
pendingUpdateExpectedVersion: string | null;
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
lastError: string | null;
};
@@ -138,6 +140,39 @@ function serializeFormForSubmit(state: ConfigState): string {
type ConfigSubmitMethod = "config.set" | "config.apply";
type ConfigSubmitBusyKey = "configSaving" | "configApplying";
function resolveUpdateStatusBanner(params: { status?: string; reason?: string }): {
tone: "danger" | "warn" | "info";
text: string;
} {
const status = (params.status ?? "error").trim() || "error";
const reason = (params.reason ?? "unexpected-error").trim() || "unexpected-error";
const tone = status === "skipped" ? "warn" : "danger";
const guidance =
{
dirty: "Commit or stash changes, then retry.",
"no-upstream": "Set an upstream branch, then retry.",
"not-git-install":
"Not a git checkout. Run `openclaw update` from the CLI for a global reinstall.",
"not-openclaw-root":
"Run the update from an OpenClaw checkout or use the CLI global reinstall path.",
"deps-install-failed": "Dependency install failed. Fix the install error and retry.",
"build-failed": "Build failed. Fix the build error and retry.",
"ui-build-failed": "The control UI rebuild failed. Fix the UI build error and retry.",
"global-install-failed":
"The global package install did not verify on disk. Retry or reinstall from the CLI.",
"restart-disabled": "The update was not applied because gateway restarts are disabled. Enable restarts in config, then retry — or run `openclaw update` from the CLI.",
"restart-unavailable":
"This global install cannot be safely replaced while restarts are disabled and no supervisor is present.",
"restart-unhealthy":
"The replacement process never became healthy. The previous process stayed up so you can recover.",
"doctor-failed": "Doctor repair failed. Run `openclaw doctor --non-interactive` and retry.",
}[reason] ?? "See the gateway logs for the exact failure and retry once the cause is fixed.";
return {
tone,
text: `Update ${status}: ${reason}. ${guidance}`,
};
}
async function submitConfigChange(
state: ConfigState,
method: ConfigSubmitMethod,
@@ -193,20 +228,27 @@ export async function runUpdate(state: ConfigState) {
}
state.updateRunning = true;
state.lastError = null;
state.updateStatusBanner = null;
try {
const res = await state.client.request<{
ok?: boolean;
result?: { status?: string; reason?: string };
result?: { status?: string; reason?: string; after?: { version?: string | null } };
}>("update.run", {
sessionKey: state.applySessionKey,
});
if (res && res.ok === false) {
const status = res.result?.status ?? "error";
const reason = res.result?.reason ?? "Update failed.";
state.lastError = `Update ${status}: ${reason}`;
const status = res.result?.status ?? (res.ok === true ? "ok" : "error");
if (status === "ok" && res.ok === true) {
state.pendingUpdateExpectedVersion = res.result?.after?.version ?? null;
return;
}
state.pendingUpdateExpectedVersion = null;
state.updateStatusBanner = resolveUpdateStatusBanner({
status,
reason: res.result?.reason,
});
} catch (err) {
state.lastError = String(err);
state.pendingUpdateExpectedVersion = null;
} finally {
state.updateRunning = false;
}