mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 16:20:43 +00:00
fix(update): require applied gateway restarts
Require Control UI updates to observe a real gateway process replacement, surface skipped/error update outcomes, and verify the running gateway version after restart.\n\nAdds update.status restart-sentinel plumbing, docs, generated protocol model updates, and changelog attribution.\n\nLocal verification:\n- pnpm test src/gateway/server-methods/update.test.ts src/cli/gateway-cli/run-loop.test.ts src/infra/restart-sentinel.test.ts src/infra/process-respawn.test.ts src/infra/update-runner.test.ts ui/src/ui/app-gateway.node.test.ts ui/src/ui/controllers/config.test.ts\n- git diff --check\n- pnpm exec oxfmt --check --threads=1 CHANGELOG.md docs/gateway/protocol.md docs/gateway/configuration.md docs/web/control-ui.md\n- pnpm docs:check-mdx
This commit is contained in:
@@ -43,6 +43,9 @@ vi.mock("./gateway.ts", async (importOriginal) => {
|
||||
readonly start = vi.fn();
|
||||
readonly stop = vi.fn();
|
||||
readonly request = vi.fn(async (method: string) => {
|
||||
if (method === "update.status") {
|
||||
return { sentinel: null };
|
||||
}
|
||||
if (method === "models.authStatus") {
|
||||
return { ts: 0, providers: [] };
|
||||
}
|
||||
@@ -154,6 +157,8 @@ function createHost(): TestGatewayHost {
|
||||
assistantAgentId: null,
|
||||
localMediaPreviewRoots: [],
|
||||
serverVersion: null,
|
||||
pendingUpdateExpectedVersion: null,
|
||||
updateStatusBanner: null,
|
||||
sessionKey: "main",
|
||||
chatMessages: [],
|
||||
chatQueue: [],
|
||||
@@ -283,6 +288,117 @@ describe("connectGateway", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("clears pending update verification when the restarted version matches", async () => {
|
||||
const host = createHost();
|
||||
host.pendingUpdateExpectedVersion = "2.0.0";
|
||||
|
||||
connectGateway(host);
|
||||
const client = gatewayClientInstances[0];
|
||||
expect(client).toBeDefined();
|
||||
client.request.mockImplementation(async (method: string) => {
|
||||
if (method === "update.status") {
|
||||
return {
|
||||
sentinel: {
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
stats: {
|
||||
after: { version: "2.0.0" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
client.emitHello({
|
||||
type: "hello-ok",
|
||||
protocol: 3,
|
||||
server: { version: "2.0.0" },
|
||||
snapshot: {},
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(host.pendingUpdateExpectedVersion).toBeNull();
|
||||
});
|
||||
expect(host.updateStatusBanner).toBeNull();
|
||||
});
|
||||
|
||||
it("shows a hard error when the restarted version does not match the expected update", async () => {
|
||||
const host = createHost();
|
||||
host.pendingUpdateExpectedVersion = "2.0.0";
|
||||
|
||||
connectGateway(host);
|
||||
const client = gatewayClientInstances[0];
|
||||
expect(client).toBeDefined();
|
||||
client.request.mockImplementation(async (method: string) => {
|
||||
if (method === "update.status") {
|
||||
return {
|
||||
sentinel: {
|
||||
kind: "update",
|
||||
status: "ok",
|
||||
stats: {
|
||||
after: { version: "1.0.0" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
client.emitHello({
|
||||
type: "hello-ok",
|
||||
protocol: 3,
|
||||
server: { version: "1.0.0" },
|
||||
snapshot: {},
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(host.pendingUpdateExpectedVersion).toBeNull();
|
||||
expect(host.updateStatusBanner?.text).toContain(
|
||||
"Update installed but running version did not change",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces post-restart sentinel failures after reconnect", async () => {
|
||||
const host = createHost();
|
||||
host.pendingUpdateExpectedVersion = "2.0.0";
|
||||
|
||||
connectGateway(host);
|
||||
const client = gatewayClientInstances[0];
|
||||
expect(client).toBeDefined();
|
||||
client.request.mockImplementation(async (method: string) => {
|
||||
if (method === "update.status") {
|
||||
return {
|
||||
sentinel: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
stats: {
|
||||
reason: "restart-unhealthy",
|
||||
after: { version: "1.0.0" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
client.emitHello({
|
||||
type: "hello-ok",
|
||||
protocol: 3,
|
||||
server: { version: "1.0.0" },
|
||||
snapshot: {},
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(host.pendingUpdateExpectedVersion).toBeNull();
|
||||
expect(host.updateStatusBanner).toEqual({
|
||||
tone: "danger",
|
||||
text: "Update error: restart-unhealthy. The replacement process never became healthy and the previous process stayed up.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores stale client onClose callbacks after reconnect", () => {
|
||||
const host = createHost();
|
||||
|
||||
|
||||
@@ -97,6 +97,8 @@ type GatewayHost = {
|
||||
assistantAvatar: string | null;
|
||||
assistantAgentId: string | null;
|
||||
serverVersion: string | null;
|
||||
pendingUpdateExpectedVersion: string | null;
|
||||
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
|
||||
sessionKey: string;
|
||||
chatRunId: string | null;
|
||||
pendingAbort?: { runId: string; sessionKey: string } | null;
|
||||
@@ -157,6 +159,94 @@ type ConnectGatewayOptions = {
|
||||
reason?: "initial" | "seq-gap";
|
||||
};
|
||||
|
||||
type UpdateRestartStatusResponse = {
|
||||
sentinel?: {
|
||||
kind?: string;
|
||||
status?: string;
|
||||
stats?: {
|
||||
reason?: string | null;
|
||||
after?: { version?: string | null } | null;
|
||||
} | null;
|
||||
} | null;
|
||||
};
|
||||
|
||||
function resolveUpdateVerificationBanner(params: {
|
||||
expectedVersion: string;
|
||||
actualVersion: string | null;
|
||||
}): { tone: "danger"; text: string } {
|
||||
const actualSuffix = params.actualVersion
|
||||
? ` Expected v${params.expectedVersion}, running v${params.actualVersion}.`
|
||||
: "";
|
||||
return {
|
||||
tone: "danger",
|
||||
text: `Update installed but running version did not change — restart may have been blocked.${actualSuffix}`,
|
||||
};
|
||||
}
|
||||
|
||||
function resolvePostRestartUpdateBanner(reason: string | null | undefined): {
|
||||
tone: "danger";
|
||||
text: string;
|
||||
} {
|
||||
const normalizedReason = reason?.trim() || "restart-unhealthy";
|
||||
const guidance =
|
||||
normalizedReason === "restart-unhealthy"
|
||||
? "The replacement process never became healthy and the previous process stayed up."
|
||||
: "Check the gateway logs for the replacement failure.";
|
||||
return {
|
||||
tone: "danger",
|
||||
text: `Update error: ${normalizedReason}. ${guidance}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyPendingUpdateVersion(
|
||||
host: GatewayHost,
|
||||
client: GatewayBrowserClient,
|
||||
): Promise<void> {
|
||||
const expectedVersion = host.pendingUpdateExpectedVersion?.trim();
|
||||
if (!expectedVersion) {
|
||||
return;
|
||||
}
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (host.client === client && host.connected && Date.now() < deadline) {
|
||||
let response: UpdateRestartStatusResponse | null = null;
|
||||
try {
|
||||
response = await client.request<UpdateRestartStatusResponse>("update.status", {});
|
||||
} catch {
|
||||
response = null;
|
||||
}
|
||||
const sentinel = response?.sentinel;
|
||||
const actualVersion = sentinel?.stats?.after?.version?.trim() || null;
|
||||
if (sentinel?.kind === "update" && actualVersion) {
|
||||
host.pendingUpdateExpectedVersion = null;
|
||||
if (sentinel.status && sentinel.status !== "ok") {
|
||||
host.updateStatusBanner = resolvePostRestartUpdateBanner(sentinel.stats?.reason ?? null);
|
||||
return;
|
||||
}
|
||||
if (actualVersion !== expectedVersion) {
|
||||
host.updateStatusBanner = resolveUpdateVerificationBanner({
|
||||
expectedVersion,
|
||||
actualVersion,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 250);
|
||||
});
|
||||
}
|
||||
if (host.client !== client || !host.connected) {
|
||||
return;
|
||||
}
|
||||
const currentVersion = host.hello?.server?.version?.trim() || null;
|
||||
host.pendingUpdateExpectedVersion = null;
|
||||
if (currentVersion !== expectedVersion) {
|
||||
host.updateStatusBanner = resolveUpdateVerificationBanner({
|
||||
expectedVersion,
|
||||
actualVersion: currentVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveControlUiClientVersion(params: {
|
||||
gatewayUrl: string;
|
||||
serverVersion: string | null;
|
||||
@@ -344,6 +434,7 @@ export function connectGateway(host: GatewayHost, options?: ConnectGatewayOption
|
||||
void refreshActiveTab(host as unknown as Parameters<typeof refreshActiveTab>[0]);
|
||||
// Re-run push reconciliation now that the gateway client is available.
|
||||
void host.reconcileWebPushState?.();
|
||||
void verifyPendingUpdateVersion(host, client);
|
||||
},
|
||||
onClose: ({ code, reason, error }) => {
|
||||
if (host.client !== client) {
|
||||
|
||||
@@ -1482,6 +1482,11 @@ export function renderApp(state: AppViewState) {
|
||||
</aside>
|
||||
</div>
|
||||
<main class="content ${isChat ? "content--chat" : ""}">
|
||||
${state.updateStatusBanner
|
||||
? html`<div class="callout ${state.updateStatusBanner.tone}" role="alert">
|
||||
${state.updateStatusBanner.text}
|
||||
</div>`
|
||||
: nothing}
|
||||
${state.updateAvailable &&
|
||||
state.updateAvailable.latestVersion !== state.updateAvailable.currentVersion &&
|
||||
!isUpdateBannerDismissed(state.updateAvailable)
|
||||
|
||||
@@ -744,7 +744,7 @@ function buildAttentionItems(host: SettingsAppHost) {
|
||||
// Use the same predicate as the Overview card so the two stay in sync.
|
||||
// Without this, a `missing` provider shows up on the card but never
|
||||
// produces the re-auth attention callout.
|
||||
const monitored = modelAuth.providers.filter(isMonitoredAuthProvider);
|
||||
const monitored = (modelAuth.providers ?? []).filter(isMonitoredAuthProvider);
|
||||
const expiredProviders = monitored.filter(
|
||||
(p) => p.status === "expired" || p.status === "missing",
|
||||
);
|
||||
|
||||
@@ -176,6 +176,8 @@ export type AppViewState = {
|
||||
configSearchQuery: string;
|
||||
configActiveSection: string | null;
|
||||
configActiveSubsection: string | null;
|
||||
pendingUpdateExpectedVersion: string | null;
|
||||
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
|
||||
communicationsFormMode: "form" | "raw";
|
||||
communicationsSearchQuery: string;
|
||||
communicationsActiveSection: string | null;
|
||||
|
||||
@@ -284,6 +284,8 @@ export class OpenClawApp extends LitElement {
|
||||
@state() configSearchQuery = "";
|
||||
@state() configActiveSection: string | null = null;
|
||||
@state() configActiveSubsection: string | null = null;
|
||||
@state() pendingUpdateExpectedVersion: string | null = null;
|
||||
@state() updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null = null;
|
||||
@state() communicationsFormMode: "form" | "raw" = "form";
|
||||
@state() communicationsSearchQuery = "";
|
||||
@state() communicationsActiveSection: string | null = null;
|
||||
|
||||
@@ -73,6 +73,8 @@ function createSaveState(): {
|
||||
configSearchQuery: "",
|
||||
configActiveSection: null,
|
||||
configActiveSubsection: null,
|
||||
pendingUpdateExpectedVersion: null,
|
||||
updateStatusBanner: null,
|
||||
lastError: null,
|
||||
},
|
||||
request,
|
||||
|
||||
@@ -37,6 +37,8 @@ function createState(): ConfigState {
|
||||
configValid: null,
|
||||
connected: false,
|
||||
lastError: null,
|
||||
pendingUpdateExpectedVersion: null,
|
||||
updateStatusBanner: null,
|
||||
updateRunning: false,
|
||||
};
|
||||
}
|
||||
@@ -554,6 +556,44 @@ describe("runUpdate", () => {
|
||||
|
||||
await runUpdate(state);
|
||||
|
||||
expect(state.lastError).toBe("Update error: network unavailable");
|
||||
expect(state.updateStatusBanner).toEqual({
|
||||
tone: "danger",
|
||||
text: "Update error: network unavailable. See the gateway logs for the exact failure and retry once the cause is fixed.",
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces skipped updates with actionable guidance", async () => {
|
||||
const request = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
result: { status: "skipped", reason: "dirty" },
|
||||
});
|
||||
const state = createState();
|
||||
state.connected = true;
|
||||
state.client = { request } as unknown as ConfigState["client"];
|
||||
|
||||
await runUpdate(state);
|
||||
|
||||
expect(state.updateStatusBanner).toEqual({
|
||||
tone: "warn",
|
||||
text: "Update skipped: dirty. Commit or stash changes, then retry.",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores the expected post-update version when update.run succeeds", async () => {
|
||||
const request = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
result: {
|
||||
status: "ok",
|
||||
after: { version: "2.0.0" },
|
||||
},
|
||||
});
|
||||
const state = createState();
|
||||
state.connected = true;
|
||||
state.client = { request } as unknown as ConfigState["client"];
|
||||
|
||||
await runUpdate(state);
|
||||
|
||||
expect(state.pendingUpdateExpectedVersion).toBe("2.0.0");
|
||||
expect(state.updateStatusBanner).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -34,6 +34,8 @@ export type ConfigState = {
|
||||
configSearchQuery: string;
|
||||
configActiveSection: string | null;
|
||||
configActiveSubsection: string | null;
|
||||
pendingUpdateExpectedVersion: string | null;
|
||||
updateStatusBanner: { tone: "danger" | "warn" | "info"; text: string } | null;
|
||||
lastError: string | null;
|
||||
};
|
||||
|
||||
@@ -138,6 +140,39 @@ function serializeFormForSubmit(state: ConfigState): string {
|
||||
type ConfigSubmitMethod = "config.set" | "config.apply";
|
||||
type ConfigSubmitBusyKey = "configSaving" | "configApplying";
|
||||
|
||||
function resolveUpdateStatusBanner(params: { status?: string; reason?: string }): {
|
||||
tone: "danger" | "warn" | "info";
|
||||
text: string;
|
||||
} {
|
||||
const status = (params.status ?? "error").trim() || "error";
|
||||
const reason = (params.reason ?? "unexpected-error").trim() || "unexpected-error";
|
||||
const tone = status === "skipped" ? "warn" : "danger";
|
||||
const guidance =
|
||||
{
|
||||
dirty: "Commit or stash changes, then retry.",
|
||||
"no-upstream": "Set an upstream branch, then retry.",
|
||||
"not-git-install":
|
||||
"Not a git checkout. Run `openclaw update` from the CLI for a global reinstall.",
|
||||
"not-openclaw-root":
|
||||
"Run the update from an OpenClaw checkout or use the CLI global reinstall path.",
|
||||
"deps-install-failed": "Dependency install failed. Fix the install error and retry.",
|
||||
"build-failed": "Build failed. Fix the build error and retry.",
|
||||
"ui-build-failed": "The control UI rebuild failed. Fix the UI build error and retry.",
|
||||
"global-install-failed":
|
||||
"The global package install did not verify on disk. Retry or reinstall from the CLI.",
|
||||
"restart-disabled": "The update was not applied because gateway restarts are disabled. Enable restarts in config, then retry — or run `openclaw update` from the CLI.",
|
||||
"restart-unavailable":
|
||||
"This global install cannot be safely replaced while restarts are disabled and no supervisor is present.",
|
||||
"restart-unhealthy":
|
||||
"The replacement process never became healthy. The previous process stayed up so you can recover.",
|
||||
"doctor-failed": "Doctor repair failed. Run `openclaw doctor --non-interactive` and retry.",
|
||||
}[reason] ?? "See the gateway logs for the exact failure and retry once the cause is fixed.";
|
||||
return {
|
||||
tone,
|
||||
text: `Update ${status}: ${reason}. ${guidance}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function submitConfigChange(
|
||||
state: ConfigState,
|
||||
method: ConfigSubmitMethod,
|
||||
@@ -193,20 +228,27 @@ export async function runUpdate(state: ConfigState) {
|
||||
}
|
||||
state.updateRunning = true;
|
||||
state.lastError = null;
|
||||
state.updateStatusBanner = null;
|
||||
try {
|
||||
const res = await state.client.request<{
|
||||
ok?: boolean;
|
||||
result?: { status?: string; reason?: string };
|
||||
result?: { status?: string; reason?: string; after?: { version?: string | null } };
|
||||
}>("update.run", {
|
||||
sessionKey: state.applySessionKey,
|
||||
});
|
||||
if (res && res.ok === false) {
|
||||
const status = res.result?.status ?? "error";
|
||||
const reason = res.result?.reason ?? "Update failed.";
|
||||
state.lastError = `Update ${status}: ${reason}`;
|
||||
const status = res.result?.status ?? (res.ok === true ? "ok" : "error");
|
||||
if (status === "ok" && res.ok === true) {
|
||||
state.pendingUpdateExpectedVersion = res.result?.after?.version ?? null;
|
||||
return;
|
||||
}
|
||||
state.pendingUpdateExpectedVersion = null;
|
||||
state.updateStatusBanner = resolveUpdateStatusBanner({
|
||||
status,
|
||||
reason: res.result?.reason,
|
||||
});
|
||||
} catch (err) {
|
||||
state.lastError = String(err);
|
||||
state.pendingUpdateExpectedVersion = null;
|
||||
} finally {
|
||||
state.updateRunning = false;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user