fix(bonjour): suppress ciao process crashes

This commit is contained in:
Peter Steinberger
2026-04-26 10:47:25 +01:00
parent 1be39ac847
commit bd95baa4f7
12 changed files with 192 additions and 32 deletions

View File

@@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- Gateway/Bonjour: suppress known @homebridge/ciao cancellation and network assertion failures through scoped process handlers so malformed mDNS packets or restricted VPS networking disable/restart Bonjour instead of crashing the gateway. Fixes #67578. Thanks @zenassist26-create.
- Discord: keep late clicks on already-resolved exec approval buttons quiet when elevated mode auto-resolved the request, while still surfacing real approval submission failures. Fixes #66906. Thanks @rlerikse.
## 2026.4.25

View File

@@ -1,5 +1,8 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime";
import {
registerUncaughtExceptionHandler,
registerUnhandledRejectionHandler,
} from "openclaw/plugin-sdk/runtime";
import { startGatewayBonjourAdvertiser } from "./src/advertiser.js";
function formatBonjourInstanceName(displayName: string) {
@@ -33,7 +36,11 @@ export default definePluginEntry({
cliPath: ctx.cliPath,
minimal: ctx.minimal,
},
{ logger: api.logger, registerUnhandledRejectionHandler },
{
logger: api.logger,
registerUncaughtExceptionHandler,
registerUnhandledRejectionHandler,
},
);
return { stop: advertiser.stop };
},

View File

@@ -5,6 +5,7 @@ const mocks = vi.hoisted(() => ({
createService: vi.fn(),
getResponder: vi.fn(),
shutdown: vi.fn(),
registerUncaughtExceptionHandler: vi.fn(),
registerUnhandledRejectionHandler: vi.fn(),
logger: {
info: vi.fn(),
@@ -12,7 +13,14 @@ const mocks = vi.hoisted(() => ({
debug: vi.fn(),
},
}));
const { createService, getResponder, shutdown, registerUnhandledRejectionHandler, logger } = mocks;
const {
createService,
getResponder,
shutdown,
registerUncaughtExceptionHandler,
registerUnhandledRejectionHandler,
logger,
} = mocks;
const asString = (value: unknown, fallback: string) =>
typeof value === "string" && value.trim() ? value : fallback;
@@ -77,6 +85,7 @@ const startAdvertiser = (
): ReturnType<StartGatewayBonjourAdvertiser> =>
startGatewayBonjourAdvertiser(opts, {
logger,
registerUncaughtExceptionHandler: (handler) => registerUncaughtExceptionHandler(handler),
registerUnhandledRejectionHandler: (handler) => registerUnhandledRejectionHandler(handler),
});
@@ -103,6 +112,7 @@ describe("gateway bonjour advertiser", () => {
createService.mockClear();
getResponder.mockReset();
shutdown.mockClear();
registerUncaughtExceptionHandler.mockClear();
registerUnhandledRejectionHandler.mockClear();
logger.info.mockClear();
logger.warn.mockClear();
@@ -220,7 +230,7 @@ describe("gateway bonjour advertiser", () => {
await started.stop();
});
it("does not install a process-level unhandled rejection handler by default", async () => {
it("does not install process-level ciao handlers by default", async () => {
enableAdvertiserUnitMode();
const destroy = vi.fn().mockResolvedValue(undefined);
@@ -237,11 +247,12 @@ describe("gateway bonjour advertiser", () => {
);
expect(processOn).not.toHaveBeenCalledWith("unhandledRejection", expect.any(Function));
expect(processOn).not.toHaveBeenCalledWith("uncaughtException", expect.any(Function));
await started.stop();
});
it("cleans up unhandled rejection handler after shutdown", async () => {
it("cleans up ciao process handlers after shutdown", async () => {
enableAdvertiserUnitMode();
const destroy = vi.fn().mockResolvedValue(undefined);
@@ -252,10 +263,14 @@ describe("gateway bonjour advertiser", () => {
});
mockCiaoService({ advertise, destroy });
const cleanup = vi.fn(() => {
order.push("cleanup");
const cleanupException = vi.fn(() => {
order.push("cleanup-exception");
});
registerUnhandledRejectionHandler.mockImplementation(() => cleanup);
const cleanupRejection = vi.fn(() => {
order.push("cleanup-rejection");
});
registerUncaughtExceptionHandler.mockImplementation(() => cleanupException);
registerUnhandledRejectionHandler.mockImplementation(() => cleanupRejection);
const started = await startAdvertiser({
gatewayPort: 18789,
@@ -264,9 +279,11 @@ describe("gateway bonjour advertiser", () => {
await started.stop();
expect(registerUncaughtExceptionHandler).toHaveBeenCalledTimes(1);
expect(registerUnhandledRejectionHandler).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(order).toEqual(["shutdown", "cleanup"]);
expect(cleanupException).toHaveBeenCalledTimes(1);
expect(cleanupRejection).toHaveBeenCalledTimes(1);
expect(order).toEqual(["shutdown", "cleanup-exception", "cleanup-rejection"]);
});
it("logs ciao handler classifications at the bonjour caller", async () => {
@@ -284,7 +301,11 @@ describe("gateway bonjour advertiser", () => {
const handler = registerUnhandledRejectionHandler.mock.calls[0]?.[0] as
| ((reason: unknown) => boolean)
| undefined;
const exceptionHandler = registerUncaughtExceptionHandler.mock.calls[0]?.[0] as
| ((reason: unknown) => boolean)
| undefined;
expect(handler).toBeTypeOf("function");
expect(exceptionHandler).toBeTypeOf("function");
expect(handler?.(new Error("CIAO PROBING CANCELLED"))).toBe(true);
expect(logger.debug).toHaveBeenCalledWith(
@@ -299,6 +320,21 @@ describe("gateway bonjour advertiser", () => {
expect.stringContaining("suppressing ciao interface assertion"),
);
logger.warn.mockClear();
expect(
exceptionHandler?.(
Object.assign(
new Error(
"IP address version must match. Netmask cannot have a version different from the address!",
),
{ name: "AssertionError" },
),
),
).toBe(true);
expect(logger.warn).toHaveBeenCalledWith(
expect.stringContaining("suppressing ciao netmask assertion"),
);
await started.stop();
});

View File

@@ -1,6 +1,6 @@
import type { PluginLogger } from "openclaw/plugin-sdk/plugin-entry";
import { isTruthyEnvValue } from "openclaw/plugin-sdk/runtime-env";
import { classifyCiaoUnhandledRejection } from "./ciao.js";
import { classifyCiaoProcessError, type CiaoProcessErrorClassification } from "./ciao.js";
import { formatBonjourError } from "./errors.js";
export type GatewayBonjourAdvertiser = {
@@ -50,6 +50,7 @@ type CiaoModule = {
type BonjourCycle = {
responder: BonjourResponder;
services: Array<{ label: string; svc: BonjourService }>;
cleanupUncaughtException?: () => void;
cleanupUnhandledRejection?: () => void;
};
@@ -59,10 +60,12 @@ type ServiceStateTracker = {
};
type ConsoleLogFn = (...args: unknown[]) => void;
type UncaughtExceptionHandler = (error: unknown) => boolean;
type UnhandledRejectionHandler = (reason: unknown) => boolean;
type BonjourAdvertiserDeps = {
logger?: Pick<PluginLogger, "info" | "warn" | "debug">;
registerUncaughtExceptionHandler?: (handler: UncaughtExceptionHandler) => () => void;
registerUnhandledRejectionHandler?: (handler: UnhandledRejectionHandler) => () => void;
};
@@ -175,19 +178,22 @@ export async function startGatewayBonjourAdvertiser(
};
const { getResponder, Protocol } = await loadCiaoModule();
const restoreConsoleLog = installCiaoConsoleNoiseFilter();
let requestCiaoRecovery: ((classification: CiaoProcessErrorClassification) => void) | undefined;
const handleCiaoUnhandledRejection = (reason: unknown): boolean => {
const classification = classifyCiaoUnhandledRejection(reason);
const handleCiaoProcessError = (reason: unknown): boolean => {
const classification = classifyCiaoProcessError(reason);
if (!classification) {
return false;
}
if (classification.kind === "interface-assertion") {
logger.warn(`bonjour: suppressing ciao interface assertion: ${classification.formatted}`);
return true;
if (classification.kind === "cancellation") {
logger.debug(`bonjour: ignoring unhandled ciao rejection: ${classification.formatted}`);
} else {
const label =
classification.kind === "netmask-assertion" ? "netmask assertion" : "interface assertion";
logger.warn(`bonjour: suppressing ciao ${label}: ${classification.formatted}`);
requestCiaoRecovery?.(classification);
}
logger.debug(`bonjour: ignoring unhandled ciao rejection: ${classification.formatted}`);
return true;
};
@@ -255,10 +261,14 @@ export async function startGatewayBonjourAdvertiser(
const cleanupUnhandledRejection =
services.length > 0 && deps.registerUnhandledRejectionHandler
? deps.registerUnhandledRejectionHandler(handleCiaoUnhandledRejection)
? deps.registerUnhandledRejectionHandler(handleCiaoProcessError)
: undefined;
const cleanupUncaughtException =
services.length > 0 && deps.registerUncaughtExceptionHandler
? deps.registerUncaughtExceptionHandler(handleCiaoProcessError)
: undefined;
return { responder, services, cleanupUnhandledRejection };
return { responder, services, cleanupUncaughtException, cleanupUnhandledRejection };
}
async function stopCycle(cycle: BonjourCycle | null, opts?: { shutdownResponder?: boolean }) {
@@ -279,6 +289,7 @@ export async function startGatewayBonjourAdvertiser(
} catch {
/* ignore */
} finally {
cycle.cleanupUncaughtException?.();
cycle.cleanupUnhandledRejection?.();
}
}
@@ -388,6 +399,9 @@ export async function startGatewayBonjourAdvertiser(
});
return recreatePromise;
};
requestCiaoRecovery = (classification) => {
void recreateAdvertiser(`ciao ${classification.kind}: ${classification.formatted}`);
};
const lastRepairAttempt = new Map<string, number>();
const watchdog = setInterval(() => {

View File

@@ -21,6 +21,23 @@ describe("bonjour-ciao", () => {
});
});
it("classifies ciao netmask assertions separately from side effects", () => {
expect(
classifyCiaoUnhandledRejection(
Object.assign(
new Error(
"IP address version must match. Netmask cannot have a version different from the address!",
),
{ name: "AssertionError" },
),
),
).toEqual({
kind: "netmask-assertion",
formatted:
"AssertionError: IP address version must match. Netmask cannot have a version different from the address!",
});
});
it("suppresses ciao announcement cancellation rejections", () => {
expect(ignoreCiaoUnhandledRejection(new Error("Ciao announcement cancelled by shutdown"))).toBe(
true,
@@ -44,6 +61,17 @@ describe("bonjour-ciao", () => {
expect(ignoreCiaoUnhandledRejection(error)).toBe(true);
});
it("suppresses ciao netmask assertion errors as non-fatal", () => {
const error = Object.assign(
new Error(
"IP address version must match. Netmask cannot have a version different from the address!",
),
{ name: "AssertionError" },
);
expect(ignoreCiaoUnhandledRejection(error)).toBe(true);
});
it("keeps unrelated rejections visible", () => {
expect(ignoreCiaoUnhandledRejection(new Error("boom"))).toBe(false);
});

View File

@@ -2,15 +2,16 @@ import { formatBonjourError } from "./errors.js";
const CIAO_CANCELLATION_MESSAGE_RE = /^CIAO (?:ANNOUNCEMENT|PROBING) CANCELLED\b/u;
const CIAO_INTERFACE_ASSERTION_MESSAGE_RE =
/REACHED ILLEGAL STATE!?\s+IPV4 ADDRESS CHANGE FROM DEFINED TO UNDEFINED!?/u;
/REACHED ILLEGAL STATE!?\s+IPV4 ADDRESS CHANGE FROM (?:DEFINED TO UNDEFINED|UNDEFINED TO DEFINED)!?/u;
const CIAO_NETMASK_ASSERTION_MESSAGE_RE =
/IP ADDRESS VERSION MUST MATCH\.\s+NETMASK CANNOT HAVE A VERSION DIFFERENT FROM THE ADDRESS!?/u;
export type CiaoUnhandledRejectionClassification =
export type CiaoProcessErrorClassification =
| { kind: "cancellation"; formatted: string }
| { kind: "interface-assertion"; formatted: string };
| { kind: "interface-assertion"; formatted: string }
| { kind: "netmask-assertion"; formatted: string };
export function classifyCiaoUnhandledRejection(
reason: unknown,
): CiaoUnhandledRejectionClassification | null {
export function classifyCiaoProcessError(reason: unknown): CiaoProcessErrorClassification | null {
const formatted = formatBonjourError(reason);
const message = formatted.toUpperCase();
if (CIAO_CANCELLATION_MESSAGE_RE.test(message)) {
@@ -19,9 +20,14 @@ export function classifyCiaoUnhandledRejection(
if (CIAO_INTERFACE_ASSERTION_MESSAGE_RE.test(message)) {
return { kind: "interface-assertion", formatted };
}
if (CIAO_NETMASK_ASSERTION_MESSAGE_RE.test(message)) {
return { kind: "netmask-assertion", formatted };
}
return null;
}
export const classifyCiaoUnhandledRejection = classifyCiaoProcessError;
export function ignoreCiaoUnhandledRejection(reason: unknown): boolean {
return classifyCiaoUnhandledRejection(reason) !== null;
return classifyCiaoProcessError(reason) !== null;
}

View File

@@ -309,7 +309,7 @@ export async function runCli(argv: string[] = process.argv) {
const [
{ buildProgram },
{ runFatalErrorHooks },
{ installUnhandledRejectionHandler },
{ installUnhandledRejectionHandler, isUncaughtExceptionHandled },
{ restoreTerminalState },
] = await Promise.all([
import("./program.js"),
@@ -324,6 +324,9 @@ export async function runCli(argv: string[] = process.argv) {
installUnhandledRejectionHandler();
process.on("uncaughtException", (error) => {
if (isUncaughtExceptionHandled(error)) {
return;
}
console.error("[openclaw] Uncaught exception:", formatUncaughtError(error));
for (const message of runFatalErrorHooks({ reason: "uncaught_exception", error })) {
console.error("[openclaw]", message);

View File

@@ -4,7 +4,10 @@ import { fileURLToPath } from "node:url";
import { formatUncaughtError } from "./infra/errors.js";
import { runFatalErrorHooks } from "./infra/fatal-error-hooks.js";
import { isMainModule } from "./infra/is-main.js";
import { installUnhandledRejectionHandler } from "./infra/unhandled-rejections.js";
import {
installUnhandledRejectionHandler,
isUncaughtExceptionHandled,
} from "./infra/unhandled-rejections.js";
type LegacyCliDeps = {
runCli: (argv: string[]) => Promise<void>;
@@ -86,6 +89,9 @@ if (isMain) {
installUnhandledRejectionHandler();
process.on("uncaughtException", (error) => {
if (isUncaughtExceptionHandled(error)) {
return;
}
console.error("[openclaw] Uncaught exception:", formatUncaughtError(error));
for (const message of runFatalErrorHooks({ reason: "uncaught_exception", error })) {
console.error("[openclaw]", message);

View File

@@ -8,7 +8,11 @@ vi.mock("../terminal/restore.js", () => ({
}));
import { resetFatalErrorHooksForTest } from "./fatal-error-hooks.js";
import { installUnhandledRejectionHandler } from "./unhandled-rejections.js";
import {
installUnhandledRejectionHandler,
isUncaughtExceptionHandled,
registerUncaughtExceptionHandler,
} from "./unhandled-rejections.js";
describe("installUnhandledRejectionHandler - fatal detection", () => {
let exitCalls: Array<string | number | null> = [];
@@ -91,6 +95,20 @@ describe("installUnhandledRejectionHandler - fatal detection", () => {
});
});
describe("scoped uncaught exception handlers", () => {
it("lets registered handlers suppress known dependency exceptions", () => {
const cleanup = registerUncaughtExceptionHandler((error) => {
return error instanceof Error && error.message === "known dependency assertion";
});
expect(isUncaughtExceptionHandled(new Error("known dependency assertion"))).toBe(true);
expect(isUncaughtExceptionHandled(new Error("unknown"))).toBe(false);
cleanup();
expect(isUncaughtExceptionHandled(new Error("known dependency assertion"))).toBe(false);
});
});
describe("configuration errors", () => {
it("exits on configuration error codes", () => {
const configurationCases = [

View File

@@ -10,11 +10,13 @@ import {
import { runFatalErrorHooks } from "./fatal-error-hooks.js";
type UnhandledRejectionHandler = (reason: unknown) => boolean;
type UncaughtExceptionHandler = (error: unknown) => boolean;
// Plugins resolve `openclaw/plugin-sdk/runtime` through their own staged
// `node_modules`, which loads a separate copy of this module. To keep registry
// state shared across instances, anchor the handlers Set on globalThis.
const HANDLERS_GLOBAL_KEY = Symbol.for("openclaw.unhandledRejection.handlers");
const EXCEPTION_HANDLERS_GLOBAL_KEY = Symbol.for("openclaw.uncaughtException.handlers");
const handlers: Set<UnhandledRejectionHandler> = (() => {
const g = globalThis as unknown as Record<symbol, Set<UnhandledRejectionHandler>>;
const existing = g[HANDLERS_GLOBAL_KEY];
@@ -25,6 +27,16 @@ const handlers: Set<UnhandledRejectionHandler> = (() => {
g[HANDLERS_GLOBAL_KEY] = created;
return created;
})();
const exceptionHandlers: Set<UncaughtExceptionHandler> = (() => {
const g = globalThis as unknown as Record<symbol, Set<UncaughtExceptionHandler>>;
const existing = g[EXCEPTION_HANDLERS_GLOBAL_KEY];
if (existing instanceof Set) {
return existing;
}
const created = new Set<UncaughtExceptionHandler>();
g[EXCEPTION_HANDLERS_GLOBAL_KEY] = created;
return created;
})();
const FATAL_ERROR_CODES = new Set([
"ERR_OUT_OF_MEMORY",
@@ -350,6 +362,29 @@ export function isUnhandledRejectionHandled(reason: unknown): boolean {
return false;
}
export function registerUncaughtExceptionHandler(handler: UncaughtExceptionHandler): () => void {
exceptionHandlers.add(handler);
return () => {
exceptionHandlers.delete(handler);
};
}
export function isUncaughtExceptionHandled(error: unknown): boolean {
for (const handler of exceptionHandlers) {
try {
if (handler(error)) {
return true;
}
} catch (err) {
console.error(
"[openclaw] Uncaught exception handler failed:",
err instanceof Error ? (err.stack ?? err.message) : err,
);
}
}
return false;
}
export function installUnhandledRejectionHandler(): void {
const exitWithTerminalRestore = (reason: string, error?: unknown, hookReason = reason) => {
for (const message of runFatalErrorHooks({ reason: hookReason, error })) {

View File

@@ -28,5 +28,8 @@ export {
} from "../infra/format-time/format-duration.ts";
export { retryAsync } from "../infra/retry.js";
export { ensureGlobalUndiciEnvProxyDispatcher } from "../infra/net/undici-global-dispatcher.js";
export { registerUnhandledRejectionHandler } from "../infra/unhandled-rejections.js";
export {
registerUncaughtExceptionHandler,
registerUnhandledRejectionHandler,
} from "../infra/unhandled-rejections.js";
export { isWSL2Sync } from "../infra/wsl.js";

View File

@@ -29,5 +29,8 @@ export {
formatPluginInstallPathIssue,
} from "../infra/plugin-install-path-warnings.js";
export { collectProviderDangerousNameMatchingScopes } from "../config/dangerous-name-matching.js";
export { registerUnhandledRejectionHandler } from "../infra/unhandled-rejections.js";
export {
registerUncaughtExceptionHandler,
registerUnhandledRejectionHandler,
} from "../infra/unhandled-rejections.js";
export { removePluginFromConfig } from "../plugins/uninstall.js";