import { spawnSync } from "node:child_process"; import { EventEmitter } from "node:events"; import fs from "node:fs"; import { createServer } from "node:http"; import os from "node:os"; import path from "node:path"; import { performance } from "node:perf_hooks"; import { describe, expect, it, vi } from "vitest"; import { testing } from "../../scripts/bench-gateway-startup.ts"; async function listenOnLoopback(handler: Parameters[0]) { const server = createServer(handler); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve()); }); const address = server.address(); if (!address || typeof address === "string") { server.close(); throw new Error("expected loopback port"); } return { port: address.port, server }; } describe("gateway startup benchmark script", () => { it("prints help without running benchmark cases", () => { const result = spawnSync( process.execPath, ["--import", "tsx", "scripts/bench-gateway-startup.ts", "--help"], { cwd: process.cwd(), encoding: "utf8", env: { ...process.env, NODE_NO_WARNINGS: "1", }, }, ); expect(result.status).toBe(0); expect(result.stdout).toContain("OpenClaw Gateway startup benchmark"); expect(result.stdout).toContain("--case "); expect(result.stdout).toContain("--cpu-prof-dir "); expect(result.stdout).toContain("default (gateway default)"); expect(result.stdout).not.toContain("[gateway-startup-bench]"); expect(result.stderr).toBe(""); }); it("rejects ambiguous benchmark CLI values before spawning Node", () => { expect(testing.parsePositiveInt("5", 1, "--runs")).toBe(5); expect(testing.parseNonNegativeInt("0", 1, "--warmup")).toBe(0); expect(() => testing.parsePositiveInt("2abc", 1, "--runs")).toThrow( /--runs must be an integer/u, ); expect(() => testing.resolveEntry("--inspect")).toThrow(/must be a file path/u); }); it("does not disable local-check policy in the child gateway environment", () => { const env = testing.sanitizedEnv("/tmp/openclaw-bench", "/tmp/openclaw-bench/config.json", { config: {}, id: "default", name: "gateway default", }); expect(env.OPENCLAW_LOCAL_CHECK).toBeUndefined(); expect(env.OPENCLAW_GATEWAY_STARTUP_TRACE).toBe("1"); }); it("classifies HTTP listen and gateway ready logs separately", () => { expect( testing.classifyGatewayReadyLog("[gateway] http server listening (0 plugins, 0.8s)"), ).toBe("http-listen"); expect(testing.classifyGatewayReadyLog("[gateway] ready (0 plugins, 0.8s)")).toBe( "gateway-ready", ); expect(testing.classifyGatewayReadyLog("[gateway] ready")).toBe("gateway-ready"); expect(testing.classifyGatewayReadyLog("[gateway] starting HTTP server...")).toBeNull(); }); it("summarizes split ready log timings without the ambiguous readyLogMs field", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { cpuCoreRatio: null, cpuMs: null, exitCode: null, firstOutputMs: 1, gatewayReadyLogLine: "[gateway] ready", gatewayReadyLogMs: 40, healthz: { firstErrorKind: "econnrefused", firstRecoveryMs: 20, ms: 20, status: 200, transitions: [], }, httpListenLogLine: "[gateway] http server listening (0 plugins)", httpListenLogMs: 10, maxRssMb: null, outputTail: "", readyz: { firstErrorKind: "http-503", firstRecoveryMs: 30, ms: 30, status: 200, transitions: [], }, signal: null, startupTrace: {}, }, ]); expect(result.summary.httpListenLogMs?.p50).toBe(10); expect(result.summary.gatewayReadyLogMs?.p50).toBe(40); expect("readyLogMs" in result.summary).toBe(false); }); it("flags samples that never produced readiness or process metrics", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { cpuCoreRatio: null, cpuMs: null, exitCode: 1, firstOutputMs: 5, gatewayReadyLogLine: null, gatewayReadyLogMs: null, healthz: { firstErrorKind: "econnrefused", firstRecoveryMs: null, ms: null, status: null, transitions: [], }, httpListenLogLine: null, httpListenLogMs: null, maxRssMb: null, outputTail: "Error: Cannot find module 'dist/entry.js'", readyz: { firstErrorKind: "econnrefused", firstRecoveryMs: null, ms: null, status: null, transitions: [], }, signal: null, startupTrace: {}, }, ]); expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([ { id: "demo", reason: "missing /healthz, /readyz, cpu, rss", sampleIndex: 1, }, ]); }); it("flags samples that become ready and then exit nonzero", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, exitCode: 1, firstOutputMs: 1, gatewayReadyLogLine: "[gateway] ready", gatewayReadyLogMs: 20, healthz: { firstErrorKind: "econnrefused", firstRecoveryMs: 10, ms: 10, status: 200, transitions: [], }, httpListenLogLine: "[gateway] http server listening (0 plugins)", httpListenLogMs: 5, maxRssMb: 120, outputTail: "ready\\nError: startup sidecar crashed", readyz: { firstErrorKind: "http-503", firstRecoveryMs: 18, ms: 18, status: 200, transitions: [], }, signal: null, startupTrace: {}, }, ]); expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([ { id: "demo", reason: "child exited 1", sampleIndex: 1, }, ]); }); it("does not flag nonzero exits from intentional teardown", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: false, exitCode: 1, firstOutputMs: 1, gatewayReadyLogLine: "[gateway] ready", gatewayReadyLogMs: 20, healthz: { firstErrorKind: "econnrefused", firstRecoveryMs: 10, ms: 10, status: 200, transitions: [], }, httpListenLogLine: "[gateway] http server listening (0 plugins)", httpListenLogMs: 5, maxRssMb: 120, outputTail: "", readyz: { firstErrorKind: "http-503", firstRecoveryMs: 18, ms: 18, status: 200, transitions: [], }, signal: null, startupTrace: {}, }, ]); expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([]); }); it("flags samples that become ready and then die from a signal", () => { const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [ { cpuCoreRatio: 0.5, cpuMs: 100, exitedBeforeTeardown: true, exitCode: null, firstOutputMs: 1, gatewayReadyLogLine: "[gateway] ready", gatewayReadyLogMs: 20, healthz: { firstErrorKind: "econnrefused", firstRecoveryMs: 10, ms: 10, status: 200, transitions: [], }, httpListenLogLine: "[gateway] http server listening (0 plugins)", httpListenLogMs: 5, maxRssMb: 120, outputTail: "ready\\nsegmentation fault", readyz: { firstErrorKind: "http-503", firstRecoveryMs: 18, ms: 18, status: 200, transitions: [], }, signal: "SIGSEGV", startupTrace: {}, }, ]); expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([ { id: "demo", reason: "child exited by SIGSEGV", sampleIndex: 1, }, ]); }); it("classifies queued child exits before sending teardown signals", async () => { const child = new EventEmitter() as EventEmitter & { exitCode: number | null; kill: ReturnType; signalCode: NodeJS.Signals | null; }; child.exitCode = null; child.signalCode = null; child.kill = vi.fn(() => true); const stopped = testing.stopChild(child as unknown as Parameters[0]); queueMicrotask(() => { child.exitCode = 7; child.emit("exit", 7, null); }); await expect(stopped).resolves.toEqual({ exitedBeforeTeardown: true, exitCode: 7, signal: null, }); expect(child.kill).not.toHaveBeenCalled(); }); it("classifies failed teardown signaling as a pre-teardown child exit", async () => { const child = new EventEmitter() as EventEmitter & { exitCode: number | null; kill: ReturnType; signalCode: NodeJS.Signals | null; }; child.exitCode = null; child.signalCode = null; child.kill = vi.fn(() => { setImmediate(() => { child.exitCode = 8; child.emit("exit", 8, null); }); return false; }); await expect( testing.stopChild(child as unknown as Parameters[0]), ).resolves.toEqual({ exitedBeforeTeardown: true, exitCode: 8, signal: null, }); expect(child.kill).toHaveBeenCalledWith("SIGTERM"); }); it("bounds teardown when the child ignores termination signals", async () => { const child = new EventEmitter() as EventEmitter & { exitCode: number | null; kill: ReturnType; signalCode: NodeJS.Signals | null; stderr: { destroy: ReturnType }; stdin: { destroy: ReturnType }; stdout: { destroy: ReturnType }; unref: ReturnType; }; child.exitCode = null; child.signalCode = null; child.kill = vi.fn(() => true); child.stderr = { destroy: vi.fn() }; child.stdin = { destroy: vi.fn() }; child.stdout = { destroy: vi.fn() }; child.unref = vi.fn(); await expect( testing.stopChild(child as unknown as Parameters[0], { killGraceMs: 1, teardownGraceMs: 1, }), ).resolves.toEqual({ exitedBeforeTeardown: false, exitCode: null, signal: "SIGKILL", }); expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM"); expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL"); expect(child.stdin.destroy).toHaveBeenCalledOnce(); expect(child.stdout.destroy).toHaveBeenCalledOnce(); expect(child.stderr.destroy).toHaveBeenCalledOnce(); expect(child.unref).toHaveBeenCalledOnce(); }); it("collects Count-suffixed startup trace metrics", () => { const startupTrace: Record = {}; testing.collectStartupTrace( "[gateway] startup trace: sidecars.acp.runtime-ready ready=1 readyCount=1 backend=acpx", startupTrace, ); expect(startupTrace["sidecars.acp.runtime-ready.ready"]).toBeUndefined(); expect(startupTrace["sidecars.acp.runtime-ready.readyCount"]).toBe(1); }); it("records probe state transitions, first error kind, and first recovery", async () => { let calls = 0; const { port, server } = await listenOnLoopback((_req, res) => { calls += 1; res.statusCode = calls === 1 ? 503 : 200; res.end("ok"); }); try { const startAt = performance.now(); const result = await testing.waitForProbe({ deadlineAt: startAt + 1_000, path: "/readyz", port, startAt, }); expect(result.status).toBe(200); expect(result.ms).toEqual(expect.any(Number)); expect(result.firstErrorKind).toBe("http-503"); expect(result.firstRecoveryMs).toEqual(expect.any(Number)); expect(result.transitions.map((transition) => transition.status)).toEqual([503, 200]); } finally { await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); }); } }); it("writes 50-plugin fixtures as a parent load path with explicit startup activation", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); try { const configPath = testing.writeConfig(root, { config: {}, id: "fiftyPlugins", name: "gateway, 50 manifest plugins", pluginActivationOnStartup: true, pluginCount: 2, }); const config = JSON.parse(fs.readFileSync(configPath, "utf8")) as { plugins?: { allow?: string[]; load?: { paths?: string[] } }; }; expect(config.plugins?.load?.paths).toEqual([path.join(root, "plugins")]); expect(config.plugins?.allow).toEqual(["bench-plugin-01", "bench-plugin-02"]); const manifest = JSON.parse( fs.readFileSync( path.join(root, "plugins", "bench-plugin-01", "openclaw.plugin.json"), "utf8", ), ) as { activation?: { onStartup?: boolean } }; expect(manifest.activation?.onStartup).toBe(true); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); it("keeps startup-lazy plugin fixtures opted out of startup activation", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-")); try { testing.writeConfig(root, { config: {}, id: "fiftyStartupLazyPlugins", name: "gateway, 50 startup-lazy manifest plugins", pluginActivationOnStartup: false, pluginCount: 1, }); const manifest = JSON.parse( fs.readFileSync( path.join(root, "plugins", "bench-plugin-01", "openclaw.plugin.json"), "utf8", ), ) as { activation?: { onStartup?: boolean } }; expect(manifest.activation?.onStartup).toBe(false); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); });