openclaw/test/scripts/bench-gateway-startup.test.ts

import { spawnSync } from "node:child_process";
import { EventEmitter } from "node:events";
import fs from "node:fs";
import { createServer } from "node:http";
import os from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { describe, expect, it, vi } from "vitest";
import { testing } from "../../scripts/bench-gateway-startup.ts";

async function listenOnLoopback(handler: Parameters<typeof createServer>[0]) {
  const server = createServer(handler);
  await new Promise<void>((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", () => resolve());
  });
  const address = server.address();
  if (!address || typeof address === "string") {
    server.close();
    throw new Error("expected loopback port");
  }
  return { port: address.port, server };
}

describe("gateway startup benchmark script", () => {
  it("prints help without running benchmark cases", () => {
    const result = spawnSync(
      process.execPath,
      ["--import", "tsx", "scripts/bench-gateway-startup.ts", "--help"],
      {
        cwd: process.cwd(),
        encoding: "utf8",
        env: {
          ...process.env,
          NODE_NO_WARNINGS: "1",
        },
      },
    );

    expect(result.status).toBe(0);
    expect(result.stdout).toContain("OpenClaw Gateway startup benchmark");
    expect(result.stdout).toContain("--case <id>");
    expect(result.stdout).toContain("--cpu-prof-dir <dir>");
    expect(result.stdout).toContain("default (gateway default)");
    expect(result.stdout).not.toContain("[gateway-startup-bench]");
    expect(result.stderr).toBe("");
  });

  it("rejects ambiguous benchmark CLI values before spawning Node", () => {
    expect(testing.parsePositiveInt("5", 1, "--runs")).toBe(5);
    expect(testing.parseNonNegativeInt("0", 1, "--warmup")).toBe(0);
    expect(() => testing.parsePositiveInt("2abc", 1, "--runs")).toThrow(
      /--runs must be an integer/u,
    );
    expect(() => testing.resolveEntry("--inspect")).toThrow(/must be a file path/u);
  });

  it("does not disable local-check policy in the child gateway environment", () => {
    const env = testing.sanitizedEnv("/tmp/openclaw-bench", "/tmp/openclaw-bench/config.json", {
      config: {},
      id: "default",
      name: "gateway default",
    });

    expect(env.OPENCLAW_LOCAL_CHECK).toBeUndefined();
    expect(env.OPENCLAW_GATEWAY_STARTUP_TRACE).toBe("1");
  });

  it("classifies HTTP listen and gateway ready logs separately", () => {
    expect(
      testing.classifyGatewayReadyLog("[gateway] http server listening (0 plugins, 0.8s)"),
    ).toBe("http-listen");
    expect(testing.classifyGatewayReadyLog("[gateway] ready (0 plugins, 0.8s)")).toBe(
      "gateway-ready",
    );
    expect(testing.classifyGatewayReadyLog("[gateway] ready")).toBe("gateway-ready");
    expect(testing.classifyGatewayReadyLog("[gateway] starting HTTP server...")).toBeNull();
  });

  it("summarizes split ready log timings without the ambiguous readyLogMs field", () => {
    const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [
      {
        cpuCoreRatio: null,
        cpuMs: null,
        exitCode: null,
        firstOutputMs: 1,
        gatewayReadyLogLine: "[gateway] ready",
        gatewayReadyLogMs: 40,
        healthz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: 20,
          ms: 20,
          status: 200,
          transitions: [],
        },
        httpListenLogLine: "[gateway] http server listening (0 plugins)",
        httpListenLogMs: 10,
        maxRssMb: null,
        outputTail: "",
        readyz: {
          firstErrorKind: "http-503",
          firstRecoveryMs: 30,
          ms: 30,
          status: 200,
          transitions: [],
        },
        signal: null,
        startupTrace: {},
      },
    ]);

    expect(result.summary.httpListenLogMs?.p50).toBe(10);
    expect(result.summary.gatewayReadyLogMs?.p50).toBe(40);
    expect("readyLogMs" in result.summary).toBe(false);
  });

  it("flags samples that never produced readiness or process metrics", () => {
    const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [
      {
        cpuCoreRatio: null,
        cpuMs: null,
        exitCode: 1,
        firstOutputMs: 5,
        gatewayReadyLogLine: null,
        gatewayReadyLogMs: null,
        healthz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: null,
          ms: null,
          status: null,
          transitions: [],
        },
        httpListenLogLine: null,
        httpListenLogMs: null,
        maxRssMb: null,
        outputTail: "Error: Cannot find module 'dist/entry.js'",
        readyz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: null,
          ms: null,
          status: null,
          transitions: [],
        },
        signal: null,
        startupTrace: {},
      },
    ]);

    expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([
      {
        id: "demo",
        reason: "missing /healthz, /readyz, cpu, rss",
        sampleIndex: 1,
      },
    ]);
  });

  it("flags samples that become ready and then exit nonzero", () => {
    const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [
      {
        cpuCoreRatio: 0.5,
        cpuMs: 100,
        exitedBeforeTeardown: true,
        exitCode: 1,
        firstOutputMs: 1,
        gatewayReadyLogLine: "[gateway] ready",
        gatewayReadyLogMs: 20,
        healthz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: 10,
          ms: 10,
          status: 200,
          transitions: [],
        },
        httpListenLogLine: "[gateway] http server listening (0 plugins)",
        httpListenLogMs: 5,
        maxRssMb: 120,
        outputTail: "ready\\nError: startup sidecar crashed",
        readyz: {
          firstErrorKind: "http-503",
          firstRecoveryMs: 18,
          ms: 18,
          status: 200,
          transitions: [],
        },
        signal: null,
        startupTrace: {},
      },
    ]);

    expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([
      {
        id: "demo",
        reason: "child exited 1",
        sampleIndex: 1,
      },
    ]);
  });

  it("does not flag nonzero exits from intentional teardown", () => {
    const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [
      {
        cpuCoreRatio: 0.5,
        cpuMs: 100,
        exitedBeforeTeardown: false,
        exitCode: 1,
        firstOutputMs: 1,
        gatewayReadyLogLine: "[gateway] ready",
        gatewayReadyLogMs: 20,
        healthz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: 10,
          ms: 10,
          status: 200,
          transitions: [],
        },
        httpListenLogLine: "[gateway] http server listening (0 plugins)",
        httpListenLogMs: 5,
        maxRssMb: 120,
        outputTail: "",
        readyz: {
          firstErrorKind: "http-503",
          firstRecoveryMs: 18,
          ms: 18,
          status: 200,
          transitions: [],
        },
        signal: null,
        startupTrace: {},
      },
    ]);

    expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([]);
  });

  it("flags samples that become ready and then die from a signal", () => {
    const result = testing.summarizeCase({ config: {}, id: "demo", name: "demo" }, [
      {
        cpuCoreRatio: 0.5,
        cpuMs: 100,
        exitedBeforeTeardown: true,
        exitCode: null,
        firstOutputMs: 1,
        gatewayReadyLogLine: "[gateway] ready",
        gatewayReadyLogMs: 20,
        healthz: {
          firstErrorKind: "econnrefused",
          firstRecoveryMs: 10,
          ms: 10,
          status: 200,
          transitions: [],
        },
        httpListenLogLine: "[gateway] http server listening (0 plugins)",
        httpListenLogMs: 5,
        maxRssMb: 120,
        outputTail: "ready\\nsegmentation fault",
        readyz: {
          firstErrorKind: "http-503",
          firstRecoveryMs: 18,
          ms: 18,
          status: 200,
          transitions: [],
        },
        signal: "SIGSEGV",
        startupTrace: {},
      },
    ]);

    expect(testing.collectResultFailures([result], { processMetricsRequired: true })).toEqual([
      {
        id: "demo",
        reason: "child exited by SIGSEGV",
        sampleIndex: 1,
      },
    ]);
  });

  it("classifies queued child exits before sending teardown signals", async () => {
    const child = new EventEmitter() as EventEmitter & {
      exitCode: number | null;
      kill: ReturnType<typeof vi.fn>;
      signalCode: NodeJS.Signals | null;
    };
    child.exitCode = null;
    child.signalCode = null;
    child.kill = vi.fn(() => true);

    const stopped = testing.stopChild(child as unknown as Parameters<typeof testing.stopChild>[0]);
    queueMicrotask(() => {
      child.exitCode = 7;
      child.emit("exit", 7, null);
    });

    await expect(stopped).resolves.toEqual({
      exitedBeforeTeardown: true,
      exitCode: 7,
      signal: null,
    });
    expect(child.kill).not.toHaveBeenCalled();
  });

  it("classifies failed teardown signaling as a pre-teardown child exit", async () => {
    const child = new EventEmitter() as EventEmitter & {
      exitCode: number | null;
      kill: ReturnType<typeof vi.fn>;
      signalCode: NodeJS.Signals | null;
    };
    child.exitCode = null;
    child.signalCode = null;
    child.kill = vi.fn(() => {
      setImmediate(() => {
        child.exitCode = 8;
        child.emit("exit", 8, null);
      });
      return false;
    });

    await expect(
      testing.stopChild(child as unknown as Parameters<typeof testing.stopChild>[0]),
    ).resolves.toEqual({
      exitedBeforeTeardown: true,
      exitCode: 8,
      signal: null,
    });
    expect(child.kill).toHaveBeenCalledWith("SIGTERM");
  });

  it("bounds teardown when the child ignores termination signals", async () => {
    const child = new EventEmitter() as EventEmitter & {
      exitCode: number | null;
      kill: ReturnType<typeof vi.fn>;
      signalCode: NodeJS.Signals | null;
      stderr: { destroy: ReturnType<typeof vi.fn> };
      stdin: { destroy: ReturnType<typeof vi.fn> };
      stdout: { destroy: ReturnType<typeof vi.fn> };
      unref: ReturnType<typeof vi.fn>;
    };
    child.exitCode = null;
    child.signalCode = null;
    child.kill = vi.fn(() => true);
    child.stderr = { destroy: vi.fn() };
    child.stdin = { destroy: vi.fn() };
    child.stdout = { destroy: vi.fn() };
    child.unref = vi.fn();

    await expect(
      testing.stopChild(child as unknown as Parameters<typeof testing.stopChild>[0], {
        killGraceMs: 1,
        teardownGraceMs: 1,
      }),
    ).resolves.toEqual({
      exitedBeforeTeardown: false,
      exitCode: null,
      signal: "SIGKILL",
    });
    expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM");
    expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL");
    expect(child.stdin.destroy).toHaveBeenCalledOnce();
    expect(child.stdout.destroy).toHaveBeenCalledOnce();
    expect(child.stderr.destroy).toHaveBeenCalledOnce();
    expect(child.unref).toHaveBeenCalledOnce();
  });

  it("collects Count-suffixed startup trace metrics", () => {
    const startupTrace: Record<string, number> = {};

    testing.collectStartupTrace(
      "[gateway] startup trace: sidecars.acp.runtime-ready ready=1 readyCount=1 backend=acpx",
      startupTrace,
    );

    expect(startupTrace["sidecars.acp.runtime-ready.ready"]).toBeUndefined();
    expect(startupTrace["sidecars.acp.runtime-ready.readyCount"]).toBe(1);
  });

  it("records probe state transitions, first error kind, and first recovery", async () => {
    let calls = 0;
    const { port, server } = await listenOnLoopback((_req, res) => {
      calls += 1;
      res.statusCode = calls === 1 ? 503 : 200;
      res.end("ok");
    });
    try {
      const startAt = performance.now();
      const result = await testing.waitForProbe({
        deadlineAt: startAt + 1_000,
        path: "/readyz",
        port,
        startAt,
      });

      expect(result.status).toBe(200);
      expect(result.ms).toEqual(expect.any(Number));
      expect(result.firstErrorKind).toBe("http-503");
      expect(result.firstRecoveryMs).toEqual(expect.any(Number));
      expect(result.transitions.map((transition) => transition.status)).toEqual([503, 200]);
    } finally {
      await new Promise<void>((resolve, reject) => {
        server.close((error) => (error ? reject(error) : resolve()));
      });
    }
  });

  it("writes 50-plugin fixtures as a parent load path with explicit startup activation", () => {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-"));
    try {
      const configPath = testing.writeConfig(root, {
        config: {},
        id: "fiftyPlugins",
        name: "gateway, 50 manifest plugins",
        pluginActivationOnStartup: true,
        pluginCount: 2,
      });
      const config = JSON.parse(fs.readFileSync(configPath, "utf8")) as {
        plugins?: { allow?: string[]; load?: { paths?: string[] } };
      };

      expect(config.plugins?.load?.paths).toEqual([path.join(root, "plugins")]);
      expect(config.plugins?.allow).toEqual(["bench-plugin-01", "bench-plugin-02"]);
      const manifest = JSON.parse(
        fs.readFileSync(
          path.join(root, "plugins", "bench-plugin-01", "openclaw.plugin.json"),
          "utf8",
        ),
      ) as { activation?: { onStartup?: boolean } };
      expect(manifest.activation?.onStartup).toBe(true);
    } finally {
      fs.rmSync(root, { recursive: true, force: true });
    }
  });

  it("keeps startup-lazy plugin fixtures opted out of startup activation", () => {
    const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-config-test-"));
    try {
      testing.writeConfig(root, {
        config: {},
        id: "fiftyStartupLazyPlugins",
        name: "gateway, 50 startup-lazy manifest plugins",
        pluginActivationOnStartup: false,
        pluginCount: 1,
      });
      const manifest = JSON.parse(
        fs.readFileSync(
          path.join(root, "plugins", "bench-plugin-01", "openclaw.plugin.json"),
          "utf8",
        ),
      ) as { activation?: { onStartup?: boolean } };
      expect(manifest.activation?.onStartup).toBe(false);
    } finally {
      fs.rmSync(root, { recursive: true, force: true });
    }
  });
});