diff --git a/docs/help/debugging.md b/docs/help/debugging.md index e8fa6619c57..60a771bcee0 100644 --- a/docs/help/debugging.md +++ b/docs/help/debugging.md @@ -246,6 +246,23 @@ Disable auto-attach while keeping tmux management: OPENCLAW_GATEWAY_WATCH_ATTACH=0 pnpm gateway:watch ``` +Profile watched Gateway CPU time when debugging startup/runtime hotspots: + +```bash +pnpm gateway:watch --benchmark +``` + +The watch wrapper consumes `--benchmark` before invoking the Gateway and writes +one V8 `.cpuprofile` per Gateway child exit under +`.artifacts/gateway-watch-profiles/`. Stop or restart the watched gateway to +flush the current profile, then open it with Chrome DevTools or Speedscope: + +```bash +npx speedscope .artifacts/gateway-watch-profiles/*.cpuprofile +``` + +Use `--benchmark-dir ` when you want profiles somewhere else. + The tmux wrapper carries common non-secret runtime selectors such as `OPENCLAW_PROFILE`, `OPENCLAW_CONFIG_PATH`, `OPENCLAW_STATE_DIR`, `OPENCLAW_GATEWAY_PORT`, and `OPENCLAW_SKIP_CHANNELS` into the pane. Put diff --git a/scripts/gateway-watch-tmux.mjs b/scripts/gateway-watch-tmux.mjs index 9e7d586e4c9..21fe0a3f024 100644 --- a/scripts/gateway-watch-tmux.mjs +++ b/scripts/gateway-watch-tmux.mjs @@ -7,6 +7,8 @@ const TMUX_DISABLE_VALUES = new Set(["0", "false", "no", "off"]); const TMUX_ATTACH_DISABLE_VALUES = new Set(["0", "false", "no", "off"]); const TMUX_ATTACH_FORCE_VALUES = new Set(["1", "true", "yes", "on"]); const DEFAULT_PROFILE_NAME = "main"; +const DEFAULT_BENCHMARK_PROFILE_DIR = ".artifacts/gateway-watch-profiles"; +const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR"; const RAW_WATCH_SCRIPT = "scripts/watch-node.mjs"; const TMUX_CWD_ENV_KEY = "OPENCLAW_GATEWAY_WATCH_CWD"; const TMUX_CWD_OPTION_KEY = "@openclaw.gateway_watch.cwd"; @@ -16,6 +18,7 @@ const TMUX_CHILD_ENV_KEYS = [ "OPENCLAW_GATEWAY_PORT", "OPENCLAW_HOME", "OPENCLAW_PROFILE", + RUN_NODE_CPU_PROF_DIR_ENV, "OPENCLAW_SKIP_CHANNELS", "OPENCLAW_STATE_DIR", ]; @@ -46,6 +49,54 @@ const readArgValue = (args, flag) => { return null; }; +const resolveGatewayWatchBenchmarkArgs = ({ args = [], env = process.env } = {}) => { + const passthroughArgs = []; + let benchmarkDir = null; + let benchmarkFlagSeen = false; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--benchmark") { + benchmarkFlagSeen = true; + benchmarkDir ??= DEFAULT_BENCHMARK_PROFILE_DIR; + continue; + } + if (typeof arg === "string" && arg.startsWith("--benchmark=")) { + benchmarkFlagSeen = true; + benchmarkDir = arg.slice("--benchmark=".length) || DEFAULT_BENCHMARK_PROFILE_DIR; + continue; + } + if (arg === "--benchmark-dir") { + benchmarkFlagSeen = true; + const next = args[index + 1]; + if (typeof next === "string" && !next.startsWith("-")) { + benchmarkDir = next; + index += 1; + } else { + benchmarkDir ??= DEFAULT_BENCHMARK_PROFILE_DIR; + } + continue; + } + if (typeof arg === "string" && arg.startsWith("--benchmark-dir=")) { + benchmarkFlagSeen = true; + benchmarkDir = arg.slice("--benchmark-dir=".length) || DEFAULT_BENCHMARK_PROFILE_DIR; + continue; + } + passthroughArgs.push(arg); + } + + const nextEnv = { ...env }; + if (benchmarkFlagSeen) { + nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] = + benchmarkDir || nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] || DEFAULT_BENCHMARK_PROFILE_DIR; + } + return { + args: passthroughArgs, + benchmarkProfileDir: nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] || null, + env: nextEnv, + }; +}; + export const resolveGatewayWatchTmuxSessionName = ({ args = [], env = process.env } = {}) => { const profile = env.OPENCLAW_PROFILE || @@ -168,10 +219,14 @@ const setTmuxSessionMetadata = ({ cwd, sessionName, spawnSyncImpl, stderr }) => }; export const runGatewayWatchTmuxMain = (params = {}) => { - const deps = { + const resolvedArgs = resolveGatewayWatchBenchmarkArgs({ args: params.args ?? process.argv.slice(2), - cwd: params.cwd ?? process.cwd(), env: params.env ? { ...params.env } : { ...process.env }, + }); + const deps = { + args: resolvedArgs.args, + cwd: params.cwd ?? process.cwd(), + env: resolvedArgs.env, nodePath: params.nodePath ?? process.execPath, spawnSync: params.spawnSync ?? spawnSync, stderr: params.stderr ?? process.stderr, @@ -180,6 +235,10 @@ export const runGatewayWatchTmuxMain = (params = {}) => { stdoutIsTTY: params.stdoutIsTTY ?? process.stdout.isTTY, }; + if (resolvedArgs.benchmarkProfileDir) { + log(deps.stderr, `gateway:watch benchmark CPU profiles: ${resolvedArgs.benchmarkProfileDir}`); + } + if (TMUX_DISABLE_VALUES.has(String(deps.env.OPENCLAW_GATEWAY_WATCH_TMUX ?? "").toLowerCase())) { return runForegroundWatcher({ args: deps.args, diff --git a/scripts/run-node.mjs b/scripts/run-node.mjs index ab8e5db04b5..655311a0406 100644 --- a/scripts/run-node.mjs +++ b/scripts/run-node.mjs @@ -432,6 +432,7 @@ const isSignalKey = (signal) => Object.hasOwn(SIGNAL_EXIT_CODES, signal); const getSignalExitCode = (signal) => (isSignalKey(signal) ? SIGNAL_EXIT_CODES[signal] : 1); const RUN_NODE_OUTPUT_LOG_ENV = "OPENCLAW_RUN_NODE_OUTPUT_LOG"; +const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR"; const RUN_NODE_BUILD_LOCK_TIMEOUT_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_TIMEOUT_MS"; const RUN_NODE_BUILD_LOCK_POLL_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_POLL_MS"; const RUN_NODE_BUILD_LOCK_STALE_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_STALE_MS"; @@ -504,6 +505,35 @@ const logRunner = (message, deps) => { deps.outputTee?.write(line); }; +const sanitizeCpuProfileNamePart = (value) => { + const normalized = String(value ?? "") + .trim() + .toLowerCase() + .replace(/[^a-z0-9_.-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return normalized || "command"; +}; + +const resolveRunNodeCpuProfileArgs = (deps) => { + const profileDir = deps.env[RUN_NODE_CPU_PROF_DIR_ENV]?.trim(); + if (!profileDir) { + return []; + } + + const absoluteProfileDir = path.resolve(deps.cwd, profileDir); + deps.fs.mkdirSync(absoluteProfileDir, { recursive: true }); + deps.env[RUN_NODE_CPU_PROF_DIR_ENV] = absoluteProfileDir; + + const commandName = sanitizeCpuProfileNamePart(deps.args[0]); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const pid = Number.isInteger(deps.process.pid) && deps.process.pid > 0 ? deps.process.pid : "pid"; + const profileName = `openclaw-${commandName}-${pid}-${timestamp}.cpuprofile`; + const profilePath = path.join(absoluteProfileDir, profileName); + const relativeProfilePath = path.relative(deps.cwd, profilePath) || profilePath; + logRunner(`Writing Node CPU profile to ${relativeProfilePath}.`, deps); + return ["--cpu-prof", `--cpu-prof-dir=${absoluteProfileDir}`, `--cpu-prof-name=${profileName}`]; +}; + const waitForSpawnedProcess = async (childProcess, deps) => { let forwardedSignal = null; let onSigInt; @@ -574,7 +604,8 @@ const getInterruptedSpawnExitCode = (res) => { }; const runOpenClaw = async (deps) => { - const nodeProcess = deps.spawn(deps.execPath, ["openclaw.mjs", ...deps.args], { + const cpuProfileArgs = resolveRunNodeCpuProfileArgs(deps); + const nodeProcess = deps.spawn(deps.execPath, [...cpuProfileArgs, "openclaw.mjs", ...deps.args], { cwd: deps.cwd, env: deps.env, stdio: deps.outputTee ? ["inherit", "pipe", "pipe"] : "inherit", diff --git a/src/infra/gateway-watch-tmux.test.ts b/src/infra/gateway-watch-tmux.test.ts index cf3ff05ada4..4e95423bf63 100644 --- a/src/infra/gateway-watch-tmux.test.ts +++ b/src/infra/gateway-watch-tmux.test.ts @@ -64,6 +64,37 @@ describe("gateway-watch tmux wrapper", () => { expect(command).toContain("'a b.jsonl'"); }); + it("consumes benchmark flags and passes the CPU profile dir to the watched child", () => { + const stdout = createOutput(); + const stderr = createOutput(); + const spawnSync = vi + .fn() + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }); + + const code = runGatewayWatchTmuxMain({ + args: ["gateway", "--force", "--benchmark"], + cwd: "/repo", + env: { SHELL: "/bin/zsh" }, + nodePath: "/node", + spawnSync, + stderr: stderr.stream, + stdout: stdout.stream, + }); + + expect(code).toBe(0); + const command = spawnSync.mock.calls[1]?.[1]?.[6] as string; + expect(command).toContain("'OPENCLAW_RUN_NODE_CPU_PROF_DIR=.artifacts/gateway-watch-profiles'"); + expect(command).not.toContain("--benchmark"); + expect(command).toContain("'gateway'"); + expect(command).toContain("'--force'"); + expect(stderr.chunks.join("")).toContain( + "gateway:watch benchmark CPU profiles: .artifacts/gateway-watch-profiles", + ); + }); + it("preserves an explicit color override for the tmux child", () => { const command = buildGatewayWatchTmuxCommand({ args: ["gateway", "--force"], diff --git a/src/infra/run-node.test.ts b/src/infra/run-node.test.ts index c87021639c8..b621158853b 100644 --- a/src/infra/run-node.test.ts +++ b/src/infra/run-node.test.ts @@ -448,6 +448,55 @@ describe("run-node script", () => { }); }); + it("adds Node CPU profiling flags to the launched OpenClaw child when requested", async () => { + await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { + await setupTrackedProject(tmp, { + files: { + [ROOT_SRC]: "export const value = 1;\n", + }, + oldPaths: [ROOT_SRC, ROOT_TSCONFIG, ROOT_PACKAGE], + buildPaths: [DIST_ENTRY, BUILD_STAMP], + }); + const profileDir = path.join(tmp, ".artifacts", "profiles"); + const spawnCalls: Array<{ args: string[]; env: Record }> = []; + const spawn = (_cmd: string, args: string[], options?: unknown) => { + const opts = options as { env?: NodeJS.ProcessEnv } | undefined; + spawnCalls.push({ args, env: { ...opts?.env } }); + return createExitedProcess(0); + }; + const { spawnSync } = createSpawnRecorder({ + gitHead: "abc123\n", + gitStatus: "", + }); + + const exitCode = await runNodeMain({ + cwd: tmp, + args: ["status"], + env: { + ...process.env, + OPENCLAW_RUNNER_LOG: "0", + OPENCLAW_RUN_NODE_CPU_PROF_DIR: ".artifacts/profiles", + }, + spawn, + spawnSync, + execPath: process.execPath, + platform: process.platform, + process: createFakeProcess(), + }); + + expect(exitCode).toBe(0); + const childArgs = spawnCalls.at(-1)?.args ?? []; + expect(childArgs[0]).toBe("--cpu-prof"); + expect(childArgs[1]).toBe(`--cpu-prof-dir=${profileDir}`); + expect(childArgs[2]).toMatch( + /^--cpu-prof-name=openclaw-status-4242-\d{4}-\d{2}-\d{2}T.*\.cpuprofile$/, + ); + expect(childArgs.slice(3)).toEqual(["openclaw.mjs", "status"]); + expect(spawnCalls.at(-1)?.env.OPENCLAW_RUN_NODE_CPU_PROF_DIR).toBe(profileDir); + expect(fsSync.existsSync(profileDir)).toBe(true); + }); + }); + it("surfaces generic output log stream errors", async () => { await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { await setupTrackedProject(tmp);