diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1fc1900bd..0e473ea9d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Gateway/watch: suppress sync-I/O trace output during `pnpm gateway:watch --benchmark` unless explicitly requested, so CPU profiling no longer floods the terminal with stack traces. +- Gateway/watch: when benchmark sync-I/O tracing is explicitly enabled, tee trace blocks to the benchmark output log and filter them from the terminal pane while keeping normal Gateway logs visible. - Agents/OpenAI: default direct OpenAI Responses models to the SSE transport instead of WebSocket auto-selection, preventing pi runtime chat turns from hanging on servers where the WebSocket path stalls while the OpenAI HTTP stream works. Thanks @vincentkoc. - Channels/plugins: key bundled package-state probes, env/config presence, and read-only command defaults by channel id instead of manifest plugin id, preserving setup and native-command detection for channel plugins whose package id differs from the channel alias. Thanks @vincentkoc. - Docker: prune package-excluded plugin dist directories from runtime images unless the build explicitly opts that plugin in, so official external plugins such as Feishu stay install-on-demand instead of shipping partial metadata without compiled runtime output. Fixes #77424. Thanks @vincentkoc. diff --git a/docs/help/debugging.md b/docs/help/debugging.md index 1e24d01dfc3..1f01cbc819c 100644 --- a/docs/help/debugging.md +++ b/docs/help/debugging.md @@ -159,7 +159,9 @@ default `--force` port cleanup and fail fast if the Gateway port is already in use. Benchmark mode suppresses sync-I/O trace spam by default. Set `OPENCLAW_TRACE_SYNC_IO=1` with `--benchmark` when you explicitly want both CPU -profiles and Node sync-I/O stack traces. +profiles and Node sync-I/O stack traces. In benchmark mode those trace blocks +are written to `gateway-watch-output.log` under the benchmark directory and +filtered from the terminal pane; normal Gateway logs remain visible. The tmux wrapper carries common non-secret runtime selectors such as `OPENCLAW_PROFILE`, `OPENCLAW_CONFIG_PATH`, `OPENCLAW_STATE_DIR`, diff --git a/scripts/gateway-watch-tmux.mjs b/scripts/gateway-watch-tmux.mjs index 0aab8cd034f..24d63ba1b57 100644 --- a/scripts/gateway-watch-tmux.mjs +++ b/scripts/gateway-watch-tmux.mjs @@ -9,6 +9,8 @@ const TMUX_ATTACH_FORCE_VALUES = new Set(["1", "true", "yes", "on"]); const DEFAULT_PROFILE_NAME = "main"; const DEFAULT_BENCHMARK_PROFILE_DIR = ".artifacts/gateway-watch-profiles"; const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR"; +const RUN_NODE_OUTPUT_LOG_ENV = "OPENCLAW_RUN_NODE_OUTPUT_LOG"; +const RUN_NODE_FILTER_SYNC_IO_STDERR_ENV = "OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR"; const RAW_WATCH_SCRIPT = "scripts/watch-node.mjs"; const TMUX_CWD_ENV_KEY = "OPENCLAW_GATEWAY_WATCH_CWD"; const TMUX_CWD_OPTION_KEY = "@openclaw.gateway_watch.cwd"; @@ -19,6 +21,8 @@ const TMUX_CHILD_ENV_KEYS = [ "OPENCLAW_HOME", "OPENCLAW_PROFILE", RUN_NODE_CPU_PROF_DIR_ENV, + RUN_NODE_FILTER_SYNC_IO_STDERR_ENV, + RUN_NODE_OUTPUT_LOG_ENV, "OPENCLAW_SKIP_CHANNELS", "OPENCLAW_STATE_DIR", "OPENCLAW_TRACE_SYNC_IO", @@ -50,6 +54,11 @@ const readArgValue = (args, flag) => { return null; }; +const joinArtifactPath = (dir, basename) => { + const normalizedDir = String(dir || DEFAULT_BENCHMARK_PROFILE_DIR).replace(/[\\/]+$/g, ""); + return `${normalizedDir || "."}/${basename}`; +}; + const resolveGatewayWatchBenchmarkArgs = ({ args = [], env = process.env } = {}) => { const passthroughArgs = []; let benchmarkDir = null; @@ -98,6 +107,13 @@ const resolveGatewayWatchBenchmarkArgs = ({ args = [], env = process.env } = {}) nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] = benchmarkDir || nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] || DEFAULT_BENCHMARK_PROFILE_DIR; nextEnv.OPENCLAW_TRACE_SYNC_IO ??= "0"; + if (nextEnv.OPENCLAW_TRACE_SYNC_IO === "1") { + nextEnv[RUN_NODE_OUTPUT_LOG_ENV] ??= joinArtifactPath( + nextEnv[RUN_NODE_CPU_PROF_DIR_ENV], + "gateway-watch-output.log", + ); + nextEnv[RUN_NODE_FILTER_SYNC_IO_STDERR_ENV] ??= "1"; + } } return { args: benchmarkNoForceSeen @@ -105,6 +121,10 @@ const resolveGatewayWatchBenchmarkArgs = ({ args = [], env = process.env } = {}) : passthroughArgs, benchmarkNoForce: benchmarkNoForceSeen, benchmarkProfileDir: nextEnv[RUN_NODE_CPU_PROF_DIR_ENV] || null, + benchmarkTraceOutputLog: + nextEnv[RUN_NODE_FILTER_SYNC_IO_STDERR_ENV] === "1" + ? nextEnv[RUN_NODE_OUTPUT_LOG_ENV] || null + : null, env: nextEnv, }; }; @@ -250,6 +270,12 @@ export const runGatewayWatchTmuxMain = (params = {}) => { if (resolvedArgs.benchmarkProfileDir) { log(deps.stderr, `gateway:watch benchmark CPU profiles: ${resolvedArgs.benchmarkProfileDir}`); } + if (resolvedArgs.benchmarkTraceOutputLog) { + log( + deps.stderr, + `gateway:watch benchmark trace output: ${resolvedArgs.benchmarkTraceOutputLog}`, + ); + } if (resolvedArgs.benchmarkNoForce) { log(deps.stderr, "gateway:watch benchmark running without --force"); } diff --git a/scripts/run-node.mjs b/scripts/run-node.mjs index e54361af211..9b2e8744b1f 100644 --- a/scripts/run-node.mjs +++ b/scripts/run-node.mjs @@ -386,6 +386,7 @@ const getSignalExitCode = (signal) => (isSignalKey(signal) ? SIGNAL_EXIT_CODES[s const RUN_NODE_OUTPUT_LOG_ENV = "OPENCLAW_RUN_NODE_OUTPUT_LOG"; const RUN_NODE_CPU_PROF_DIR_ENV = "OPENCLAW_RUN_NODE_CPU_PROF_DIR"; +const RUN_NODE_FILTER_SYNC_IO_STDERR_ENV = "OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR"; const RUN_NODE_BUILD_LOCK_TIMEOUT_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_TIMEOUT_MS"; const RUN_NODE_BUILD_LOCK_POLL_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_POLL_MS"; const RUN_NODE_BUILD_LOCK_STALE_ENV = "OPENCLAW_RUN_NODE_BUILD_LOCK_STALE_MS"; @@ -585,14 +586,78 @@ const pipeSpawnedOutput = (childProcess, deps) => { if (!deps.outputTee) { return; } + const stderrFilter = + deps.env[RUN_NODE_FILTER_SYNC_IO_STDERR_ENV] === "1" + ? createSyncIoTraceStderrFilter(deps) + : null; childProcess.stdout?.on("data", (chunk) => { deps.stdout.write(chunk); deps.outputTee.write(chunk); }); childProcess.stderr?.on("data", (chunk) => { - deps.stderr.write(chunk); + if (stderrFilter) { + stderrFilter.write(chunk); + } else { + deps.stderr.write(chunk); + } deps.outputTee.write(chunk); }); + childProcess.stderr?.on("end", () => { + stderrFilter?.flush(); + }); +}; + +const createSyncIoTraceStderrFilter = (deps) => { + let buffer = ""; + let inSyncIoTrace = false; + + const shouldSuppressLine = (line) => { + const text = line.replace(/\r?\n$/, ""); + if (/^\(node:\d+\) WARNING: Detected use of sync API/.test(text)) { + inSyncIoTrace = true; + return true; + } + if (!inSyncIoTrace) { + return false; + } + if (text.trim() === "") { + inSyncIoTrace = false; + return true; + } + if (/^\s+at\b/.test(text)) { + return true; + } + inSyncIoTrace = false; + return false; + }; + + const writeLine = (line) => { + if (!shouldSuppressLine(line)) { + deps.stderr.write(line); + } + }; + + return { + write(chunk) { + buffer += String(chunk); + while (true) { + const newlineIndex = buffer.indexOf("\n"); + if (newlineIndex === -1) { + break; + } + const line = buffer.slice(0, newlineIndex + 1); + buffer = buffer.slice(newlineIndex + 1); + writeLine(line); + } + }, + flush() { + if (!buffer) { + return; + } + writeLine(buffer); + buffer = ""; + }, + }; }; const closeRunNodeOutputTee = async (deps, exitCode) => { diff --git a/src/infra/gateway-watch-tmux.test.ts b/src/infra/gateway-watch-tmux.test.ts index 99045a54cc1..6c21233c535 100644 --- a/src/infra/gateway-watch-tmux.test.ts +++ b/src/infra/gateway-watch-tmux.test.ts @@ -121,6 +121,13 @@ describe("gateway-watch tmux wrapper", () => { expect(code).toBe(0); const command = spawnSync.mock.calls[1]?.[1]?.[6] as string; expect(command).toContain("'OPENCLAW_TRACE_SYNC_IO=1'"); + expect(command).toContain( + "'OPENCLAW_RUN_NODE_OUTPUT_LOG=.artifacts/gateway-watch-profiles/gateway-watch-output.log'", + ); + expect(command).toContain("'OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR=1'"); + expect(stderr.chunks.join("")).toContain( + "gateway:watch benchmark trace output: .artifacts/gateway-watch-profiles/gateway-watch-output.log", + ); }); it("can remove --force from benchmarked watch runs", () => { diff --git a/src/infra/run-node.test.ts b/src/infra/run-node.test.ts index b38e944a929..25ff7b966d1 100644 --- a/src/infra/run-node.test.ts +++ b/src/infra/run-node.test.ts @@ -448,6 +448,59 @@ describe("run-node script", () => { }); }); + it("routes sync I/O trace stderr blocks to the output log without flooding stderr", async () => { + await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { + await setupTrackedProject(tmp); + const outputPath = path.join(tmp, ".artifacts", "gateway-watch-profiles", "output.log"); + const childStderr = [ + "normal before\n", + "(node:12345) WARNING: Detected use of sync API\n", + " at statSync (node:fs:1739:25)\n", + " at loadConfig (/repo/src/config.ts:1:1)\n", + "\n", + "normal after\n", + ].join(""); + const spawn = (_cmd: string, args: string[]) => + createPipedExitedProcess({ + stderr: args[0] === "openclaw.mjs" ? childStderr : "", + }); + const stderrChunks: string[] = []; + const stderr = { + write: (chunk: string | Buffer) => { + stderrChunks.push(String(chunk)); + return true; + }, + } as unknown as NodeJS.WriteStream; + const stdout = { + write: () => true, + } as unknown as NodeJS.WriteStream; + + const exitCode = await runNodeMain({ + cwd: tmp, + args: ["status"], + env: { + ...process.env, + OPENCLAW_RUNNER_LOG: "0", + OPENCLAW_RUN_NODE_FILTER_SYNC_IO_STDERR: "1", + OPENCLAW_RUN_NODE_OUTPUT_LOG: outputPath, + }, + spawn, + stderr, + stdout, + execPath: process.execPath, + platform: process.platform, + } as Parameters[0] & { stdout: NodeJS.WriteStream }); + + expect(exitCode).toBe(0); + const terminalStderr = stderrChunks.join(""); + expect(terminalStderr).toContain("normal before\n"); + expect(terminalStderr).toContain("normal after\n"); + expect(terminalStderr).not.toContain("Detected use of sync API"); + expect(terminalStderr).not.toContain("statSync"); + await expect(fs.readFile(outputPath, "utf-8")).resolves.toContain(childStderr); + }); + }); + it("adds Node CPU profiling flags to the launched OpenClaw child when requested", async () => { await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { await setupTrackedProject(tmp, {