fix(process): UTF-8 command output corrupts at Windows byte limits (#105274)

* fix(process): preserve byte-capped UTF-8 output on Windows

* fix(process): skip codepage probe for UTF-8 output

Co-authored-by: 杨浩宇0668001029 <yang.haoyu@xydigit.com>

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>

package.json

@@ -1998,7 +1998,7 @@
     "test:unit:fast:audit": "node scripts/test-unit-fast-audit.mjs",
     "test:voicecall:closedloop": "node scripts/test-voicecall-closedloop.mjs",
     "test:watch": "node scripts/test-projects.mjs --watch",
-    "test:windows:ci": "node scripts/test-projects.mjs src/shared/runtime-import.test.ts src/config/sessions/store.session-lifecycle-mutation.test.ts src/agents/sessions/windows-git-bash-path.test.ts src/process/exec.windows.test.ts src/process/windows-command.test.ts src/infra/windows-install-roots.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/daemon/schtasks.startup-fallback.test.ts extensions/lobster/src/lobster-runner.test.ts test/scripts/format-generated-module.test.ts test/scripts/npm-runner.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ts-topology.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts",
+    "test:windows:ci": "node scripts/test-projects.mjs src/shared/runtime-import.test.ts src/config/sessions/store.session-lifecycle-mutation.test.ts src/agents/sessions/windows-git-bash-path.test.ts src/process/exec.windows.test.ts src/process/exec.windows.integration.test.ts src/process/windows-command.test.ts src/infra/windows-install-roots.test.ts src/node-host/invoke-system-run-allowlist.test.ts src/daemon/schtasks.startup-fallback.test.ts extensions/lobster/src/lobster-runner.test.ts test/scripts/format-generated-module.test.ts test/scripts/npm-runner.test.ts test/scripts/openclaw-cross-os-installer.windows.test.ts test/scripts/openclaw-cross-os-release-workflow.test.ts test/scripts/pnpm-runner.test.ts test/scripts/run-with-env.test.ts test/scripts/ts-topology.test.ts test/scripts/ui.test.ts test/scripts/vitest-process-group.test.ts",
     "tool-display:check": "node --import tsx scripts/tool-display.ts --check",
     "tool-display:write": "node --import tsx scripts/tool-display.ts --write",
     "ts-topology": "node --import tsx scripts/ts-topology.ts",

src/gateway/node-pairing-ssh-verify.runtime.ts

@@ -1,7 +1,7 @@
 // SSH probe execution for SSH-verified node pairing.
 // Kept as a narrow runtime boundary so gateway tests can mock the probe
 // without touching the eligibility/verification policy.
-import { runCommandWithTimeout } from "../process/exec.js";
+import { runUtf8CommandWithTimeout } from "../process/exec.js";
 
 export type NodeIdentityProbeParams = {
   user: string;
@@ -70,7 +70,7 @@ export async function runNodeIdentityProbe(
   try {
     // PATH-resolved `ssh` keeps Windows OpenSSH working; the gateway process
     // environment is operator-owned, so PATH lookup is not an injection risk.
-    const result = await runCommandWithTimeout(["ssh", ...args], {
+    const result = await runUtf8CommandWithTimeout(["ssh", ...args], {
       maxOutputBytes: MAX_PROBE_OUTPUT_BYTES,
       outputCapture: "head",
       timeoutMs: Math.max(250, params.timeoutMs),

src/process/exec-output.ts

@@ -113,8 +113,9 @@ function trimTruncatedUtf8Boundary(
   buffer: Buffer,
   mode: CommandOutputCaptureMode,
   truncatedBytes: number,
+  forceUtf8: boolean,
 ): Buffer {
-  if (truncatedBytes === 0 || buffer.length === 0 || process.platform === "win32") {
+  if (truncatedBytes === 0 || buffer.length === 0 || (process.platform === "win32" && !forceUtf8)) {
     return buffer;
   }
   if (mode === "tail") {
@@ -140,9 +141,10 @@ function trimTruncatedUtf8Boundary(
 export function finalizeCapturedOutput(
   capture: CapturedOutputBuffers,
   mode: CommandOutputCaptureMode,
+  forceUtf8 = false,
 ): Buffer {
   const buffered = Buffer.concat(capture.chunks, capture.bytes);
-  const trimmed = trimTruncatedUtf8Boundary(buffered, mode, capture.truncatedBytes);
+  const trimmed = trimTruncatedUtf8Boundary(buffered, mode, capture.truncatedBytes, forceUtf8);
   capture.truncatedBytes += buffered.byteLength - trimmed.byteLength;
   return trimmed;
 }

src/process/exec-runner.ts

@@ -68,6 +68,22 @@ export type CommandOptions = {
 export async function runCommandWithTimeout(
   argv: string[],
   optionsOrTimeout: number | CommandOptions,
+): Promise<SpawnResult> {
+  return await runCommandWithOutputEncoding(argv, optionsOrTimeout, false);
+}
+
+/** Run a command whose stdout and stderr are defined to be UTF-8 on every platform. */
+export async function runUtf8CommandWithTimeout(
+  argv: string[],
+  optionsOrTimeout: number | CommandOptions,
+): Promise<SpawnResult> {
+  return await runCommandWithOutputEncoding(argv, optionsOrTimeout, true);
+}
+
+async function runCommandWithOutputEncoding(
+  argv: string[],
+  optionsOrTimeout: number | CommandOptions,
+  forceUtf8: boolean,
 ): Promise<SpawnResult> {
   const options: CommandOptions =
     typeof optionsOrTimeout === "number" ? { timeoutMs: optionsOrTimeout } : optionsOrTimeout;
@@ -123,7 +139,7 @@ export async function runCommandWithTimeout(
     MAX_PRESERVED_PENDING_LINE_BYTES,
   );
   const maxPreservedOutputLines = Math.max(0, Math.floor(options.maxPreservedOutputLines ?? 16));
-  const windowsEncoding = resolveWindowsConsoleEncoding();
+  const windowsEncoding = forceUtf8 ? null : resolveWindowsConsoleEncoding();
   const cancelController = new AbortController();
   let termination: CommandTerminationReason | undefined;
   let childExitState: { code: number | null; signal: NodeJS.Signals | null } | undefined;
@@ -447,16 +463,20 @@ export async function runCommandWithTimeout(
     }
   }
 
+  const decodeCapturedOutput = (
+    capture: CapturedOutputBuffers,
+    captureMode: CommandOutputCaptureMode,
+  ): string => {
+    const buffer = finalizeCapturedOutput(capture, captureMode, forceUtf8);
+    return forceUtf8
+      ? buffer.toString("utf8")
+      : decodeWindowsOutputBuffer({ buffer, windowsEncoding });
+  };
+
   return {
     pid: child.pid,
-    stdout: decodeWindowsOutputBuffer({
-      buffer: finalizeCapturedOutput(stdoutCapture, stdoutCaptureMode),
-      windowsEncoding,
-    }),
-    stderr: decodeWindowsOutputBuffer({
-      buffer: finalizeCapturedOutput(stderrCapture, stderrCaptureMode),
-      windowsEncoding,
-    }),
+    stdout: decodeCapturedOutput(stdoutCapture, stdoutCaptureMode),
+    stderr: decodeCapturedOutput(stderrCapture, stderrCaptureMode),
     stdoutTruncatedBytes: stdoutCapture.truncatedBytes || undefined,
     stderrTruncatedBytes: stderrCapture.truncatedBytes || undefined,
     preservedStdoutLines:

src/process/exec.ts

@@ -10,7 +10,7 @@ import { releaseChildProcessOutputAfterExit } from "./child-process.js";
 import { resolveMaxOutputBytes, type CommandOutputStream } from "./exec-output.js";
 import { runCommandWithTimeout } from "./exec-runner.js";
 import { COMMAND_PROCESS_TREE_KILL_GRACE_MS, spawnCommand } from "./exec-spawn.js";
-export { runCommandWithTimeout } from "./exec-runner.js";
+export { runCommandWithTimeout, runUtf8CommandWithTimeout } from "./exec-runner.js";
 export type { CommandOptions } from "./exec-runner.js";
 export { isPlainCommandExitFailure, resolveProcessExitCode } from "./exec-result.js";
 export type { SpawnResult } from "./exec-result.js";

src/process/exec.windows.integration.test.ts

@@ -0,0 +1,24 @@
+import process from "node:process";
+import { describe, expect, it } from "vitest";
+import { runUtf8CommandWithTimeout } from "./exec.js";
+
+describe("runUtf8CommandWithTimeout Windows integration", () => {
+  it.runIf(process.platform === "win32")(
+    "keeps truncated UTF-8 head output on a code point boundary",
+    async () => {
+      const result = await runUtf8CommandWithTimeout(
+        [process.execPath, "-e", "process.stdout.write('a😀z'); process.stderr.write('b😀y')"],
+        {
+          maxOutputBytes: 3,
+          outputCapture: "head",
+          timeoutMs: 3_000,
+        },
+      );
+
+      expect(result.stdout).toBe("a");
+      expect(result.stderr).toBe("b");
+      expect(result.stdoutTruncatedBytes).toBe(5);
+      expect(result.stderrTruncatedBytes).toBe(5);
+    },
+  );
+});

src/process/exec.windows.test.ts

@@ -126,6 +126,7 @@ function expectCmdWrappedInvocation(call: ExecaCall, commandFragment = "pnpm.cmd
 }
 
 let runCommandWithTimeout: typeof import("./exec.js").runCommandWithTimeout;
+let runUtf8CommandWithTimeout: typeof import("./exec.js").runUtf8CommandWithTimeout;
 let runExec: typeof import("./exec.js").runExec;
 let spawnCommand: typeof import("./exec.js").spawnCommand;
 let getWindowsInstallRoots: typeof import("../infra/windows-install-roots.js").getWindowsInstallRoots;
@@ -160,7 +161,8 @@ describe("Windows command execution", () => {
     });
     ({ getWindowsInstallRoots, getWindowsSystem32ExePath } =
       await import("../infra/windows-install-roots.js"));
-    ({ runCommandWithTimeout, runExec, spawnCommand } = await import("./exec.js"));
+    ({ runCommandWithTimeout, runExec, runUtf8CommandWithTimeout, spawnCommand } =
+      await import("./exec.js"));
   });
 
   afterAll(() => {
@@ -529,6 +531,48 @@ describe("Windows command execution", () => {
     });
   });
 
+  it("keeps truncated UTF-8 head output on a code point boundary", async () => {
+    execaMock.mockImplementationOnce(() =>
+      createMockSubprocess({
+        stdoutChunks: [Buffer.from("a😀z", "utf8")],
+        stderrChunks: [Buffer.from("b😀y", "utf8")],
+      }),
+    );
+    await withMockedWindowsPlatform(async () => {
+      await expect(
+        runUtf8CommandWithTimeout(["node", "utf8-output.js"], {
+          maxOutputBytes: 3,
+          outputCapture: "head",
+          timeoutMs: 1_000,
+        }),
+      ).resolves.toMatchObject({
+        stdout: "a",
+        stderr: "b",
+        stdoutTruncatedBytes: 5,
+        stderrTruncatedBytes: 5,
+      });
+      expect(spawnSyncMock).not.toHaveBeenCalled();
+    });
+  });
+
+  it("preserves complete legacy-code-page characters in truncated head output", async () => {
+    execaMock.mockImplementationOnce(() =>
+      createMockSubprocess({ stdoutChunks: [Buffer.from([0x61, 0xb2, 0xe2, 0xca, 0xd4])] }),
+    );
+    await withMockedWindowsPlatform(async () => {
+      await expect(
+        runCommandWithTimeout(["node", "gbk-output.js"], {
+          maxOutputBytes: 3,
+          outputCapture: "head",
+          timeoutMs: 1_000,
+        }),
+      ).resolves.toMatchObject({
+        stdout: "a测",
+        stdoutTruncatedBytes: 2,
+      });
+    });
+  });
+
   it("decodes split GBK chunks after complete output capture", async () => {
     execaMock.mockImplementationOnce(() =>
       createMockSubprocess({