fix(qa): bound gateway log cleanup (#116431)

This commit is contained in:
Dallin Romney
2026-07-31 14:23:36 +08:00
committed by GitHub
parent 5c061507d3
commit b13b3d9a0f
2 changed files with 56 additions and 8 deletions

View File

@@ -4,6 +4,7 @@ import { EventEmitter, once } from "node:events";
import { lstat, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Writable } from "node:stream";
import { pathToFileURL } from "node:url";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
@@ -1360,6 +1361,25 @@ describe("buildQaRuntimeEnv", () => {
});
});
it("force-closes a gateway log stream whose final flush never settles", async () => {
const stream = new Writable({
write(_chunk, _encoding, callback) {
callback();
},
final() {
// Simulate the stalled filesystem flush observed in the release profile.
},
});
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
await testing.closeWriteStream(stream as never, "stdout", 1);
expect(stream.destroyed).toBe(true);
expect(stderr).toHaveBeenCalledWith(
"[qa-suite] stdout gateway log flush exceeded 1ms; forcing close\n",
);
});
it.runIf(process.platform !== "win32")(
"fails closed when forced gateway process-group shutdown times out",
async () => {

View File

@@ -12,6 +12,7 @@ import fs from "node:fs/promises";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { finished } from "node:stream/promises";
import { setTimeout as sleep } from "node:timers/promises";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
@@ -86,6 +87,7 @@ const QA_GATEWAY_CHILD_RESTART_BOUNDARY_TIMEOUT_MS = 90_000;
const QA_GATEWAY_CHILD_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 30_000;
// Loaded Docker runners can take several seconds to reap a force-killed process group.
const QA_GATEWAY_CHILD_FORCE_SHUTDOWN_TIMEOUT_MS = 10_000;
const QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS = 5_000;
const QA_MOCK_OPENAI_API_KEY = ["qa", "mock", "openai", "key"].join("-");
const QA_GATEWAY_CHILD_BLOCKED_SECRET_ENV_VARS = Object.freeze([
"OPENCLAW_QA_CONVEX_SECRET_CI",
@@ -250,10 +252,29 @@ async function getFreePort() {
});
}
async function closeWriteStream(stream: WriteStream) {
await new Promise<void>((resolve) => {
stream.end(() => resolve());
});
async function closeWriteStream(
stream: WriteStream,
label: "stderr" | "stdout",
timeoutMs = QA_GATEWAY_LOG_CLOSE_TIMEOUT_MS,
) {
if (stream.destroyed) {
return;
}
stream.end();
const signal = AbortSignal.timeout(timeoutMs);
try {
await finished(stream, { cleanup: true, signal });
} catch (error) {
if (!signal.aborted) {
throw error;
}
// Gateway logs are diagnostic only. Never let a stuck filesystem flush
// retain the stopped child runtime and its live transport credentials.
process.stderr.write(
`[qa-suite] ${label} gateway log flush exceeded ${timeoutMs}ms; forcing close\n`,
);
stream.destroy();
}
}
async function writeSanitizedQaGatewayDebugLog(params: { sourcePath: string; targetPath: string }) {
@@ -666,6 +687,7 @@ export const testing = {
resolveQaGatewayChildStopTimeouts,
stopQaGatewayChildProcessTree,
classifyLinuxProcessGroupStats,
closeWriteStream,
};
function hasChildExited(child: ChildProcess) {
@@ -1718,9 +1740,12 @@ export async function startQaGatewayChild(params: {
processStopped = false;
cleanupErrors.push(error);
}
for (const stream of [stdoutLog, stderrLog]) {
for (const [label, stream] of [
["stdout", stdoutLog],
["stderr", stderrLog],
] as const) {
try {
await closeWriteStream(stream);
await closeWriteStream(stream, label);
} catch (error) {
cleanupErrors.push(error);
}
@@ -1786,9 +1811,12 @@ export async function startQaGatewayChild(params: {
cleanupErrors.push(cleanupError);
}
}
for (const stream of [stdoutLog, stderrLog]) {
for (const [label, stream] of [
["stdout", stdoutLog],
["stderr", stderrLog],
] as const) {
try {
await closeWriteStream(stream);
await closeWriteStream(stream, label);
} catch (cleanupError) {
cleanupErrors.push(cleanupError);
}