fix(ci): harden test gating under load

This commit is contained in:
Vincent Koc
2026-03-19 11:01:16 -07:00
parent 51519b4086
commit feb9a3b5b2
11 changed files with 103 additions and 17 deletions

View File

@@ -1,9 +1,10 @@
const DEFAULT_OUTPUT_CAPTURE_LIMIT = 200_000;
const fatalOutputPatterns = [
/FATAL ERROR:.*heap out of memory/i,
/Allocation failed - JavaScript heap out of memory/i,
/FATAL ERROR:/i,
/JavaScript heap out of memory/i,
/node::OOMErrorHandler/i,
/ERR_WORKER_OUT_OF_MEMORY/i,
];
export function appendCapturedOutput(current, chunk, limit = DEFAULT_OUTPUT_CAPTURE_LIMIT) {
@@ -21,14 +22,17 @@ export function hasFatalTestRunOutput(output) {
return fatalOutputPatterns.some((pattern) => pattern.test(output));
}
export function resolveTestRunExitCode({ code, signal, output }) {
export function resolveTestRunExitCode({ code, signal, output, fatalSeen = false, childError }) {
if (typeof code === "number" && code !== 0) {
return code;
}
if (childError) {
return 1;
}
if (signal) {
return 1;
}
if (hasFatalTestRunOutput(output)) {
if (fatalSeen || hasFatalTestRunOutput(output)) {
return 1;
}
return code ?? 0;

View File

@@ -4,7 +4,11 @@ import os from "node:os";
import path from "node:path";
import { channelTestPrefixes } from "../vitest.channel-paths.mjs";
import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs";
import { appendCapturedOutput, resolveTestRunExitCode } from "./test-parallel-utils.mjs";
import {
appendCapturedOutput,
hasFatalTestRunOutput,
resolveTestRunExitCode,
} from "./test-parallel-utils.mjs";
import {
loadTestRunnerBehavior,
loadUnitTimingManifest,
@@ -742,6 +746,8 @@ const runOnce = (entry, extraArgs = []) =>
? `${nextNodeOptions} ${heapFlag}`.trim()
: nextNodeOptions;
let output = "";
let fatalSeen = false;
let childError = null;
let child;
try {
child = spawn(pnpm, args, {
@@ -757,20 +763,23 @@ const runOnce = (entry, extraArgs = []) =>
children.add(child);
child.stdout?.on("data", (chunk) => {
const text = chunk.toString();
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
output = appendCapturedOutput(output, text);
process.stdout.write(chunk);
});
child.stderr?.on("data", (chunk) => {
const text = chunk.toString();
fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
output = appendCapturedOutput(output, text);
process.stderr.write(chunk);
});
child.on("error", (err) => {
childError = err;
console.error(`[test-parallel] child error: ${String(err)}`);
});
child.on("close", (code, signal) => {
children.delete(child);
const resolvedCode = resolveTestRunExitCode({ code, signal, output });
const resolvedCode = resolveTestRunExitCode({ code, signal, output, fatalSeen, childError });
console.log(
`[test-parallel] done ${entry.name} code=${String(resolvedCode)} elapsed=${formatElapsedMs(Date.now() - startedAt)}`,
);