vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-22 22:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add collect-all test failure planning

Browse Source
This commit is contained in:
Tak Hoffman
2026-03-27 23:20:41 -05:00
parent 39829b5dc6
commit d6fafb8af9
7 changed files with 497 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/media-understanding-core/src/runtime.ts
View File

@@ -50,6 +50,10 @@ export async function runMediaUnderstandingFile(
if (attachments.length === 0) {
return { text: undefined };
}
const config = params.cfg.tools?.media?.[params.capability];
if (config?.enabled === false) {
return { text: undefined };
}
const config = params.cfg.tools?.media?.[params.capability];
if (config?.enabled === false) {

30
scripts/test-parallel.mjs
View File

@@ -13,6 +13,7 @@ import {
const parseCliArgs = (args) => {
const wrapper = {
ciManifest: false,
failurePolicy: null,
plan: false,
explain: null,
mode: null,
@@ -25,6 +26,27 @@ const parseCliArgs = (args) => {
let passthroughMode = false;
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === "--collect-failures") {
wrapper.failurePolicy = "collect-all";
continue;
}
if (arg === "--failure-policy") {
const nextValue = args[index + 1] ?? "";
if (nextValue === "fail-fast" || nextValue === "collect-all") {
wrapper.failurePolicy = nextValue;
index += 1;
continue;
}
throw new Error(`Invalid --failure-policy value: ${String(nextValue || "<missing>")}`);
}
if (arg.startsWith("--failure-policy=")) {
const value = arg.slice("--failure-policy=".length);
if (value === "fail-fast" || value === "collect-all") {
wrapper.failurePolicy = value;
continue;
}
throw new Error(`Invalid --failure-policy value: ${String(value || "<missing>")}`);
}
if (passthroughMode) {
wrapper.passthroughArgs.push(arg);
continue;
@@ -121,12 +143,15 @@ if (rawCli.showHelp) {
" --files <pattern> Add targeted files or path patterns (repeatable)",
" --mode <ci|local> Override runtime mode",
" --profile <name> Override execution intent: normal, max, serial",
" --failure-policy <name> Override execution failure policy: fail-fast, collect-all",
" --collect-failures Shortcut for --failure-policy collect-all",
" --help Show this help text",
"",
"Examples:",
" node scripts/test-parallel.mjs",
" node scripts/test-parallel.mjs --plan --surface unit --surface extensions",
" node scripts/test-parallel.mjs --explain src/auto-reply/reply/followup-runner.test.ts",
" node scripts/test-parallel.mjs --collect-failures --surface unit",
" node scripts/test-parallel.mjs --files src/foo.test.ts -- --reporter=dot",
"",
"Environment:",
@@ -138,6 +163,7 @@ if (rawCli.showHelp) {
}
const request = {
failurePolicy: rawCli.failurePolicy,
mode: rawCli.mode,
profile: rawCli.profile,
surfaces: rawCli.surfaces,
@@ -177,5 +203,5 @@ if (process.env.OPENCLAW_TEST_LIST_LANES === "1" || rawCli.plan) {
exitWithCleanup(artifacts, 0);
}
const exitCode = await executePlan(plan, { env: process.env, artifacts });
process.exit(exitCode);
const result = await executePlan(plan, { env: process.env, artifacts });
process.exit(typeof result === "number" ? result : result.exitCode);

372
scripts/test-planner/executor.mjs
View File

@@ -67,6 +67,78 @@ const formatMemoryKb = (rssKb) =>
const formatMemoryDeltaKb = (rssKb) =>
`${rssKb >= 0 ? "+" : "-"}${formatMemoryKb(Math.abs(rssKb))}`;
const extractFailedTestFiles = (output) => {
const failureFiles = new Set();
const pattern = /^\s*❯\s+([^\s(][^(]*?\.(?:test|spec)\.[cm]?[jt]sx?)/gmu;
for (const match of output.matchAll(pattern)) {
const file = match[1]?.trim();
if (file) {
failureFiles.add(file);
}
}
return [...failureFiles];
};
const classifyRunResult = ({ resolvedCode, signal, fatalSeen, childError, failedTestFiles }) => {
if (resolvedCode === 0) {
return "pass";
}
if (childError || signal || fatalSeen || failedTestFiles.length === 0) {
return "infra-failure";
}
return "test-failure";
};
const formatRunLabel = (result) =>
`unit=${result.unitId}${result.shardLabel ? ` shard=${result.shardLabel}` : ""}`;
const buildFinalRunReport = (results) => {
const failedResults = results.filter((result) => result.exitCode !== 0);
const failedUnits = new Set(failedResults.map((result) => result.unitId));
const failedTestFiles = new Set(failedResults.flatMap((result) => result.failedTestFiles ?? []));
const infraFailures = failedResults.filter((result) => result.classification === "infra-failure");
return {
exitCode: failedResults.length > 0 ? 1 : 0,
results,
summary: {
failedRunCount: failedResults.length,
failedUnitCount: failedUnits.size,
failedTestFileCount: failedTestFiles.size,
infraFailureCount: infraFailures.length,
},
};
};
const printFinalRunSummary = (plan, report, reportArtifactPath) => {
console.log(
`[test-parallel] summary failurePolicy=${plan.failurePolicy} failedUnits=${String(
report.summary.failedUnitCount,
)} failedTestFiles=${String(report.summary.failedTestFileCount)} infraFailures=${String(
report.summary.infraFailureCount,
)}`,
);
if (report.summary.failedTestFileCount > 0) {
console.error("[test-parallel] failing tests");
const failedTestFiles = report.results
.flatMap((result) => (result.classification === "test-failure" ? result.failedTestFiles : []))
.filter((file) => typeof file === "string");
for (const file of new Set(failedTestFiles)) {
console.error(`- ${String(file)}`);
}
}
if (report.summary.infraFailureCount > 0) {
console.error("[test-parallel] infrastructure failures");
for (const result of report.results.filter(
(entry) => entry.classification === "infra-failure",
)) {
console.error(
`- ${formatRunLabel(result)} code=${String(result.exitCode)} signal=${result.signal ?? "none"} log=${result.logPath}${result.failureArtifactPath ? ` meta=${result.failureArtifactPath}` : ""}`,
);
}
}
console.error(`[test-parallel] summary artifact ${reportArtifactPath}`);
};
export function createExecutionArtifacts(env = process.env) {
let tempArtifactDir = null;
const ensureTempArtifactDir = () => {
@@ -157,7 +229,7 @@ export const resolveVitestFsModuleCachePath = ({
export function formatPlanOutput(plan) {
return [
`runtime=${plan.runtimeCapabilities.runtimeProfileName} mode=${plan.runtimeCapabilities.mode} intent=${plan.runtimeCapabilities.intentProfile} memoryBand=${plan.runtimeCapabilities.memoryBand} loadBand=${plan.runtimeCapabilities.loadBand} vitestMaxWorkers=${String(plan.executionBudget.vitestMaxWorkers ?? "default")} topLevelParallel=${plan.topLevelParallelEnabled ? String(plan.topLevelParallelLimit) : "off"}`,
`runtime=${plan.runtimeCapabilities.runtimeProfileName} mode=${plan.runtimeCapabilities.mode} intent=${plan.runtimeCapabilities.intentProfile} memoryBand=${plan.runtimeCapabilities.memoryBand} loadBand=${plan.runtimeCapabilities.loadBand} failurePolicy=${plan.failurePolicy} vitestMaxWorkers=${String(plan.executionBudget.vitestMaxWorkers ?? "default")} topLevelParallel=${plan.topLevelParallelEnabled ? String(plan.topLevelParallelLimit) : "off"}`,
...plan.selectedUnits.map(
(unit) =>
`${unit.id} filters=${String(countExplicitEntryFilters(unit.args) ?? "all")} maxWorkers=${String(
@@ -308,6 +380,8 @@ export async function executePlan(plan, options = {}) {
process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("exit", artifacts.cleanupTempArtifacts);
const shouldCollectAllFailures = plan.failurePolicy === "collect-all";
const runOnce = (unit, extraArgs = []) =>
new Promise((resolve) => {
const startedAt = Date.now();
@@ -362,7 +436,19 @@ export async function executePlan(plan, options = {}) {
console.error(
`[test-parallel] failed to create heap snapshot dir ${heapSnapshotDir}: ${String(err)}`,
);
resolve(1);
resolve({
unitId: unit.id,
shardLabel,
classification: "infra-failure",
exitCode: 1,
signal: null,
elapsedMs: Date.now() - startedAt,
failedTestFiles: [...explicitEntryFilters],
explicitEntryFilters,
failureArtifactPath: null,
logPath: laneLogPath,
outputTail: "",
});
return;
}
resolvedNodeOptions = ensureNodeOptionFlag(
@@ -384,6 +470,8 @@ export async function executePlan(plan, options = {}) {
let memoryPollTimer = null;
let heapSnapshotTimer = null;
let closeFallbackTimer = null;
let failureArtifactPath = null;
let failureTail = "";
const memoryFileRecords = [];
let initialTreeSample = null;
let latestTreeSample = null;
@@ -540,8 +628,8 @@ export async function executePlan(plan, options = {}) {
const elapsedMs = Date.now() - startedAt;
logMemoryTraceSummary();
if (resolvedCode !== 0) {
const failureTail = formatCapturedOutputTail(output);
const failureArtifactPath = artifacts.writeTempJsonArtifact(`${artifactStem}-failure`, {
failureTail = formatCapturedOutputTail(output);
failureArtifactPath = artifacts.writeTempJsonArtifact(`${artifactStem}-failure`, {
entry: unit.id,
command: [pnpmInvocation.command, ...spawnArgs],
elapsedMs,
@@ -573,7 +661,27 @@ export async function executePlan(plan, options = {}) {
console.log(
`[test-parallel] done ${unit.id} code=${String(resolvedCode)} elapsed=${formatElapsedMs(elapsedMs)}`,
);
resolve(resolvedCode);
const failedTestFiles = extractFailedTestFiles(output);
const classification = classifyRunResult({
resolvedCode,
signal,
fatalSeen,
childError,
failedTestFiles,
});
resolve({
unitId: unit.id,
shardLabel,
classification,
exitCode: resolvedCode,
signal: signal ?? null,
elapsedMs,
failedTestFiles,
explicitEntryFilters,
failureArtifactPath,
logPath: laneLogPath,
outputTail: failureTail,
});
};
try {
const childEnv = {
@@ -610,7 +718,19 @@ export async function executePlan(plan, options = {}) {
} catch (err) {
laneLogStream.end();
console.error(`[test-parallel] spawn failed: ${String(err)}`);
resolve(1);
resolve({
unitId: unit.id,
shardLabel: getShardLabel(extraArgs),
classification: "infra-failure",
exitCode: 1,
signal: null,
elapsedMs: Date.now() - startedAt,
failedTestFiles: [...explicitEntryFilters],
explicitEntryFilters,
failureArtifactPath: null,
logPath: laneLogPath,
outputTail: String(err),
});
return;
}
children.add(child);
@@ -652,19 +772,22 @@ export async function executePlan(plan, options = {}) {
});
const runUnit = async (unit, extraArgs = []) => {
const results = [];
if (unit.fixedShardIndex !== undefined) {
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== unit.fixedShardIndex) {
return 0;
return results;
}
return runOnce(unit, extraArgs);
results.push(await runOnce(unit, extraArgs));
return results;
}
const explicitFilterCount = countExplicitEntryFilters(unit.args);
const topLevelAssignedShard = plan.topLevelSingleShardAssignments.get(unit);
if (topLevelAssignedShard !== undefined) {
if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== topLevelAssignedShard) {
return 0;
return results;
}
return runOnce(unit, extraArgs);
results.push(await runOnce(unit, extraArgs));
return results;
}
const effectiveShardCount =
explicitFilterCount === null
@@ -672,67 +795,73 @@ export async function executePlan(plan, options = {}) {
: Math.min(plan.shardCount, Math.max(1, explicitFilterCount - 1));
if (effectiveShardCount <= 1) {
if (plan.shardIndexOverride !== null && plan.shardIndexOverride > effectiveShardCount) {
return 0;
return results;
}
return runOnce(unit, extraArgs);
results.push(await runOnce(unit, extraArgs));
return results;
}
if (plan.shardIndexOverride !== null) {
if (plan.shardIndexOverride > effectiveShardCount) {
return 0;
return results;
}
return runOnce(unit, [
"--shard",
`${plan.shardIndexOverride}/${effectiveShardCount}`,
...extraArgs,
]);
results.push(
await runOnce(unit, [
"--shard",
`${plan.shardIndexOverride}/${effectiveShardCount}`,
...extraArgs,
]),
);
return results;
}
for (let shardIndex = 1; shardIndex <= effectiveShardCount; shardIndex += 1) {
// eslint-disable-next-line no-await-in-loop
const code = await runOnce(unit, [
"--shard",
`${shardIndex}/${effectiveShardCount}`,
...extraArgs,
]);
if (code !== 0) {
return code;
results.push(
// eslint-disable-next-line no-await-in-loop
await runOnce(unit, ["--shard", `${shardIndex}/${effectiveShardCount}`, ...extraArgs]),
);
if (!shouldCollectAllFailures && results.at(-1)?.exitCode !== 0) {
return results;
}
}
return 0;
return results;
};
const runUnitsWithLimit = async (units, extraArgs = [], concurrency = 1) => {
const results = [];
if (units.length === 0) {
return undefined;
return results;
}
const normalizedConcurrency = Math.max(1, Math.floor(concurrency));
if (normalizedConcurrency <= 1) {
for (const unit of units) {
// eslint-disable-next-line no-await-in-loop
const code = await runUnit(unit, extraArgs);
if (code !== 0) {
return code;
results.push(
// eslint-disable-next-line no-await-in-loop
...(await runUnit(unit, extraArgs)),
);
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
return results;
}
}
return undefined;
return results;
}
let nextIndex = 0;
let firstFailure;
let stopScheduling = false;
const worker = async () => {
while (firstFailure === undefined) {
while (!stopScheduling) {
const unitIndex = nextIndex;
nextIndex += 1;
if (unitIndex >= units.length) {
return;
}
const code = await runUnit(units[unitIndex], extraArgs);
if (code !== 0 && firstFailure === undefined) {
firstFailure = code;
const unitResults = await runUnit(units[unitIndex], extraArgs);
results.push(...unitResults);
if (!shouldCollectAllFailures && unitResults.some((result) => result.exitCode !== 0)) {
stopScheduling = true;
}
}
};
const workerCount = Math.min(normalizedConcurrency, units.length);
await Promise.all(Array.from({ length: workerCount }, () => worker()));
return firstFailure;
return results;
};
const runUnits = async (units, extraArgs = []) => {
@@ -743,14 +872,19 @@ export async function executePlan(plan, options = {}) {
};
if (plan.passthroughMetadataOnly) {
return runOnce(
{
id: "vitest-meta",
args: ["vitest", "run"],
maxWorkers: null,
},
plan.passthroughOptionArgs,
);
const report = buildFinalRunReport([
await runOnce(
{
id: "vitest-meta",
args: ["vitest", "run"],
maxWorkers: null,
},
plan.passthroughOptionArgs,
),
]);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
if (plan.targetedUnits.length > 0) {
@@ -758,29 +892,60 @@ export async function executePlan(plan, options = {}) {
console.error(
"[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time.",
);
return 2;
return {
exitCode: 2,
results: [],
summary: {
failedRunCount: 0,
failedUnitCount: 0,
failedTestFileCount: 0,
infraFailureCount: 0,
},
};
}
const failedTargetedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
if (failedTargetedParallel !== undefined) {
return failedTargetedParallel;
const results = [];
results.push(...(await runUnits(plan.parallelUnits, plan.passthroughOptionArgs)));
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
for (const unit of plan.serialUnits) {
// eslint-disable-next-line no-await-in-loop
const code = await runUnit(unit, plan.passthroughOptionArgs);
if (code !== 0) {
return code;
results.push(
// eslint-disable-next-line no-await-in-loop
...(await runUnit(unit, plan.passthroughOptionArgs)),
);
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
return 0;
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
if (plan.passthroughRequiresSingleRun && plan.passthroughOptionArgs.length > 0) {
console.error(
"[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter.",
);
return 2;
return {
exitCode: 2,
results: [],
summary: {
failedRunCount: 0,
failedUnitCount: 0,
failedTestFileCount: 0,
infraFailureCount: 0,
},
};
}
const results = [];
if (plan.serialPrefixUnits.length > 0) {
const orderedSegments = buildOrderedParallelSegments(plan.parallelUnits);
let pendingDeferredSegment = null;
@@ -854,58 +1019,101 @@ export async function executePlan(plan, options = {}) {
}
pendingDeferredSegment = null;
// eslint-disable-next-line no-await-in-loop
const failedSerialPhase = await runUnits(segment.units, plan.passthroughOptionArgs);
if (failedSerialPhase !== undefined) {
return failedSerialPhase;
const serialPhaseResults = await runUnits(segment.units, plan.passthroughOptionArgs);
results.push(...serialPhaseResults);
if (!shouldCollectAllFailures && serialPhaseResults.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
if (deferredCarryPromise !== null && deferredCarrySurface === segment.phase) {
// eslint-disable-next-line no-await-in-loop
const failedCarriedDeferred = await deferredCarryPromise;
if (failedCarriedDeferred !== undefined) {
return failedCarriedDeferred;
const carriedDeferredResults =
// eslint-disable-next-line no-await-in-loop
await deferredCarryPromise;
results.push(...carriedDeferredResults);
if (
!shouldCollectAllFailures &&
carriedDeferredResults.some((result) => result.exitCode !== 0)
) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
deferredCarryPromise = null;
deferredCarrySurface = null;
}
if (deferredPromise !== null) {
// eslint-disable-next-line no-await-in-loop
const failedDeferredPhase = await deferredPromise;
if (failedDeferredPhase !== undefined) {
return failedDeferredPhase;
const deferredResults =
// eslint-disable-next-line no-await-in-loop
await deferredPromise;
results.push(...deferredResults);
if (!shouldCollectAllFailures && deferredResults.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
carriedDeferredPromise = deferredCarryPromise;
carriedDeferredSurface = deferredCarrySurface;
}
if (pendingDeferredSegment !== null) {
const failedDeferredParallel = await runUnitsWithLimit(
const deferredParallelResults = await runUnitsWithLimit(
pendingDeferredSegment.units,
plan.passthroughOptionArgs,
plan.deferredRunConcurrency ?? 1,
);
if (failedDeferredParallel !== undefined) {
return failedDeferredParallel;
results.push(...deferredParallelResults);
if (
!shouldCollectAllFailures &&
deferredParallelResults.some((result) => result.exitCode !== 0)
) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
if (carriedDeferredPromise !== null) {
const failedCarriedDeferred = await carriedDeferredPromise;
if (failedCarriedDeferred !== undefined) {
return failedCarriedDeferred;
const carriedDeferredResults = await carriedDeferredPromise;
results.push(...carriedDeferredResults);
if (
!shouldCollectAllFailures &&
carriedDeferredResults.some((result) => result.exitCode !== 0)
) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
} else {
const failedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
if (failedParallel !== undefined) {
return failedParallel;
const parallelResults = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs);
results.push(...parallelResults);
if (!shouldCollectAllFailures && parallelResults.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
for (const unit of plan.serialUnits) {
// eslint-disable-next-line no-await-in-loop
const code = await runUnit(unit, plan.passthroughOptionArgs);
if (code !== 0) {
return code;
results.push(
// eslint-disable-next-line no-await-in-loop
...(await runUnit(unit, plan.passthroughOptionArgs)),
);
if (!shouldCollectAllFailures && results.some((result) => result.exitCode !== 0)) {
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}
}
return 0;
const report = buildFinalRunReport(results);
const reportArtifactPath = artifacts.writeTempJsonArtifact("summary-report", report);
printFinalRunSummary(plan, report, reportArtifactPath);
return report;
}

59
scripts/test-planner/planner.mjs
View File

@@ -97,7 +97,14 @@ const normalizeSurfaces = (values = []) => [
),
];
const EXPLICIT_PLAN_SURFACES = new Set(["unit", "extensions", "channels", "contracts", "gateway"]);
const EXPLICIT_PLAN_SURFACES = new Set([
"unit",
"extensions",
"channels",
"contracts",
"gateway",
]);
const FAILURE_POLICIES = new Set(["fail-fast", "collect-all"]);
const validateExplicitSurfaces = (surfaces) => {
const invalidSurfaces = surfaces.filter((surface) => !EXPLICIT_PLAN_SURFACES.has(surface));
@@ -134,6 +141,48 @@ const buildRequestedSurfaces = (request, env) => {
return surfaces;
};
const normalizeFailurePolicy = (requestFailurePolicy, optionArgs) => {
if (requestFailurePolicy !== null && requestFailurePolicy !== undefined) {
if (!FAILURE_POLICIES.has(requestFailurePolicy)) {
throw new Error(
`Unsupported failure policy "${String(requestFailurePolicy)}". Supported values: fail-fast, collect-all.`,
);
}
return { failurePolicy: requestFailurePolicy, passthroughOptionArgs: optionArgs };
}
const normalizedOptionArgs = [];
let failurePolicy = "fail-fast";
for (let index = 0; index < optionArgs.length; index += 1) {
const arg = optionArgs[index];
if (arg === "--bail") {
const nextValue = optionArgs[index + 1] ?? "";
if (nextValue === "0") {
failurePolicy = "collect-all";
index += 1;
continue;
}
throw new Error(
`Unsupported wrapper-level --bail value: ${String(nextValue || "<missing>")}. Use --bail=0, --collect-failures, or --failure-policy=collect-all.`,
);
}
if (arg.startsWith("--bail=")) {
const value = arg.slice("--bail=".length);
if (value === "0") {
failurePolicy = "collect-all";
continue;
}
throw new Error(
`Unsupported wrapper-level --bail value: ${String(value || "<missing>")}. Use --bail=0, --collect-failures, or --failure-policy=collect-all.`,
);
}
normalizedOptionArgs.push(arg);
}
return { failurePolicy, passthroughOptionArgs: normalizedOptionArgs };
};
const createPlannerContext = (request, options = {}) => {
const env = options.env ?? process.env;
const runtime = resolveRuntimeCapabilities(env, {
@@ -1364,6 +1413,7 @@ export function buildExecutionPlan(request, options = {}) {
const { fileFilters: passthroughFileFilters, optionArgs } = parsePassthroughArgs(
request.passthroughArgs ?? [],
);
const normalizedFailurePolicy = normalizeFailurePolicy(request.failurePolicy ?? null, optionArgs);
const fileFilters = [...explicitFileFilters, ...passthroughFileFilters];
const passthroughMetadataFlags = new Set(["-h", "--help", "--listTags", "--clearCache"]);
const passthroughMetadataOnly =
@@ -1376,7 +1426,7 @@ export function buildExecutionPlan(request, options = {}) {
const [flag] = arg.split("=", 1);
return passthroughMetadataFlags.has(flag);
});
const passthroughRequiresSingleRun = optionArgs.some((arg) => {
const passthroughRequiresSingleRun = normalizedFailurePolicy.passthroughOptionArgs.some((arg) => {
if (!arg.startsWith("-")) {
return false;
}
@@ -1387,7 +1437,7 @@ export function buildExecutionPlan(request, options = {}) {
{
...request,
fileFilters,
passthroughOptionArgs: optionArgs,
passthroughOptionArgs: normalizedFailurePolicy.passthroughOptionArgs,
},
options,
);
@@ -1465,7 +1515,8 @@ export function buildExecutionPlan(request, options = {}) {
return {
runtimeCapabilities: context.runtime,
executionBudget: context.executionBudget,
passthroughOptionArgs: optionArgs,
failurePolicy: normalizedFailurePolicy.failurePolicy,
passthroughOptionArgs: normalizedFailurePolicy.passthroughOptionArgs,
passthroughRequiresSingleRun,
passthroughMetadataOnly,
fileFilters,

18
test/scripts/test-parallel.test.ts
View File

@@ -471,6 +471,24 @@ describe("scripts/test-parallel lane planning", () => {
expect(output).toContain("unit-deliver-isolated filters=1");
});
it("prints collect-all failure policy in planner output for wrapper-native flag", () => {
const output = runPlannerPlan(["--plan", "--collect-failures", "--surface", "unit"]);
expect(output).toContain("failurePolicy=collect-all");
});
it("maps --bail=0 to collect-all failure policy in planner output", () => {
const output = runPlannerPlan(["--plan", "--surface", "unit", "--", "--bail=0"]);
expect(output).toContain("failurePolicy=collect-all");
});
it("rejects wrapper-level positive --bail values", () => {
expect(() => runPlannerPlan(["--plan", "--surface", "unit", "--", "--bail=2"])).toThrowError(
/Unsupported wrapper-level --bail value/u,
);
});
it("rejects removed machine-name profiles", () => {
expect(() => runPlannerPlan(["--plan", "--profile", "macmini"])).toThrowError(
/Unsupported test profile "macmini"/u,

83
test/scripts/test-planner.executor-fallback.test.ts
View File

@@ -36,6 +36,7 @@ describe("test planner executor", () => {
const artifacts = createExecutionArtifacts({ OPENCLAW_TEST_CLOSE_GRACE_MS: "10" });
const executePromise = executePlan(
{
failurePolicy: "fail-fast",
passthroughMetadataOnly: true,
passthroughOptionArgs: [],
runtimeCapabilities: { isWindowsCi: false, isCI: false, isWindows: false },
@@ -46,9 +47,89 @@ describe("test planner executor", () => {
},
);
await expect(executePromise).resolves.toBe(0);
await expect(executePromise).resolves.toMatchObject({
exitCode: 0,
summary: {
failedRunCount: 0,
},
});
expect(spawnMock).toHaveBeenCalledTimes(1);
artifacts.cleanupTempArtifacts();
});
it("collects failures across planned units when failure policy is collect-all", async () => {
vi.useRealTimers();
const children = [1, 2].map((pid, index) => {
const stdout = new PassThrough();
const stderr = new PassThrough();
return Object.assign(new EventEmitter(), {
stdout,
stderr,
pid,
kill: vi.fn(),
index,
});
});
let childIndex = 0;
const spawnMock = vi.fn(() => {
const child = children[childIndex];
childIndex += 1;
setTimeout(() => {
child.stdout.write(
child.index === 0
? " ❯ src/alpha.test.ts (1 test | 1 failed)\n"
: " ❯ src/beta.test.ts (1 test | 1 failed)\n",
);
child.emit("exit", 1, null);
child.emit("close", 1, null);
}, 0);
return child;
});
vi.doMock("node:child_process", () => ({
spawn: spawnMock,
}));
const { executePlan, createExecutionArtifacts } = await importFreshModule<
typeof import("../../scripts/test-planner/executor.mjs")
>(import.meta.url, "../../scripts/test-planner/executor.mjs?scope=collect-all");
const artifacts = createExecutionArtifacts({});
const report = await executePlan(
{
failurePolicy: "collect-all",
passthroughMetadataOnly: false,
passthroughOptionArgs: [],
targetedUnits: [],
parallelUnits: [
{ id: "unit-a", args: ["vitest", "run", "src/alpha.test.ts"] },
{ id: "unit-b", args: ["vitest", "run", "src/beta.test.ts"] },
],
serialUnits: [],
serialPrefixUnits: [],
shardCount: 1,
shardIndexOverride: null,
topLevelSingleShardAssignments: new Map(),
runtimeCapabilities: { isWindowsCi: false, isCI: false, isWindows: false },
topLevelParallelEnabled: false,
topLevelParallelLimit: 1,
deferredRunConcurrency: 1,
passthroughRequiresSingleRun: false,
},
{
env: {},
artifacts,
},
);
expect(spawnMock).toHaveBeenCalledTimes(2);
expect(report.exitCode).toBe(1);
expect(report.summary.failedRunCount).toBe(2);
expect(report.summary.failedTestFileCount).toBe(2);
expect(report.results.map((result) => result.classification)).toEqual([
"test-failure",
"test-failure",
]);
artifacts.cleanupTempArtifacts();
});
});

20
test/scripts/test-planner.test.ts
View File

@@ -40,6 +40,7 @@ describe("test planner", () => {
);
expect(plan.runtimeCapabilities.runtimeProfileName).toBe("local-darwin");
expect(plan.failurePolicy).toBe("fail-fast");
expect(plan.runtimeCapabilities.memoryBand).toBe("mid");
expect(plan.executionBudget.unitSharedWorkers).toBe(4);
expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8);
@@ -177,6 +178,25 @@ describe("test planner", () => {
artifacts.cleanupTempArtifacts();
});
it("normalizes --bail=0 into collect-all failure policy", () => {
const artifacts = createExecutionArtifacts({});
const plan = buildExecutionPlan(
{
mode: "local",
surfaces: ["unit"],
passthroughArgs: ["--bail=0"],
},
{
env: {},
writeTempJsonArtifact: artifacts.writeTempJsonArtifact,
},
);
expect(plan.failurePolicy).toBe("collect-all");
expect(plan.passthroughOptionArgs).not.toContain("--bail=0");
artifacts.cleanupTempArtifacts();
});
it("explains runtime truth using the same catalog and worker policy", () => {
const explanation = explainExecutionTarget(
{