From b460ee48a61bacf7c80eb8bc2179de306fe5d348 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 27 May 2026 10:16:13 +0200 Subject: [PATCH] fix(test): fail empty plugin gauntlet runs --- scripts/check-plugin-gateway-gauntlet.mjs | 29 ++++++++++- test/scripts/plugin-gateway-gauntlet.test.ts | 51 +++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/scripts/check-plugin-gateway-gauntlet.mjs b/scripts/check-plugin-gateway-gauntlet.mjs index 472d59bf4cf..3df4aeb2fc2 100644 --- a/scripts/check-plugin-gateway-gauntlet.mjs +++ b/scripts/check-plugin-gateway-gauntlet.mjs @@ -58,6 +58,7 @@ function parseArgs(argv) { commandTimeoutMs: 120_000, buildTimeoutMs: 600_000, qaTimeoutMs: 900_000, + allowEmpty: false, keepRunRoot: process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_KEEP_RUN_ROOT === "1", }; const envIds = normalizeCsv(process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS); @@ -156,6 +157,9 @@ function parseArgs(argv) { case "--keep-run-root": options.keepRunRoot = true; break; + case "--allow-empty": + options.allowEmpty = true; + break; case "--help": printHelp(); process.exit(0); @@ -191,6 +195,7 @@ Options: --skip-lifecycle Skip plugin install/inspect/disable/enable/doctor/uninstall --skip-qa Skip QA Lab RPC conversation runs --skip-slash-help Skip CLI help probes for plugin-declared command aliases + --allow-empty Allow zero-command runs when every active phase is skipped --keep-run-root Preserve isolated HOME/state/log temp root after success `); } @@ -611,6 +616,10 @@ export function runMeasuredCommandLive(params) { }); } +export function hasGauntletWorkRows(rows) { + return rows.some((row) => row.phase !== "prebuild"); +} + function runPluginLifecycle(params) { for (const plugin of params.plugins) { const commands = [ @@ -814,7 +823,18 @@ async function main() { const failures = rows.filter( (row) => row.status !== 0 || row.timedOut || row.diagnosticFailure, ); - preserveRunRoot = preserveRunRoot || failures.length > 0; + const guardFailures = + !hasGauntletWorkRows(rows) && !options.allowEmpty + ? [ + { + kind: "empty-run", + message: + "No lifecycle, slash-help, or QA gauntlet commands ran; remove a skip flag or pass --allow-empty for intentional dry runs.", + }, + ] + : []; + const hasFailures = failures.length > 0 || guardFailures.length > 0; + preserveRunRoot = preserveRunRoot || hasFailures; let cleanupError = null; if (!preserveRunRoot) { try { @@ -841,6 +861,7 @@ async function main() { qaScenarios: options.qaScenarios, qaPluginChunkSize: options.qaPluginChunkSize, qaBaseline: options.qaBaseline, + allowEmpty: options.allowEmpty, keepRunRoot: options.keepRunRoot, skipLifecycle: options.skipLifecycle, skipQa: options.skipQa, @@ -861,6 +882,7 @@ async function main() { rows, observations: [...metricObservations, ...qaBaselineObservations, ...gatewayObservations], failures, + guardFailures, }; const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json"); fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); @@ -876,10 +898,13 @@ async function main() { `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? ""} status=${failure.status} timedOut=${failure.timedOut} diagnostic=${failure.diagnosticFailure ?? ""} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`, ); } + for (const failure of guardFailures) { + process.stdout.write(`[plugin-gauntlet] failure ${failure.kind}: ${failure.message}\n`); + } for (const observation of summary.observations.slice(0, 20)) { process.stdout.write(`[plugin-gauntlet] observation ${JSON.stringify(observation)}\n`); } - if (failures.length > 0) { + if (hasFailures) { process.exitCode = 1; } } catch (error) { diff --git a/test/scripts/plugin-gateway-gauntlet.test.ts b/test/scripts/plugin-gateway-gauntlet.test.ts index 3385158421e..93a752d6c34 100644 --- a/test/scripts/plugin-gateway-gauntlet.test.ts +++ b/test/scripts/plugin-gateway-gauntlet.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createGauntletPrebuildCommand, + hasGauntletWorkRows, parseTimedMetrics, runMeasuredCommand, runMeasuredCommandLive, @@ -328,6 +329,16 @@ describe("plugin gateway gauntlet helpers", () => { }); }); + it("does not count prebuild setup as gauntlet work", () => { + expect(hasGauntletWorkRows([])).toBe(false); + expect(hasGauntletWorkRows([{ phase: "prebuild" }])).toBe(false); + expect(hasGauntletWorkRows([{ phase: "prebuild" }, { phase: "lifecycle:install" }])).toBe( + true, + ); + expect(hasGauntletWorkRows([{ phase: "slash:help" }])).toBe(true); + expect(hasGauntletWorkRows([{ phase: "qa:rpc" }])).toBe(true); + }); + it("parses macOS time -l metrics from strict trailing lines", () => { const metrics = parseTimedMetrics( [ @@ -457,7 +468,7 @@ describe("plugin gateway gauntlet helpers", () => { await expect(fs.readFile(markerPath, "utf8")).resolves.toBe(afterReturn); }); - it("cleans the isolated run root after a successful dry run", async () => { + it("fails dry runs that do not execute any gauntlet commands", async () => { const outputDir = path.join(repoRoot, "artifacts"); const result = spawnSync( process.execPath, @@ -478,10 +489,48 @@ describe("plugin gateway gauntlet helpers", () => { }, ); + expect(result.status).toBe(1); + expect(result.stdout).toContain("No lifecycle, slash-help, or QA gauntlet commands ran"); + const summary = JSON.parse( + await fs.readFile(path.join(outputDir, "plugin-gateway-gauntlet-summary.json"), "utf8"), + ); + expect(summary.guardFailures).toEqual([ + expect.objectContaining({ + kind: "empty-run", + }), + ]); + expect(summary.isolatedRunRootPreserved).toBe(true); + await expect(fs.stat(summary.isolatedRunRoot)).resolves.toBeTruthy(); + await fs.rm(summary.isolatedRunRoot, { recursive: true, force: true }); + }); + + it("cleans the isolated run root after an explicitly empty dry run", async () => { + const outputDir = path.join(repoRoot, "artifacts"); + const result = spawnSync( + process.execPath, + [ + path.resolve("scripts/check-plugin-gateway-gauntlet.mjs"), + "--repo-root", + repoRoot, + "--output-dir", + outputDir, + "--skip-prebuild", + "--skip-lifecycle", + "--skip-slash-help", + "--skip-qa", + "--allow-empty", + ], + { + cwd: path.resolve("."), + encoding: "utf8", + }, + ); + expect(result.status, result.stderr).toBe(0); const summary = JSON.parse( await fs.readFile(path.join(outputDir, "plugin-gateway-gauntlet-summary.json"), "utf8"), ); + expect(summary.guardFailures).toEqual([]); expect(summary.isolatedRunRootPreserved).toBe(false); await expect(fs.stat(summary.isolatedRunRoot)).rejects.toHaveProperty("code", "ENOENT"); });