import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const SCHEMA = { report: "kova.report.v1", gate: "kova.gate.v1", performance: "kova.performance.v1", profiling: "kova.profiling.v1", baseline: "kova.baselineComparison.v1", gateBaseline: "kova.gateBaselineSummary.v1", }; const PROFILED_INTERPRETATION = "instrumented run; CPU/RSS can include profiler and diagnostic overhead"; const RSS_METRICS = ["peakRssMb", "resourcePeakGatewayRssMb"]; const CPU_METRICS = ["cpuPercentMax"]; const DIRECT_VIOLATIONS = new Set(["cpuPercentMax", "peakRssMb"]); const SOAK_VIOLATIONS = new Set(["gatewayRssGrowthMb", "rssGrowthMb"]); const ROLE_VIOLATION = /^resourceByRole\.([^.]+)\.(maxCpuPercent|peakRssMb)$/u; function check(condition, reason) { if (!condition) { throw new Error(reason); } } function object(value, label) { check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`); return value; } function array(value, label) { check(Array.isArray(value), `invalid ${label}`); return value; } function recordViolations(record) { if (!Object.hasOwn(record, "violations")) { return []; } return array(record.violations, "record violations"); } function count(value, label, { positive = false } = {}) { check(Number.isSafeInteger(value) && value >= (positive ? 1 : 0), `invalid ${label}`); return value; } function text(value, label) { check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`); return value; } function finite(value, label) { check(typeof value === "number" && Number.isFinite(value), `invalid ${label}`); return value; } function stateId(record) { return text(object(record.state, "record state").id, "record state id"); } function recordKey(record) { return [ text(record.scenario, "record scenario"), text(record.surface, "record surface"), stateId(record), ].join("\u0000"); } function statusCounts(records) { const statuses = {}; for (const record of records) { text(record.status, "record status"); statuses[record.status] = (statuses[record.status] ?? 0) + 1; } return statuses; } function exactCounts(actualValue, expected, label) { const actual = object(actualValue, label); check(Object.keys(actual).length === Object.keys(expected).length, `${label} keys did not match`); for (const [key, expectedCount] of Object.entries(expected)) { check(count(actual[key], `${label}.${key}`) === expectedCount, `${label}.${key} did not match`); } } function exactStrings(actualValue, expected, label) { const actual = array(actualValue, label); check( actual.length === expected.length && actual.every((value, index) => value === expected[index]), `${label} did not match`, ); } function validateCommandResult(value, label) { const result = object(value, label); check(result.status === 0 && result.timedOut === false, `${label} failed`); } function validateCollectorOnlyPhase(phase) { check( phase.driverKind === "none" && phase.collectionIntent === "post-ready-health", "commandless phase was not a collector-only phase", ); const evidence = array(phase.evidence, "collector-only phase evidence"); check( evidence.length > 0 && evidence.every((entry) => typeof entry === "string" && entry.trim().length > 0), "collector-only phase evidence was invalid", ); check( object(phase.metrics, "collector-only phase metrics").schemaVersion === "kova.envMetrics.v1", "collector-only phase metrics schema was invalid", ); } function alreadyAbsent(result, noun) { check( Number.isSafeInteger(result.status) && result.status !== 0, `${noun} cleanup status was invalid`, ); check(result.timedOut === false, `${noun} cleanup timed out`); const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; const missing = new RegExp(`\\b${noun}\\b[\\s\\S]*\\b(?:does not exist|not found)\\b`, "iu").test( output, ); check(missing, `${noun} cleanup lacked already-absent evidence`); } function validateCleanup(status, resultValue, successStatus, noun, label) { const result = object(resultValue, `${label} result`); if (status === successStatus) { validateCommandResult(result, `${label} result`); return; } check(status === "already-absent", `${label} was not completed`); alreadyAbsent(result, noun); } function validateRecord(recordValue) { const record = object(recordValue, "record"); recordKey(record); object(record.measurements, "record measurements"); const profiling = object(record.profiling, "record profiling"); check(profiling.schemaVersion === SCHEMA.profiling, "wrong profiling schema"); const phases = array(record.phases, "record phases"); check(phases.length > 0, "record had no phases"); for (const phaseValue of phases) { const phase = object(phaseValue, "phase"); const commands = array(phase.commands, "phase commands"); const results = array(phase.results, "phase results"); check(commands.length === results.length, "phase command/result counts did not match"); if (commands.length === 0) { validateCollectorOnlyPhase(phase); continue; } results.forEach((resultValue, index) => { const result = object(resultValue, `phase result ${index}`); check( text(result.command, `phase result ${index} command`) === text(commands[index], `phase command ${index}`), `phase command ${index} did not match its result`, ); validateCommandResult(result, `phase result ${index}`); }); } validateCleanup( record.cleanup, record.cleanupResult, "destroyed", "environment", "record cleanup", ); return record; } function validateTargetCleanup(report) { if (!text(report.target, "report target").startsWith("local-build:")) { check(report.targetCleanup === null, "non-local target had cleanup metadata"); return; } const cleanup = object(report.targetCleanup, "target cleanup"); validateCleanup(cleanup.status, cleanup.result, "removed", "runtime", "target cleanup"); } function validateBaselines(report, gate) { const reportBaseline = report.baseline; const gateBaseline = gate.baseline; check( (reportBaseline === null || reportBaseline === undefined) === (gateBaseline === null || gateBaseline === undefined), "baseline evidence was one-sided", ); if (reportBaseline === null || reportBaseline === undefined) { return; } const comparison = object( object(reportBaseline, "report baseline").comparison, "baseline comparison", ); const summary = object(gateBaseline, "gate baseline"); check(comparison.schemaVersion === SCHEMA.baseline, "wrong baseline comparison schema"); check(summary.schemaVersion === SCHEMA.gateBaseline, "wrong gate baseline schema"); for (const [field, listField] of [ ["regressionCount", "regressions"], ["missingBaselineCount", "missing"], ]) { check(count(comparison[field], `baseline ${field}`) === 0, `baseline ${field} was nonzero`); check( array(comparison[listField], `baseline ${listField}`).length === 0, `baseline ${listField} present`, ); check( count(summary[field], `gate baseline ${field}`) === 0, `gate baseline ${field} was nonzero`, ); const gateList = field === "regressionCount" ? "regressedGroups" : listField; check( array(summary[gateList], `gate baseline ${gateList}`).length === 0, `gate baseline ${gateList} present`, ); } check(comparison.ok === true && summary.ok === true, "baseline was not ok"); check( count(comparison.baselineEntryCount, "baseline entry count") === count(summary.baselineEntryCount, "gate baseline entry count"), "baseline entry counts did not match", ); } function validateSummary(report, records) { const summary = object(report.summary, "report summary"); check( count(summary.total, "summary total") === records.length, "summary total did not match records", ); exactCounts(summary.statuses, statusCounts(records), "summary statuses"); } function validateMetric(metricValue, sampleCount, label) { const metric = object(metricValue, label); const samples = array(metric.samples, `${label} samples`); const metricCount = count(metric.count, `${label} count`, { positive: true }); check(metricCount === samples.length && metricCount <= sampleCount, `${label} count drift`); check( metric.classification === "stable" || metric.classification === "unstable", `${label} classification was invalid`, ); samples.forEach((sample) => finite(sample, `${label} sample`)); return samples; } function sampledMetric(metrics, ids, sampleCount, label) { for (const id of ids) { if (metrics[id] !== undefined) { const samples = validateMetric(metrics[id], sampleCount, `${label}.${id}`); check(samples.length === sampleCount, `${label}.${id} did not cover every record`); return { id, samples }; } } throw new Error(`${label} was not sampled`); } function measured(record, ids, label) { for (const id of ids) { if (record.measurements[id] !== undefined) { return finite(record.measurements[id], `${label}.${id}`); } } throw new Error(`${label} was not measured`); } function validatePerformance(report, records, repeat) { const performance = object(report.performance, "performance"); check(performance.schemaVersion === SCHEMA.performance, "wrong performance schema"); check(performance.repeat === repeat, "performance repeat did not match controls"); const groups = array(performance.groups, "performance groups"); check( count(performance.groupCount, "performance group count") === groups.length, "group count drift", ); const recordsByKey = new Map(); for (const record of records) { const key = recordKey(record); const matching = recordsByKey.get(key) ?? []; matching.push(record); recordsByKey.set(key, matching); } check(groups.length === recordsByKey.size, "performance groups did not map to records"); const groupsByKey = new Map(); let profiledCount = 0; for (const groupValue of groups) { const group = object(groupValue, "performance group"); const identity = [ text(group.scenario, "group scenario"), text(group.surface, "group surface"), text(group.state, "group state"), ]; const key = identity.join("\u0000"); check( group.key === identity.join("|") && !groupsByKey.has(key), "performance group identity was invalid", ); const matching = recordsByKey.get(key); check(matching?.length > 0, "performance group had no records"); check( count(group.sampleCount, "group sample count", { positive: true }) === matching.length, "sample count drift", ); exactCounts(group.statuses, statusCounts(matching), "group statuses"); const matchingProfiled = matching.filter((record) => record.profiling.enabled === true).length; check( count(group.profiledRunCount, "group profiled count") === matchingProfiled, "profile count drift", ); check( group.resourceInterpretation === (matchingProfiled > 0 ? "instrumented" : "normal"), "group resource interpretation was invalid", ); profiledCount += matchingProfiled; const metrics = object(group.metrics, "group metrics"); Object.entries(metrics).forEach(([id, metric]) => validateMetric(metric, group.sampleCount, `group metric ${id}`), ); const rss = sampledMetric(metrics, RSS_METRICS, group.sampleCount, "group RSS"); const cpu = sampledMetric(metrics, CPU_METRICS, group.sampleCount, "group CPU"); for (const record of matching) { check( rss.samples.includes(measured(record, RSS_METRICS, "record RSS")), "record RSS was not sampled", ); check( cpu.samples.includes(measured(record, CPU_METRICS, "record CPU")), "record CPU was not sampled", ); } groupsByKey.set(key, group); } check( count(performance.profiledRunCount, "performance profiled count") === profiledCount, "profile total drift", ); const unstableCount = groups.filter((group) => Object.values(group.metrics).some((metric) => metric.classification === "unstable"), ).length; check( count(performance.unstableGroupCount, "unstable group count") === unstableCount, "unstable group count drift", ); return groupsByKey; } function validateGateCards(gate) { const cards = array(gate.cards, "gate cards"); const severities = { blocking: 0, warning: 0, info: 0 }; for (const cardValue of cards) { const card = object(cardValue, "gate card"); const severity = card.severity; check(Object.hasOwn(severities, severity), "unknown gate card severity"); if (severity === "info") { check( (card.kind === "filtered-required-scenario" || card.kind === "filtered-required-coverage") && card.status === "MISSING", "unexpected info gate card", ); } if (severity === "warning") { check( card.kind === "missing-required-coverage" && card.status === "MISSING", "unexpected warning gate card", ); } severities[severity] += 1; } check( count(gate.blockingCount, "blocking count") === severities.blocking, "blocking count drift", ); check(count(gate.warningCount, "warning count") === severities.warning, "warning count drift"); check(count(gate.infoCount, "info count") === severities.info, "info count drift"); check( count(gate.missingRequiredCount, "missing required count") === severities.info, "missing count drift", ); return cards; } function validateEnvelope(reportValue) { const report = object(reportValue, "report"); const gate = object(report.gate, "gate"); const controls = object(report.controls, "controls"); check( report.schemaVersion === SCHEMA.report && report.mode === "execution", "wrong report schema or mode", ); check( gate.schemaVersion === SCHEMA.gate && gate.enabled === true, "wrong or disabled gate schema", ); check(controls.gate === true, "gate controls were disabled"); const filters = [ ...array(controls.include, "include filters"), ...array(controls.exclude, "exclude filters"), ]; check( filters.length > 0 && filters.every((filter) => typeof filter === "string" && filter.trim().length > 0), "report filters were invalid", ); const repeat = count(controls.repeat, "repeat", { positive: true }); check( gate.partial === true && gate.complete === false && gate.ok === false, "gate metadata was not partial", ); const records = array(report.records, "records").map(validateRecord); check(records.length > 0, "report had no records"); const cards = validateGateCards(gate); validateBaselines(report, gate); validateSummary(report, records); validateTargetCleanup(report); const groups = validatePerformance(report, records, repeat); return { report, gate, records, cards, groups }; } function deepProfiled(record) { const profiling = record.profiling; const measurements = record.measurements; return [ [profiling.enabled, true], [profiling.deepProfile, true], [profiling.nodeProfile, true], [profiling.heapSnapshot, true], [profiling.diagnosticReport, true], [profiling.profileOnFailure, false], [profiling.affectsResourceMeasurements, true], [profiling.baselineEligible, false], [profiling.interpretation, PROFILED_INTERPRETATION], [measurements.profilingEnabled, true], [measurements.profilingAffectsResourceMeasurements, true], [measurements.profilingBaselineEligible, false], [measurements.profilingResourceInterpretation, PROFILED_INTERPRETATION], ].every(([actual, expected]) => actual === expected); } function violationMeasurement(record, violation) { const metric = text(violation.metric, "violation metric"); if (violation.kind === "threshold" && DIRECT_VIOLATIONS.has(metric)) { return finite(record.measurements[metric], `measurement ${metric}`); } if (violation.kind === "soak" && SOAK_VIOLATIONS.has(metric)) { return finite(record.measurements[metric], `measurement ${metric}`); } const match = violation.kind === "resource" ? ROLE_VIOLATION.exec(metric) : null; check(match !== null && violation.role === match[1], "role violation identity was invalid"); const role = object( object(record.measurements.resourceByRole, "role measurements")[match[1]], "role measurement", ); return finite(role[match[2]], `role measurement ${metric}`); } function validateProfiledFailure(record, card, group) { check(deepProfiled(record), "failed record was not canonical deep profiling"); check( group.resourceInterpretation === "instrumented", "failed record group was not instrumented", ); const violations = recordViolations(record); check(violations.length > 0, "failed record had no violations"); for (const violationValue of violations) { const violation = object(violationValue, "violation"); const actual = finite(violation.actual, "violation actual"); check(actual === violationMeasurement(record, violation), "violation evidence drift"); if (violation.kind === "threshold") { const metric = text(violation.metric, "violation metric"); const samples = validateMetric( object(group.metrics, "group metrics")[metric], group.sampleCount, `violation group metric ${metric}`, ); check(samples.includes(actual), "violation was not sampled in its performance group"); } text(violation.expected, "violation expectation"); text(violation.message, "violation message"); } const messages = violations.map((violation) => violation.message); check( card.kind === "openclaw-failure" && card.status === "FAIL" && card.failedCommand === null && card.scenario === record.scenario && card.state === stateId(record), "blocking card identity was invalid", ); check(card.summary === messages[0], "blocking card summary did not match"); exactStrings(card.violations, messages, "blocking card violations"); const cardMeasurements = object(card.measurements, "blocking card measurements"); check( cardMeasurements.peakRssMb === record.measurements.peakRssMb && cardMeasurements.cpuPercentMax === record.measurements.cpuPercentMax, "blocking card measurements did not match", ); } function evaluate(evaluator) { try { evaluator(); return { ok: true }; } catch (error) { return { ok: false, reason: error instanceof Error ? error.message : String(error) }; } } export function evaluateToleratedPartialKovaReport(report) { return evaluate(() => { const { gate, records, cards } = validateEnvelope(report); check(gate.verdict === "PARTIAL", "gate verdict was not PARTIAL"); check(gate.blockingCount === 0, "PARTIAL gate had blocking cards"); check( cards.every((card) => card.severity !== "blocking"), "PARTIAL gate had a blocking card", ); check( records.every((record) => record.status === "PASS"), "PARTIAL report had a non-PASS record", ); check( records.every((record) => recordViolations(record).length === 0), "PARTIAL report had violations", ); }); } export function evaluateToleratedProfiledKovaReport(report) { return evaluate(() => { const { gate, records, cards, groups } = validateEnvelope(report); check(gate.verdict === "DO_NOT_SHIP", "gate verdict was not DO_NOT_SHIP"); check( records.every((record) => record.status === "PASS" || record.status === "FAIL"), "invalid record status", ); check( records .filter((record) => record.status === "PASS") .every((record) => recordViolations(record).length === 0), "PASS record had violations", ); const failed = records.filter((record) => record.status === "FAIL"); const blocking = cards.filter((card) => card.severity === "blocking"); check( failed.length > 0 && blocking.length === failed.length, "failure/card counts did not match", ); const remaining = [...blocking]; for (const record of failed) { const index = remaining.findIndex( (card) => card.scenario === record.scenario && card.state === stateId(record), ); check(index >= 0, "blocking cards did not map one-to-one"); const card = remaining.splice(index, 1)[0]; validateProfiledFailure(record, card, groups.get(recordKey(record))); } check(remaining.length === 0, "blocking cards did not map one-to-one"); }); } export function evaluateToleratedKovaReport(report) { const partial = evaluateToleratedPartialKovaReport(report); if (partial.ok) { return { ok: true, classification: "filtered-partial" }; } const profiled = evaluateToleratedProfiledKovaReport(report); if (profiled.ok) { return { ok: true, classification: "profiled-resource-only" }; } return { ok: false, reason: `partial: ${partial.reason}; profiled: ${profiled.reason}` }; } function readCliReportPath() { const reportPath = process.argv[2] || process.env.REPORT_JSON; if (!reportPath) { throw new Error("usage: node scripts/lib/kova-report-gate.mjs "); } return reportPath; } const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url)); const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(process.argv[1])) : ""; if (modulePath === invokedPath) { try { const report = JSON.parse(fs.readFileSync(readCliReportPath(), "utf8")); const result = evaluateToleratedKovaReport(report); if (!result.ok) { console.error(`Kova verdict is not tolerable: ${result.reason}`); process.exit(1); } console.log(`Tolerated Kova verdict: ${result.classification}`); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } }