From dfd52e72b3785a8ff951fa9d5e299ea412f0b189 Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Sat, 6 Jun 2026 21:13:08 +0200
Subject: [PATCH] fix(test): reject empty Kova summaries

---
 scripts/kova-ci-summary.mjs          | 28 +++++++++++
 test/scripts/kova-ci-summary.test.ts | 69 ++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)

diff --git a/scripts/kova-ci-summary.mjs b/scripts/kova-ci-summary.mjs
index d85e72b120b..548552e2b93 100644
--- a/scripts/kova-ci-summary.mjs
+++ b/scripts/kova-ci-summary.mjs
@@ -32,6 +32,11 @@ const keyMetricIds = [
 
 const reportPath = path.resolve(args.report);
 const report = JSON.parse(await readFile(reportPath, "utf8"));
+const invalidReport = validateKovaSummaryReport(report);
+if (invalidReport) {
+  console.error(`error: invalid Kova report: ${invalidReport}`);
+  process.exit(1);
+}
 const markdown = renderSummary(report, {
   lane: args.lane || "kova",
   reportUrl: args.reportUrl || "",
@@ -152,6 +157,29 @@ function renderSummary(reportLocal, options) {
   return `${lines.join("\n").trimEnd()}\n`;
 }
 
+function validateKovaSummaryReport(reportLocal) {
+  if (!reportLocal || typeof reportLocal !== "object" || Array.isArray(reportLocal)) {
+    return "report must be an object";
+  }
+  const statuses = reportLocal.summary?.statuses;
+  if (
+    !statuses ||
+    typeof statuses !== "object" ||
+    Array.isArray(statuses) ||
+    Object.keys(statuses).length === 0
+  ) {
+    return "missing summary.statuses";
+  }
+  const records = Array.isArray(reportLocal.records) ? reportLocal.records : [];
+  const groups = Array.isArray(reportLocal.performance?.groups)
+    ? reportLocal.performance.groups
+    : [];
+  if (records.length === 0 && groups.length === 0) {
+    return "missing records or performance groups";
+  }
+  return null;
+}
+
 function collectViolations(records) {
   if (!Array.isArray(records)) {
     return [];
diff --git a/test/scripts/kova-ci-summary.test.ts b/test/scripts/kova-ci-summary.test.ts
index d5871475c01..0bfaa09704d 100644
--- a/test/scripts/kova-ci-summary.test.ts
+++ b/test/scripts/kova-ci-summary.test.ts
@@ -1,7 +1,31 @@
 // Kova Ci Summary tests cover kova ci summary script behavior.
 import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, it } from "vitest";
 
+function runSummary(report: unknown) {
+  const root = mkdtempSync(join(tmpdir(), "openclaw-kova-summary-"));
+  const reportPath = join(root, "report.json");
+  const outputPath = join(root, "summary.md");
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
+  const result = spawnSync(
+    process.execPath,
+    ["scripts/kova-ci-summary.mjs", "--report", reportPath, "--output", outputPath],
+    {
+      cwd: process.cwd(),
+      encoding: "utf8",
+    },
+  );
+  let output = "";
+  try {
+    output = readFileSync(outputPath, "utf8");
+  } catch {}
+  rmSync(root, { force: true, recursive: true });
+  return { output, result };
+}
+
 describe("scripts/kova-ci-summary", () => {
   it("prints help without treating --help as a valued option", () => {
     const result = spawnSync(process.execPath, ["scripts/kova-ci-summary.mjs", "--help"], {
@@ -13,4 +37,49 @@ describe("scripts/kova-ci-summary", () => {
     expect(result.stderr).toBe("");
     expect(result.stdout).toContain("usage: node scripts/kova-ci-summary.mjs --report");
   });
+
+  it("rejects empty Kova reports instead of rendering unknown summaries", () => {
+    const empty = runSummary({});
+    expect(empty.result.status).toBe(1);
+    expect(empty.result.stderr).toContain("invalid Kova report: missing summary.statuses");
+
+    const noEvidence = runSummary({ summary: { statuses: { pass: 1 } } });
+    expect(noEvidence.result.status).toBe(1);
+    expect(noEvidence.result.stderr).toContain(
+      "invalid Kova report: missing records or performance groups",
+    );
+  });
+
+  it("renders a Kova summary when status and evidence are present", () => {
+    const { output, result } = runSummary({
+      generatedAt: "2026-06-06T00:00:00.000Z",
+      performance: {
+        repeat: 1,
+        groups: [
+          {
+            metrics: {
+              timeToHealthReadyMs: {
+                count: 1,
+                max: 30,
+                median: 20,
+                p95: 30,
+                title: "Health ready",
+                unit: "ms",
+              },
+            },
+            scenario: "gateway",
+            state: "clean",
+          },
+        ],
+      },
+      records: [{ scenario: "gateway", state: "clean", status: "pass" }],
+      runId: "run-1",
+      summary: { statuses: { pass: 1 } },
+      target: "main",
+    });
+
+    expect(result.status).toBe(0);
+    expect(output).toContain("- Statuses: pass: 1");
+    expect(output).toContain("| gateway | clean | Health ready | 20 ms | 30 ms | 30 ms |");
+  });
 });