fix(ci): verify live provider traffic in performance runs (#103073)

* fix(diagnostics): emit provider request timeline events

* fix(ci): verify Kova live provider evidence

.github/workflows/openclaw-performance.yml

@@ -247,6 +247,27 @@ jobs:
           chmod 0755 "$HOME/.local/bin/kova"
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
+      - name: Allow live auth for OpenAI candidate state
+        if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          node - <<'NODE'
+          const fs = require("node:fs");
+          const path = require("node:path");
+          const file = path.join(process.env.KOVA_SRC, "states/mock-openai-provider.json");
+          const state = JSON.parse(fs.readFileSync(file, "utf8"));
+          if (state.auth?.mode !== "mock") {
+            throw new Error(`expected mock-openai-provider auth.mode=mock, got ${state.auth?.mode}`);
+          }
+
+          // The release profile pairs the live agent scenario with this otherwise mock-only state.
+          // This ephemeral checkout must honor the lane's explicit --auth live selection.
+          state.auth.mode = "default";
+          state.auth.reason = "Honor the workflow lane's explicit run-level auth selection.";
+          fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+          NODE
+
       - name: Configure OCM local workspace dependencies
         if: steps.lane.outputs.run == 'true'
         shell: bash
@@ -292,12 +313,24 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
+          repeat="$REQUESTED_REPEAT"
+          if [[ "$MATRIX_REPEAT" != "input" ]]; then
+            repeat="$MATRIX_REPEAT"
+          fi
+          plan_dir="${RUNNER_TEMP}/kova-plans"
+          plan_json="${plan_dir}/${LANE_ID}.json"
+          mkdir -p "$plan_dir"
+
           kova version --json
           kova matrix plan \
             --profile "$PROFILE" \
             --target "local-build:${GITHUB_WORKSPACE}" \
-            --include scenario:fresh-install \
-            --json >/tmp/kova-plan.json
+            --include "$INCLUDE_FILTERS" \
+            --parallel 1 \
+            --repeat "$repeat" \
+            --json >"$plan_json"
+          echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"
+          echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"
 
       - name: Configure live OpenAI auth
         if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
@@ -326,15 +359,13 @@ jobs:
           set -euo pipefail
           mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR"
 
-          repeat="$REQUESTED_REPEAT"
-          if [[ "$MATRIX_REPEAT" != "input" ]]; then
-            repeat="$MATRIX_REPEAT"
-          fi
+          repeat="$KOVA_LANE_REPEAT"
 
           args=(
             matrix run
             --profile "$PROFILE"
             --target "local-build:${GITHUB_WORKSPACE}"
+            --include "$INCLUDE_FILTERS"
             --auth "$AUTH_MODE"
             --parallel 1
             --repeat "$repeat"
@@ -344,8 +375,6 @@ jobs:
             --json
           )
 
-          args+=(--include "$INCLUDE_FILTERS")
-
           if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then
             args+=(--deep-profile)
           fi
@@ -365,6 +394,15 @@ jobs:
             exit 1
           fi
           report_md="${report_json%.json}.md"
+          node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \
+            --plan "$KOVA_PLAN_JSON" \
+            --report "$report_json" \
+            --profile "$PROFILE" \
+            --target "local-build:${GITHUB_WORKSPACE}" \
+            --repeat "$repeat" \
+            --include "$INCLUDE_FILTERS" \
+            --auth "$AUTH_MODE"
+
           effective_status="$status"
           if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
             if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"

scripts/lib/kova-workflow-evidence.mjs

@@ -0,0 +1,269 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const AUTH_MODES = new Set(["live", "mock"]);
+const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]);
+
+function check(condition, reason) {
+  if (!condition) {
+    throw new Error(reason);
+  }
+}
+
+function object(value, label) {
+  check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`);
+  return value;
+}
+
+function array(value, label) {
+  check(Array.isArray(value), `invalid ${label}`);
+  return value;
+}
+
+function text(value, label) {
+  check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`);
+  return value;
+}
+
+function positiveInteger(value, label) {
+  check(Number.isSafeInteger(value) && value > 0, `invalid ${label}`);
+  return value;
+}
+
+function exactStrings(value, expected, label) {
+  const actual = array(value, label);
+  check(
+    actual.length === expected.length && actual.every((item, index) => item === expected[index]),
+    `${label} did not match`,
+  );
+}
+
+function pairKey(scenario, state, label) {
+  return `${text(scenario, `${label} scenario`)}\u0000${text(state, `${label} state`)}`;
+}
+
+function displayPair(key) {
+  return key.replace("\u0000", "/");
+}
+
+function profileId(value, label) {
+  return text(object(value, label).id, `${label} id`);
+}
+
+function repeatIndexesFor(selectedPairs) {
+  return new Map([...selectedPairs].map((key) => [key, new Set()]));
+}
+
+function validateLiveRecord(record, key) {
+  const auth = object(record.auth, `record ${displayPair(key)} auth`);
+  const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`);
+  check(
+    auth.environmentDependent === true,
+    `live record ${displayPair(key)} auth was not environment-dependent`,
+  );
+  check(provider.authMode === "live", `live record ${displayPair(key)} provider auth was not live`);
+  check(
+    provider.environmentDependent === true,
+    `live record ${displayPair(key)} provider evidence was not environment-dependent`,
+  );
+  check(
+    provider.source === "openclaw-timeline",
+    `live record ${displayPair(key)} provider source was ${provider.source}`,
+  );
+  check(
+    provider.available === true,
+    `live record ${displayPair(key)} provider evidence was unavailable`,
+  );
+  positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`);
+}
+
+export function validateKovaWorkflowEvidence({
+  plan,
+  report,
+  profile,
+  target,
+  repeat,
+  includeFilters,
+  authMode,
+}) {
+  const expectedProfile = text(profile, "expected profile");
+  const expectedTarget = text(target, "expected target");
+  const expectedRepeat = positiveInteger(repeat, "expected repeat");
+  const expectedFilters = array(includeFilters, "expected include filters").map((value, index) =>
+    text(value, `expected include filter ${index}`),
+  );
+  const expectedAuth = text(authMode, "expected auth mode");
+  check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`);
+  check(expectedFilters.length > 0, "expected include filters were empty");
+
+  const lanePlan = object(plan, "lane plan");
+  const laneReport = object(report, "lane report");
+  check(lanePlan.schemaVersion === "kova.matrix.plan.v1", "unexpected lane plan schema");
+  check(laneReport.schemaVersion === "kova.report.v1", "unexpected lane report schema");
+  check(laneReport.mode === "execution", `lane report mode was ${laneReport.mode}`);
+  check(
+    profileId(lanePlan.profile, "lane plan profile") === expectedProfile,
+    "lane plan profile did not match",
+  );
+  check(
+    profileId(laneReport.profile, "lane report profile") === expectedProfile,
+    "lane report profile did not match",
+  );
+  check(lanePlan.target === expectedTarget, "lane plan target did not match");
+  check(laneReport.target === expectedTarget, "lane report target did not match");
+
+  const planControls = object(lanePlan.controls, "lane plan controls");
+  const reportControls = object(laneReport.controls, "lane report controls");
+  exactStrings(planControls.include, expectedFilters, "lane plan include filters");
+  exactStrings(reportControls.include, expectedFilters, "lane report include filters");
+  check(planControls.repeat === expectedRepeat, "lane plan repeat did not match");
+  check(reportControls.repeat === expectedRepeat, "lane report repeat did not match");
+
+  const selectedPairs = new Set();
+  for (const [index, value] of array(lanePlan.entries, "lane plan entries").entries()) {
+    const entry = object(value, `lane plan entry ${index}`);
+    const status = text(entry.status, `lane plan entry ${index} status`);
+    check(status === "SELECTED" || status === "SKIPPED", `invalid lane plan entry ${index} status`);
+    if (status === "SKIPPED") {
+      continue;
+    }
+    const key = pairKey(
+      object(entry.scenario, `lane plan entry ${index} scenario`).id,
+      object(entry.state, `lane plan entry ${index} state`).id,
+      `lane plan entry ${index}`,
+    );
+    check(!selectedPairs.has(key), `lane plan selected duplicate ${displayPair(key)}`);
+    selectedPairs.add(key);
+  }
+  check(selectedPairs.size > 0, "lane plan selected no scenario/state pairs");
+
+  const reportAuth = object(laneReport.auth, "lane report auth");
+  check(reportAuth.requestedMode === expectedAuth, "lane report requested auth did not match");
+  if (expectedAuth === "live") {
+    check(
+      object(reportAuth.live, "lane report live auth").environmentDependent === true,
+      "lane report live auth was not environment-dependent",
+    );
+  }
+
+  const records = array(laneReport.records, "lane report records");
+  const repeatIndexes = repeatIndexesFor(selectedPairs);
+  for (const [index, value] of records.entries()) {
+    const record = object(value, `lane report record ${index}`);
+    const state = object(record.state, `lane report record ${index} state`);
+    const key = pairKey(record.scenario, state.id, `lane report record ${index}`);
+    check(selectedPairs.has(key), `lane report contained unexpected ${displayPair(key)}`);
+    const status = text(record.status, `record ${displayPair(key)} status`);
+    check(
+      ["BLOCKED", "FAIL", "PASS"].includes(status),
+      `invalid record ${displayPair(key)} status`,
+    );
+
+    const recordRepeat = object(record.repeat, `record ${displayPair(key)} repeat`);
+    check(
+      recordRepeat.total === expectedRepeat,
+      `record ${displayPair(key)} repeat total did not match`,
+    );
+    const repeatIndex = positiveInteger(
+      recordRepeat.index,
+      `record ${displayPair(key)} repeat index`,
+    );
+    check(
+      repeatIndex <= expectedRepeat,
+      `record ${displayPair(key)} repeat index exceeded ${expectedRepeat}`,
+    );
+    const indexes = repeatIndexes.get(key);
+    check(!indexes.has(repeatIndex), `record ${displayPair(key)} repeated index ${repeatIndex}`);
+    indexes.add(repeatIndex);
+
+    const recordAuth = object(record.auth, `record ${displayPair(key)} auth`);
+    check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`);
+    if (expectedAuth === "live") {
+      validateLiveRecord(record, key);
+    }
+  }
+
+  for (const [key, indexes] of repeatIndexes) {
+    check(
+      indexes.size === expectedRepeat,
+      `lane report collapsed ${displayPair(key)} coverage to ${indexes.size}/${expectedRepeat} repeats`,
+    );
+    for (let index = 1; index <= expectedRepeat; index += 1) {
+      check(indexes.has(index), `lane report missed ${displayPair(key)} repeat ${index}`);
+    }
+  }
+  check(
+    records.length === selectedPairs.size * expectedRepeat,
+    "lane report record count did not match selected coverage",
+  );
+
+  return {
+    authMode: expectedAuth,
+    pairCount: selectedPairs.size,
+    recordCount: records.length,
+    repeat: expectedRepeat,
+  };
+}
+
+function parseCliArgs(argv) {
+  const flags = {};
+  for (let index = 0; index < argv.length; index += 2) {
+    const name = argv[index];
+    const value = argv[index + 1];
+    check(name?.startsWith("--") && value !== undefined, "invalid CLI arguments");
+    const key = name.slice(2);
+    check(CLI_KEYS.has(key), `unknown --${key}`);
+    check(!Object.hasOwn(flags, key), `duplicate --${key}`);
+    flags[key] = value;
+  }
+  for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) {
+    text(flags[key], `--${key}`);
+  }
+  return flags;
+}
+
+function readJson(pathValue, label) {
+  const file = text(pathValue, label);
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
+      { cause: error },
+    );
+  }
+}
+
+function runCli() {
+  const flags = parseCliArgs(process.argv.slice(2));
+  const result = validateKovaWorkflowEvidence({
+    plan: readJson(flags.plan, "--plan"),
+    report: readJson(flags.report, "--report"),
+    profile: flags.profile,
+    target: flags.target,
+    repeat: Number(flags.repeat),
+    includeFilters: flags.include
+      .split(",")
+      .map((value) => value.trim())
+      .filter(Boolean),
+    authMode: flags.auth,
+  });
+  console.log(
+    `Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`,
+  );
+}
+
+const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url));
+const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(process.argv[1])) : "";
+
+if (modulePath === invokedPath) {
+  try {
+    runCli();
+  } catch (error) {
+    console.error(
+      `Kova evidence validation failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+    process.exit(1);
+  }
+}

src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts

@@ -1,6 +1,9 @@
 // Coverage for model-call diagnostic events around attempt stream functions.
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { useAutoCleanupTempDirTracker } from "../../../../test/helpers/temp-dir.js";
 import {
   onInternalDiagnosticEvent,
   onTrustedInternalDiagnosticEvent,
@@ -20,8 +23,11 @@ import {
   resetGlobalHookRunner,
 } from "../../../plugins/hook-runner-global.js";
 import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-helpers.js";
+import { withEnvAsync } from "../../../test-utils/env.js";
 import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js";
 
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
+
 async function collectModelCallEvents(run: () => Promise<void>): Promise<DiagnosticEventPayload[]> {
   // Diagnostics are emitted asynchronously; collect only public model-call
   // events and flush one tick after the stream completes.
@@ -112,6 +118,24 @@ function requireMockRecordArg(
   return requireRecord(mock.mock.calls[callIndex]?.[argIndex], label);
 }
 
+async function collectProviderTimelineEvents(run: () => Promise<void>) {
+  const root = tempDirs.make("openclaw-provider-timeline-");
+  const timelinePath = join(root, "timeline.jsonl");
+  await withEnvAsync(
+    {
+      OPENCLAW_DIAGNOSTICS: "1",
+      OPENCLAW_DIAGNOSTICS_TIMELINE_PATH: timelinePath,
+    },
+    run,
+  );
+  return readFileSync(timelinePath, "utf8")
+    .trim()
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => requireRecord(JSON.parse(line), "provider timeline event"))
+    .filter((event) => event.type === "provider.request");
+}
+
 describe("wrapStreamFnWithDiagnosticModelCallEvents", () => {
   beforeEach(() => {
     resetDiagnosticEventsForTest();
@@ -204,6 +228,101 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => {
     expect(JSON.stringify(events)).not.toContain("sk-test-secret-value");
   });
 
+  it("emits one successful provider timeline event for result and iterator completion", async () => {
+    let now = Date.parse("2026-07-09T18:30:00.000Z");
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    async function* stream() {
+      yield { type: "text", text: "ok" };
+    }
+    const originalStream = stream() as unknown as AsyncIterable<unknown> & {
+      result: () => Promise<string>;
+    };
+    originalStream.result = async () => {
+      now += 125;
+      return "kept";
+    };
+    const wrapped = wrapStreamFnWithDiagnosticModelCallEvents(
+      (() => originalStream) as unknown as StreamFn,
+      {
+        runId: "run-timeline-success",
+        provider: "openai",
+        model: "gpt-5.5",
+        api: "openai-responses",
+        transport: "http",
+        trace: createDiagnosticTraceContext(),
+        nextCallId: () => "call-timeline-success",
+      },
+    );
+
+    const events = await collectProviderTimelineEvents(async () => {
+      const returned = wrapped(
+        {} as never,
+        {} as never,
+        {} as never,
+      ) as unknown as typeof originalStream;
+      await returned.result();
+      await drain(returned);
+    });
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      type: "provider.request",
+      name: "provider.request",
+      timestamp: "2026-07-09T18:30:00.000Z",
+      runId: "run-timeline-success",
+      spanId: "call-timeline-success",
+      durationMs: 125,
+      provider: "openai",
+      operation: "openai-responses",
+      ok: true,
+      attributes: {
+        model: "gpt-5.5",
+        api: "openai-responses",
+        transport: "http",
+      },
+    });
+  });
+
+  it("emits one failed provider timeline event for a thrown model call", async () => {
+    let now = Date.parse("2026-07-09T18:31:00.000Z");
+    vi.spyOn(Date, "now").mockImplementation(() => now);
+    const wrapped = wrapStreamFnWithDiagnosticModelCallEvents(
+      (() => {
+        now += 75;
+        throw new Error("provider failed");
+      }) as unknown as StreamFn,
+      {
+        runId: "run-timeline-error",
+        provider: "anthropic",
+        model: "claude-sonnet-4-6",
+        transport: "sse",
+        trace: createDiagnosticTraceContext(),
+        nextCallId: () => "call-timeline-error",
+      },
+    );
+
+    const events = await collectProviderTimelineEvents(async () => {
+      expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("provider failed");
+    });
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      type: "provider.request",
+      name: "provider.request",
+      timestamp: "2026-07-09T18:31:00.000Z",
+      runId: "run-timeline-error",
+      spanId: "call-timeline-error",
+      durationMs: 75,
+      provider: "anthropic",
+      operation: "sse",
+      ok: false,
+      attributes: {
+        model: "claude-sonnet-4-6",
+        transport: "sse",
+      },
+    });
+  });
+
   it("updates diagnostic run activity from throttled stream chunks", async () => {
     let now = 1_000_000;
     vi.spyOn(Date, "now").mockImplementation(() => now);

src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts

@@ -26,6 +26,7 @@ import {
   formatDiagnosticTraceparent,
   type DiagnosticTraceContext,
 } from "../../../infra/diagnostic-trace-context.js";
+import { emitDiagnosticsTimelineEvent } from "../../../infra/diagnostics-timeline.js";
 import { markDiagnosticRunProgress } from "../../../logging/diagnostic-run-activity.js";
 import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js";
 import type {
@@ -99,6 +100,7 @@ const MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS = 30_000;
 const MODEL_CALL_STREAM_PROGRESS_REASON = "model_call:stream_progress";
 const MODEL_CALL_STREAM_RETURN_TIMEOUT_MS = 1000;
 const TRACEPARENT_HEADER_NAME = "traceparent";
+const TIMELINE_ATTRIBUTE_MAX_LENGTH = 256;
 type ModelCallStreamOptions = Parameters<StreamFn>[2];
 
 function utf8JsonByteLength(value: unknown): number | undefined {
@@ -417,6 +419,39 @@ function modelCallUsageField(state: ModelCallObservationState) {
   return state.usage ? { usage: state.usage } : {};
 }
 
+function boundedTimelineAttribute(value: string | undefined): string | undefined {
+  const normalized = value?.trim();
+  return normalized ? normalized.slice(0, TIMELINE_ATTRIBUTE_MAX_LENGTH) : undefined;
+}
+
+function emitProviderRequestTimelineEvent(
+  eventBase: ModelCallEventBase,
+  startedAt: number,
+  durationMs: number,
+  ok: boolean,
+): void {
+  const provider = boundedTimelineAttribute(eventBase.provider);
+  const model = boundedTimelineAttribute(eventBase.model);
+  const api = boundedTimelineAttribute(eventBase.api);
+  const transport = boundedTimelineAttribute(eventBase.transport);
+  emitDiagnosticsTimelineEvent({
+    type: "provider.request",
+    name: "provider.request",
+    timestamp: new Date(startedAt).toISOString(),
+    runId: eventBase.runId,
+    spanId: eventBase.callId,
+    durationMs,
+    provider,
+    operation: api ?? transport ?? "model.call",
+    ok,
+    attributes: {
+      ...(model ? { model } : {}),
+      ...(api ? { api } : {}),
+      ...(transport ? { transport } : {}),
+    },
+  });
+}
+
 function modelCallErrorFields(err: unknown): ModelCallErrorFields {
   const upstreamRequestIdHash = diagnosticProviderRequestIdHash(err);
   const failureKind = diagnosticErrorFailureKind(err);
@@ -537,6 +572,7 @@ function emitModelCallCompleted(
   state.terminalEventEmitted = true;
   const durationMs = Date.now() - startedAt;
   const sizeTimingFields = modelCallSizeTimingFields(state);
+  emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, true);
   emitTrustedDiagnosticEventWithPrivateData(
     {
       type: "model.call.completed",
@@ -566,6 +602,7 @@ function emitModelCallError(
   state.terminalEventEmitted = true;
   const durationMs = Date.now() - startedAt;
   const sizeTimingFields = modelCallSizeTimingFields(state);
+  emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, false);
   emitTrustedDiagnosticEventWithPrivateData(
     {
       type: "model.call.error",

test/scripts/kova-workflow-evidence.test.ts

@@ -0,0 +1,304 @@
+import { spawnSync } from "node:child_process";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { validateKovaWorkflowEvidence } from "../../scripts/lib/kova-workflow-evidence.mjs";
+import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
+
+type JsonObject = Record<string, unknown>;
+type Pair = {
+  scenario: string;
+  state: string;
+};
+
+const PROFILE = "release";
+const TARGET = "local-build:/work/openclaw";
+const INCLUDE_FILTERS = ["scenario:scenario-a", "scenario:scenario-b"];
+const FIRST_PAIR: Pair = { scenario: "scenario-a", state: "state-a" };
+const PAIRS: Pair[] = [FIRST_PAIR, { scenario: "scenario-b", state: "state-b" }];
+const SCRIPT_PATH = "scripts/lib/kova-workflow-evidence.mjs";
+const tempRoots = useAutoCleanupTempDirTracker(afterEach);
+
+function plan(
+  pairs: Pair[] = PAIRS,
+  repeat = 2,
+  includeFilters: string[] = INCLUDE_FILTERS,
+): JsonObject {
+  return {
+    schemaVersion: "kova.matrix.plan.v1",
+    profile: { id: PROFILE },
+    target: TARGET,
+    controls: {
+      include: includeFilters,
+      repeat,
+    },
+    entries: pairs.map((pair) => ({
+      scenario: { id: pair.scenario },
+      state: { id: pair.state },
+      status: "SELECTED",
+    })),
+  };
+}
+
+function record(pair: Pair, repeatIndex: number, repeat: number, authMode: "live" | "mock") {
+  return {
+    scenario: pair.scenario,
+    state: { id: pair.state },
+    status: "PASS",
+    repeat: {
+      index: repeatIndex,
+      total: repeat,
+    },
+    auth: {
+      mode: authMode,
+      environmentDependent: authMode === "live",
+    },
+    providerEvidence:
+      authMode === "live"
+        ? {
+            authMode: "live",
+            available: true,
+            environmentDependent: true,
+            requestCount: 1,
+            source: "openclaw-timeline",
+          }
+        : {
+            authMode: "mock",
+            available: true,
+            environmentDependent: false,
+            requestCount: 1,
+            source: "mock-provider-log",
+          },
+  };
+}
+
+function report({
+  authMode = "mock",
+  includeFilters = INCLUDE_FILTERS,
+  pairs = PAIRS,
+  repeat = 2,
+}: {
+  authMode?: "live" | "mock";
+  includeFilters?: string[];
+  pairs?: Pair[];
+  repeat?: number;
+} = {}): JsonObject {
+  return {
+    schemaVersion: "kova.report.v1",
+    mode: "execution",
+    profile: { id: PROFILE },
+    target: TARGET,
+    controls: {
+      include: includeFilters,
+      repeat,
+    },
+    auth: {
+      requestedMode: authMode,
+      live: {
+        environmentDependent: authMode === "live",
+      },
+    },
+    records: pairs.flatMap((pair) =>
+      Array.from({ length: repeat }, (_, index) => record(pair, index + 1, repeat, authMode)),
+    ),
+  };
+}
+
+function validate(
+  lanePlan: JsonObject,
+  laneReport: JsonObject,
+  options: { authMode?: "live" | "mock"; includeFilters?: string[]; repeat?: number } = {},
+) {
+  return validateKovaWorkflowEvidence({
+    plan: lanePlan,
+    report: laneReport,
+    profile: PROFILE,
+    target: TARGET,
+    repeat: options.repeat ?? 2,
+    includeFilters: options.includeFilters ?? INCLUDE_FILTERS,
+    authMode: options.authMode ?? "mock",
+  });
+}
+
+function recordsOf(laneReport: JsonObject): JsonObject[] {
+  return laneReport.records as JsonObject[];
+}
+
+function firstRecordOf(laneReport: JsonObject): JsonObject {
+  const first = recordsOf(laneReport)[0];
+  if (!first) {
+    throw new Error("fixture report has no records");
+  }
+  return first;
+}
+
+function runCli({
+  lanePlan = JSON.stringify(plan()),
+  laneReport = JSON.stringify(report()),
+  repeat = "2",
+  includeAuth = true,
+}: {
+  lanePlan?: string;
+  laneReport?: string;
+  repeat?: string;
+  includeAuth?: boolean;
+} = {}) {
+  const root = tempRoots.make("openclaw-kova-evidence-");
+  const planPath = join(root, "plan.json");
+  const reportPath = join(root, "report.json");
+  writeFileSync(planPath, lanePlan);
+  writeFileSync(reportPath, laneReport);
+  const args = [
+    SCRIPT_PATH,
+    "--plan",
+    planPath,
+    "--report",
+    reportPath,
+    "--profile",
+    PROFILE,
+    "--target",
+    TARGET,
+    "--repeat",
+    repeat,
+    "--include",
+    INCLUDE_FILTERS.join(","),
+  ];
+  if (includeAuth) {
+    args.push("--auth", "mock");
+  }
+  return spawnSync(process.execPath, args, {
+    cwd: process.cwd(),
+    encoding: "utf8",
+  });
+}
+
+describe("Kova workflow evidence", () => {
+  it("accepts exact selected-pair coverage for every repeat", () => {
+    expect(validate(plan(), report())).toEqual({
+      authMode: "mock",
+      pairCount: 2,
+      recordCount: 4,
+      repeat: 2,
+    });
+  });
+
+  it("rejects collapsed repeat coverage", () => {
+    const laneReport = report();
+    recordsOf(laneReport).pop();
+
+    expect(() => validate(plan(), laneReport)).toThrow(
+      "lane report collapsed scenario-b/state-b coverage to 1/2 repeats",
+    );
+  });
+
+  it("rejects report pairs outside the selected plan", () => {
+    const laneReport = report();
+    const firstRecord = firstRecordOf(laneReport);
+    firstRecord.scenario = "scenario-extra";
+
+    expect(() => validate(plan(), laneReport)).toThrow(
+      "lane report contained unexpected scenario-extra/state-a",
+    );
+  });
+
+  it("rejects live records backed by mock-provider evidence", () => {
+    const liveFilters = ["scenario:scenario-a"];
+    const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
+    const laneReport = report({
+      authMode: "live",
+      includeFilters: liveFilters,
+      pairs: [FIRST_PAIR],
+      repeat: 1,
+    });
+    const firstRecord = firstRecordOf(laneReport);
+    (firstRecord.providerEvidence as JsonObject).source = "mock-provider-log";
+
+    expect(() =>
+      validate(lanePlan, laneReport, {
+        authMode: "live",
+        includeFilters: liveFilters,
+        repeat: 1,
+      }),
+    ).toThrow("live record scenario-a/state-a provider source was mock-provider-log");
+  });
+
+  it("accepts live timeline evidence with observed provider requests", () => {
+    const liveFilters = ["scenario:scenario-a"];
+    const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
+    const laneReport = report({
+      authMode: "live",
+      includeFilters: liveFilters,
+      pairs: [FIRST_PAIR],
+      repeat: 1,
+    });
+
+    expect(
+      validate(lanePlan, laneReport, {
+        authMode: "live",
+        includeFilters: liveFilters,
+        repeat: 1,
+      }),
+    ).toEqual({
+      authMode: "live",
+      pairCount: 1,
+      recordCount: 1,
+      repeat: 1,
+    });
+  });
+
+  it("rejects plan and report schema drift", () => {
+    const badPlan = plan();
+    badPlan.schemaVersion = "kova.matrix.plan.v0";
+    expect(() => validate(badPlan, report())).toThrow("unexpected lane plan schema");
+
+    const badReport = report();
+    badReport.schemaVersion = "kova.report.v0";
+    expect(() => validate(plan(), badReport)).toThrow("unexpected lane report schema");
+  });
+
+  it("rejects plan and report include-filter drift", () => {
+    const badPlan = plan();
+    (badPlan.controls as JsonObject).include = ["scenario:scenario-a"];
+    expect(() => validate(badPlan, report())).toThrow("lane plan include filters did not match");
+
+    const badReport = report();
+    (badReport.controls as JsonObject).include = ["scenario:scenario-a"];
+    expect(() => validate(plan(), badReport)).toThrow("lane report include filters did not match");
+  });
+
+  it("rejects requested and record auth drift", () => {
+    const laneReport = report();
+    (laneReport.auth as JsonObject).requestedMode = "live";
+    expect(() => validate(plan(), laneReport)).toThrow("lane report requested auth did not match");
+
+    const recordAuthDrift = report();
+    (firstRecordOf(recordAuthDrift).auth as JsonObject).mode = "live";
+    expect(() => validate(plan(), recordAuthDrift)).toThrow(
+      "record scenario-a/state-a auth mode did not match",
+    );
+  });
+
+  it("rejects malformed record state", () => {
+    const laneReport = report();
+    firstRecordOf(laneReport).state = "state-a";
+
+    expect(() => validate(plan(), laneReport)).toThrow("invalid lane report record 0 state");
+  });
+
+  it("rejects malformed JSON through the CLI", () => {
+    const result = runCli({ lanePlan: "{not-json" });
+
+    expect(result.status).toBe(1);
+    expect(result.stderr).toContain("--plan is not valid JSON");
+  });
+
+  it("rejects zero repeat and missing CLI arguments", () => {
+    const zeroRepeat = runCli({ repeat: "0" });
+    expect(zeroRepeat.status).toBe(1);
+    expect(zeroRepeat.stderr).toContain("invalid expected repeat");
+
+    const missingAuth = runCli({ includeAuth: false });
+    expect(missingAuth.status).toBe(1);
+    expect(missingAuth.stderr).toContain("invalid --auth");
+  });
+});

test/scripts/openclaw-performance-workflow.test.ts

@@ -18,7 +18,7 @@ type WorkflowJob = {
   steps?: WorkflowStep[];
   strategy?: {
     matrix?: {
-      include?: Array<{ include_filters?: string }>;
+      include?: Array<Record<string, string>>;
     };
   };
 };
@@ -38,6 +38,10 @@ function findStep(name: string): WorkflowStep {
   return step as WorkflowStep;
 }
 
+function kovaMatrixEntries(): Array<Record<string, string>> {
+  return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? [];
+}
+
 describe("OpenClaw performance workflow", () => {
   it("uses an optional dispatch identifier to name parent-owned runs", () => {
     const workflow = readFileSync(WORKFLOW, "utf8");
@@ -112,22 +116,58 @@ describe("OpenClaw performance workflow", () => {
     );
   });
 
-  it("passes every configured scenario through one Kova include flag", () => {
-    const workflow = readWorkflow();
-    const includeFilters = workflow.jobs?.kova?.strategy?.matrix?.include?.map(
-      (lane) => lane.include_filters,
-    );
+  it("passes one comma-delimited include set to the lane plan and run", () => {
+    const plan = findStep("Kova version and plan sanity");
     const runKova = findStep("Run Kova");
+    const includeFilters = kovaMatrixEntries().map((entry) => entry.include_filters);
 
     expect(includeFilters).toEqual([
       "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:bundled-runtime-deps,scenario:agent-cold-warm-message",
       "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message",
       "scenario:agent-cold-warm-message",
     ]);
-    expect(runKova.run).toContain('args+=(--include "$INCLUDE_FILTERS")');
+    expect(includeFilters.every((filters) => !filters.includes(" "))).toBe(true);
+    expect(plan.run).toContain('plan_dir="${RUNNER_TEMP}/kova-plans"');
+    expect(plan.run).toContain('--include "$INCLUDE_FILTERS"');
+    expect(plan.run).toContain('--repeat "$repeat"');
+    expect(plan.run).toContain('echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"');
+    expect(plan.run).not.toContain("$REPORT_DIR");
+    expect(runKova.run).toContain('--include "$INCLUDE_FILTERS"');
     expect(runKova.run).not.toContain("for filter in $INCLUDE_FILTERS");
   });
 
+  it("makes the live lane honor explicit live auth in the ephemeral Kova checkout", () => {
+    const override = findStep("Allow live auth for OpenAI candidate state");
+
+    expect(override.if).toContain("matrix.live == 'true'");
+    expect(override.run).toContain("states/mock-openai-provider.json");
+    expect(override.run).toContain('state.auth?.mode !== "mock"');
+    expect(override.run).toContain('state.auth.mode = "default"');
+    expect(override.run).toContain(
+      "This ephemeral checkout must honor the lane's explicit --auth live selection.",
+    );
+    expect(override.run).toContain(
+      'state.auth.reason = "Honor the workflow lane\'s explicit run-level auth selection."',
+    );
+  });
+
+  it("runs the trusted lane evidence validator before tolerating gate failures", () => {
+    const runKova = findStep("Run Kova");
+    const run = runKova.run ?? "";
+    const evidenceValidator = run.indexOf("scripts/lib/kova-workflow-evidence.mjs");
+    const trustedGateAdapter = run.indexOf("scripts/lib/kova-report-gate.mjs");
+
+    expect(evidenceValidator).toBeGreaterThan(-1);
+    expect(trustedGateAdapter).toBeGreaterThan(evidenceValidator);
+    expect(run).toContain('--plan "$KOVA_PLAN_JSON"');
+    expect(run).toContain('--report "$report_json"');
+    expect(run).toContain('--profile "$PROFILE"');
+    expect(run).toContain('--target "local-build:${GITHUB_WORKSPACE}"');
+    expect(run).toContain('--repeat "$repeat"');
+    expect(run).toContain('--include "$INCLUDE_FILTERS"');
+    expect(run).toContain('--auth "$AUTH_MODE"');
+  });
+
   it("installs local workspace packages beside the OCM root tarball", () => {
     const configure = findStep("Configure OCM local workspace dependencies");