From ee9d9ea6dbb70bf7674544f657072897bdb37dfe Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 9 Jul 2026 12:39:41 -0700 Subject: [PATCH] fix(ci): verify live provider traffic in performance runs (#103073) * fix(diagnostics): emit provider request timeline events * fix(ci): verify Kova live provider evidence --- .github/workflows/openclaw-performance.yml | 54 +++- scripts/lib/kova-workflow-evidence.mjs | 269 ++++++++++++++++ .../attempt.model-diagnostic-events.test.ts | 119 +++++++ .../run/attempt.model-diagnostic-events.ts | 37 +++ test/scripts/kova-workflow-evidence.test.ts | 304 ++++++++++++++++++ .../openclaw-performance-workflow.test.ts | 54 +++- 6 files changed, 822 insertions(+), 15 deletions(-) create mode 100644 scripts/lib/kova-workflow-evidence.mjs create mode 100644 test/scripts/kova-workflow-evidence.test.ts diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index 362d22b83df2..02fe5198941e 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -247,6 +247,27 @@ jobs: chmod 0755 "$HOME/.local/bin/kova" echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Allow live auth for OpenAI candidate state + if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }} + shell: bash + run: | + set -euo pipefail + node - <<'NODE' + const fs = require("node:fs"); + const path = require("node:path"); + const file = path.join(process.env.KOVA_SRC, "states/mock-openai-provider.json"); + const state = JSON.parse(fs.readFileSync(file, "utf8")); + if (state.auth?.mode !== "mock") { + throw new Error(`expected mock-openai-provider auth.mode=mock, got ${state.auth?.mode}`); + } + + // The release profile pairs the live agent scenario with this otherwise mock-only state. + // This ephemeral checkout must honor the lane's explicit --auth live selection. + state.auth.mode = "default"; + state.auth.reason = "Honor the workflow lane's explicit run-level auth selection."; + fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + NODE + - name: Configure OCM local workspace dependencies if: steps.lane.outputs.run == 'true' shell: bash @@ -292,12 +313,24 @@ jobs: shell: bash run: | set -euo pipefail + repeat="$REQUESTED_REPEAT" + if [[ "$MATRIX_REPEAT" != "input" ]]; then + repeat="$MATRIX_REPEAT" + fi + plan_dir="${RUNNER_TEMP}/kova-plans" + plan_json="${plan_dir}/${LANE_ID}.json" + mkdir -p "$plan_dir" + kova version --json kova matrix plan \ --profile "$PROFILE" \ --target "local-build:${GITHUB_WORKSPACE}" \ - --include scenario:fresh-install \ - --json >/tmp/kova-plan.json + --include "$INCLUDE_FILTERS" \ + --parallel 1 \ + --repeat "$repeat" \ + --json >"$plan_json" + echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV" + echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV" - name: Configure live OpenAI auth if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }} @@ -326,15 +359,13 @@ jobs: set -euo pipefail mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR" - repeat="$REQUESTED_REPEAT" - if [[ "$MATRIX_REPEAT" != "input" ]]; then - repeat="$MATRIX_REPEAT" - fi + repeat="$KOVA_LANE_REPEAT" args=( matrix run --profile "$PROFILE" --target "local-build:${GITHUB_WORKSPACE}" + --include "$INCLUDE_FILTERS" --auth "$AUTH_MODE" --parallel 1 --repeat "$repeat" @@ -344,8 +375,6 @@ jobs: --json ) - args+=(--include "$INCLUDE_FILTERS") - if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then args+=(--deep-profile) fi @@ -365,6 +394,15 @@ jobs: exit 1 fi report_md="${report_json%.json}.md" + node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \ + --plan "$KOVA_PLAN_JSON" \ + --report "$report_json" \ + --profile "$PROFILE" \ + --target "local-build:${GITHUB_WORKSPACE}" \ + --repeat "$repeat" \ + --include "$INCLUDE_FILTERS" \ + --auth "$AUTH_MODE" + effective_status="$status" if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json" diff --git a/scripts/lib/kova-workflow-evidence.mjs b/scripts/lib/kova-workflow-evidence.mjs new file mode 100644 index 000000000000..9e37a31a3448 --- /dev/null +++ b/scripts/lib/kova-workflow-evidence.mjs @@ -0,0 +1,269 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const AUTH_MODES = new Set(["live", "mock"]); +const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]); + +function check(condition, reason) { + if (!condition) { + throw new Error(reason); + } +} + +function object(value, label) { + check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`); + return value; +} + +function array(value, label) { + check(Array.isArray(value), `invalid ${label}`); + return value; +} + +function text(value, label) { + check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`); + return value; +} + +function positiveInteger(value, label) { + check(Number.isSafeInteger(value) && value > 0, `invalid ${label}`); + return value; +} + +function exactStrings(value, expected, label) { + const actual = array(value, label); + check( + actual.length === expected.length && actual.every((item, index) => item === expected[index]), + `${label} did not match`, + ); +} + +function pairKey(scenario, state, label) { + return `${text(scenario, `${label} scenario`)}\u0000${text(state, `${label} state`)}`; +} + +function displayPair(key) { + return key.replace("\u0000", "/"); +} + +function profileId(value, label) { + return text(object(value, label).id, `${label} id`); +} + +function repeatIndexesFor(selectedPairs) { + return new Map([...selectedPairs].map((key) => [key, new Set()])); +} + +function validateLiveRecord(record, key) { + const auth = object(record.auth, `record ${displayPair(key)} auth`); + const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`); + check( + auth.environmentDependent === true, + `live record ${displayPair(key)} auth was not environment-dependent`, + ); + check(provider.authMode === "live", `live record ${displayPair(key)} provider auth was not live`); + check( + provider.environmentDependent === true, + `live record ${displayPair(key)} provider evidence was not environment-dependent`, + ); + check( + provider.source === "openclaw-timeline", + `live record ${displayPair(key)} provider source was ${provider.source}`, + ); + check( + provider.available === true, + `live record ${displayPair(key)} provider evidence was unavailable`, + ); + positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`); +} + +export function validateKovaWorkflowEvidence({ + plan, + report, + profile, + target, + repeat, + includeFilters, + authMode, +}) { + const expectedProfile = text(profile, "expected profile"); + const expectedTarget = text(target, "expected target"); + const expectedRepeat = positiveInteger(repeat, "expected repeat"); + const expectedFilters = array(includeFilters, "expected include filters").map((value, index) => + text(value, `expected include filter ${index}`), + ); + const expectedAuth = text(authMode, "expected auth mode"); + check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`); + check(expectedFilters.length > 0, "expected include filters were empty"); + + const lanePlan = object(plan, "lane plan"); + const laneReport = object(report, "lane report"); + check(lanePlan.schemaVersion === "kova.matrix.plan.v1", "unexpected lane plan schema"); + check(laneReport.schemaVersion === "kova.report.v1", "unexpected lane report schema"); + check(laneReport.mode === "execution", `lane report mode was ${laneReport.mode}`); + check( + profileId(lanePlan.profile, "lane plan profile") === expectedProfile, + "lane plan profile did not match", + ); + check( + profileId(laneReport.profile, "lane report profile") === expectedProfile, + "lane report profile did not match", + ); + check(lanePlan.target === expectedTarget, "lane plan target did not match"); + check(laneReport.target === expectedTarget, "lane report target did not match"); + + const planControls = object(lanePlan.controls, "lane plan controls"); + const reportControls = object(laneReport.controls, "lane report controls"); + exactStrings(planControls.include, expectedFilters, "lane plan include filters"); + exactStrings(reportControls.include, expectedFilters, "lane report include filters"); + check(planControls.repeat === expectedRepeat, "lane plan repeat did not match"); + check(reportControls.repeat === expectedRepeat, "lane report repeat did not match"); + + const selectedPairs = new Set(); + for (const [index, value] of array(lanePlan.entries, "lane plan entries").entries()) { + const entry = object(value, `lane plan entry ${index}`); + const status = text(entry.status, `lane plan entry ${index} status`); + check(status === "SELECTED" || status === "SKIPPED", `invalid lane plan entry ${index} status`); + if (status === "SKIPPED") { + continue; + } + const key = pairKey( + object(entry.scenario, `lane plan entry ${index} scenario`).id, + object(entry.state, `lane plan entry ${index} state`).id, + `lane plan entry ${index}`, + ); + check(!selectedPairs.has(key), `lane plan selected duplicate ${displayPair(key)}`); + selectedPairs.add(key); + } + check(selectedPairs.size > 0, "lane plan selected no scenario/state pairs"); + + const reportAuth = object(laneReport.auth, "lane report auth"); + check(reportAuth.requestedMode === expectedAuth, "lane report requested auth did not match"); + if (expectedAuth === "live") { + check( + object(reportAuth.live, "lane report live auth").environmentDependent === true, + "lane report live auth was not environment-dependent", + ); + } + + const records = array(laneReport.records, "lane report records"); + const repeatIndexes = repeatIndexesFor(selectedPairs); + for (const [index, value] of records.entries()) { + const record = object(value, `lane report record ${index}`); + const state = object(record.state, `lane report record ${index} state`); + const key = pairKey(record.scenario, state.id, `lane report record ${index}`); + check(selectedPairs.has(key), `lane report contained unexpected ${displayPair(key)}`); + const status = text(record.status, `record ${displayPair(key)} status`); + check( + ["BLOCKED", "FAIL", "PASS"].includes(status), + `invalid record ${displayPair(key)} status`, + ); + + const recordRepeat = object(record.repeat, `record ${displayPair(key)} repeat`); + check( + recordRepeat.total === expectedRepeat, + `record ${displayPair(key)} repeat total did not match`, + ); + const repeatIndex = positiveInteger( + recordRepeat.index, + `record ${displayPair(key)} repeat index`, + ); + check( + repeatIndex <= expectedRepeat, + `record ${displayPair(key)} repeat index exceeded ${expectedRepeat}`, + ); + const indexes = repeatIndexes.get(key); + check(!indexes.has(repeatIndex), `record ${displayPair(key)} repeated index ${repeatIndex}`); + indexes.add(repeatIndex); + + const recordAuth = object(record.auth, `record ${displayPair(key)} auth`); + check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`); + if (expectedAuth === "live") { + validateLiveRecord(record, key); + } + } + + for (const [key, indexes] of repeatIndexes) { + check( + indexes.size === expectedRepeat, + `lane report collapsed ${displayPair(key)} coverage to ${indexes.size}/${expectedRepeat} repeats`, + ); + for (let index = 1; index <= expectedRepeat; index += 1) { + check(indexes.has(index), `lane report missed ${displayPair(key)} repeat ${index}`); + } + } + check( + records.length === selectedPairs.size * expectedRepeat, + "lane report record count did not match selected coverage", + ); + + return { + authMode: expectedAuth, + pairCount: selectedPairs.size, + recordCount: records.length, + repeat: expectedRepeat, + }; +} + +function parseCliArgs(argv) { + const flags = {}; + for (let index = 0; index < argv.length; index += 2) { + const name = argv[index]; + const value = argv[index + 1]; + check(name?.startsWith("--") && value !== undefined, "invalid CLI arguments"); + const key = name.slice(2); + check(CLI_KEYS.has(key), `unknown --${key}`); + check(!Object.hasOwn(flags, key), `duplicate --${key}`); + flags[key] = value; + } + for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) { + text(flags[key], `--${key}`); + } + return flags; +} + +function readJson(pathValue, label) { + const file = text(pathValue, label); + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + throw new Error( + `${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } +} + +function runCli() { + const flags = parseCliArgs(process.argv.slice(2)); + const result = validateKovaWorkflowEvidence({ + plan: readJson(flags.plan, "--plan"), + report: readJson(flags.report, "--report"), + profile: flags.profile, + target: flags.target, + repeat: Number(flags.repeat), + includeFilters: flags.include + .split(",") + .map((value) => value.trim()) + .filter(Boolean), + authMode: flags.auth, + }); + console.log( + `Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`, + ); +} + +const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url)); +const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(process.argv[1])) : ""; + +if (modulePath === invokedPath) { + try { + runCli(); + } catch (error) { + console.error( + `Kova evidence validation failed: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(1); + } +} diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts index bfdce521543b..c27634291836 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.test.ts @@ -1,6 +1,9 @@ // Coverage for model-call diagnostic events around attempt stream functions. +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../../../test/helpers/temp-dir.js"; import { onInternalDiagnosticEvent, onTrustedInternalDiagnosticEvent, @@ -20,8 +23,11 @@ import { resetGlobalHookRunner, } from "../../../plugins/hook-runner-global.js"; import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-helpers.js"; +import { withEnvAsync } from "../../../test-utils/env.js"; import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + async function collectModelCallEvents(run: () => Promise): Promise { // Diagnostics are emitted asynchronously; collect only public model-call // events and flush one tick after the stream completes. @@ -112,6 +118,24 @@ function requireMockRecordArg( return requireRecord(mock.mock.calls[callIndex]?.[argIndex], label); } +async function collectProviderTimelineEvents(run: () => Promise) { + const root = tempDirs.make("openclaw-provider-timeline-"); + const timelinePath = join(root, "timeline.jsonl"); + await withEnvAsync( + { + OPENCLAW_DIAGNOSTICS: "1", + OPENCLAW_DIAGNOSTICS_TIMELINE_PATH: timelinePath, + }, + run, + ); + return readFileSync(timelinePath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => requireRecord(JSON.parse(line), "provider timeline event")) + .filter((event) => event.type === "provider.request"); +} + describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { beforeEach(() => { resetDiagnosticEventsForTest(); @@ -204,6 +228,101 @@ describe("wrapStreamFnWithDiagnosticModelCallEvents", () => { expect(JSON.stringify(events)).not.toContain("sk-test-secret-value"); }); + it("emits one successful provider timeline event for result and iterator completion", async () => { + let now = Date.parse("2026-07-09T18:30:00.000Z"); + vi.spyOn(Date, "now").mockImplementation(() => now); + async function* stream() { + yield { type: "text", text: "ok" }; + } + const originalStream = stream() as unknown as AsyncIterable & { + result: () => Promise; + }; + originalStream.result = async () => { + now += 125; + return "kept"; + }; + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => originalStream) as unknown as StreamFn, + { + runId: "run-timeline-success", + provider: "openai", + model: "gpt-5.5", + api: "openai-responses", + transport: "http", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-success", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + const returned = wrapped( + {} as never, + {} as never, + {} as never, + ) as unknown as typeof originalStream; + await returned.result(); + await drain(returned); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + name: "provider.request", + timestamp: "2026-07-09T18:30:00.000Z", + runId: "run-timeline-success", + spanId: "call-timeline-success", + durationMs: 125, + provider: "openai", + operation: "openai-responses", + ok: true, + attributes: { + model: "gpt-5.5", + api: "openai-responses", + transport: "http", + }, + }); + }); + + it("emits one failed provider timeline event for a thrown model call", async () => { + let now = Date.parse("2026-07-09T18:31:00.000Z"); + vi.spyOn(Date, "now").mockImplementation(() => now); + const wrapped = wrapStreamFnWithDiagnosticModelCallEvents( + (() => { + now += 75; + throw new Error("provider failed"); + }) as unknown as StreamFn, + { + runId: "run-timeline-error", + provider: "anthropic", + model: "claude-sonnet-4-6", + transport: "sse", + trace: createDiagnosticTraceContext(), + nextCallId: () => "call-timeline-error", + }, + ); + + const events = await collectProviderTimelineEvents(async () => { + expect(() => wrapped({} as never, {} as never, {} as never)).toThrow("provider failed"); + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "provider.request", + name: "provider.request", + timestamp: "2026-07-09T18:31:00.000Z", + runId: "run-timeline-error", + spanId: "call-timeline-error", + durationMs: 75, + provider: "anthropic", + operation: "sse", + ok: false, + attributes: { + model: "claude-sonnet-4-6", + transport: "sse", + }, + }); + }); + it("updates diagnostic run activity from throttled stream chunks", async () => { let now = 1_000_000; vi.spyOn(Date, "now").mockImplementation(() => now); diff --git a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts index db1b66f058fe..e60c12461881 100644 --- a/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts +++ b/src/agents/embedded-agent-runner/run/attempt.model-diagnostic-events.ts @@ -26,6 +26,7 @@ import { formatDiagnosticTraceparent, type DiagnosticTraceContext, } from "../../../infra/diagnostic-trace-context.js"; +import { emitDiagnosticsTimelineEvent } from "../../../infra/diagnostics-timeline.js"; import { markDiagnosticRunProgress } from "../../../logging/diagnostic-run-activity.js"; import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; import type { @@ -99,6 +100,7 @@ const MODEL_CALL_STREAM_PROGRESS_INTERVAL_MS = 30_000; const MODEL_CALL_STREAM_PROGRESS_REASON = "model_call:stream_progress"; const MODEL_CALL_STREAM_RETURN_TIMEOUT_MS = 1000; const TRACEPARENT_HEADER_NAME = "traceparent"; +const TIMELINE_ATTRIBUTE_MAX_LENGTH = 256; type ModelCallStreamOptions = Parameters[2]; function utf8JsonByteLength(value: unknown): number | undefined { @@ -417,6 +419,39 @@ function modelCallUsageField(state: ModelCallObservationState) { return state.usage ? { usage: state.usage } : {}; } +function boundedTimelineAttribute(value: string | undefined): string | undefined { + const normalized = value?.trim(); + return normalized ? normalized.slice(0, TIMELINE_ATTRIBUTE_MAX_LENGTH) : undefined; +} + +function emitProviderRequestTimelineEvent( + eventBase: ModelCallEventBase, + startedAt: number, + durationMs: number, + ok: boolean, +): void { + const provider = boundedTimelineAttribute(eventBase.provider); + const model = boundedTimelineAttribute(eventBase.model); + const api = boundedTimelineAttribute(eventBase.api); + const transport = boundedTimelineAttribute(eventBase.transport); + emitDiagnosticsTimelineEvent({ + type: "provider.request", + name: "provider.request", + timestamp: new Date(startedAt).toISOString(), + runId: eventBase.runId, + spanId: eventBase.callId, + durationMs, + provider, + operation: api ?? transport ?? "model.call", + ok, + attributes: { + ...(model ? { model } : {}), + ...(api ? { api } : {}), + ...(transport ? { transport } : {}), + }, + }); +} + function modelCallErrorFields(err: unknown): ModelCallErrorFields { const upstreamRequestIdHash = diagnosticProviderRequestIdHash(err); const failureKind = diagnosticErrorFailureKind(err); @@ -537,6 +572,7 @@ function emitModelCallCompleted( state.terminalEventEmitted = true; const durationMs = Date.now() - startedAt; const sizeTimingFields = modelCallSizeTimingFields(state); + emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, true); emitTrustedDiagnosticEventWithPrivateData( { type: "model.call.completed", @@ -566,6 +602,7 @@ function emitModelCallError( state.terminalEventEmitted = true; const durationMs = Date.now() - startedAt; const sizeTimingFields = modelCallSizeTimingFields(state); + emitProviderRequestTimelineEvent(eventBase, startedAt, durationMs, false); emitTrustedDiagnosticEventWithPrivateData( { type: "model.call.error", diff --git a/test/scripts/kova-workflow-evidence.test.ts b/test/scripts/kova-workflow-evidence.test.ts new file mode 100644 index 000000000000..5ec2163eeaf1 --- /dev/null +++ b/test/scripts/kova-workflow-evidence.test.ts @@ -0,0 +1,304 @@ +import { spawnSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { validateKovaWorkflowEvidence } from "../../scripts/lib/kova-workflow-evidence.mjs"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +type JsonObject = Record; +type Pair = { + scenario: string; + state: string; +}; + +const PROFILE = "release"; +const TARGET = "local-build:/work/openclaw"; +const INCLUDE_FILTERS = ["scenario:scenario-a", "scenario:scenario-b"]; +const FIRST_PAIR: Pair = { scenario: "scenario-a", state: "state-a" }; +const PAIRS: Pair[] = [FIRST_PAIR, { scenario: "scenario-b", state: "state-b" }]; +const SCRIPT_PATH = "scripts/lib/kova-workflow-evidence.mjs"; +const tempRoots = useAutoCleanupTempDirTracker(afterEach); + +function plan( + pairs: Pair[] = PAIRS, + repeat = 2, + includeFilters: string[] = INCLUDE_FILTERS, +): JsonObject { + return { + schemaVersion: "kova.matrix.plan.v1", + profile: { id: PROFILE }, + target: TARGET, + controls: { + include: includeFilters, + repeat, + }, + entries: pairs.map((pair) => ({ + scenario: { id: pair.scenario }, + state: { id: pair.state }, + status: "SELECTED", + })), + }; +} + +function record(pair: Pair, repeatIndex: number, repeat: number, authMode: "live" | "mock") { + return { + scenario: pair.scenario, + state: { id: pair.state }, + status: "PASS", + repeat: { + index: repeatIndex, + total: repeat, + }, + auth: { + mode: authMode, + environmentDependent: authMode === "live", + }, + providerEvidence: + authMode === "live" + ? { + authMode: "live", + available: true, + environmentDependent: true, + requestCount: 1, + source: "openclaw-timeline", + } + : { + authMode: "mock", + available: true, + environmentDependent: false, + requestCount: 1, + source: "mock-provider-log", + }, + }; +} + +function report({ + authMode = "mock", + includeFilters = INCLUDE_FILTERS, + pairs = PAIRS, + repeat = 2, +}: { + authMode?: "live" | "mock"; + includeFilters?: string[]; + pairs?: Pair[]; + repeat?: number; +} = {}): JsonObject { + return { + schemaVersion: "kova.report.v1", + mode: "execution", + profile: { id: PROFILE }, + target: TARGET, + controls: { + include: includeFilters, + repeat, + }, + auth: { + requestedMode: authMode, + live: { + environmentDependent: authMode === "live", + }, + }, + records: pairs.flatMap((pair) => + Array.from({ length: repeat }, (_, index) => record(pair, index + 1, repeat, authMode)), + ), + }; +} + +function validate( + lanePlan: JsonObject, + laneReport: JsonObject, + options: { authMode?: "live" | "mock"; includeFilters?: string[]; repeat?: number } = {}, +) { + return validateKovaWorkflowEvidence({ + plan: lanePlan, + report: laneReport, + profile: PROFILE, + target: TARGET, + repeat: options.repeat ?? 2, + includeFilters: options.includeFilters ?? INCLUDE_FILTERS, + authMode: options.authMode ?? "mock", + }); +} + +function recordsOf(laneReport: JsonObject): JsonObject[] { + return laneReport.records as JsonObject[]; +} + +function firstRecordOf(laneReport: JsonObject): JsonObject { + const first = recordsOf(laneReport)[0]; + if (!first) { + throw new Error("fixture report has no records"); + } + return first; +} + +function runCli({ + lanePlan = JSON.stringify(plan()), + laneReport = JSON.stringify(report()), + repeat = "2", + includeAuth = true, +}: { + lanePlan?: string; + laneReport?: string; + repeat?: string; + includeAuth?: boolean; +} = {}) { + const root = tempRoots.make("openclaw-kova-evidence-"); + const planPath = join(root, "plan.json"); + const reportPath = join(root, "report.json"); + writeFileSync(planPath, lanePlan); + writeFileSync(reportPath, laneReport); + const args = [ + SCRIPT_PATH, + "--plan", + planPath, + "--report", + reportPath, + "--profile", + PROFILE, + "--target", + TARGET, + "--repeat", + repeat, + "--include", + INCLUDE_FILTERS.join(","), + ]; + if (includeAuth) { + args.push("--auth", "mock"); + } + return spawnSync(process.execPath, args, { + cwd: process.cwd(), + encoding: "utf8", + }); +} + +describe("Kova workflow evidence", () => { + it("accepts exact selected-pair coverage for every repeat", () => { + expect(validate(plan(), report())).toEqual({ + authMode: "mock", + pairCount: 2, + recordCount: 4, + repeat: 2, + }); + }); + + it("rejects collapsed repeat coverage", () => { + const laneReport = report(); + recordsOf(laneReport).pop(); + + expect(() => validate(plan(), laneReport)).toThrow( + "lane report collapsed scenario-b/state-b coverage to 1/2 repeats", + ); + }); + + it("rejects report pairs outside the selected plan", () => { + const laneReport = report(); + const firstRecord = firstRecordOf(laneReport); + firstRecord.scenario = "scenario-extra"; + + expect(() => validate(plan(), laneReport)).toThrow( + "lane report contained unexpected scenario-extra/state-a", + ); + }); + + it("rejects live records backed by mock-provider evidence", () => { + const liveFilters = ["scenario:scenario-a"]; + const lanePlan = plan([FIRST_PAIR], 1, liveFilters); + const laneReport = report({ + authMode: "live", + includeFilters: liveFilters, + pairs: [FIRST_PAIR], + repeat: 1, + }); + const firstRecord = firstRecordOf(laneReport); + (firstRecord.providerEvidence as JsonObject).source = "mock-provider-log"; + + expect(() => + validate(lanePlan, laneReport, { + authMode: "live", + includeFilters: liveFilters, + repeat: 1, + }), + ).toThrow("live record scenario-a/state-a provider source was mock-provider-log"); + }); + + it("accepts live timeline evidence with observed provider requests", () => { + const liveFilters = ["scenario:scenario-a"]; + const lanePlan = plan([FIRST_PAIR], 1, liveFilters); + const laneReport = report({ + authMode: "live", + includeFilters: liveFilters, + pairs: [FIRST_PAIR], + repeat: 1, + }); + + expect( + validate(lanePlan, laneReport, { + authMode: "live", + includeFilters: liveFilters, + repeat: 1, + }), + ).toEqual({ + authMode: "live", + pairCount: 1, + recordCount: 1, + repeat: 1, + }); + }); + + it("rejects plan and report schema drift", () => { + const badPlan = plan(); + badPlan.schemaVersion = "kova.matrix.plan.v0"; + expect(() => validate(badPlan, report())).toThrow("unexpected lane plan schema"); + + const badReport = report(); + badReport.schemaVersion = "kova.report.v0"; + expect(() => validate(plan(), badReport)).toThrow("unexpected lane report schema"); + }); + + it("rejects plan and report include-filter drift", () => { + const badPlan = plan(); + (badPlan.controls as JsonObject).include = ["scenario:scenario-a"]; + expect(() => validate(badPlan, report())).toThrow("lane plan include filters did not match"); + + const badReport = report(); + (badReport.controls as JsonObject).include = ["scenario:scenario-a"]; + expect(() => validate(plan(), badReport)).toThrow("lane report include filters did not match"); + }); + + it("rejects requested and record auth drift", () => { + const laneReport = report(); + (laneReport.auth as JsonObject).requestedMode = "live"; + expect(() => validate(plan(), laneReport)).toThrow("lane report requested auth did not match"); + + const recordAuthDrift = report(); + (firstRecordOf(recordAuthDrift).auth as JsonObject).mode = "live"; + expect(() => validate(plan(), recordAuthDrift)).toThrow( + "record scenario-a/state-a auth mode did not match", + ); + }); + + it("rejects malformed record state", () => { + const laneReport = report(); + firstRecordOf(laneReport).state = "state-a"; + + expect(() => validate(plan(), laneReport)).toThrow("invalid lane report record 0 state"); + }); + + it("rejects malformed JSON through the CLI", () => { + const result = runCli({ lanePlan: "{not-json" }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("--plan is not valid JSON"); + }); + + it("rejects zero repeat and missing CLI arguments", () => { + const zeroRepeat = runCli({ repeat: "0" }); + expect(zeroRepeat.status).toBe(1); + expect(zeroRepeat.stderr).toContain("invalid expected repeat"); + + const missingAuth = runCli({ includeAuth: false }); + expect(missingAuth.status).toBe(1); + expect(missingAuth.stderr).toContain("invalid --auth"); + }); +}); diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index eb00fb6b77e8..3a07b8b7dac5 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -18,7 +18,7 @@ type WorkflowJob = { steps?: WorkflowStep[]; strategy?: { matrix?: { - include?: Array<{ include_filters?: string }>; + include?: Array>; }; }; }; @@ -38,6 +38,10 @@ function findStep(name: string): WorkflowStep { return step as WorkflowStep; } +function kovaMatrixEntries(): Array> { + return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? []; +} + describe("OpenClaw performance workflow", () => { it("uses an optional dispatch identifier to name parent-owned runs", () => { const workflow = readFileSync(WORKFLOW, "utf8"); @@ -112,22 +116,58 @@ describe("OpenClaw performance workflow", () => { ); }); - it("passes every configured scenario through one Kova include flag", () => { - const workflow = readWorkflow(); - const includeFilters = workflow.jobs?.kova?.strategy?.matrix?.include?.map( - (lane) => lane.include_filters, - ); + it("passes one comma-delimited include set to the lane plan and run", () => { + const plan = findStep("Kova version and plan sanity"); const runKova = findStep("Run Kova"); + const includeFilters = kovaMatrixEntries().map((entry) => entry.include_filters); expect(includeFilters).toEqual([ "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:bundled-runtime-deps,scenario:agent-cold-warm-message", "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message", "scenario:agent-cold-warm-message", ]); - expect(runKova.run).toContain('args+=(--include "$INCLUDE_FILTERS")'); + expect(includeFilters.every((filters) => !filters.includes(" "))).toBe(true); + expect(plan.run).toContain('plan_dir="${RUNNER_TEMP}/kova-plans"'); + expect(plan.run).toContain('--include "$INCLUDE_FILTERS"'); + expect(plan.run).toContain('--repeat "$repeat"'); + expect(plan.run).toContain('echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"'); + expect(plan.run).not.toContain("$REPORT_DIR"); + expect(runKova.run).toContain('--include "$INCLUDE_FILTERS"'); expect(runKova.run).not.toContain("for filter in $INCLUDE_FILTERS"); }); + it("makes the live lane honor explicit live auth in the ephemeral Kova checkout", () => { + const override = findStep("Allow live auth for OpenAI candidate state"); + + expect(override.if).toContain("matrix.live == 'true'"); + expect(override.run).toContain("states/mock-openai-provider.json"); + expect(override.run).toContain('state.auth?.mode !== "mock"'); + expect(override.run).toContain('state.auth.mode = "default"'); + expect(override.run).toContain( + "This ephemeral checkout must honor the lane's explicit --auth live selection.", + ); + expect(override.run).toContain( + 'state.auth.reason = "Honor the workflow lane\'s explicit run-level auth selection."', + ); + }); + + it("runs the trusted lane evidence validator before tolerating gate failures", () => { + const runKova = findStep("Run Kova"); + const run = runKova.run ?? ""; + const evidenceValidator = run.indexOf("scripts/lib/kova-workflow-evidence.mjs"); + const trustedGateAdapter = run.indexOf("scripts/lib/kova-report-gate.mjs"); + + expect(evidenceValidator).toBeGreaterThan(-1); + expect(trustedGateAdapter).toBeGreaterThan(evidenceValidator); + expect(run).toContain('--plan "$KOVA_PLAN_JSON"'); + expect(run).toContain('--report "$report_json"'); + expect(run).toContain('--profile "$PROFILE"'); + expect(run).toContain('--target "local-build:${GITHUB_WORKSPACE}"'); + expect(run).toContain('--repeat "$repeat"'); + expect(run).toContain('--include "$INCLUDE_FILTERS"'); + expect(run).toContain('--auth "$AUTH_MODE"'); + }); + it("installs local workspace packages beside the OCM root tarball", () => { const configure = findStep("Configure OCM local workspace dependencies");