mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-17 13:01:35 +00:00
fix(ci): verify live provider traffic in performance runs (#103073)
* fix(diagnostics): emit provider request timeline events * fix(ci): verify Kova live provider evidence
This commit is contained in:
Vincent Koc
304
test/scripts/kova-workflow-evidence.test.ts
Normal file
304
test/scripts/kova-workflow-evidence.test.ts
Normal file
@@ -0,0 +1,304 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { validateKovaWorkflowEvidence } from "../../scripts/lib/kova-workflow-evidence.mjs";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
type JsonObject = Record<string, unknown>;
|
||||
type Pair = {
|
||||
scenario: string;
|
||||
state: string;
|
||||
};
|
||||
|
||||
const PROFILE = "release";
|
||||
const TARGET = "local-build:/work/openclaw";
|
||||
const INCLUDE_FILTERS = ["scenario:scenario-a", "scenario:scenario-b"];
|
||||
const FIRST_PAIR: Pair = { scenario: "scenario-a", state: "state-a" };
|
||||
const PAIRS: Pair[] = [FIRST_PAIR, { scenario: "scenario-b", state: "state-b" }];
|
||||
const SCRIPT_PATH = "scripts/lib/kova-workflow-evidence.mjs";
|
||||
const tempRoots = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function plan(
|
||||
pairs: Pair[] = PAIRS,
|
||||
repeat = 2,
|
||||
includeFilters: string[] = INCLUDE_FILTERS,
|
||||
): JsonObject {
|
||||
return {
|
||||
schemaVersion: "kova.matrix.plan.v1",
|
||||
profile: { id: PROFILE },
|
||||
target: TARGET,
|
||||
controls: {
|
||||
include: includeFilters,
|
||||
repeat,
|
||||
},
|
||||
entries: pairs.map((pair) => ({
|
||||
scenario: { id: pair.scenario },
|
||||
state: { id: pair.state },
|
||||
status: "SELECTED",
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function record(pair: Pair, repeatIndex: number, repeat: number, authMode: "live" | "mock") {
|
||||
return {
|
||||
scenario: pair.scenario,
|
||||
state: { id: pair.state },
|
||||
status: "PASS",
|
||||
repeat: {
|
||||
index: repeatIndex,
|
||||
total: repeat,
|
||||
},
|
||||
auth: {
|
||||
mode: authMode,
|
||||
environmentDependent: authMode === "live",
|
||||
},
|
||||
providerEvidence:
|
||||
authMode === "live"
|
||||
? {
|
||||
authMode: "live",
|
||||
available: true,
|
||||
environmentDependent: true,
|
||||
requestCount: 1,
|
||||
source: "openclaw-timeline",
|
||||
}
|
||||
: {
|
||||
authMode: "mock",
|
||||
available: true,
|
||||
environmentDependent: false,
|
||||
requestCount: 1,
|
||||
source: "mock-provider-log",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function report({
|
||||
authMode = "mock",
|
||||
includeFilters = INCLUDE_FILTERS,
|
||||
pairs = PAIRS,
|
||||
repeat = 2,
|
||||
}: {
|
||||
authMode?: "live" | "mock";
|
||||
includeFilters?: string[];
|
||||
pairs?: Pair[];
|
||||
repeat?: number;
|
||||
} = {}): JsonObject {
|
||||
return {
|
||||
schemaVersion: "kova.report.v1",
|
||||
mode: "execution",
|
||||
profile: { id: PROFILE },
|
||||
target: TARGET,
|
||||
controls: {
|
||||
include: includeFilters,
|
||||
repeat,
|
||||
},
|
||||
auth: {
|
||||
requestedMode: authMode,
|
||||
live: {
|
||||
environmentDependent: authMode === "live",
|
||||
},
|
||||
},
|
||||
records: pairs.flatMap((pair) =>
|
||||
Array.from({ length: repeat }, (_, index) => record(pair, index + 1, repeat, authMode)),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function validate(
|
||||
lanePlan: JsonObject,
|
||||
laneReport: JsonObject,
|
||||
options: { authMode?: "live" | "mock"; includeFilters?: string[]; repeat?: number } = {},
|
||||
) {
|
||||
return validateKovaWorkflowEvidence({
|
||||
plan: lanePlan,
|
||||
report: laneReport,
|
||||
profile: PROFILE,
|
||||
target: TARGET,
|
||||
repeat: options.repeat ?? 2,
|
||||
includeFilters: options.includeFilters ?? INCLUDE_FILTERS,
|
||||
authMode: options.authMode ?? "mock",
|
||||
});
|
||||
}
|
||||
|
||||
function recordsOf(laneReport: JsonObject): JsonObject[] {
|
||||
return laneReport.records as JsonObject[];
|
||||
}
|
||||
|
||||
function firstRecordOf(laneReport: JsonObject): JsonObject {
|
||||
const first = recordsOf(laneReport)[0];
|
||||
if (!first) {
|
||||
throw new Error("fixture report has no records");
|
||||
}
|
||||
return first;
|
||||
}
|
||||
|
||||
function runCli({
|
||||
lanePlan = JSON.stringify(plan()),
|
||||
laneReport = JSON.stringify(report()),
|
||||
repeat = "2",
|
||||
includeAuth = true,
|
||||
}: {
|
||||
lanePlan?: string;
|
||||
laneReport?: string;
|
||||
repeat?: string;
|
||||
includeAuth?: boolean;
|
||||
} = {}) {
|
||||
const root = tempRoots.make("openclaw-kova-evidence-");
|
||||
const planPath = join(root, "plan.json");
|
||||
const reportPath = join(root, "report.json");
|
||||
writeFileSync(planPath, lanePlan);
|
||||
writeFileSync(reportPath, laneReport);
|
||||
const args = [
|
||||
SCRIPT_PATH,
|
||||
"--plan",
|
||||
planPath,
|
||||
"--report",
|
||||
reportPath,
|
||||
"--profile",
|
||||
PROFILE,
|
||||
"--target",
|
||||
TARGET,
|
||||
"--repeat",
|
||||
repeat,
|
||||
"--include",
|
||||
INCLUDE_FILTERS.join(","),
|
||||
];
|
||||
if (includeAuth) {
|
||||
args.push("--auth", "mock");
|
||||
}
|
||||
return spawnSync(process.execPath, args, {
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
describe("Kova workflow evidence", () => {
|
||||
it("accepts exact selected-pair coverage for every repeat", () => {
|
||||
expect(validate(plan(), report())).toEqual({
|
||||
authMode: "mock",
|
||||
pairCount: 2,
|
||||
recordCount: 4,
|
||||
repeat: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects collapsed repeat coverage", () => {
|
||||
const laneReport = report();
|
||||
recordsOf(laneReport).pop();
|
||||
|
||||
expect(() => validate(plan(), laneReport)).toThrow(
|
||||
"lane report collapsed scenario-b/state-b coverage to 1/2 repeats",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects report pairs outside the selected plan", () => {
|
||||
const laneReport = report();
|
||||
const firstRecord = firstRecordOf(laneReport);
|
||||
firstRecord.scenario = "scenario-extra";
|
||||
|
||||
expect(() => validate(plan(), laneReport)).toThrow(
|
||||
"lane report contained unexpected scenario-extra/state-a",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects live records backed by mock-provider evidence", () => {
|
||||
const liveFilters = ["scenario:scenario-a"];
|
||||
const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
|
||||
const laneReport = report({
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
pairs: [FIRST_PAIR],
|
||||
repeat: 1,
|
||||
});
|
||||
const firstRecord = firstRecordOf(laneReport);
|
||||
(firstRecord.providerEvidence as JsonObject).source = "mock-provider-log";
|
||||
|
||||
expect(() =>
|
||||
validate(lanePlan, laneReport, {
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
repeat: 1,
|
||||
}),
|
||||
).toThrow("live record scenario-a/state-a provider source was mock-provider-log");
|
||||
});
|
||||
|
||||
it("accepts live timeline evidence with observed provider requests", () => {
|
||||
const liveFilters = ["scenario:scenario-a"];
|
||||
const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
|
||||
const laneReport = report({
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
pairs: [FIRST_PAIR],
|
||||
repeat: 1,
|
||||
});
|
||||
|
||||
expect(
|
||||
validate(lanePlan, laneReport, {
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
repeat: 1,
|
||||
}),
|
||||
).toEqual({
|
||||
authMode: "live",
|
||||
pairCount: 1,
|
||||
recordCount: 1,
|
||||
repeat: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects plan and report schema drift", () => {
|
||||
const badPlan = plan();
|
||||
badPlan.schemaVersion = "kova.matrix.plan.v0";
|
||||
expect(() => validate(badPlan, report())).toThrow("unexpected lane plan schema");
|
||||
|
||||
const badReport = report();
|
||||
badReport.schemaVersion = "kova.report.v0";
|
||||
expect(() => validate(plan(), badReport)).toThrow("unexpected lane report schema");
|
||||
});
|
||||
|
||||
it("rejects plan and report include-filter drift", () => {
|
||||
const badPlan = plan();
|
||||
(badPlan.controls as JsonObject).include = ["scenario:scenario-a"];
|
||||
expect(() => validate(badPlan, report())).toThrow("lane plan include filters did not match");
|
||||
|
||||
const badReport = report();
|
||||
(badReport.controls as JsonObject).include = ["scenario:scenario-a"];
|
||||
expect(() => validate(plan(), badReport)).toThrow("lane report include filters did not match");
|
||||
});
|
||||
|
||||
it("rejects requested and record auth drift", () => {
|
||||
const laneReport = report();
|
||||
(laneReport.auth as JsonObject).requestedMode = "live";
|
||||
expect(() => validate(plan(), laneReport)).toThrow("lane report requested auth did not match");
|
||||
|
||||
const recordAuthDrift = report();
|
||||
(firstRecordOf(recordAuthDrift).auth as JsonObject).mode = "live";
|
||||
expect(() => validate(plan(), recordAuthDrift)).toThrow(
|
||||
"record scenario-a/state-a auth mode did not match",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects malformed record state", () => {
|
||||
const laneReport = report();
|
||||
firstRecordOf(laneReport).state = "state-a";
|
||||
|
||||
expect(() => validate(plan(), laneReport)).toThrow("invalid lane report record 0 state");
|
||||
});
|
||||
|
||||
it("rejects malformed JSON through the CLI", () => {
|
||||
const result = runCli({ lanePlan: "{not-json" });
|
||||
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("--plan is not valid JSON");
|
||||
});
|
||||
|
||||
it("rejects zero repeat and missing CLI arguments", () => {
|
||||
const zeroRepeat = runCli({ repeat: "0" });
|
||||
expect(zeroRepeat.status).toBe(1);
|
||||
expect(zeroRepeat.stderr).toContain("invalid expected repeat");
|
||||
|
||||
const missingAuth = runCli({ includeAuth: false });
|
||||
expect(missingAuth.status).toBe(1);
|
||||
expect(missingAuth.stderr).toContain("invalid --auth");
|
||||
});
|
||||
});
|
||||
@@ -18,7 +18,7 @@ type WorkflowJob = {
|
||||
steps?: WorkflowStep[];
|
||||
strategy?: {
|
||||
matrix?: {
|
||||
include?: Array<{ include_filters?: string }>;
|
||||
include?: Array<Record<string, string>>;
|
||||
};
|
||||
};
|
||||
};
|
||||
@@ -38,6 +38,10 @@ function findStep(name: string): WorkflowStep {
|
||||
return step as WorkflowStep;
|
||||
}
|
||||
|
||||
function kovaMatrixEntries(): Array<Record<string, string>> {
|
||||
return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? [];
|
||||
}
|
||||
|
||||
describe("OpenClaw performance workflow", () => {
|
||||
it("uses an optional dispatch identifier to name parent-owned runs", () => {
|
||||
const workflow = readFileSync(WORKFLOW, "utf8");
|
||||
@@ -112,22 +116,58 @@ describe("OpenClaw performance workflow", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("passes every configured scenario through one Kova include flag", () => {
|
||||
const workflow = readWorkflow();
|
||||
const includeFilters = workflow.jobs?.kova?.strategy?.matrix?.include?.map(
|
||||
(lane) => lane.include_filters,
|
||||
);
|
||||
it("passes one comma-delimited include set to the lane plan and run", () => {
|
||||
const plan = findStep("Kova version and plan sanity");
|
||||
const runKova = findStep("Run Kova");
|
||||
const includeFilters = kovaMatrixEntries().map((entry) => entry.include_filters);
|
||||
|
||||
expect(includeFilters).toEqual([
|
||||
"scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:bundled-runtime-deps,scenario:agent-cold-warm-message",
|
||||
"scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message",
|
||||
"scenario:agent-cold-warm-message",
|
||||
]);
|
||||
expect(runKova.run).toContain('args+=(--include "$INCLUDE_FILTERS")');
|
||||
expect(includeFilters.every((filters) => !filters.includes(" "))).toBe(true);
|
||||
expect(plan.run).toContain('plan_dir="${RUNNER_TEMP}/kova-plans"');
|
||||
expect(plan.run).toContain('--include "$INCLUDE_FILTERS"');
|
||||
expect(plan.run).toContain('--repeat "$repeat"');
|
||||
expect(plan.run).toContain('echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"');
|
||||
expect(plan.run).not.toContain("$REPORT_DIR");
|
||||
expect(runKova.run).toContain('--include "$INCLUDE_FILTERS"');
|
||||
expect(runKova.run).not.toContain("for filter in $INCLUDE_FILTERS");
|
||||
});
|
||||
|
||||
it("makes the live lane honor explicit live auth in the ephemeral Kova checkout", () => {
|
||||
const override = findStep("Allow live auth for OpenAI candidate state");
|
||||
|
||||
expect(override.if).toContain("matrix.live == 'true'");
|
||||
expect(override.run).toContain("states/mock-openai-provider.json");
|
||||
expect(override.run).toContain('state.auth?.mode !== "mock"');
|
||||
expect(override.run).toContain('state.auth.mode = "default"');
|
||||
expect(override.run).toContain(
|
||||
"This ephemeral checkout must honor the lane's explicit --auth live selection.",
|
||||
);
|
||||
expect(override.run).toContain(
|
||||
'state.auth.reason = "Honor the workflow lane\'s explicit run-level auth selection."',
|
||||
);
|
||||
});
|
||||
|
||||
it("runs the trusted lane evidence validator before tolerating gate failures", () => {
|
||||
const runKova = findStep("Run Kova");
|
||||
const run = runKova.run ?? "";
|
||||
const evidenceValidator = run.indexOf("scripts/lib/kova-workflow-evidence.mjs");
|
||||
const trustedGateAdapter = run.indexOf("scripts/lib/kova-report-gate.mjs");
|
||||
|
||||
expect(evidenceValidator).toBeGreaterThan(-1);
|
||||
expect(trustedGateAdapter).toBeGreaterThan(evidenceValidator);
|
||||
expect(run).toContain('--plan "$KOVA_PLAN_JSON"');
|
||||
expect(run).toContain('--report "$report_json"');
|
||||
expect(run).toContain('--profile "$PROFILE"');
|
||||
expect(run).toContain('--target "local-build:${GITHUB_WORKSPACE}"');
|
||||
expect(run).toContain('--repeat "$repeat"');
|
||||
expect(run).toContain('--include "$INCLUDE_FILTERS"');
|
||||
expect(run).toContain('--auth "$AUTH_MODE"');
|
||||
});
|
||||
|
||||
it("installs local workspace packages beside the OCM root tarball", () => {
|
||||
const configure = findStep("Configure OCM local workspace dependencies");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user