mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-19 21:41:37 +00:00
fix(ci): verify live provider traffic in performance runs (#103073)
* fix(diagnostics): emit provider request timeline events * fix(ci): verify Kova live provider evidence
This commit is contained in:
Vincent Koc
269
scripts/lib/kova-workflow-evidence.mjs
Normal file
269
scripts/lib/kova-workflow-evidence.mjs
Normal file
@@ -0,0 +1,269 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const AUTH_MODES = new Set(["live", "mock"]);
|
||||
const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]);
|
||||
|
||||
function check(condition, reason) {
|
||||
if (!condition) {
|
||||
throw new Error(reason);
|
||||
}
|
||||
}
|
||||
|
||||
function object(value, label) {
|
||||
check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function array(value, label) {
|
||||
check(Array.isArray(value), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function text(value, label) {
|
||||
check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInteger(value, label) {
|
||||
check(Number.isSafeInteger(value) && value > 0, `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function exactStrings(value, expected, label) {
|
||||
const actual = array(value, label);
|
||||
check(
|
||||
actual.length === expected.length && actual.every((item, index) => item === expected[index]),
|
||||
`${label} did not match`,
|
||||
);
|
||||
}
|
||||
|
||||
function pairKey(scenario, state, label) {
|
||||
return `${text(scenario, `${label} scenario`)}\u0000${text(state, `${label} state`)}`;
|
||||
}
|
||||
|
||||
function displayPair(key) {
|
||||
return key.replace("\u0000", "/");
|
||||
}
|
||||
|
||||
function profileId(value, label) {
|
||||
return text(object(value, label).id, `${label} id`);
|
||||
}
|
||||
|
||||
function repeatIndexesFor(selectedPairs) {
|
||||
return new Map([...selectedPairs].map((key) => [key, new Set()]));
|
||||
}
|
||||
|
||||
function validateLiveRecord(record, key) {
|
||||
const auth = object(record.auth, `record ${displayPair(key)} auth`);
|
||||
const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`);
|
||||
check(
|
||||
auth.environmentDependent === true,
|
||||
`live record ${displayPair(key)} auth was not environment-dependent`,
|
||||
);
|
||||
check(provider.authMode === "live", `live record ${displayPair(key)} provider auth was not live`);
|
||||
check(
|
||||
provider.environmentDependent === true,
|
||||
`live record ${displayPair(key)} provider evidence was not environment-dependent`,
|
||||
);
|
||||
check(
|
||||
provider.source === "openclaw-timeline",
|
||||
`live record ${displayPair(key)} provider source was ${provider.source}`,
|
||||
);
|
||||
check(
|
||||
provider.available === true,
|
||||
`live record ${displayPair(key)} provider evidence was unavailable`,
|
||||
);
|
||||
positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`);
|
||||
}
|
||||
|
||||
export function validateKovaWorkflowEvidence({
|
||||
plan,
|
||||
report,
|
||||
profile,
|
||||
target,
|
||||
repeat,
|
||||
includeFilters,
|
||||
authMode,
|
||||
}) {
|
||||
const expectedProfile = text(profile, "expected profile");
|
||||
const expectedTarget = text(target, "expected target");
|
||||
const expectedRepeat = positiveInteger(repeat, "expected repeat");
|
||||
const expectedFilters = array(includeFilters, "expected include filters").map((value, index) =>
|
||||
text(value, `expected include filter ${index}`),
|
||||
);
|
||||
const expectedAuth = text(authMode, "expected auth mode");
|
||||
check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`);
|
||||
check(expectedFilters.length > 0, "expected include filters were empty");
|
||||
|
||||
const lanePlan = object(plan, "lane plan");
|
||||
const laneReport = object(report, "lane report");
|
||||
check(lanePlan.schemaVersion === "kova.matrix.plan.v1", "unexpected lane plan schema");
|
||||
check(laneReport.schemaVersion === "kova.report.v1", "unexpected lane report schema");
|
||||
check(laneReport.mode === "execution", `lane report mode was ${laneReport.mode}`);
|
||||
check(
|
||||
profileId(lanePlan.profile, "lane plan profile") === expectedProfile,
|
||||
"lane plan profile did not match",
|
||||
);
|
||||
check(
|
||||
profileId(laneReport.profile, "lane report profile") === expectedProfile,
|
||||
"lane report profile did not match",
|
||||
);
|
||||
check(lanePlan.target === expectedTarget, "lane plan target did not match");
|
||||
check(laneReport.target === expectedTarget, "lane report target did not match");
|
||||
|
||||
const planControls = object(lanePlan.controls, "lane plan controls");
|
||||
const reportControls = object(laneReport.controls, "lane report controls");
|
||||
exactStrings(planControls.include, expectedFilters, "lane plan include filters");
|
||||
exactStrings(reportControls.include, expectedFilters, "lane report include filters");
|
||||
check(planControls.repeat === expectedRepeat, "lane plan repeat did not match");
|
||||
check(reportControls.repeat === expectedRepeat, "lane report repeat did not match");
|
||||
|
||||
const selectedPairs = new Set();
|
||||
for (const [index, value] of array(lanePlan.entries, "lane plan entries").entries()) {
|
||||
const entry = object(value, `lane plan entry ${index}`);
|
||||
const status = text(entry.status, `lane plan entry ${index} status`);
|
||||
check(status === "SELECTED" || status === "SKIPPED", `invalid lane plan entry ${index} status`);
|
||||
if (status === "SKIPPED") {
|
||||
continue;
|
||||
}
|
||||
const key = pairKey(
|
||||
object(entry.scenario, `lane plan entry ${index} scenario`).id,
|
||||
object(entry.state, `lane plan entry ${index} state`).id,
|
||||
`lane plan entry ${index}`,
|
||||
);
|
||||
check(!selectedPairs.has(key), `lane plan selected duplicate ${displayPair(key)}`);
|
||||
selectedPairs.add(key);
|
||||
}
|
||||
check(selectedPairs.size > 0, "lane plan selected no scenario/state pairs");
|
||||
|
||||
const reportAuth = object(laneReport.auth, "lane report auth");
|
||||
check(reportAuth.requestedMode === expectedAuth, "lane report requested auth did not match");
|
||||
if (expectedAuth === "live") {
|
||||
check(
|
||||
object(reportAuth.live, "lane report live auth").environmentDependent === true,
|
||||
"lane report live auth was not environment-dependent",
|
||||
);
|
||||
}
|
||||
|
||||
const records = array(laneReport.records, "lane report records");
|
||||
const repeatIndexes = repeatIndexesFor(selectedPairs);
|
||||
for (const [index, value] of records.entries()) {
|
||||
const record = object(value, `lane report record ${index}`);
|
||||
const state = object(record.state, `lane report record ${index} state`);
|
||||
const key = pairKey(record.scenario, state.id, `lane report record ${index}`);
|
||||
check(selectedPairs.has(key), `lane report contained unexpected ${displayPair(key)}`);
|
||||
const status = text(record.status, `record ${displayPair(key)} status`);
|
||||
check(
|
||||
["BLOCKED", "FAIL", "PASS"].includes(status),
|
||||
`invalid record ${displayPair(key)} status`,
|
||||
);
|
||||
|
||||
const recordRepeat = object(record.repeat, `record ${displayPair(key)} repeat`);
|
||||
check(
|
||||
recordRepeat.total === expectedRepeat,
|
||||
`record ${displayPair(key)} repeat total did not match`,
|
||||
);
|
||||
const repeatIndex = positiveInteger(
|
||||
recordRepeat.index,
|
||||
`record ${displayPair(key)} repeat index`,
|
||||
);
|
||||
check(
|
||||
repeatIndex <= expectedRepeat,
|
||||
`record ${displayPair(key)} repeat index exceeded ${expectedRepeat}`,
|
||||
);
|
||||
const indexes = repeatIndexes.get(key);
|
||||
check(!indexes.has(repeatIndex), `record ${displayPair(key)} repeated index ${repeatIndex}`);
|
||||
indexes.add(repeatIndex);
|
||||
|
||||
const recordAuth = object(record.auth, `record ${displayPair(key)} auth`);
|
||||
check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`);
|
||||
if (expectedAuth === "live") {
|
||||
validateLiveRecord(record, key);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, indexes] of repeatIndexes) {
|
||||
check(
|
||||
indexes.size === expectedRepeat,
|
||||
`lane report collapsed ${displayPair(key)} coverage to ${indexes.size}/${expectedRepeat} repeats`,
|
||||
);
|
||||
for (let index = 1; index <= expectedRepeat; index += 1) {
|
||||
check(indexes.has(index), `lane report missed ${displayPair(key)} repeat ${index}`);
|
||||
}
|
||||
}
|
||||
check(
|
||||
records.length === selectedPairs.size * expectedRepeat,
|
||||
"lane report record count did not match selected coverage",
|
||||
);
|
||||
|
||||
return {
|
||||
authMode: expectedAuth,
|
||||
pairCount: selectedPairs.size,
|
||||
recordCount: records.length,
|
||||
repeat: expectedRepeat,
|
||||
};
|
||||
}
|
||||
|
||||
function parseCliArgs(argv) {
|
||||
const flags = {};
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const name = argv[index];
|
||||
const value = argv[index + 1];
|
||||
check(name?.startsWith("--") && value !== undefined, "invalid CLI arguments");
|
||||
const key = name.slice(2);
|
||||
check(CLI_KEYS.has(key), `unknown --${key}`);
|
||||
check(!Object.hasOwn(flags, key), `duplicate --${key}`);
|
||||
flags[key] = value;
|
||||
}
|
||||
for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) {
|
||||
text(flags[key], `--${key}`);
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
function readJson(pathValue, label) {
|
||||
const file = text(pathValue, label);
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`${label} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function runCli() {
|
||||
const flags = parseCliArgs(process.argv.slice(2));
|
||||
const result = validateKovaWorkflowEvidence({
|
||||
plan: readJson(flags.plan, "--plan"),
|
||||
report: readJson(flags.report, "--report"),
|
||||
profile: flags.profile,
|
||||
target: flags.target,
|
||||
repeat: Number(flags.repeat),
|
||||
includeFilters: flags.include
|
||||
.split(",")
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean),
|
||||
authMode: flags.auth,
|
||||
});
|
||||
console.log(
|
||||
`Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`,
|
||||
);
|
||||
}
|
||||
|
||||
const modulePath = fs.realpathSync.native(fileURLToPath(import.meta.url));
|
||||
const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(process.argv[1])) : "";
|
||||
|
||||
if (modulePath === invokedPath) {
|
||||
try {
|
||||
runCli();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Kova evidence validation failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user