mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 09:50:42 +00:00
test(plugins): add gateway gauntlet
This commit is contained in:
Vincent Koc
@@ -163,7 +163,7 @@ function collectObservations(params) {
|
||||
const observations = [];
|
||||
for (const result of params.startup?.results ?? []) {
|
||||
const cpuCoreMax = result.summary?.cpuCoreRatio?.max;
|
||||
const wallMax = result.summary?.readyz?.max ?? result.summary?.healthz?.max;
|
||||
const wallMax = result.summary?.readyzMs?.max ?? result.summary?.healthzMs?.max;
|
||||
if (
|
||||
typeof cpuCoreMax === "number" &&
|
||||
typeof wallMax === "number" &&
|
||||
|
||||
578
scripts/check-plugin-gateway-gauntlet.mjs
Normal file
578
scripts/check-plugin-gateway-gauntlet.mjs
Normal file
@@ -0,0 +1,578 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import {
|
||||
collectGatewayCpuObservations,
|
||||
collectMetricObservations,
|
||||
discoverBundledPluginManifests,
|
||||
selectPluginEntries,
|
||||
} from "./lib/plugin-gateway-gauntlet.mjs";
|
||||
|
||||
const DEFAULT_QA_SCENARIOS = [
|
||||
"channel-chat-baseline",
|
||||
"memory-failure-fallback",
|
||||
"gateway-restart-inflight-run",
|
||||
];
|
||||
const DEFAULT_CPU_CORE_WARN = 0.9;
|
||||
const DEFAULT_HOT_WALL_WARN_MS = 30_000;
|
||||
const DEFAULT_MAX_RSS_WARN_MB = 1536;
|
||||
const DEFAULT_QA_PLUGIN_CHUNK_SIZE = 12;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: path.join(
|
||||
process.cwd(),
|
||||
".artifacts",
|
||||
"plugin-gateway-gauntlet",
|
||||
new Date().toISOString().replace(/[:.]/g, "-"),
|
||||
),
|
||||
pluginIds: [],
|
||||
shardTotal: readOptionalPositiveIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_TOTAL") ?? 1,
|
||||
shardIndex: readOptionalNonNegativeIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_INDEX") ?? 0,
|
||||
limit: undefined,
|
||||
skipPrebuild: false,
|
||||
skipLifecycle: false,
|
||||
skipQa: false,
|
||||
skipSlashHelp: false,
|
||||
qaScenarios: [],
|
||||
qaPluginChunkSize: DEFAULT_QA_PLUGIN_CHUNK_SIZE,
|
||||
cpuCoreWarn: DEFAULT_CPU_CORE_WARN,
|
||||
hotWallWarnMs: DEFAULT_HOT_WALL_WARN_MS,
|
||||
maxRssWarnMb: DEFAULT_MAX_RSS_WARN_MB,
|
||||
wallAnomalyMultiplier: 3,
|
||||
rssAnomalyMultiplier: 2.5,
|
||||
commandTimeoutMs: 120_000,
|
||||
buildTimeoutMs: 600_000,
|
||||
qaTimeoutMs: 900_000,
|
||||
};
|
||||
const envIds = normalizeCsv(process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS);
|
||||
options.pluginIds.push(...envIds);
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const readValue = () => {
|
||||
const value = argv[index + 1];
|
||||
if (!value) {
|
||||
throw new Error(`Missing value for ${arg}`);
|
||||
}
|
||||
index += 1;
|
||||
return value;
|
||||
};
|
||||
switch (arg) {
|
||||
case "--":
|
||||
break;
|
||||
case "--repo-root":
|
||||
options.repoRoot = path.resolve(readValue());
|
||||
break;
|
||||
case "--output-dir":
|
||||
options.outputDir = path.resolve(readValue());
|
||||
break;
|
||||
case "--plugin":
|
||||
options.pluginIds.push(readValue());
|
||||
break;
|
||||
case "--shard-total":
|
||||
options.shardTotal = parsePositiveInt(readValue(), "--shard-total");
|
||||
break;
|
||||
case "--shard-index":
|
||||
options.shardIndex = parseNonNegativeInt(readValue(), "--shard-index");
|
||||
break;
|
||||
case "--limit":
|
||||
options.limit = parsePositiveInt(readValue(), "--limit");
|
||||
break;
|
||||
case "--qa-scenario":
|
||||
options.qaScenarios.push(readValue());
|
||||
break;
|
||||
case "--qa-plugin-chunk-size":
|
||||
options.qaPluginChunkSize = parsePositiveInt(readValue(), "--qa-plugin-chunk-size");
|
||||
break;
|
||||
case "--cpu-core-warn":
|
||||
options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn");
|
||||
break;
|
||||
case "--hot-wall-warn-ms":
|
||||
options.hotWallWarnMs = parsePositiveInt(readValue(), "--hot-wall-warn-ms");
|
||||
break;
|
||||
case "--max-rss-warn-mb":
|
||||
options.maxRssWarnMb = parsePositiveNumber(readValue(), "--max-rss-warn-mb");
|
||||
break;
|
||||
case "--wall-anomaly-multiplier":
|
||||
options.wallAnomalyMultiplier = parsePositiveNumber(
|
||||
readValue(),
|
||||
"--wall-anomaly-multiplier",
|
||||
);
|
||||
break;
|
||||
case "--rss-anomaly-multiplier":
|
||||
options.rssAnomalyMultiplier = parsePositiveNumber(readValue(), "--rss-anomaly-multiplier");
|
||||
break;
|
||||
case "--command-timeout-ms":
|
||||
options.commandTimeoutMs = parsePositiveInt(readValue(), "--command-timeout-ms");
|
||||
break;
|
||||
case "--build-timeout-ms":
|
||||
options.buildTimeoutMs = parsePositiveInt(readValue(), "--build-timeout-ms");
|
||||
break;
|
||||
case "--qa-timeout-ms":
|
||||
options.qaTimeoutMs = parsePositiveInt(readValue(), "--qa-timeout-ms");
|
||||
break;
|
||||
case "--skip-prebuild":
|
||||
options.skipPrebuild = true;
|
||||
break;
|
||||
case "--skip-lifecycle":
|
||||
options.skipLifecycle = true;
|
||||
break;
|
||||
case "--skip-qa":
|
||||
options.skipQa = true;
|
||||
break;
|
||||
case "--skip-slash-help":
|
||||
options.skipSlashHelp = true;
|
||||
break;
|
||||
case "--help":
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
if (options.qaScenarios.length === 0) {
|
||||
options.qaScenarios = [...DEFAULT_QA_SCENARIOS];
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: pnpm test:plugins:gateway-gauntlet [options]
|
||||
|
||||
Runs a shardable bundled-plugin lifecycle, slash inventory, and QA gateway perf gauntlet.
|
||||
|
||||
Options:
|
||||
--plugin <id> Plugin id to include, repeatable
|
||||
--shard-total <count> Total plugin shards (default: env or 1)
|
||||
--shard-index <index> Zero-based shard index (default: env or 0)
|
||||
--limit <count> Limit selected plugins after sharding
|
||||
--output-dir <path> Artifact directory
|
||||
--qa-scenario <id> QA Lab scenario id, repeatable
|
||||
--qa-plugin-chunk-size <count> Plugins enabled per QA run (default: 12)
|
||||
--cpu-core-warn <ratio> Hot CPU threshold (default: 0.9)
|
||||
--hot-wall-warn-ms <ms> Minimum wall time for hot CPU observations (default: 30000)
|
||||
--max-rss-warn-mb <mb> Maximum RSS warning threshold (default: 1536)
|
||||
--skip-prebuild Skip the upfront build used to avoid per-command rebuild noise
|
||||
--skip-lifecycle Skip plugin install/inspect/disable/enable/doctor/uninstall
|
||||
--skip-qa Skip QA Lab RPC conversation runs
|
||||
--skip-slash-help Skip CLI help probes for plugin-declared command aliases
|
||||
`);
|
||||
}
|
||||
|
||||
function normalizeCsv(raw) {
|
||||
return raw
|
||||
? raw
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry.length > 0)
|
||||
: [];
|
||||
}
|
||||
|
||||
function readOptionalPositiveIntEnv(name) {
|
||||
const raw = process.env[name];
|
||||
return raw ? parsePositiveInt(raw, name) : undefined;
|
||||
}
|
||||
|
||||
function readOptionalNonNegativeIntEnv(name) {
|
||||
const raw = process.env[name];
|
||||
return raw ? parseNonNegativeInt(raw, name) : undefined;
|
||||
}
|
||||
|
||||
function parsePositiveInt(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseNonNegativeInt(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 0) {
|
||||
throw new Error(`${label} must be a non-negative integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error(`${label} must be a positive number`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function pnpmCommand() {
|
||||
return process.platform === "win32" ? "pnpm.cmd" : "pnpm";
|
||||
}
|
||||
|
||||
function openclawCommand(repoRoot, args) {
|
||||
return {
|
||||
command: process.execPath,
|
||||
args: [path.join(repoRoot, "scripts", "run-node.mjs"), ...args],
|
||||
};
|
||||
}
|
||||
|
||||
function chunkArray(values, chunkSize) {
|
||||
const chunks = [];
|
||||
for (let index = 0; index < values.length; index += chunkSize) {
|
||||
chunks.push(values.slice(index, index + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function toRepoRelativePath(repoRoot, absolutePath) {
|
||||
const relativePath = path.relative(repoRoot, absolutePath);
|
||||
if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
|
||||
throw new Error(`Output path must stay inside repo root: ${absolutePath}`);
|
||||
}
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
function createIsolatedEnv(repoRoot, runRoot) {
|
||||
const home = path.join(runRoot, "home");
|
||||
const stateDir = path.join(runRoot, "state");
|
||||
fs.mkdirSync(home, { recursive: true });
|
||||
fs.mkdirSync(stateDir, { recursive: true });
|
||||
return {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
XDG_CONFIG_HOME: path.join(home, ".config"),
|
||||
XDG_CACHE_HOME: path.join(home, ".cache"),
|
||||
XDG_DATA_HOME: path.join(home, ".local", "share"),
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
|
||||
OPENCLAW_LOG_DIR: path.join(runRoot, "logs"),
|
||||
OPENCLAW_QA_SUITE_PROGRESS: process.env.OPENCLAW_QA_SUITE_PROGRESS ?? "1",
|
||||
PATH: process.env.PATH,
|
||||
PWD: repoRoot,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsrBinTime() {
|
||||
return fs.existsSync("/usr/bin/time");
|
||||
}
|
||||
|
||||
function timeWrapperArgs(command, args) {
|
||||
if (!hasUsrBinTime()) {
|
||||
return { command, args, mode: "none" };
|
||||
}
|
||||
if (process.platform === "darwin") {
|
||||
return { command: "/usr/bin/time", args: ["-l", command, ...args], mode: "bsd" };
|
||||
}
|
||||
return { command: "/usr/bin/time", args: ["-v", command, ...args], mode: "gnu" };
|
||||
}
|
||||
|
||||
function parseTimedMetrics(stderr, wallMs, mode) {
|
||||
let userSeconds = null;
|
||||
let systemSeconds = null;
|
||||
let maxRssMb = null;
|
||||
if (mode === "gnu") {
|
||||
userSeconds = parseFirstFloat(stderr, /User time \(seconds\):\s*([0-9.]+)/u);
|
||||
systemSeconds = parseFirstFloat(stderr, /System time \(seconds\):\s*([0-9.]+)/u);
|
||||
const maxRssKb = parseFirstFloat(stderr, /Maximum resident set size \(kbytes\):\s*([0-9.]+)/u);
|
||||
maxRssMb = maxRssKb == null ? null : maxRssKb / 1024;
|
||||
} else if (mode === "bsd") {
|
||||
userSeconds = parseFirstFloat(stderr, /[0-9.]+\s+real\s+([0-9.]+)\s+user/u);
|
||||
systemSeconds = parseFirstFloat(stderr, /([0-9.]+)\s+sys/u);
|
||||
const maxRssBytes = parseFirstFloat(stderr, /([0-9]+)\s+maximum resident set size/u);
|
||||
maxRssMb = maxRssBytes == null ? null : maxRssBytes / 1024 / 1024;
|
||||
}
|
||||
const cpuMs =
|
||||
userSeconds == null && systemSeconds == null
|
||||
? null
|
||||
: ((userSeconds ?? 0) + (systemSeconds ?? 0)) * 1000;
|
||||
return {
|
||||
wallMs,
|
||||
cpuMs,
|
||||
cpuCoreRatio: cpuMs == null || wallMs <= 0 ? null : cpuMs / wallMs,
|
||||
maxRssMb,
|
||||
};
|
||||
}
|
||||
|
||||
function parseFirstFloat(value, pattern) {
|
||||
const match = value.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number(match[1]);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function stripAnsi(value) {
|
||||
return value.replace(/\u001B\[[0-9;]*m/gu, "");
|
||||
}
|
||||
|
||||
function writeCommandLog(params) {
|
||||
const { logDir, label, stdout, stderr } = params;
|
||||
fs.mkdirSync(logDir, { recursive: true });
|
||||
const safeLabel = label.replace(/[^a-zA-Z0-9_.-]+/gu, "_");
|
||||
const logPath = path.join(logDir, `${safeLabel}.log`);
|
||||
fs.writeFileSync(
|
||||
logPath,
|
||||
[`$ ${params.command.join(" ")}`, "", stripAnsi(stdout), stripAnsi(stderr)].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
return logPath;
|
||||
}
|
||||
|
||||
function runMeasuredCommand(params) {
|
||||
const { command, args, mode } = timeWrapperArgs(params.command, params.args);
|
||||
const started = process.hrtime.bigint();
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: params.cwd,
|
||||
env: params.env,
|
||||
encoding: "utf8",
|
||||
timeout: params.timeoutMs,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
const wallMs = Number(process.hrtime.bigint() - started) / 1_000_000;
|
||||
const status = result.status ?? (result.signal ? 1 : 0);
|
||||
const stdout = result.stdout ?? "";
|
||||
const stderr = result.stderr ?? "";
|
||||
const logPath = writeCommandLog({
|
||||
logDir: params.logDir,
|
||||
label: params.label,
|
||||
command: [params.command, ...params.args],
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
return {
|
||||
label: params.label,
|
||||
phase: params.phase,
|
||||
pluginId: params.pluginId ?? null,
|
||||
status,
|
||||
signal: result.signal ?? null,
|
||||
timedOut: result.error?.code === "ETIMEDOUT",
|
||||
logPath,
|
||||
...parseTimedMetrics(stderr, wallMs, mode),
|
||||
};
|
||||
}
|
||||
|
||||
function runPluginLifecycle(params) {
|
||||
for (const plugin of params.plugins) {
|
||||
const commands = [
|
||||
["install", ["install", plugin.id]],
|
||||
["inspect", ["inspect", plugin.id, "--json"]],
|
||||
["disable", ["disable", plugin.id]],
|
||||
["enable", ["enable", plugin.id]],
|
||||
["doctor", ["doctor"]],
|
||||
["uninstall", ["uninstall", plugin.id, "--force"]],
|
||||
];
|
||||
for (const [phase, args] of commands) {
|
||||
process.stderr.write(`[plugin-gauntlet] ${plugin.id} ${phase}\n`);
|
||||
params.rows.push(
|
||||
runMeasuredCommand({
|
||||
cwd: params.repoRoot,
|
||||
env: params.env,
|
||||
logDir: path.join(params.outputDir, "logs", "lifecycle"),
|
||||
...openclawCommand(params.repoRoot, ["plugins", ...args]),
|
||||
label: `${plugin.id}-${phase}`,
|
||||
phase: `lifecycle:${phase}`,
|
||||
pluginId: plugin.id,
|
||||
timeoutMs: params.commandTimeoutMs,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runSlashHelpProbes(params) {
|
||||
for (const plugin of params.plugins) {
|
||||
for (const alias of plugin.cliCommandAliases) {
|
||||
const command = alias.cliCommand ?? alias.name;
|
||||
process.stderr.write(`[plugin-gauntlet] ${plugin.id} slash-help /${alias.name}\n`);
|
||||
params.rows.push(
|
||||
runMeasuredCommand({
|
||||
cwd: params.repoRoot,
|
||||
env: params.env,
|
||||
logDir: path.join(params.outputDir, "logs", "slash-help"),
|
||||
...openclawCommand(params.repoRoot, [command, "--help"]),
|
||||
label: `${plugin.id}-slash-${alias.name}`,
|
||||
phase: "slash:help",
|
||||
pluginId: plugin.id,
|
||||
timeoutMs: params.commandTimeoutMs,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runQaChunks(params) {
|
||||
const chunks = chunkArray(params.plugins, params.qaPluginChunkSize);
|
||||
const summaries = [];
|
||||
for (let index = 0; index < chunks.length; index += 1) {
|
||||
const chunk = chunks[index];
|
||||
const outputDir = path.join(
|
||||
params.outputDir,
|
||||
"qa-suite",
|
||||
`chunk-${String(index).padStart(2, "0")}`,
|
||||
);
|
||||
const outputArg = toRepoRelativePath(params.repoRoot, outputDir);
|
||||
const pluginIds = chunk.map((plugin) => plugin.id);
|
||||
process.stderr.write(
|
||||
`[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIds.join(",")}\n`,
|
||||
);
|
||||
const row = runMeasuredCommand({
|
||||
cwd: params.repoRoot,
|
||||
env: params.env,
|
||||
logDir: path.join(params.outputDir, "logs", "qa-suite"),
|
||||
...openclawCommand(params.repoRoot, [
|
||||
"qa",
|
||||
"suite",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--concurrency",
|
||||
"1",
|
||||
"--output-dir",
|
||||
outputArg,
|
||||
...params.qaScenarios.flatMap((scenario) => ["--scenario", scenario]),
|
||||
...pluginIds.flatMap((pluginId) => ["--enable-plugin", pluginId]),
|
||||
]),
|
||||
label: `qa-chunk-${String(index).padStart(2, "0")}`,
|
||||
phase: "qa:rpc",
|
||||
timeoutMs: params.qaTimeoutMs,
|
||||
});
|
||||
params.rows.push({ ...row, pluginId: pluginIds.join(",") });
|
||||
const summaryPath = path.join(outputDir, "qa-suite-summary.json");
|
||||
if (fs.existsSync(summaryPath)) {
|
||||
summaries.push(JSON.parse(fs.readFileSync(summaryPath, "utf8")));
|
||||
}
|
||||
}
|
||||
return summaries;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const repoRoot = path.resolve(options.repoRoot);
|
||||
fs.mkdirSync(options.outputDir, { recursive: true });
|
||||
const runRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-gauntlet-"));
|
||||
const env = createIsolatedEnv(repoRoot, runRoot);
|
||||
const matrix = discoverBundledPluginManifests(repoRoot);
|
||||
const selectedPlugins = selectPluginEntries(matrix, {
|
||||
ids: options.pluginIds,
|
||||
shardTotal: options.shardTotal,
|
||||
shardIndex: options.shardIndex,
|
||||
limit: options.limit,
|
||||
});
|
||||
const rows = [];
|
||||
if (!options.skipPrebuild && (selectedPlugins.length > 0 || !options.skipQa)) {
|
||||
process.stderr.write("[plugin-gauntlet] prebuild\n");
|
||||
rows.push(
|
||||
runMeasuredCommand({
|
||||
cwd: repoRoot,
|
||||
env,
|
||||
logDir: path.join(options.outputDir, "logs", "prebuild"),
|
||||
command: pnpmCommand(),
|
||||
args: ["build"],
|
||||
label: "prebuild",
|
||||
phase: "prebuild",
|
||||
timeoutMs: options.buildTimeoutMs,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const prebuildFailed = rows.some(
|
||||
(row) => row.phase === "prebuild" && (row.status !== 0 || row.timedOut),
|
||||
);
|
||||
if (!prebuildFailed && !options.skipLifecycle) {
|
||||
runPluginLifecycle({
|
||||
repoRoot,
|
||||
outputDir: options.outputDir,
|
||||
env,
|
||||
plugins: selectedPlugins,
|
||||
rows,
|
||||
commandTimeoutMs: options.commandTimeoutMs,
|
||||
});
|
||||
}
|
||||
if (!prebuildFailed && !options.skipSlashHelp) {
|
||||
runSlashHelpProbes({
|
||||
repoRoot,
|
||||
outputDir: options.outputDir,
|
||||
env,
|
||||
plugins: selectedPlugins,
|
||||
rows,
|
||||
commandTimeoutMs: options.commandTimeoutMs,
|
||||
});
|
||||
}
|
||||
const qaSummaries =
|
||||
options.skipQa || prebuildFailed
|
||||
? []
|
||||
: runQaChunks({
|
||||
repoRoot,
|
||||
outputDir: options.outputDir,
|
||||
env,
|
||||
plugins: selectedPlugins,
|
||||
rows,
|
||||
qaScenarios: options.qaScenarios,
|
||||
qaPluginChunkSize: options.qaPluginChunkSize,
|
||||
qaTimeoutMs: options.qaTimeoutMs,
|
||||
});
|
||||
const metricObservations = collectMetricObservations(rows, {
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
maxRssWarnMb: options.maxRssWarnMb,
|
||||
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
|
||||
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
|
||||
});
|
||||
const gatewayObservations = qaSummaries.flatMap((qa) =>
|
||||
collectGatewayCpuObservations({
|
||||
startup: null,
|
||||
qa,
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
}),
|
||||
);
|
||||
const failures = rows.filter((row) => row.status !== 0 || row.timedOut);
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
repoRoot,
|
||||
outputDir: options.outputDir,
|
||||
isolatedRunRoot: runRoot,
|
||||
selectedPluginCount: selectedPlugins.length,
|
||||
totalPluginCount: matrix.length,
|
||||
options: {
|
||||
pluginIds: options.pluginIds,
|
||||
shardTotal: options.shardTotal,
|
||||
shardIndex: options.shardIndex,
|
||||
limit: options.limit ?? null,
|
||||
qaScenarios: options.qaScenarios,
|
||||
qaPluginChunkSize: options.qaPluginChunkSize,
|
||||
skipLifecycle: options.skipLifecycle,
|
||||
skipQa: options.skipQa,
|
||||
skipSlashHelp: options.skipSlashHelp,
|
||||
skipPrebuild: options.skipPrebuild,
|
||||
thresholds: {
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
maxRssWarnMb: options.maxRssWarnMb,
|
||||
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
|
||||
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
|
||||
},
|
||||
},
|
||||
matrix,
|
||||
selectedPlugins,
|
||||
rows,
|
||||
observations: [...metricObservations, ...gatewayObservations],
|
||||
failures,
|
||||
};
|
||||
const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json");
|
||||
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
||||
process.stdout.write(`[plugin-gauntlet] summary: ${summaryPath}\n`);
|
||||
process.stdout.write(
|
||||
`[plugin-gauntlet] plugins=${selectedPlugins.length}/${matrix.length} rows=${rows.length} failures=${failures.length} observations=${summary.observations.length}\n`,
|
||||
);
|
||||
if (failures.length > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
326
scripts/lib/plugin-gateway-gauntlet.mjs
Normal file
326
scripts/lib/plugin-gateway-gauntlet.mjs
Normal file
@@ -0,0 +1,326 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import JSON5 from "json5";
|
||||
|
||||
const MANIFEST_NAMES = ["openclaw.plugin.json", "openclaw.plugin.json5"];
|
||||
|
||||
function isPlainObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function normalizeString(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function normalizeStringArray(value) {
|
||||
return Array.isArray(value)
|
||||
? value.map((entry) => normalizeString(entry)).filter((entry) => entry.length > 0)
|
||||
: [];
|
||||
}
|
||||
|
||||
function readPluginManifest(manifestPath) {
|
||||
const raw = fs.readFileSync(manifestPath, "utf8");
|
||||
const parsed = manifestPath.endsWith(".json5") ? JSON5.parse(raw) : JSON.parse(raw);
|
||||
if (!isPlainObject(parsed)) {
|
||||
throw new Error(`Plugin manifest must be an object: ${manifestPath}`);
|
||||
}
|
||||
const id = normalizeString(parsed.id);
|
||||
if (!id) {
|
||||
throw new Error(`Plugin manifest is missing id: ${manifestPath}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function schemaHasRequiredFields(schema, seen = new Set()) {
|
||||
if (!isPlainObject(schema) || seen.has(schema)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(schema);
|
||||
if (Array.isArray(schema.required) && schema.required.length > 0) {
|
||||
return true;
|
||||
}
|
||||
for (const key of ["properties", "patternProperties", "$defs", "definitions"]) {
|
||||
const children = schema[key];
|
||||
if (!isPlainObject(children)) {
|
||||
continue;
|
||||
}
|
||||
for (const child of Object.values(children)) {
|
||||
if (schemaHasRequiredFields(child, seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of ["items", "additionalProperties", "contains", "not", "if", "then", "else"]) {
|
||||
if (schemaHasRequiredFields(schema[key], seen)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for (const key of ["allOf", "anyOf", "oneOf", "prefixItems"]) {
|
||||
const children = schema[key];
|
||||
if (!Array.isArray(children)) {
|
||||
continue;
|
||||
}
|
||||
if (children.some((child) => schemaHasRequiredFields(child, seen))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function collectCommandAliasRecords(manifest) {
|
||||
const aliases = Array.isArray(manifest.commandAliases) ? manifest.commandAliases : [];
|
||||
return aliases
|
||||
.map((alias) => {
|
||||
if (typeof alias === "string") {
|
||||
const name = normalizeString(alias);
|
||||
return name ? { name, kind: "runtime-slash", cliCommand: null } : null;
|
||||
}
|
||||
if (!isPlainObject(alias)) {
|
||||
return null;
|
||||
}
|
||||
const name = normalizeString(alias.name);
|
||||
if (!name) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
name,
|
||||
kind: normalizeString(alias.kind) || "runtime-slash",
|
||||
cliCommand: normalizeString(alias.cliCommand) || null,
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function collectAuthMethods(manifest) {
|
||||
const auth = Array.isArray(manifest.auth) ? manifest.auth : [];
|
||||
return auth
|
||||
.map((entry) => (isPlainObject(entry) ? normalizeString(entry.method) : ""))
|
||||
.filter((method) => method.length > 0);
|
||||
}
|
||||
|
||||
function collectOnboardingScopes(manifest) {
|
||||
const scopes = new Set();
|
||||
const addScopes = (value) => {
|
||||
for (const scope of normalizeStringArray(value)) {
|
||||
scopes.add(scope);
|
||||
}
|
||||
};
|
||||
addScopes(manifest.onboardingScopes);
|
||||
if (Array.isArray(manifest.auth)) {
|
||||
for (const entry of manifest.auth) {
|
||||
if (isPlainObject(entry)) {
|
||||
addScopes(entry.onboardingScopes);
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...scopes];
|
||||
}
|
||||
|
||||
function buildPluginMatrixEntry(params) {
|
||||
const { repoRoot, manifestPath, manifest } = params;
|
||||
const relativeManifestPath = path.relative(repoRoot, manifestPath);
|
||||
const commandAliases = collectCommandAliasRecords(manifest);
|
||||
return {
|
||||
id: manifest.id,
|
||||
name: normalizeString(manifest.name) || manifest.id,
|
||||
dir: path.relative(repoRoot, path.dirname(manifestPath)),
|
||||
manifestPath: relativeManifestPath,
|
||||
enabledByDefault: manifest.enabledByDefault === true,
|
||||
activation: isPlainObject(manifest.activation) ? manifest.activation : {},
|
||||
providers: normalizeStringArray(manifest.providers),
|
||||
channels: normalizeStringArray(manifest.channels),
|
||||
skills: normalizeStringArray(manifest.skills),
|
||||
authMethods: collectAuthMethods(manifest),
|
||||
onboardingScopes: collectOnboardingScopes(manifest),
|
||||
hasConfigSchema: isPlainObject(manifest.configSchema),
|
||||
hasRequiredConfigFields: schemaHasRequiredFields(manifest.configSchema),
|
||||
commandAliases,
|
||||
cliCommandAliases: commandAliases.filter((alias) => alias.cliCommand),
|
||||
runtimeSlashAliases: commandAliases.filter((alias) => alias.kind === "runtime-slash"),
|
||||
};
|
||||
}
|
||||
|
||||
function discoverBundledPluginManifests(repoRoot) {
|
||||
const extensionsDir = path.join(repoRoot, "extensions");
|
||||
const entries = fs
|
||||
.readdirSync(extensionsDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.flatMap((entry) => {
|
||||
const pluginDir = path.join(extensionsDir, entry.name);
|
||||
const manifestName = MANIFEST_NAMES.find((name) => fs.existsSync(path.join(pluginDir, name)));
|
||||
if (!manifestName) {
|
||||
return [];
|
||||
}
|
||||
const manifestPath = path.join(pluginDir, manifestName);
|
||||
const manifest = readPluginManifest(manifestPath);
|
||||
return [buildPluginMatrixEntry({ repoRoot, manifestPath, manifest })];
|
||||
});
|
||||
return entries.sort((left, right) => left.id.localeCompare(right.id));
|
||||
}
|
||||
|
||||
function selectPluginEntries(entries, options = {}) {
|
||||
const ids = new Set(normalizeStringArray(options.ids));
|
||||
let selected = ids.size > 0 ? entries.filter((entry) => ids.has(entry.id)) : [...entries];
|
||||
const missingIds = [...ids].filter((id) => !entries.some((entry) => entry.id === id));
|
||||
if (missingIds.length > 0) {
|
||||
throw new Error(`Unknown bundled plugin id(s): ${missingIds.join(", ")}`);
|
||||
}
|
||||
const shardTotal = options.shardTotal ?? 1;
|
||||
const shardIndex = options.shardIndex ?? 0;
|
||||
if (!Number.isInteger(shardTotal) || shardTotal < 1) {
|
||||
throw new Error("--shard-total must be a positive integer");
|
||||
}
|
||||
if (!Number.isInteger(shardIndex) || shardIndex < 0 || shardIndex >= shardTotal) {
|
||||
throw new Error("--shard-index must be in range [0, shard-total)");
|
||||
}
|
||||
selected = selected.filter((_, index) => index % shardTotal === shardIndex);
|
||||
if (options.limit !== undefined) {
|
||||
if (!Number.isInteger(options.limit) || options.limit < 1) {
|
||||
throw new Error("--limit must be a positive integer");
|
||||
}
|
||||
selected = selected.slice(0, options.limit);
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function median(values) {
|
||||
const sorted = values
|
||||
.filter((value) => typeof value === "number" && Number.isFinite(value))
|
||||
.sort((left, right) => left - right);
|
||||
if (sorted.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const midpoint = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 === 1 ? sorted[midpoint] : (sorted[midpoint - 1] + sorted[midpoint]) / 2;
|
||||
}
|
||||
|
||||
function groupByPhase(rows) {
|
||||
const phases = new Map();
|
||||
for (const row of rows) {
|
||||
const phase = normalizeString(row.phase) || "unknown";
|
||||
const current = phases.get(phase) ?? [];
|
||||
current.push(row);
|
||||
phases.set(phase, current);
|
||||
}
|
||||
return phases;
|
||||
}
|
||||
|
||||
function collectMetricObservations(rows, thresholds = {}) {
|
||||
const cpuCoreWarn = thresholds.cpuCoreWarn ?? 0.9;
|
||||
const hotWallWarnMs = thresholds.hotWallWarnMs ?? 30_000;
|
||||
const wallAnomalyMultiplier = thresholds.wallAnomalyMultiplier ?? 3;
|
||||
const maxRssWarnMb = thresholds.maxRssWarnMb ?? null;
|
||||
const rssAnomalyMultiplier = thresholds.rssAnomalyMultiplier ?? 2.5;
|
||||
const observations = [];
|
||||
for (const [phase, phaseRows] of groupByPhase(rows)) {
|
||||
const wallMedianMs = median(phaseRows.map((row) => row.wallMs));
|
||||
const rssMedianMb = median(phaseRows.map((row) => row.maxRssMb));
|
||||
for (const row of phaseRows) {
|
||||
if (
|
||||
typeof row.cpuCoreRatio === "number" &&
|
||||
typeof row.wallMs === "number" &&
|
||||
row.cpuCoreRatio >= cpuCoreWarn &&
|
||||
row.wallMs >= hotWallWarnMs
|
||||
) {
|
||||
observations.push({
|
||||
kind: "phase-cpu-hot",
|
||||
pluginId: row.pluginId ?? null,
|
||||
phase,
|
||||
cpuCoreRatio: row.cpuCoreRatio,
|
||||
wallMs: row.wallMs,
|
||||
});
|
||||
}
|
||||
if (
|
||||
wallMedianMs !== null &&
|
||||
phaseRows.length >= 3 &&
|
||||
typeof row.wallMs === "number" &&
|
||||
row.wallMs >= wallMedianMs * wallAnomalyMultiplier
|
||||
) {
|
||||
observations.push({
|
||||
kind: "phase-wall-anomaly",
|
||||
pluginId: row.pluginId ?? null,
|
||||
phase,
|
||||
wallMs: row.wallMs,
|
||||
medianWallMs: wallMedianMs,
|
||||
multiplier: wallAnomalyMultiplier,
|
||||
});
|
||||
}
|
||||
if (
|
||||
typeof maxRssWarnMb === "number" &&
|
||||
typeof row.maxRssMb === "number" &&
|
||||
row.maxRssMb >= maxRssWarnMb
|
||||
) {
|
||||
observations.push({
|
||||
kind: "phase-rss-high",
|
||||
pluginId: row.pluginId ?? null,
|
||||
phase,
|
||||
maxRssMb: row.maxRssMb,
|
||||
thresholdMb: maxRssWarnMb,
|
||||
});
|
||||
}
|
||||
if (
|
||||
rssMedianMb !== null &&
|
||||
rssMedianMb > 0 &&
|
||||
phaseRows.length >= 3 &&
|
||||
typeof row.maxRssMb === "number" &&
|
||||
row.maxRssMb >= rssMedianMb * rssAnomalyMultiplier
|
||||
) {
|
||||
observations.push({
|
||||
kind: "phase-rss-anomaly",
|
||||
pluginId: row.pluginId ?? null,
|
||||
phase,
|
||||
maxRssMb: row.maxRssMb,
|
||||
medianRssMb: rssMedianMb,
|
||||
multiplier: rssAnomalyMultiplier,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
function collectGatewayCpuObservations(params) {
|
||||
const observations = [];
|
||||
for (const result of params.startup?.results ?? []) {
|
||||
const cpuCoreMax = result.summary?.cpuCoreRatio?.max;
|
||||
const wallMax = result.summary?.readyzMs?.max ?? result.summary?.healthzMs?.max;
|
||||
if (
|
||||
typeof cpuCoreMax === "number" &&
|
||||
typeof wallMax === "number" &&
|
||||
cpuCoreMax >= params.cpuCoreWarn &&
|
||||
wallMax >= params.hotWallWarnMs
|
||||
) {
|
||||
observations.push({
|
||||
kind: "startup-cpu-hot",
|
||||
id: result.id,
|
||||
cpuCoreRatioMax: cpuCoreMax,
|
||||
wallMsMax: wallMax,
|
||||
});
|
||||
}
|
||||
}
|
||||
const qaCpuCoreRatio = params.qa?.metrics?.gatewayCpuCoreRatio;
|
||||
const qaWallMs = params.qa?.metrics?.wallMs;
|
||||
if (
|
||||
typeof qaCpuCoreRatio === "number" &&
|
||||
typeof qaWallMs === "number" &&
|
||||
qaCpuCoreRatio >= params.cpuCoreWarn &&
|
||||
qaWallMs >= params.hotWallWarnMs
|
||||
) {
|
||||
observations.push({
|
||||
kind: "qa-cpu-hot",
|
||||
id: "qa-suite",
|
||||
cpuCoreRatio: qaCpuCoreRatio,
|
||||
wallMs: qaWallMs,
|
||||
});
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
export {
|
||||
collectCommandAliasRecords,
|
||||
collectGatewayCpuObservations,
|
||||
collectMetricObservations,
|
||||
discoverBundledPluginManifests,
|
||||
schemaHasRequiredFields,
|
||||
selectPluginEntries,
|
||||
};
|
||||
Reference in New Issue
Block a user