diff --git a/scripts/check-gateway-cpu-scenarios.mjs b/scripts/check-gateway-cpu-scenarios.mjs index b917d973d62..c3f14dca107 100644 --- a/scripts/check-gateway-cpu-scenarios.mjs +++ b/scripts/check-gateway-cpu-scenarios.mjs @@ -5,12 +5,12 @@ import fs from "node:fs"; import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; +import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mjs"; import { parseNonNegativeInt, parsePositiveInt, parsePositiveNumber, } from "./lib/numeric-options.mjs"; -import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mjs"; import { collectGatewayCpuObservations } from "./lib/plugin-gateway-gauntlet.mjs"; import { createPnpmRunnerSpawnSpec } from "./pnpm-runner.mjs"; @@ -22,6 +22,10 @@ const DEFAULT_QA_SCENARIOS = [ ]; const DEFAULT_CPU_CORE_WARN = 0.9; const DEFAULT_HOT_WALL_WARN_MS = 30_000; +const PRIVATE_QA_REQUIRED_DIST_ENTRIES = [ + "dist/plugin-sdk/qa-lab.js", + "dist/plugin-sdk/qa-runtime.js", +]; function parseArgs(argv) { const args = stripLeadingPackageManagerSeparator(argv); @@ -128,8 +132,8 @@ function runStep(name, command, args, options = {}, params = {}) { console.error(`[gateway-cpu] start ${name}`); const spawn = params.spawnSync ?? defaultSpawnSync; const result = spawn(command, args, { - cwd: process.cwd(), - env: process.env, + cwd: params.cwd ?? process.cwd(), + env: params.env ?? process.env, stdio: "inherit", ...options, }); @@ -138,29 +142,51 @@ function runStep(name, command, args, options = {}, params = {}) { return { name, status, signal: result.signal ?? null }; } -function pnpmCommand(args) { +function pnpmCommand(args, params = {}) { return createPnpmRunnerSpawnSpec({ - cwd: process.cwd(), - env: process.env, + cwd: params.cwd ?? process.cwd(), + env: params.env ?? process.env, pnpmArgs: args, stdio: "inherit", }); } -function toRepoRelativePath(absolutePath) { - const relativePath = path.relative(process.cwd(), absolutePath); +function toRepoRelativePath(repoRoot, absolutePath) { + const relativePath = path.relative(repoRoot, absolutePath); if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) { throw new Error(`Output path must stay inside the repo root: ${absolutePath}`); } return relativePath; } +function hasPrivateQaDist(repoRoot, fsImpl = fs) { + return PRIVATE_QA_REQUIRED_DIST_ENTRIES.every((relativePath) => { + try { + return fsImpl.statSync(path.join(repoRoot, relativePath)).isFile(); + } catch { + return false; + } + }); +} + +function buildPrivateQaEnv(env) { + return { + ...env, + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1", + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: env.OPENCLAW_RUN_NODE_SKIP_DTS_BUILD ?? "1", + }; +} + async function runGatewayCpuScenarios(options, params = {}) { + const repoRoot = params.cwd ?? process.cwd(); + const baseEnv = params.env ?? process.env; + const qaBuildEnv = buildPrivateQaEnv(baseEnv); fs.mkdirSync(options.outputDir, { recursive: true }); const startupOutput = path.join(options.outputDir, "gateway-startup-bench.json"); const qaOutputDir = path.join(options.outputDir, "qa-suite"); - const qaOutputArg = toRepoRelativePath(qaOutputDir); + const qaOutputArg = toRepoRelativePath(repoRoot, qaOutputDir); const steps = []; if (!options.skipStartup) { @@ -196,20 +222,40 @@ async function runGatewayCpuScenarios(options, params = {}) { ); } + let privateQaBuildFailed = false; + if (!options.skipQa && !hasPrivateQaDist(repoRoot, params.fs ?? fs)) { + const privateQaBuild = runStep( + "private QA build", + process.execPath, + ["scripts/build-all.mjs", "cliStartup"], + { env: qaBuildEnv }, + params, + ); + steps.push(privateQaBuild); + privateQaBuildFailed = privateQaBuild.status !== 0; + } + if (!options.skipQa) { - const qaCommand = pnpmCommand([ - "openclaw", - "qa", - "suite", - "--provider-mode", - "mock-openai", - "--concurrency", - "1", - "--output-dir", - qaOutputArg, - ...options.qaScenarios.flatMap((id) => ["--scenario", id]), - ]); - steps.push(runStep("qa suite", qaCommand.command, qaCommand.args, qaCommand.options, params)); + const qaCommand = pnpmCommand( + [ + "openclaw", + "qa", + "suite", + "--provider-mode", + "mock-openai", + "--concurrency", + "1", + "--output-dir", + qaOutputArg, + ...options.qaScenarios.flatMap((id) => ["--scenario", id]), + ], + { cwd: repoRoot, env: qaBuildEnv }, + ); + steps.push( + privateQaBuildFailed + ? { name: "qa suite", signal: null, status: 1 } + : runStep("qa suite", qaCommand.command, qaCommand.args, qaCommand.options, params), + ); } const startup = readJsonIfExists(startupOutput); @@ -264,6 +310,7 @@ async function main(params = {}) { } export const testing = { + hasPrivateQaDist, parseArgs, runGatewayCpuScenarios, }; diff --git a/test/scripts/check-gateway-cpu-scenarios.test.ts b/test/scripts/check-gateway-cpu-scenarios.test.ts index 78a919fad22..73d1e31175d 100644 --- a/test/scripts/check-gateway-cpu-scenarios.test.ts +++ b/test/scripts/check-gateway-cpu-scenarios.test.ts @@ -47,12 +47,12 @@ describe("gateway CPU scenario guard", () => { }); it("rejects non-decimal numeric options", () => { - expect(() => - testing.parseArgs(["--output-dir", makeTempRoot(), "--runs", "1e3"]), - ).toThrow("--runs must be a positive integer"); - expect(() => - testing.parseArgs(["--output-dir", makeTempRoot(), "--warmup", "0x10"]), - ).toThrow("--warmup must be a non-negative integer"); + expect(() => testing.parseArgs(["--output-dir", makeTempRoot(), "--runs", "1e3"])).toThrow( + "--runs must be a positive integer", + ); + expect(() => testing.parseArgs(["--output-dir", makeTempRoot(), "--warmup", "0x10"])).toThrow( + "--warmup must be a non-negative integer", + ); expect(() => testing.parseArgs(["--output-dir", makeTempRoot(), "--cpu-core-warn", "1e3"]), ).toThrow("--cpu-core-warn must be a positive number"); @@ -108,6 +108,74 @@ describe("gateway CPU scenario guard", () => { ]); }); + it("prebuilds private QA dist before running QA scenarios when it is missing", async () => { + const cwd = makeTempRoot(); + const outputDir = path.join(cwd, "out"); + const calls: Array<{ args: string[]; env?: Record }> = []; + const options = testing.parseArgs([ + "--output-dir", + outputDir, + "--skip-startup", + "--qa-scenario", + "channel-chat-baseline", + ]); + + const result = await testing.runGatewayCpuScenarios(options, { + cwd, + silent: true, + spawnSync: (_command: string, args: string[], opts?: { env?: Record }) => { + calls.push({ args, env: opts?.env }); + if (args[0] === "scripts/build-all.mjs") { + const pluginSdkDist = path.join(cwd, "dist", "plugin-sdk"); + mkdirSync(pluginSdkDist, { recursive: true }); + writeFileSync(path.join(pluginSdkDist, "qa-lab.js"), "export {};\n"); + writeFileSync(path.join(pluginSdkDist, "qa-runtime.js"), "export {};\n"); + } + return { status: 0 }; + }, + }); + + expect(result.exitCode).toBe(0); + expect(result.summary.steps.map((step) => step.name)).toEqual(["private QA build", "qa suite"]); + expect(calls[0]?.args).toEqual(["scripts/build-all.mjs", "cliStartup"]); + expect(calls[0]?.env).toMatchObject({ + OPENCLAW_BUILD_PRIVATE_QA: "1", + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1", + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "1", + }); + expect(calls[0]?.env?.OPENCLAW_BUNDLED_PLUGIN_BUILD_IDS).toBeUndefined(); + }); + + it("does not prebuild private QA dist when the required entries already exist", async () => { + const cwd = makeTempRoot(); + const outputDir = path.join(cwd, "out"); + const pluginSdkDist = path.join(cwd, "dist", "plugin-sdk"); + mkdirSync(pluginSdkDist, { recursive: true }); + writeFileSync(path.join(pluginSdkDist, "qa-lab.js"), "export {};\n"); + writeFileSync(path.join(pluginSdkDist, "qa-runtime.js"), "export {};\n"); + const calls: string[][] = []; + const options = testing.parseArgs([ + "--output-dir", + outputDir, + "--skip-startup", + "--qa-scenario", + "channel-chat-baseline", + ]); + + const result = await testing.runGatewayCpuScenarios(options, { + cwd, + silent: true, + spawnSync: (_command: string, args: string[]) => { + calls.push(args); + return { status: 0 }; + }, + }); + + expect(result.exitCode).toBe(0); + expect(result.summary.steps.map((step) => step.name)).toEqual(["qa suite"]); + expect(calls.some((args) => args[0] === "scripts/build-all.mjs")).toBe(false); + }); + it("fails when completed runs report hot gateway CPU observations", async () => { const outputDir = makeTempRoot(); const startupOutput = path.join(outputDir, "gateway-startup-bench.json");