diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b84ca6da4b0..f69c7ae2698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,9 +200,28 @@ jobs: - name: Setup Node environment uses: ./.github/actions/setup-node-env + - name: Configure vitest JSON reports + if: matrix.task == 'test' && matrix.runtime == 'node' + run: echo "OPENCLAW_VITEST_REPORT_DIR=$RUNNER_TEMP/vitest-reports" >> "$GITHUB_ENV" + - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) run: ${{ matrix.command }} + - name: Summarize slowest tests + if: matrix.task == 'test' && matrix.runtime == 'node' + run: | + node scripts/vitest-slowest.mjs --dir "$OPENCLAW_VITEST_REPORT_DIR" --top 50 --out "$RUNNER_TEMP/vitest-slowest.md" > /dev/null + echo "Slowest test summary written to $RUNNER_TEMP/vitest-slowest.md" + + - name: Upload vitest reports + if: matrix.task == 'test' && matrix.runtime == 'node' + uses: actions/upload-artifact@v4 + with: + name: vitest-reports-${{ runner.os }}-${{ matrix.runtime }} + path: | + ${{ env.OPENCLAW_VITEST_REPORT_DIR }} + ${{ runner.temp }}/vitest-slowest.md + # Types, lint, and format check. check: name: "check" @@ -364,9 +383,28 @@ jobs: pnpm -v pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + - name: Configure vitest JSON reports + if: matrix.task == 'test' + run: echo "OPENCLAW_VITEST_REPORT_DIR=$RUNNER_TEMP/vitest-reports" >> "$GITHUB_ENV" + - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) run: ${{ matrix.command }} + - name: Summarize slowest tests + if: matrix.task == 'test' + run: | + node scripts/vitest-slowest.mjs --dir "$OPENCLAW_VITEST_REPORT_DIR" --top 50 --out "$RUNNER_TEMP/vitest-slowest.md" > /dev/null + echo "Slowest test summary written to $RUNNER_TEMP/vitest-slowest.md" + + - name: Upload vitest reports + if: matrix.task == 'test' + uses: actions/upload-artifact@v4 + with: + name: vitest-reports-${{ runner.os }}-${{ matrix.runtime }} + path: | + ${{ env.OPENCLAW_VITEST_REPORT_DIR }} + ${{ runner.temp }}/vitest-slowest.md + # Consolidated macOS job: runs TS tests + Swift lint/build/test sequentially # on a single runner. GitHub limits macOS concurrent jobs to 5 per org; # running 4 separate jobs per PR (as before) starved the queue. One job diff --git a/package.json b/package.json index e64862a82b5..88d2cee8d71 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh", "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh", "test:e2e": "vitest run --config vitest.e2e.config.ts", + "test:fast": "vitest run --config vitest.unit.config.ts", "test:force": "node --import tsx scripts/test-force.ts", "test:install:e2e": "bash scripts/test-install-sh-e2e-docker.sh", "test:install:e2e:anthropic": "OPENCLAW_E2E_MODELS=anthropic CLAWDBOT_E2E_MODELS=anthropic bash scripts/test-install-sh-e2e-docker.sh", diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index da06ebdd7df..4d4c9282291 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -1,5 +1,7 @@ import { spawn } from "node:child_process"; +import fs from "node:fs"; import os from "node:os"; +import path from "node:path"; const pnpm = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; @@ -70,12 +72,59 @@ const WARNING_SUPPRESSION_FLAGS = [ "--disable-warning=DEP0060", ]; +function resolveReportDir() { + const raw = process.env.OPENCLAW_VITEST_REPORT_DIR?.trim(); + if (!raw) { + return null; + } + try { + fs.mkdirSync(raw, { recursive: true }); + } catch { + return null; + } + return raw; +} + +function buildReporterArgs(entry, extraArgs) { + const reportDir = resolveReportDir(); + if (!reportDir) { + return []; + } + + // Vitest supports both `--shard 1/2` and `--shard=1/2`. We use it in the + // split-arg form, so we need to read the next arg to avoid overwriting reports. + const shardIndex = extraArgs.findIndex((arg) => arg === "--shard"); + const inlineShardArg = extraArgs.find( + (arg) => typeof arg === "string" && arg.startsWith("--shard="), + ); + const shardValue = + shardIndex >= 0 && typeof extraArgs[shardIndex + 1] === "string" + ? extraArgs[shardIndex + 1] + : typeof inlineShardArg === "string" + ? inlineShardArg.slice("--shard=".length) + : ""; + const shardSuffix = shardValue + ? `-shard${String(shardValue).replaceAll("/", "of").replaceAll(" ", "")}` + : ""; + + const outputFile = path.join(reportDir, `vitest-${entry.name}${shardSuffix}.json`); + return ["--reporter=default", "--reporter=json", "--outputFile", outputFile]; +} + const runOnce = (entry, extraArgs = []) => new Promise((resolve) => { const maxWorkers = maxWorkersForRun(entry.name); + const reporterArgs = buildReporterArgs(entry, extraArgs); const args = maxWorkers - ? [...entry.args, "--maxWorkers", String(maxWorkers), ...windowsCiArgs, ...extraArgs] - : [...entry.args, ...windowsCiArgs, ...extraArgs]; + ? [ + ...entry.args, + "--maxWorkers", + String(maxWorkers), + ...reporterArgs, + ...windowsCiArgs, + ...extraArgs, + ] + : [...entry.args, ...reporterArgs, ...windowsCiArgs, ...extraArgs]; const nodeOptions = process.env.NODE_OPTIONS ?? ""; const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce( (acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()), @@ -117,6 +166,7 @@ process.on("SIGINT", () => shutdown("SIGINT")); process.on("SIGTERM", () => shutdown("SIGTERM")); if (passthroughArgs.length > 0) { + const maxWorkers = maxWorkersForRun("unit"); const args = maxWorkers ? ["vitest", "run", "--maxWorkers", String(maxWorkers), ...windowsCiArgs, ...passthroughArgs] : ["vitest", "run", ...windowsCiArgs, ...passthroughArgs]; diff --git a/scripts/vitest-slowest.mjs b/scripts/vitest-slowest.mjs new file mode 100644 index 00000000000..21de70325f9 --- /dev/null +++ b/scripts/vitest-slowest.mjs @@ -0,0 +1,160 @@ +import fs from "node:fs"; +import path from "node:path"; + +function parseArgs(argv) { + const out = { + dir: "", + top: 50, + outFile: "", + }; + for (let i = 2; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--dir") { + out.dir = argv[i + 1] ?? ""; + i += 1; + continue; + } + if (arg === "--top") { + out.top = Number.parseInt(argv[i + 1] ?? "", 10); + if (!Number.isFinite(out.top) || out.top <= 0) { + out.top = 50; + } + i += 1; + continue; + } + if (arg === "--out") { + out.outFile = argv[i + 1] ?? ""; + i += 1; + continue; + } + } + return out; +} + +function readJson(filePath) { + const raw = fs.readFileSync(filePath, "utf8"); + return JSON.parse(raw); +} + +function toMs(value) { + if (typeof value !== "number" || !Number.isFinite(value)) { + return 0; + } + return value; +} + +function safeRel(baseDir, filePath) { + try { + const rel = path.relative(baseDir, filePath); + return rel.startsWith("..") ? filePath : rel; + } catch { + return filePath; + } +} + +function main() { + const args = parseArgs(process.argv); + const dir = args.dir?.trim(); + if (!dir) { + console.error( + "usage: node scripts/vitest-slowest.mjs --dir [--top 50] [--out out.md]", + ); + process.exit(2); + } + if (!fs.existsSync(dir)) { + console.error(`vitest report dir not found: ${dir}`); + process.exit(2); + } + + const entries = fs + .readdirSync(dir) + .filter((name) => name.endsWith(".json")) + .map((name) => path.join(dir, name)); + if (entries.length === 0) { + console.error(`no vitest json reports in ${dir}`); + process.exit(2); + } + + const fileRows = []; + const testRows = []; + + for (const filePath of entries) { + let payload; + try { + payload = readJson(filePath); + } catch (err) { + fileRows.push({ + kind: "report", + name: safeRel(dir, filePath), + ms: 0, + note: `failed to parse: ${String(err)}`, + }); + continue; + } + const suiteResults = Array.isArray(payload.testResults) ? payload.testResults : []; + for (const suite of suiteResults) { + const suiteName = typeof suite?.name === "string" ? suite.name : "(unknown)"; + const startTime = toMs(suite?.startTime); + const endTime = toMs(suite?.endTime); + const suiteMs = Math.max(0, endTime - startTime); + fileRows.push({ + kind: "file", + name: safeRel(process.cwd(), suiteName), + ms: suiteMs, + note: safeRel(dir, filePath), + }); + + const assertions = Array.isArray(suite?.assertionResults) ? suite.assertionResults : []; + for (const assertion of assertions) { + const title = typeof assertion?.title === "string" ? assertion.title : "(unknown)"; + const duration = toMs(assertion?.duration); + testRows.push({ + name: `${safeRel(process.cwd(), suiteName)} :: ${title}`, + ms: duration, + suite: safeRel(process.cwd(), suiteName), + title, + }); + } + } + } + + fileRows.sort((a, b) => b.ms - a.ms); + testRows.sort((a, b) => b.ms - a.ms); + + const topFiles = fileRows.slice(0, args.top); + const topTests = testRows.slice(0, args.top); + + const lines = []; + lines.push(`# Vitest Slowest (${new Date().toISOString()})`); + lines.push(""); + lines.push(`Reports: ${entries.length}`); + lines.push(""); + lines.push("## Slowest Files"); + lines.push(""); + lines.push("| ms | file | report |"); + lines.push("|---:|:-----|:-------|"); + for (const row of topFiles) { + lines.push(`| ${Math.round(row.ms)} | \`${row.name}\` | \`${row.note}\` |`); + } + lines.push(""); + lines.push("## Slowest Tests"); + lines.push(""); + lines.push("| ms | test |"); + lines.push("|---:|:-----|"); + for (const row of topTests) { + lines.push(`| ${Math.round(row.ms)} | \`${row.name}\` |`); + } + lines.push(""); + lines.push( + `Notes: file times are (endTime-startTime) per suite; test times come from assertion duration (may exclude setup/import).`, + ); + lines.push(""); + + const outText = lines.join("\n"); + if (args.outFile?.trim()) { + fs.writeFileSync(args.outFile, outText, "utf8"); + } + process.stdout.write(outText); +} + +main();