diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7bacc8504f..c55d9d21dc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,13 +252,94 @@ jobs: if: matrix.runtime != 'bun' || github.event_name != 'pull_request' run: ${{ matrix.command }} - extension-fast: - name: "extension-fast (${{ matrix.extension }})" + extension-fast-precheck: + name: "extension-fast-precheck" needs: [docs-scope, changed-scope, changed-extensions] if: needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + use-sticky-disk: "false" + + - name: Select representative extension-fast files + id: representative + run: | + node --input-type=module <<'EOF' + import { appendFileSync } from "node:fs"; + import { listAvailableExtensionIds, resolveExtensionTestPlan } from "./scripts/test-extension.mjs"; + + let extensionFile = ""; + let channelFile = ""; + + for (const extensionId of listAvailableExtensionIds()) { + const plan = resolveExtensionTestPlan({ targetArg: extensionId, cwd: process.cwd() }); + if (plan.testFiles.length === 0) { + continue; + } + const firstFile = plan.testFiles[0] ?? ""; + if (!extensionFile && plan.config === "vitest.extensions.config.ts") { + extensionFile = firstFile; + } + if (!channelFile && plan.config === "vitest.channels.config.ts") { + channelFile = firstFile; + } + if (extensionFile && channelFile) { + break; + } + } + + appendFileSync(process.env.GITHUB_OUTPUT, `extension_file=${extensionFile}\n`, "utf8"); + appendFileSync(process.env.GITHUB_OUTPUT, `channel_file=${channelFile}\n`, "utf8"); + EOF + + - name: Run extension-fast import precheck + env: + EXTENSION_FILE: ${{ steps.representative.outputs.extension_file }} + CHANNEL_FILE: ${{ steps.representative.outputs.channel_file }} + run: | + set -euo pipefail + precheck_start="$(date +%s)" + + if [ -n "$EXTENSION_FILE" ]; then + echo "Running extensions precheck: $EXTENSION_FILE" + pnpm exec vitest run --config vitest.extensions.config.ts --pool=forks --maxWorkers=1 --bail=1 "$EXTENSION_FILE" + fi + + if [ -n "$CHANNEL_FILE" ]; then + echo "Running channels precheck: $CHANNEL_FILE" + pnpm exec vitest run --config vitest.channels.config.ts --pool=forks --maxWorkers=1 --bail=1 "$CHANNEL_FILE" + fi + + if [ -z "$EXTENSION_FILE" ] && [ -z "$CHANNEL_FILE" ]; then + echo "::warning::extension-fast precheck found no representative test files." + fi + + precheck_end="$(date +%s)" + precheck_duration="$((precheck_end - precheck_start))" + { + echo "### extension-fast-precheck" + echo "- extension file: ${EXTENSION_FILE:-none}" + echo "- channel file: ${CHANNEL_FILE:-none}" + echo "- duration: ${precheck_duration}s" + } >> "$GITHUB_STEP_SUMMARY" + + extension-fast: + name: "extension-fast (${{ matrix.extension }})" + needs: [docs-scope, changed-scope, changed-extensions, extension-fast-precheck] + if: needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 25 strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'pull_request' }} matrix: ${{ fromJson(needs.changed-extensions.outputs.changed_extensions_matrix) }} steps: - name: Checkout @@ -272,10 +353,222 @@ jobs: install-bun: "false" use-sticky-disk: "false" - - name: Run changed extension tests + - name: Show extension-fast test plan + id: plan env: OPENCLAW_CHANGED_EXTENSION: ${{ matrix.extension }} - run: pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION" + run: | + set -euo pipefail + plan_json="$(pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION" --allow-empty --dry-run --json)" + config="$(printf '%s' "$plan_json" | jq -r '.config')" + roots="$(printf '%s' "$plan_json" | jq -r '.roots | join(", ")')" + tests="$(printf '%s' "$plan_json" | jq -r '.testFiles | length')" + { + echo "config=$config" + echo "tests=$tests" + } >> "$GITHUB_OUTPUT" + echo "extension-fast plan: config=$config tests=$tests roots=$roots" + { + echo "### extension-fast (${OPENCLAW_CHANGED_EXTENSION}) plan" + echo "- config: \`$config\`" + echo "- roots: \`$roots\`" + echo "- test files: \`$tests\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run changed extension tests (timed) + env: + OPENCLAW_CHANGED_EXTENSION: ${{ matrix.extension }} + PLAN_CONFIG: ${{ steps.plan.outputs.config }} + PLAN_TESTS: ${{ steps.plan.outputs.tests }} + run: | + set -euo pipefail + test_start="$(date +%s)" + set +e + pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION" --allow-empty -- --pool=forks --maxWorkers=1 --bail=1 + test_status=$? + set -e + test_end="$(date +%s)" + test_duration="$((test_end - test_start))" + + metrics_dir="${RUNNER_TEMP}/extension-fast-metrics" + mkdir -p "$metrics_dir" + metrics_json="${metrics_dir}/${OPENCLAW_CHANGED_EXTENSION}.json" + metrics_tsv="${metrics_dir}/${OPENCLAW_CHANGED_EXTENSION}.tsv" + + jq -n \ + --arg extension "$OPENCLAW_CHANGED_EXTENSION" \ + --arg config "${PLAN_CONFIG:-unknown}" \ + --argjson tests "${PLAN_TESTS:-0}" \ + --arg runId "$GITHUB_RUN_ID" \ + --arg runAttempt "$GITHUB_RUN_ATTEMPT" \ + --arg sha "$GITHUB_SHA" \ + --arg ref "$GITHUB_REF" \ + --arg status "$test_status" \ + --argjson durationSeconds "$test_duration" \ + '{ + extension: $extension, + config: $config, + tests: $tests, + runId: $runId, + runAttempt: $runAttempt, + sha: $sha, + ref: $ref, + status: $status, + durationSeconds: $durationSeconds + }' > "$metrics_json" + + printf "extension\tconfig\ttests\tstatus\tduration_seconds\trun_id\trun_attempt\tsha\tref\n" > "$metrics_tsv" + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$OPENCLAW_CHANGED_EXTENSION" \ + "${PLAN_CONFIG:-unknown}" \ + "${PLAN_TESTS:-0}" \ + "$test_status" \ + "$test_duration" \ + "$GITHUB_RUN_ID" \ + "$GITHUB_RUN_ATTEMPT" \ + "$GITHUB_SHA" \ + "$GITHUB_REF" >> "$metrics_tsv" + + echo "extension-fast test duration: ${test_duration}s" + { + echo "### extension-fast (${OPENCLAW_CHANGED_EXTENSION}) runtime" + echo "- duration: ${test_duration}s" + echo "- exit code: ${test_status}" + echo "- metrics json: \`${metrics_json}\`" + echo "- metrics tsv: \`${metrics_tsv}\`" + } >> "$GITHUB_STEP_SUMMARY" + exit "$test_status" + + - name: Upload extension-fast timing artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: extension-fast-metrics-${{ matrix.extension }} + path: | + ${{ runner.temp }}/extension-fast-metrics/${{ matrix.extension }}.json + ${{ runner.temp }}/extension-fast-metrics/${{ matrix.extension }}.tsv + if-no-files-found: warn + retention-days: 7 + + extension-fast-metrics-summary: + name: "extension-fast-metrics-summary" + needs: [docs-scope, changed-scope, changed-extensions, extension-fast] + if: always() && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2404 + steps: + - name: Download extension-fast timing artifacts + uses: actions/download-artifact@v8 + with: + pattern: extension-fast-metrics-* + merge-multiple: true + path: extension-fast-metrics + + - name: Summarize extension-fast timing + run: | + node --input-type=module <<'EOF' + import { readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs"; + import path from "node:path"; + + const metricsDir = path.resolve("extension-fast-metrics"); + const summaryPath = path.resolve("extension-fast-summary.json"); + const summaryTsvPath = path.resolve("extension-fast-summary.tsv"); + const slaP95Seconds = 900; + const slaMaxSeconds = 1500; + + if (!existsSync(metricsDir)) { + console.log("::warning::No extension-fast timing artifacts found."); + process.exit(0); + } + + const rows = readdirSync(metricsDir) + .filter((entry) => entry.endsWith(".json")) + .map((entry) => { + const fullPath = path.join(metricsDir, entry); + return JSON.parse(readFileSync(fullPath, "utf8")); + }) + .filter((row) => typeof row.durationSeconds === "number"); + + if (rows.length === 0) { + console.log("::warning::No extension-fast timing JSON rows were found."); + process.exit(0); + } + + const durations = rows.map((row) => row.durationSeconds).toSorted((a, b) => a - b); + const pick = (p) => durations[Math.max(0, Math.min(durations.length - 1, Math.ceil(p * durations.length) - 1))]; + const p50 = pick(0.5); + const p95 = pick(0.95); + const max = durations[durations.length - 1]; + const failed = rows.filter((row) => String(row.status) !== "0").length; + + const summary = { + count: rows.length, + failed, + p50Seconds: p50, + p95Seconds: p95, + maxSeconds: max, + rows, + }; + writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + + const tsvLines = [ + "extension\tconfig\ttests\tstatus\tduration_seconds\trun_id\trun_attempt\tsha\tref", + ...rows.map((row) => + [ + row.extension, + row.config, + row.tests, + row.status, + row.durationSeconds, + row.runId, + row.runAttempt, + row.sha, + row.ref, + ].join("\t"), + ), + ]; + writeFileSync(summaryTsvPath, `${tsvLines.join("\n")}\n`, "utf8"); + + const markdown = [ + "### extension-fast timing summary", + `- lanes: \`${rows.length}\``, + `- failed lanes: \`${failed}\``, + `- p50: \`${p50}s\``, + `- p95: \`${p95}s\``, + `- max: \`${max}s\``, + "", + "| extension | config | tests | status | duration (s) |", + "| --- | --- | ---: | ---: | ---: |", + ...rows + .toSorted((a, b) => b.durationSeconds - a.durationSeconds) + .map( + (row) => + `| ${row.extension} | ${row.config} | ${row.tests} | ${row.status} | ${row.durationSeconds} |`, + ), + ].join("\n"); + writeFileSync(process.env.GITHUB_STEP_SUMMARY, `${markdown}\n`, { flag: "a" }); + + if (p95 > slaP95Seconds) { + console.log( + `::warning::extension-fast p95 ${p95}s exceeds SLA target ${slaP95Seconds}s.`, + ); + } + if (max > slaMaxSeconds) { + console.log( + `::warning::extension-fast max ${max}s exceeds SLA ceiling ${slaMaxSeconds}s.`, + ); + } + EOF + + - name: Upload extension-fast timing summary artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: extension-fast-metrics-summary + path: | + extension-fast-summary.json + extension-fast-summary.tsv + if-no-files-found: warn + retention-days: 7 # Types, lint, and format check. check: diff --git a/scripts/test-extension.mjs b/scripts/test-extension.mjs index 6442556c778..aa8eda2761e 100644 --- a/scripts/test-extension.mjs +++ b/scripts/test-extension.mjs @@ -181,6 +181,7 @@ export function resolveExtensionTestPlan(params = {}) { function printUsage() { console.error("Usage: pnpm test:extension [vitest args...]"); console.error(" node scripts/test-extension.mjs [extension-name|path] [vitest args...]"); + console.error(" node scripts/test-extension.mjs --allow-empty"); console.error(" node scripts/test-extension.mjs --list"); console.error( " node scripts/test-extension.mjs --list-changed --base [--head ]", @@ -191,6 +192,7 @@ async function run() { const rawArgs = process.argv.slice(2); const dryRun = rawArgs.includes("--dry-run"); const json = rawArgs.includes("--json"); + const allowEmpty = rawArgs.includes("--allow-empty"); const list = rawArgs.includes("--list"); const listChanged = rawArgs.includes("--list-changed"); const args = rawArgs.filter( @@ -198,6 +200,7 @@ async function run() { arg !== "--" && arg !== "--dry-run" && arg !== "--json" && + arg !== "--allow-empty" && arg !== "--list" && arg !== "--list-changed", ); @@ -271,13 +274,22 @@ async function run() { process.exit(1); } - if (plan.testFiles.length === 0) { + if (plan.testFiles.length === 0 && !allowEmpty) { console.error( `No tests found for ${plan.extensionDir}. Run "pnpm test:extension ${plan.extensionId} -- --dry-run" to inspect the resolved roots.`, ); process.exit(1); } + if (plan.testFiles.length === 0 && allowEmpty && !dryRun) { + const message = `[test-extension] Skipping ${plan.extensionId}: no test files were found under ${plan.roots.join(", ")}`; + console.warn(message); + if (process.env.GITHUB_ACTIONS === "true") { + console.log(`::warning::${message}`); + } + return; + } + if (dryRun) { if (json) { process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`); diff --git a/test/scripts/test-extension.test.ts b/test/scripts/test-extension.test.ts index 8919130c19a..8265f1c8661 100644 --- a/test/scripts/test-extension.test.ts +++ b/test/scripts/test-extension.test.ts @@ -17,6 +17,16 @@ function readPlan(args: string[], cwd = process.cwd()) { return JSON.parse(stdout) as ReturnType; } +function findZeroTestExtensionId(): string | undefined { + for (const extensionId of listAvailableExtensionIds()) { + const plan = resolveExtensionTestPlan({ targetArg: extensionId, cwd: process.cwd() }); + if (plan.testFiles.length === 0) { + return extensionId; + } + } + return undefined; +} + describe("scripts/test-extension.mjs", () => { it("resolves channel-root extensions onto the channel vitest config", () => { const plan = resolveExtensionTestPlan({ targetArg: "slack", cwd: process.cwd() }); @@ -72,4 +82,13 @@ describe("scripts/test-extension.mjs", () => { [...extensionIds].toSorted((left, right) => left.localeCompare(right)), ); }); + + it("permits zero-test extensions when --allow-empty is passed", () => { + const extensionId = findZeroTestExtensionId(); + expect(extensionId).toBeTruthy(); + + const plan = readPlan([extensionId!, "--allow-empty"]); + expect(plan.extensionId).toBe(extensionId); + expect(plan.testFiles).toHaveLength(0); + }); });