CI: stabilize extension-fast and publish timing metrics

.github/workflows/ci.yml

@@ -252,13 +252,94 @@ jobs:
         if: matrix.runtime != 'bun' || github.event_name != 'pull_request'
         run: ${{ matrix.command }}
 
-  extension-fast:
-    name: "extension-fast (${{ matrix.extension }})"
+  extension-fast-precheck:
+    name: "extension-fast-precheck"
     needs: [docs-scope, changed-scope, changed-extensions]
     if: needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true'
     runs-on: blacksmith-16vcpu-ubuntu-2404
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: false
+
+      - name: Setup Node environment
+        uses: ./.github/actions/setup-node-env
+        with:
+          install-bun: "false"
+          use-sticky-disk: "false"
+
+      - name: Select representative extension-fast files
+        id: representative
+        run: |
+          node --input-type=module <<'EOF'
+          import { appendFileSync } from "node:fs";
+          import { listAvailableExtensionIds, resolveExtensionTestPlan } from "./scripts/test-extension.mjs";
+
+          let extensionFile = "";
+          let channelFile = "";
+
+          for (const extensionId of listAvailableExtensionIds()) {
+            const plan = resolveExtensionTestPlan({ targetArg: extensionId, cwd: process.cwd() });
+            if (plan.testFiles.length === 0) {
+              continue;
+            }
+            const firstFile = plan.testFiles[0] ?? "";
+            if (!extensionFile && plan.config === "vitest.extensions.config.ts") {
+              extensionFile = firstFile;
+            }
+            if (!channelFile && plan.config === "vitest.channels.config.ts") {
+              channelFile = firstFile;
+            }
+            if (extensionFile && channelFile) {
+              break;
+            }
+          }
+
+          appendFileSync(process.env.GITHUB_OUTPUT, `extension_file=${extensionFile}\n`, "utf8");
+          appendFileSync(process.env.GITHUB_OUTPUT, `channel_file=${channelFile}\n`, "utf8");
+          EOF
+
+      - name: Run extension-fast import precheck
+        env:
+          EXTENSION_FILE: ${{ steps.representative.outputs.extension_file }}
+          CHANNEL_FILE: ${{ steps.representative.outputs.channel_file }}
+        run: |
+          set -euo pipefail
+          precheck_start="$(date +%s)"
+
+          if [ -n "$EXTENSION_FILE" ]; then
+            echo "Running extensions precheck: $EXTENSION_FILE"
+            pnpm exec vitest run --config vitest.extensions.config.ts --pool=forks --maxWorkers=1 --bail=1 "$EXTENSION_FILE"
+          fi
+
+          if [ -n "$CHANNEL_FILE" ]; then
+            echo "Running channels precheck: $CHANNEL_FILE"
+            pnpm exec vitest run --config vitest.channels.config.ts --pool=forks --maxWorkers=1 --bail=1 "$CHANNEL_FILE"
+          fi
+
+          if [ -z "$EXTENSION_FILE" ] && [ -z "$CHANNEL_FILE" ]; then
+            echo "::warning::extension-fast precheck found no representative test files."
+          fi
+
+          precheck_end="$(date +%s)"
+          precheck_duration="$((precheck_end - precheck_start))"
+          {
+            echo "### extension-fast-precheck"
+            echo "- extension file: ${EXTENSION_FILE:-none}"
+            echo "- channel file: ${CHANNEL_FILE:-none}"
+            echo "- duration: ${precheck_duration}s"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  extension-fast:
+    name: "extension-fast (${{ matrix.extension }})"
+    needs: [docs-scope, changed-scope, changed-extensions, extension-fast-precheck]
+    if: needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true'
+    runs-on: blacksmith-16vcpu-ubuntu-2404
+    timeout-minutes: 25
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
       matrix: ${{ fromJson(needs.changed-extensions.outputs.changed_extensions_matrix) }}
     steps:
       - name: Checkout
@@ -272,10 +353,222 @@ jobs:
           install-bun: "false"
           use-sticky-disk: "false"
 
-      - name: Run changed extension tests
+      - name: Show extension-fast test plan
+        id: plan
         env:
           OPENCLAW_CHANGED_EXTENSION: ${{ matrix.extension }}
-        run: pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION"
+        run: |
+          set -euo pipefail
+          plan_json="$(pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION" --allow-empty --dry-run --json)"
+          config="$(printf '%s' "$plan_json" | jq -r '.config')"
+          roots="$(printf '%s' "$plan_json" | jq -r '.roots | join(", ")')"
+          tests="$(printf '%s' "$plan_json" | jq -r '.testFiles | length')"
+          {
+            echo "config=$config"
+            echo "tests=$tests"
+          } >> "$GITHUB_OUTPUT"
+          echo "extension-fast plan: config=$config tests=$tests roots=$roots"
+          {
+            echo "### extension-fast (${OPENCLAW_CHANGED_EXTENSION}) plan"
+            echo "- config: \`$config\`"
+            echo "- roots: \`$roots\`"
+            echo "- test files: \`$tests\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Run changed extension tests (timed)
+        env:
+          OPENCLAW_CHANGED_EXTENSION: ${{ matrix.extension }}
+          PLAN_CONFIG: ${{ steps.plan.outputs.config }}
+          PLAN_TESTS: ${{ steps.plan.outputs.tests }}
+        run: |
+          set -euo pipefail
+          test_start="$(date +%s)"
+          set +e
+          pnpm test:extension "$OPENCLAW_CHANGED_EXTENSION" --allow-empty -- --pool=forks --maxWorkers=1 --bail=1
+          test_status=$?
+          set -e
+          test_end="$(date +%s)"
+          test_duration="$((test_end - test_start))"
+
+          metrics_dir="${RUNNER_TEMP}/extension-fast-metrics"
+          mkdir -p "$metrics_dir"
+          metrics_json="${metrics_dir}/${OPENCLAW_CHANGED_EXTENSION}.json"
+          metrics_tsv="${metrics_dir}/${OPENCLAW_CHANGED_EXTENSION}.tsv"
+
+          jq -n \
+            --arg extension "$OPENCLAW_CHANGED_EXTENSION" \
+            --arg config "${PLAN_CONFIG:-unknown}" \
+            --argjson tests "${PLAN_TESTS:-0}" \
+            --arg runId "$GITHUB_RUN_ID" \
+            --arg runAttempt "$GITHUB_RUN_ATTEMPT" \
+            --arg sha "$GITHUB_SHA" \
+            --arg ref "$GITHUB_REF" \
+            --arg status "$test_status" \
+            --argjson durationSeconds "$test_duration" \
+            '{
+              extension: $extension,
+              config: $config,
+              tests: $tests,
+              runId: $runId,
+              runAttempt: $runAttempt,
+              sha: $sha,
+              ref: $ref,
+              status: $status,
+              durationSeconds: $durationSeconds
+            }' > "$metrics_json"
+
+          printf "extension\tconfig\ttests\tstatus\tduration_seconds\trun_id\trun_attempt\tsha\tref\n" > "$metrics_tsv"
+          printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \
+            "$OPENCLAW_CHANGED_EXTENSION" \
+            "${PLAN_CONFIG:-unknown}" \
+            "${PLAN_TESTS:-0}" \
+            "$test_status" \
+            "$test_duration" \
+            "$GITHUB_RUN_ID" \
+            "$GITHUB_RUN_ATTEMPT" \
+            "$GITHUB_SHA" \
+            "$GITHUB_REF" >> "$metrics_tsv"
+
+          echo "extension-fast test duration: ${test_duration}s"
+          {
+            echo "### extension-fast (${OPENCLAW_CHANGED_EXTENSION}) runtime"
+            echo "- duration: ${test_duration}s"
+            echo "- exit code: ${test_status}"
+            echo "- metrics json: \`${metrics_json}\`"
+            echo "- metrics tsv: \`${metrics_tsv}\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+          exit "$test_status"
+
+      - name: Upload extension-fast timing artifacts
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: extension-fast-metrics-${{ matrix.extension }}
+          path: |
+            ${{ runner.temp }}/extension-fast-metrics/${{ matrix.extension }}.json
+            ${{ runner.temp }}/extension-fast-metrics/${{ matrix.extension }}.tsv
+          if-no-files-found: warn
+          retention-days: 7
+
+  extension-fast-metrics-summary:
+    name: "extension-fast-metrics-summary"
+    needs: [docs-scope, changed-scope, changed-extensions, extension-fast]
+    if: always() && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_node == 'true' && needs.changed-extensions.outputs.has_changed_extensions == 'true'
+    runs-on: blacksmith-16vcpu-ubuntu-2404
+    steps:
+      - name: Download extension-fast timing artifacts
+        uses: actions/download-artifact@v8
+        with:
+          pattern: extension-fast-metrics-*
+          merge-multiple: true
+          path: extension-fast-metrics
+
+      - name: Summarize extension-fast timing
+        run: |
+          node --input-type=module <<'EOF'
+          import { readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
+          import path from "node:path";
+
+          const metricsDir = path.resolve("extension-fast-metrics");
+          const summaryPath = path.resolve("extension-fast-summary.json");
+          const summaryTsvPath = path.resolve("extension-fast-summary.tsv");
+          const slaP95Seconds = 900;
+          const slaMaxSeconds = 1500;
+
+          if (!existsSync(metricsDir)) {
+            console.log("::warning::No extension-fast timing artifacts found.");
+            process.exit(0);
+          }
+
+          const rows = readdirSync(metricsDir)
+            .filter((entry) => entry.endsWith(".json"))
+            .map((entry) => {
+              const fullPath = path.join(metricsDir, entry);
+              return JSON.parse(readFileSync(fullPath, "utf8"));
+            })
+            .filter((row) => typeof row.durationSeconds === "number");
+
+          if (rows.length === 0) {
+            console.log("::warning::No extension-fast timing JSON rows were found.");
+            process.exit(0);
+          }
+
+          const durations = rows.map((row) => row.durationSeconds).toSorted((a, b) => a - b);
+          const pick = (p) => durations[Math.max(0, Math.min(durations.length - 1, Math.ceil(p * durations.length) - 1))];
+          const p50 = pick(0.5);
+          const p95 = pick(0.95);
+          const max = durations[durations.length - 1];
+          const failed = rows.filter((row) => String(row.status) !== "0").length;
+
+          const summary = {
+            count: rows.length,
+            failed,
+            p50Seconds: p50,
+            p95Seconds: p95,
+            maxSeconds: max,
+            rows,
+          };
+          writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+
+          const tsvLines = [
+            "extension\tconfig\ttests\tstatus\tduration_seconds\trun_id\trun_attempt\tsha\tref",
+            ...rows.map((row) =>
+              [
+                row.extension,
+                row.config,
+                row.tests,
+                row.status,
+                row.durationSeconds,
+                row.runId,
+                row.runAttempt,
+                row.sha,
+                row.ref,
+              ].join("\t"),
+            ),
+          ];
+          writeFileSync(summaryTsvPath, `${tsvLines.join("\n")}\n`, "utf8");
+
+          const markdown = [
+            "### extension-fast timing summary",
+            `- lanes: \`${rows.length}\``,
+            `- failed lanes: \`${failed}\``,
+            `- p50: \`${p50}s\``,
+            `- p95: \`${p95}s\``,
+            `- max: \`${max}s\``,
+            "",
+            "| extension | config | tests | status | duration (s) |",
+            "| --- | --- | ---: | ---: | ---: |",
+            ...rows
+              .toSorted((a, b) => b.durationSeconds - a.durationSeconds)
+              .map(
+                (row) =>
+                  `| ${row.extension} | ${row.config} | ${row.tests} | ${row.status} | ${row.durationSeconds} |`,
+              ),
+          ].join("\n");
+          writeFileSync(process.env.GITHUB_STEP_SUMMARY, `${markdown}\n`, { flag: "a" });
+
+          if (p95 > slaP95Seconds) {
+            console.log(
+              `::warning::extension-fast p95 ${p95}s exceeds SLA target ${slaP95Seconds}s.`,
+            );
+          }
+          if (max > slaMaxSeconds) {
+            console.log(
+              `::warning::extension-fast max ${max}s exceeds SLA ceiling ${slaMaxSeconds}s.`,
+            );
+          }
+          EOF
+
+      - name: Upload extension-fast timing summary artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: extension-fast-metrics-summary
+          path: |
+            extension-fast-summary.json
+            extension-fast-summary.tsv
+          if-no-files-found: warn
+          retention-days: 7
 
   # Types, lint, and format check.
   check:

scripts/test-extension.mjs

@@ -181,6 +181,7 @@ export function resolveExtensionTestPlan(params = {}) {
 function printUsage() {
   console.error("Usage: pnpm test:extension <extension-name|path> [vitest args...]");
   console.error("       node scripts/test-extension.mjs [extension-name|path] [vitest args...]");
+  console.error("       node scripts/test-extension.mjs <extension-name|path> --allow-empty");
   console.error("       node scripts/test-extension.mjs --list");
   console.error(
     "       node scripts/test-extension.mjs --list-changed --base <git-ref> [--head <git-ref>]",
@@ -191,6 +192,7 @@ async function run() {
   const rawArgs = process.argv.slice(2);
   const dryRun = rawArgs.includes("--dry-run");
   const json = rawArgs.includes("--json");
+  const allowEmpty = rawArgs.includes("--allow-empty");
   const list = rawArgs.includes("--list");
   const listChanged = rawArgs.includes("--list-changed");
   const args = rawArgs.filter(
@@ -198,6 +200,7 @@ async function run() {
       arg !== "--" &&
       arg !== "--dry-run" &&
       arg !== "--json" &&
+      arg !== "--allow-empty" &&
       arg !== "--list" &&
       arg !== "--list-changed",
   );
@@ -271,13 +274,22 @@ async function run() {
     process.exit(1);
   }
 
-  if (plan.testFiles.length === 0) {
+  if (plan.testFiles.length === 0 && !allowEmpty) {
     console.error(
       `No tests found for ${plan.extensionDir}. Run "pnpm test:extension ${plan.extensionId} -- --dry-run" to inspect the resolved roots.`,
     );
     process.exit(1);
   }
 
+  if (plan.testFiles.length === 0 && allowEmpty && !dryRun) {
+    const message = `[test-extension] Skipping ${plan.extensionId}: no test files were found under ${plan.roots.join(", ")}`;
+    console.warn(message);
+    if (process.env.GITHUB_ACTIONS === "true") {
+      console.log(`::warning::${message}`);
+    }
+    return;
+  }
+
   if (dryRun) {
     if (json) {
       process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`);

test/scripts/test-extension.test.ts

@@ -17,6 +17,16 @@ function readPlan(args: string[], cwd = process.cwd()) {
   return JSON.parse(stdout) as ReturnType<typeof resolveExtensionTestPlan>;
 }
 
+function findZeroTestExtensionId(): string | undefined {
+  for (const extensionId of listAvailableExtensionIds()) {
+    const plan = resolveExtensionTestPlan({ targetArg: extensionId, cwd: process.cwd() });
+    if (plan.testFiles.length === 0) {
+      return extensionId;
+    }
+  }
+  return undefined;
+}
+
 describe("scripts/test-extension.mjs", () => {
   it("resolves channel-root extensions onto the channel vitest config", () => {
     const plan = resolveExtensionTestPlan({ targetArg: "slack", cwd: process.cwd() });
@@ -72,4 +82,13 @@ describe("scripts/test-extension.mjs", () => {
       [...extensionIds].toSorted((left, right) => left.localeCompare(right)),
     );
   });
+
+  it("permits zero-test extensions when --allow-empty is passed", () => {
+    const extensionId = findZeroTestExtensionId();
+    expect(extensionId).toBeTruthy();
+
+    const plan = readPlan([extensionId!, "--allow-empty"]);
+    expect(plan.extensionId).toBe(extensionId);
+    expect(plan.testFiles).toHaveLength(0);
+  });
 });