diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index c459a9899f0..ac78dfae36d 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -244,8 +244,8 @@ jobs: run: | set -euo pipefail if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "OPENAI_API_KEY is not configured; live GPT 5.5 lane will be skipped." >> "$GITHUB_STEP_SUMMARY" - exit 0 + echo "OPENAI_API_KEY is not configured; live GPT 5.5 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY" + exit 1 fi kova setup --ci --json kova setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --json @@ -262,11 +262,6 @@ jobs: set -euo pipefail mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR" - if [[ "$MATRIX_LIVE" == "true" && -z "${OPENAI_API_KEY:-}" ]]; then - echo "skipped=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - repeat="$REQUESTED_REPEAT" if [[ "$MATRIX_REPEAT" != "input" ]]; then repeat="$MATRIX_REPEAT" @@ -360,6 +355,28 @@ jobs: exit "$effective_status" fi + - name: Validate Kova evidence + if: ${{ always() && steps.lane.outputs.run == 'true' }} + shell: bash + run: | + set -euo pipefail + missing=0 + if ! find "$REPORT_DIR" -maxdepth 1 -type f -name '*.json' -size +0c -print -quit | grep -q .; then + echo "::error::Kova JSON report is missing for ${LANE_ID}." + missing=1 + fi + if [[ ! -s "$BUNDLE_DIR/bundle.json" ]]; then + echo "::error::Kova bundle evidence is missing for ${LANE_ID}." + missing=1 + fi + if [[ ! -s "$SUMMARY_DIR/${LANE_ID}.md" ]]; then + echo "::error::Kova summary evidence is missing for ${LANE_ID}." + missing=1 + fi + if [[ "$missing" != "0" ]]; then + exit 1 + fi + - name: Fetch previous source performance baseline if: ${{ steps.lane.outputs.run == 'true' && matrix.lane == 'mock-provider' && steps.clawgrit.outputs.present == 'true' }} env: @@ -530,7 +547,7 @@ jobs: .artifacts/kova/bundles/${{ matrix.lane }} .artifacts/kova/summaries/${{ matrix.lane }}.md .artifacts/openclaw-performance/source/${{ matrix.lane }} - if-no-files-found: ignore + if-no-files-found: error retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }} - name: Prepare clawgrit reports checkout diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 6e32a33ef65..51a4cd8c5ae 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -10,6 +10,7 @@ type WorkflowStep = { if?: string; run?: string; env?: Record; + with?: Record; }; type WorkflowJob = { @@ -67,4 +68,30 @@ describe("OpenClaw performance workflow", () => { ); expect(runKova.run).not.toContain("report.summary?.statuses ?? {}"); }); + + it("fails selected live Kova lanes when live auth is missing", () => { + const configureAuth = findStep("Configure live OpenAI auth"); + const runKova = findStep("Run Kova"); + + expect(configureAuth.if).toContain("matrix.live == 'true'"); + expect(configureAuth.env?.OPENAI_API_KEY).toBe("${{ secrets.OPENAI_API_KEY }}"); + expect(configureAuth.run).toContain('if [[ -z "${OPENAI_API_KEY:-}" ]]; then'); + expect(configureAuth.run).toContain("cannot run without live evidence"); + expect(configureAuth.run).toContain("exit 1"); + expect(configureAuth.run).not.toContain("will be skipped"); + expect(runKova.run).not.toContain('echo "skipped=true" >> "$GITHUB_OUTPUT"'); + }); + + it("requires Kova evidence before uploading selected lane artifacts", () => { + const validateEvidence = findStep("Validate Kova evidence"); + const upload = findStep("Upload Kova artifacts"); + + expect(validateEvidence.if).toContain("always()"); + expect(validateEvidence.if).toContain("steps.lane.outputs.run == 'true'"); + expect(validateEvidence.run).toContain('"$REPORT_DIR" -maxdepth 1 -type f -name'); + expect(validateEvidence.run).toContain('"$BUNDLE_DIR/bundle.json"'); + expect(validateEvidence.run).toContain('"$SUMMARY_DIR/${LANE_ID}.md"'); + expect(validateEvidence.run).toContain("exit 1"); + expect(upload.with?.["if-no-files-found"]).toBe("error"); + }); });