From d8624a4e1b22fa9038bf261cd7e6325f52c2cfb2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 01:27:26 +0100 Subject: [PATCH] fix: fail release performance when reports do not publish (#103126) * fix(ci): harden performance report publishing * ci: use scoped app token for reports * ci: pin Kova readiness probe fix * ci: pin latest Kova readiness checks * ci: pin Kova startup sequencing fix * ci: pin calibrated Kova release policy --- .github/workflows/openclaw-performance.yml | 515 +++++++++++++----- docs/ci.md | 2 +- docs/help/testing.md | 11 +- .../openclaw-performance-workflow.test.ts | 428 ++++++++++++++- 4 files changed, 789 insertions(+), 167 deletions(-) diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index e4a30f09da7d..b00de9e54142 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -45,7 +45,7 @@ on: kova_ref: description: openclaw/Kova Git ref to install required: false - default: 886a0005269de56632491cfac89bf55256fff778 + default: a18f4c018151f5885d980804d863643bc78933b3 type: string dispatch_id: description: Optional parent workflow dispatch identifier @@ -69,8 +69,51 @@ env: KOVA_SCENARIO_TIMEOUT_MS: "300000" jobs: + resolve_target: + name: Resolve target ref + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + checkout_ref: ${{ steps.resolve.outputs.checkout_ref }} + tested_ref: ${{ steps.resolve.outputs.tested_ref }} + tested_sha: ${{ steps.resolve.outputs.tested_sha }} + steps: + - name: Resolve OpenClaw target ref + id: resolve + env: + GH_TOKEN: ${{ github.token }} + TARGET_REF_INPUT: ${{ inputs.target_ref }} + shell: bash + run: | + set -euo pipefail + requested="${TARGET_REF_INPUT:-}" + if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then + echo "::error::target_ref must be a single line." + exit 1 + fi + if [[ -z "$requested" ]]; then + echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + echo "tested_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")" + if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then + echo "::error::Unable to resolve OpenClaw target_ref '${requested}'." + exit 1 + fi + if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'." + exit 1 + fi + echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT" + echo "tested_ref=$requested" >> "$GITHUB_OUTPUT" + echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT" + kova: name: ${{ matrix.title }} + needs: resolve_target runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 240 strategy: @@ -105,7 +148,7 @@ jobs: include_filters: "scenario:agent-cold-warm-message" expected_release_entries: "agent-cold-warm-message:mock-openai-provider" env: - KOVA_REF: ${{ inputs.kova_ref || '886a0005269de56632491cfac89bf55256fff778' }} + KOVA_REF: ${{ inputs.kova_ref || 'a18f4c018151f5885d980804d863643bc78933b3' }} KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }} PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }} @@ -113,7 +156,8 @@ jobs: SUMMARY_DIR: ${{ github.workspace }}/.artifacts/kova/summaries SOURCE_PERF_DIR: ${{ github.workspace }}/.artifacts/openclaw-performance/source/${{ matrix.lane }} LANE_ID: ${{ matrix.lane }} - TARGET_REF: ${{ inputs.target_ref || github.ref_name }} + TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }} + EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }} PROFILE: ${{ inputs.profile || 'diagnostic' }} REQUESTED_REPEAT: ${{ inputs.repeat || '3' }} FAIL_ON_REGRESSION: ${{ inputs.fail_on_regression || 'false' }} @@ -144,53 +188,11 @@ jobs: echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY" fi - - name: Detect clawgrit report token - id: clawgrit - if: steps.lane.outputs.run == 'true' - env: - CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }} - shell: bash - run: | - set -euo pipefail - if [[ -n "${CLAWGRIT_REPORTS_TOKEN:-}" ]]; then - echo "present=true" >> "$GITHUB_OUTPUT" - else - echo "present=false" >> "$GITHUB_OUTPUT" - fi - - - name: Resolve OpenClaw target ref - id: target - if: steps.lane.outputs.run == 'true' - env: - GH_TOKEN: ${{ github.token }} - TARGET_REF_INPUT: ${{ inputs.target_ref }} - shell: bash - run: | - set -euo pipefail - requested="${TARGET_REF_INPUT:-}" - if [[ -z "$requested" ]]; then - echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" - exit 0 - fi - - encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")" - if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then - echo "::error::Unable to resolve OpenClaw target_ref '${requested}'." >&2 - exit 1 - fi - if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'." >&2 - exit 1 - fi - echo "checkout_ref=${resolved_sha}" >> "$GITHUB_OUTPUT" - echo "tested_ref=${requested}" >> "$GITHUB_OUTPUT" - - name: Checkout OpenClaw if: steps.lane.outputs.run == 'true' uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: - ref: ${{ steps.target.outputs.checkout_ref }} + ref: ${{ needs.resolve_target.outputs.checkout_ref }} fetch-depth: 1 persist-credentials: false @@ -209,11 +211,15 @@ jobs: run: | set -euo pipefail tested_sha="$(git rev-parse HEAD)" + if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then + echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}." + exit 1 + fi echo "TESTED_REF=${TARGET_REF}" >> "$GITHUB_ENV" - echo "TESTED_SHA=${tested_sha}" >> "$GITHUB_ENV" + echo "TESTED_SHA=${EXPECTED_TESTED_SHA}" >> "$GITHUB_ENV" { echo "Tested ref: ${TARGET_REF}" - echo "Tested SHA: ${tested_sha}" + echo "Tested SHA: ${EXPECTED_TESTED_SHA}" echo "Workflow ref: ${GITHUB_REF_NAME}" echo "Workflow SHA: ${GITHUB_SHA}" } >> "$GITHUB_STEP_SUMMARY" @@ -437,7 +443,6 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} - CLAWGRIT_REPORTS_TOKEN_PRESENT: ${{ steps.clawgrit.outputs.present || 'false' }} shell: bash run: | set -euo pipefail @@ -505,17 +510,8 @@ jobs: kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json" - ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')" - run_slug="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - report_url="" - if [[ "${CLAWGRIT_REPORTS_TOKEN_PRESENT:-false}" == "true" ]]; then - report_url="https://github.com/openclaw/clawgrit-reports/tree/main/openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}" - fi summary_path="$SUMMARY_DIR/${LANE_ID}.md" summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID") - if [[ -n "$report_url" ]]; then - summary_args+=(--report-url "$report_url") - fi "${summary_args[@]}" cat >> "$summary_path" <> "$GITHUB_STEP_SUMMARY" exit 0 @@ -743,56 +737,221 @@ jobs: if-no-files-found: error retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }} - - name: Prepare clawgrit reports checkout - id: clawgrit_reports - if: ${{ steps.kova.outputs.report_json != '' && steps.clawgrit.outputs.present == 'true' }} + publish: + name: Publish ${{ matrix.title }} report + needs: [resolve_target, kova] + if: ${{ always() && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - lane: mock-provider + title: mock provider + - lane: mock-deep-profile + title: mock deep profile + - lane: live-openai-candidate + title: live OpenAI candidate + env: + LANE_ID: ${{ matrix.lane }} + REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }} + steps: + - name: Decide report publication lane + id: lane + shell: bash + run: | + set -euo pipefail + run_lane=true + if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then + run_lane=false + fi + if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then + run_lane=false + fi + echo "run=$run_lane" >> "$GITHUB_OUTPUT" + + - name: Resolve Kova artifact + id: artifact + if: steps.lane.outputs.run == 'true' + continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} env: - CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }} + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)" + prefix="openclaw-performance-${LANE_ID}-${GITHUB_RUN_ID}-" + if ! artifacts="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" --jq '.artifacts[] | select(.expired == false) | [.id, .name] | @tsv')"; then + echo "::${annotation}::Unable to list Kova artifacts for this workflow run." + exit 1 + fi + + artifact_id="" + producer_attempt=0 + while IFS=$'\t' read -r candidate_id candidate_name; do + [[ "$candidate_name" == "$prefix"* ]] || continue + candidate_attempt="${candidate_name#"$prefix"}" + [[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue + if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > producer_attempt )); then + artifact_id="$candidate_id" + producer_attempt="$candidate_attempt" + fi + done <<< "$artifacts" + + if [[ -z "$artifact_id" ]]; then + echo "::${annotation}::No Kova artifact is available for ${LANE_ID} through attempt ${GITHUB_RUN_ATTEMPT}." + exit 1 + fi + echo "id=$artifact_id" >> "$GITHUB_OUTPUT" + echo "producer_attempt=$producer_attempt" >> "$GITHUB_OUTPUT" + + - name: Create isolated publisher paths + id: paths + if: steps.artifact.outputs.id != '' + shell: bash + run: | + set -euo pipefail + input_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX")" + reports_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX")" + chmod 0700 "$input_root" "$reports_root" + echo "input_root=$input_root" >> "$GITHUB_OUTPUT" + echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT" + + - name: Download Kova artifacts + id: download + if: steps.artifact.outputs.id != '' + continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.artifact.outputs.id }} + path: ${{ steps.paths.outputs.input_root }} + + - name: Prepare clawgrit report commit + id: prepare + if: steps.download.outcome == 'success' + continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} + env: + INPUT_ROOT: ${{ steps.paths.outputs.input_root }} + REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }} + TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }} + TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }} + PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.producer_attempt }} shell: bash run: | set -euo pipefail echo "ready=false" >> "$GITHUB_OUTPUT" - reports_root=".artifacts/clawgrit-reports" - mkdir -p "$reports_root" - git -C "$reports_root" init -b main - git -C "$reports_root" remote add origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git" - if timeout 60s git -C "$reports_root" ls-remote --exit-code --heads origin main >/dev/null 2>&1; then - if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then - echo "::warning::Skipping optional clawgrit report publish because the reports checkout fetch timed out or failed." - exit 0 - fi - git -C "$reports_root" checkout -B main FETCH_HEAD - else - git -C "$reports_root" checkout -B main + annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)" + input_root="$(realpath "$INPUT_ROOT")" + reports_root="$(realpath "$REPORTS_ROOT")" + if [[ "$input_root" != "$RUNNER_TEMP/"* || "$reports_root" != "$RUNNER_TEMP/"* ]]; then + echo "::${annotation}::Publisher paths escaped RUNNER_TEMP." + exit 1 + fi + if [[ ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then + echo "::${annotation}::Kova artifact attempt metadata is invalid." + exit 1 fi - echo "ready=true" >> "$GITHUB_OUTPUT" - - name: Publish to clawgrit reports - if: ${{ steps.kova.outputs.report_json != '' && steps.clawgrit.outputs.present == 'true' && steps.clawgrit_reports.outputs.ready == 'true' }} - env: - CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }} - shell: bash - run: | - set -euo pipefail - reports_root=".artifacts/clawgrit-reports" - git -C "$reports_root" remote set-url origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git" - ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')" - run_slug="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - dest="${reports_root}/openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}" + mapfile -d '' report_jsons < <(find "$input_root" -type f -path "*/kova/reports/${LANE_ID}/*.json" -print0) + if [[ "${#report_jsons[@]}" != "1" ]]; then + echo "::${annotation}::Expected exactly one Kova JSON report for ${LANE_ID}; found ${#report_jsons[@]}." + exit 1 + fi + report_json="$(realpath "${report_jsons[0]}")" + if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then + echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty." + exit 1 + fi + report_md_path="${report_json%.json}.md" + if [[ -e "$report_md_path" ]]; then + if [[ ! -f "$report_md_path" || -L "$report_md_path" ]]; then + echo "::${annotation}::Kova Markdown report is not a canonical artifact file." + exit 1 + fi + report_md="$(realpath "$report_md_path")" + if [[ "$report_md" != "$input_root/"* ]]; then + echo "::${annotation}::Kova Markdown report escaped the downloaded artifact." + exit 1 + fi + else + report_md="" + fi + + mapfile -d '' summaries < <(find "$input_root" -type f -path "*/kova/summaries/${LANE_ID}.md" -print0) + mapfile -d '' bundles < <(find "$input_root" -type d -path "*/kova/bundles/${LANE_ID}" -print0) + if [[ "${#summaries[@]}" != "1" || "${#bundles[@]}" != "1" ]]; then + echo "::${annotation}::Kova summary or bundle is missing for ${LANE_ID}." + exit 1 + fi + summary="$(realpath "${summaries[0]}")" + bundle="$(realpath "${bundles[0]}")" + if [[ "$summary" != "$input_root/"* || "$bundle" != "$input_root/"* || ! -s "$summary" ]]; then + echo "::${annotation}::Kova summary or bundle escaped the downloaded artifact." + exit 1 + fi + if find "$bundle" -type l -print -quit | grep -q . || find "$bundle" ! -type d ! -type f -print -quit | grep -q .; then + echo "::${annotation}::Kova bundle contains a symlink or special file." + exit 1 + fi + + source_dir="" + mapfile -d '' sources < <(find "$input_root" -type d -path "*/openclaw-performance/source/${LANE_ID}" -print0) + if [[ "${#sources[@]}" == "1" ]]; then + source_dir="$(realpath "${sources[0]}")" + if [[ "$source_dir" != "$input_root/"* ]] || find "$source_dir" -type l -print -quit | grep -q . || find "$source_dir" ! -type d ! -type f -print -quit | grep -q .; then + echo "::${annotation}::Source probe artifact contains an unsafe path." + exit 1 + fi + elif [[ "${#sources[@]}" != "0" ]]; then + echo "::${annotation}::Expected at most one source probe directory for ${LANE_ID}." + exit 1 + fi + + git -C "$reports_root" init -b main + git -C "$reports_root" config core.hooksPath /dev/null + git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git" + if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then + echo "::${annotation}::Unable to fetch openclaw/clawgrit-reports before publishing." + exit 1 + fi + git -C "$reports_root" checkout -B main FETCH_HEAD + + ref_slug="$(printf '%s' "$TESTED_REF" | tr -c 'A-Za-z0-9._-' '-')" + if [[ -z "$ref_slug" || "$ref_slug" == "." || "$ref_slug" == ".." ]]; then + echo "::${annotation}::Tested ref does not produce a safe report path." + exit 1 + fi + run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}" + dest_rel="openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}" + report_url="https://github.com/openclaw/clawgrit-reports/tree/main/${dest_rel}" + if git -C "$reports_root" cat-file -e "HEAD:${dest_rel}/report.json" 2>/dev/null; then + { + echo "### Clawgrit report already published" + echo + echo "- Published report: ${report_url}" + } >> "$GITHUB_STEP_SUMMARY" + echo "already_published=true" >> "$GITHUB_OUTPUT" + echo "report_url=$report_url" >> "$GITHUB_OUTPUT" + echo "ready=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + dest="${reports_root}/${dest_rel}" mkdir -p "$dest" - cp "${{ steps.kova.outputs.report_json }}" "$dest/report.json" - if [[ -f "${{ steps.kova.outputs.report_md }}" ]]; then - cp "${{ steps.kova.outputs.report_md }}" "$dest/report.md" + install -m 0644 "$report_json" "$dest/report.json" + if [[ -n "$report_md" ]]; then + install -m 0644 "$report_md" "$dest/report.md" fi - cp "$SUMMARY_DIR/${LANE_ID}.md" "$dest/index.md" - if [[ -d "$BUNDLE_DIR" ]]; then - mkdir -p "$dest/bundles" - cp -R "$BUNDLE_DIR"/. "$dest/bundles/" - fi - if [[ -d "$SOURCE_PERF_DIR" ]]; then + install -m 0644 "$summary" "$dest/index.md" + mkdir -p "$dest/bundles" + cp -R "$bundle"/. "$dest/bundles/" + if [[ -n "$source_dir" ]]; then mkdir -p "$dest/source" - cp -R "$SOURCE_PERF_DIR"/. "$dest/source/" - if [[ -f "$SOURCE_PERF_DIR/index.md" ]]; then + cp -R "$source_dir"/. "$dest/source/" + if [[ -f "$source_dir/index.md" ]]; then cat >> "$dest/index.md" <<'EOF' ## Source probes @@ -801,49 +960,137 @@ jobs: EOF fi fi - cat > "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json" < "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json" git -C "$reports_root" config user.name "openclaw-performance[bot]" git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com" - git -C "$reports_root" add openclaw-performance - if git -C "$reports_root" diff --cached --quiet; then - echo "No clawgrit report changes to publish." - exit 0 + git -C "$reports_root" add -- openclaw-performance + if ! git -C "$reports_root" diff --cached --quiet; then + git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}" fi - git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}" + report_commit="$(git -C "$reports_root" rev-parse HEAD)" + echo "already_published=false" >> "$GITHUB_OUTPUT" + echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT" + echo "report_commit=$report_commit" >> "$GITHUB_OUTPUT" + echo "dest_rel=$dest_rel" >> "$GITHUB_OUTPUT" + echo "report_url=$report_url" >> "$GITHUB_OUTPUT" + echo "ready=true" >> "$GITHUB_OUTPUT" + + - name: Create clawgrit reports app token + id: clawgrit_app_token + if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} + continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: Iv23liOECG0slfuhz093 + private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }} + owner: openclaw + repositories: clawgrit-reports + permission-contents: write + + - name: Publish to clawgrit reports + if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }} + continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }} + env: + CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }} + REPORTS_ROOT: ${{ steps.prepare.outputs.reports_root }} + REPORT_COMMIT: ${{ steps.prepare.outputs.report_commit }} + DEST_REL: ${{ steps.prepare.outputs.dest_rel }} + REPORT_URL: ${{ steps.prepare.outputs.report_url }} + shell: bash + run: | + set -euo pipefail + annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)" + if [[ -z "${CLAWGRIT_REPORTS_APP_TOKEN:-}" ]]; then + { + echo "### Clawgrit report publish unavailable" + echo + echo "The repository-scoped ClawSweeper GitHub App token could not be created." + } >> "$GITHUB_STEP_SUMMARY" + echo "::${annotation}::ClawSweeper GitHub App token is unavailable; Kova artifacts were uploaded, but the report cannot be published." + exit 1 + fi + token="$CLAWGRIT_REPORTS_APP_TOKEN" + unset CLAWGRIT_REPORTS_APP_TOKEN + auth_header="$(printf 'x-access-token:%s' "$token" | base64 | tr -d '\n')" + unset token + echo "::add-mask::${auth_header}" + + reports_root="$(realpath "$REPORTS_ROOT")" + if [[ "$reports_root" != "$RUNNER_TEMP/"* || ! "$REPORT_COMMIT" =~ ^[0-9a-f]{40}$ || ! "$DEST_REL" =~ ^openclaw-performance/[A-Za-z0-9._-]+/[0-9]+-[0-9]+/[A-Za-z0-9._-]+$ ]]; then + echo "::${annotation}::Prepared report metadata failed validation." + exit 1 + fi + if [[ "$REPORT_URL" != "https://github.com/openclaw/clawgrit-reports/tree/main/${DEST_REL}" ]]; then + echo "::${annotation}::Prepared report URL failed validation." + exit 1 + fi + + git_local() { + git -C "$reports_root" -c core.hooksPath=/dev/null "$@" + } + push_report() { + GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=http.https://github.com/.extraheader GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}" timeout 120s git -C "$reports_root" push origin HEAD:main + } + + if [[ "$(git_local config --local --get core.hooksPath)" != "/dev/null" || "$(git_local remote get-url origin)" != "https://github.com/openclaw/clawgrit-reports.git" ]]; then + echo "::${annotation}::Prepared reports checkout failed integrity validation." + exit 1 + fi + git_local cat-file -e "${REPORT_COMMIT}^{commit}" + report_commit="$REPORT_COMMIT" + for attempt in 1 2 3 4 5; do - if git -C "$reports_root" push origin HEAD:main; then + if push_report; then + { + echo "### Clawgrit report published" + echo + echo "- Published report: ${REPORT_URL}" + } >> "$GITHUB_STEP_SUMMARY" exit 0 fi if [[ "$attempt" == "5" ]]; then { - echo "### Clawgrit report publish skipped" + echo "### Clawgrit report publish failed" echo - echo "Kova artifacts were uploaded, but publishing the optional clawgrit report failed after ${attempt} attempts." - echo "Check the \`CLAWGRIT_REPORTS_TOKEN\` secret or the reports repository permissions." + echo "Kova artifacts were uploaded, but publishing the clawgrit report failed after ${attempt} attempts." + echo "Check the ClawSweeper GitHub App installation and contents permission for openclaw/clawgrit-reports." } >> "$GITHUB_STEP_SUMMARY" - echo "::warning::Kova artifacts uploaded, but optional clawgrit report publish failed after ${attempt} attempts." - exit 0 + echo "::${annotation}::Kova artifacts uploaded, but clawgrit report publish failed after ${attempt} attempts." + exit 1 fi + sleep $((attempt * 2)) - timeout 120s git -C "$reports_root" fetch --depth=1 origin main || { - echo "::warning::Skipping optional clawgrit report rebase because the reports fetch timed out or failed." - exit 0 - } - git -C "$reports_root" rebase FETCH_HEAD + if ! timeout 120s git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main; then + echo "::warning::Unable to refresh openclaw/clawgrit-reports after publish attempt ${attempt}; retrying." + continue + fi + if git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json" 2>/dev/null; then + # A timed-out push may have succeeded. Preserve any newer latest pointer. + git_local checkout --detach FETCH_HEAD + continue + fi + + git_local checkout --detach FETCH_HEAD + # The run path is unique; only the shared latest pointer can conflict. + # Replaying with the current commit wins that pointer deterministically. + if ! git_local cherry-pick -X theirs "$report_commit"; then + git_local cherry-pick --abort || true + echo "::${annotation}::Unable to replay the clawgrit report after a concurrent publish." + exit 1 + fi + report_commit="$(git_local rev-parse HEAD)" done diff --git a/docs/ci.md b/docs/ci.md index 4037e60a3091..652c7f5f4327 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -234,7 +234,7 @@ The workflow installs OCM from a pinned release and Kova from `openclaw/Kova` at The mock-provider lane also runs OpenClaw-native source probes after the Kova pass: gateway boot timing and memory across default, skipped-channel, internal-hook, and fifty-plugin startup cases; bundled plugin import RSS, repeated mock-OpenAI `channel-chat-baseline` hello loops, CLI startup commands against the booted gateway, and the SQLite state smoke performance probe. When the previous published mock-provider source report is available for the tested ref, the source summary compares current RSS and heap values against that baseline and marks large RSS increases as `watch`. The source probe Markdown summary lives at `source/index.md` in the report bundle, with raw JSON beside it. -Every lane uploads GitHub artifacts. When `CLAWGRIT_REPORTS_TOKEN` is configured, the workflow also commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts into `openclaw/clawgrit-reports` under `openclaw-performance//-//`. The current tested-ref pointer is written as `openclaw-performance//latest-.json`. +Every lane uploads GitHub artifacts. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts under `openclaw-performance//-//`; the current tested-ref pointer is `openclaw-performance//latest-.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication. ## Full Release Validation diff --git a/docs/help/testing.md b/docs/help/testing.md index 540de332c986..fcc7fd0e07b9 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -86,10 +86,13 @@ When debugging real providers/models (requires real creds): - Runtime performance reports: dispatch `OpenClaw Performance` with `live_openai_candidate=true` for a real `openai/gpt-5.5` agent turn or `deep_profile=true` for Kova CPU/heap/trace artifacts. Daily scheduled runs - publish mock-provider, deep-profile, and GPT 5.5 lane artifacts to - `openclaw/clawgrit-reports` when `CLAWGRIT_REPORTS_TOKEN` is configured. The - mock-provider report also includes source-level gateway boot, memory, - plugin-pressure, repeated fake-model hello-loop, and CLI startup numbers. + publish mock-provider, deep-profile, and GPT 5.5 lane reports to + `openclaw/clawgrit-reports` from a separate artifact-consuming publisher job; + missing or invalid publisher authentication fails scheduled and + `profile=release` runs. Manual non-release dispatches keep the GitHub artifacts + and treat report publication as advisory. The mock-provider report also + includes source-level gateway boot, memory, plugin-pressure, repeated + fake-model hello-loop, and CLI startup numbers. - Docker live model sweep: `pnpm test:docker:live-models` - Each selected model runs a text turn plus a small file-read-style probe. Models whose metadata advertises `image` input also run a tiny image turn. diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index ffb53adf9894..6e405362e981 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -1,5 +1,17 @@ // Openclaw Performance Workflow tests cover openclaw performance workflow script behavior. -import { readFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parse } from "yaml"; @@ -11,10 +23,17 @@ type WorkflowStep = { if?: string; run?: string; env?: Record; + uses?: string; with?: Record; + "continue-on-error"?: boolean | string; }; type WorkflowJob = { + env?: Record; + if?: string; + needs?: string | string[]; + permissions?: Record; + "runs-on"?: string; steps?: WorkflowStep[]; strategy?: { matrix?: { @@ -31,13 +50,21 @@ function readWorkflow(): Workflow { return parse(readFileSync(WORKFLOW, "utf8")) as Workflow; } -function findStep(name: string): WorkflowStep { - const steps = readWorkflow().jobs?.kova?.steps ?? []; +function findStep(name: string, job = "kova"): WorkflowStep { + const steps = readWorkflow().jobs?.[job]?.steps ?? []; const step = steps.find((candidate) => candidate.name === name); expect(step).toBeDefined(); return step as WorkflowStep; } +function runGit(cwd: string, args: string[]): string { + return execFileSync("git", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} + function kovaMatrixEntries(): Array> { return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? []; } @@ -55,53 +82,398 @@ describe("OpenClaw performance workflow", () => { it("pins the Kova evaluator that reads agent payloads", () => { const workflow = readFileSync(WORKFLOW, "utf8"); - const kovaRef = "886a0005269de56632491cfac89bf55256fff778"; + const kovaRef = "a18f4c018151f5885d980804d863643bc78933b3"; expect(workflow).toContain(`default: ${kovaRef}`); expect(workflow).toContain(`inputs.kova_ref || '${kovaRef}'`); }); - it("resolves dispatch target refs before checkout", () => { - const resolveTarget = findStep("Resolve OpenClaw target ref"); + it("resolves each target once before benchmark and publication fan out", () => { + const workflow = readWorkflow(); + const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target"); const checkout = findStep("Checkout OpenClaw"); + const record = findStep("Record tested revision"); - expect(resolveTarget.id).toBe("target"); - expect(resolveTarget.if).toBe("steps.lane.outputs.run == 'true'"); + expect(workflow.jobs?.kova?.needs).toBe("resolve_target"); + expect(resolveTarget.id).toBe("resolve"); expect(resolveTarget.env?.GH_TOKEN).toBe("${{ github.token }}"); expect(resolveTarget.env?.TARGET_REF_INPUT).toBe("${{ inputs.target_ref }}"); expect(resolveTarget.run).toContain("encodeURIComponent"); expect(resolveTarget.run).toContain( 'gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}"', ); - expect(resolveTarget.run).toContain("checkout_ref=${resolved_sha}"); - expect(checkout.with?.ref).toBe("${{ steps.target.outputs.checkout_ref }}"); + expect(resolveTarget.run).toContain("checkout_ref=$resolved_sha"); + expect(resolveTarget.run).toContain("tested_sha=$resolved_sha"); + expect(checkout.with?.ref).toBe("${{ needs.resolve_target.outputs.checkout_ref }}"); + expect(record.run).toContain('[[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]'); + expect( + Object.values(workflow.jobs ?? {}) + .flatMap((job) => job.steps ?? []) + .filter((step) => step.name === "Resolve OpenClaw target ref"), + ).toHaveLength(1); }); - it("uses the clawgrit reports token for every report repo push path", () => { - const prepare = findStep("Prepare clawgrit reports checkout"); - const publish = findStep("Publish to clawgrit reports"); + it("fetches the public clawgrit baseline without publisher credentials", () => { + const workflowText = readFileSync(WORKFLOW, "utf8"); + const baseline = findStep("Fetch previous source performance baseline"); - expect(prepare.env?.CLAWGRIT_REPORTS_TOKEN).toBe("${{ secrets.CLAWGRIT_REPORTS_TOKEN }}"); - expect(publish.env?.CLAWGRIT_REPORTS_TOKEN).toBe("${{ secrets.CLAWGRIT_REPORTS_TOKEN }}"); + expect(baseline.if).toBe( + "${{ steps.lane.outputs.run == 'true' && matrix.lane == 'mock-provider' }}", + ); + expect(baseline.env?.CLAWGRIT_REPORTS_TOKEN).toBeUndefined(); + expect(baseline.run).toContain( + 'remote add origin "https://github.com/openclaw/clawgrit-reports.git"', + ); + expect(workflowText).not.toContain("https://x-access-token:"); + }); + + it("isolates required publication in a fresh artifact-consuming job", () => { + const workflow = readWorkflow(); + const publisher = workflow.jobs?.publish; + const kovaSteps = workflow.jobs?.kova?.steps ?? []; + const publishSteps = publisher?.steps ?? []; + const appTokenIndex = publishSteps.findIndex( + (step) => step.name === "Create clawgrit reports app token", + ); + const artifactIndex = publishSteps.findIndex((step) => step.name === "Resolve Kova artifact"); + const downloadIndex = publishSteps.findIndex((step) => step.name === "Download Kova artifacts"); + const prepareIndex = publishSteps.findIndex( + (step) => step.name === "Prepare clawgrit report commit", + ); + const pushIndex = publishSteps.findIndex((step) => step.name === "Publish to clawgrit reports"); + + expect(publisher?.needs).toEqual(["resolve_target", "kova"]); + expect(publisher?.if).toBe( + "${{ always() && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' }}", + ); + expect(publisher?.["runs-on"]).toBe("ubuntu-24.04"); + expect(publisher?.permissions?.actions).toBe("read"); + expect(publisher?.env?.REPORT_PUBLISH_REQUIRED).toBe( + "${{ github.event_name == 'schedule' || inputs.profile == 'release' }}", + ); + expect(kovaSteps.some((step) => step.name === "Upload Kova artifacts")).toBe(true); + expect(JSON.stringify(kovaSteps)).not.toContain("CLAWSWEEPER_APP_PRIVATE_KEY"); + expect(artifactIndex).toBeGreaterThanOrEqual(0); + expect(downloadIndex).toBeGreaterThan(artifactIndex); + expect(prepareIndex).toBeGreaterThan(downloadIndex); + expect(appTokenIndex).toBeGreaterThan(prepareIndex); + expect(pushIndex).toBeGreaterThan(appTokenIndex); + }); + + it("mints only a short-lived repo-scoped ClawSweeper app token", () => { + const workflowText = readFileSync(WORKFLOW, "utf8"); + const publisher = readWorkflow().jobs?.publish; + const publishSteps = publisher?.steps ?? []; + const appToken = findStep("Create clawgrit reports app token", "publish"); + const publish = findStep("Publish to clawgrit reports", "publish"); + const appTokenOutput = "${{ steps.clawgrit_app_token.outputs.token }}"; + const tokenConsumers = publishSteps.filter((step) => + Object.values(step.env ?? {}).includes(appTokenOutput), + ); + + expect(appToken.id).toBe("clawgrit_app_token"); + expect(appToken.if).toBe( + "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", + ); + expect(appToken.uses).toBe( + "actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3", + ); + expect(appToken.with).toEqual({ + "client-id": "Iv23liOECG0slfuhz093", + "private-key": "${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}", + owner: "openclaw", + repositories: "clawgrit-reports", + "permission-contents": "write", + }); + expect(appToken.with?.["skip-token-revoke"]).toBeUndefined(); + expect(tokenConsumers.map((step) => step.name)).toEqual(["Publish to clawgrit reports"]); + expect(publish.env?.CLAWGRIT_REPORTS_APP_TOKEN).toBe(appTokenOutput); + expect(workflowText.split(appTokenOutput)).toHaveLength(2); + expect(workflowText.split("${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}")).toHaveLength(2); + expect(publish.if).toBe( + "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}", + ); + expect(workflowText).not.toContain("CLAWGRIT_REPORTS_TOKEN"); + expect(workflowText).not.toContain("secrets.GH_APP_PRIVATE_KEY"); + expect(workflowText).not.toContain('app-id: "2729701"'); + }); + + it("keeps manual non-release publication advisory", () => { + const continuation = "${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}"; + const steps = [ + findStep("Create clawgrit reports app token", "publish"), + findStep("Resolve Kova artifact", "publish"), + findStep("Download Kova artifacts", "publish"), + findStep("Prepare clawgrit report commit", "publish"), + findStep("Publish to clawgrit reports", "publish"), + ]; + + for (const step of steps) { + expect(step["continue-on-error"]).toBe(continuation); + } + for (const step of steps.filter((candidate) => candidate.run)) { + expect(step.run).toContain( + 'annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"', + ); + } + }); + + it("keeps app credentials out of artifact processing and scopes them to Git push", () => { + const workflow = readWorkflow(); + const kovaJob = workflow.jobs?.kova; + const artifact = findStep("Resolve Kova artifact", "publish"); + const paths = findStep("Create isolated publisher paths", "publish"); + const download = findStep("Download Kova artifacts", "publish"); + const prepare = findStep("Prepare clawgrit report commit", "publish"); + const publish = findStep("Publish to clawgrit reports", "publish"); + + expect(JSON.stringify(kovaJob)).not.toContain("CLAWSWEEPER_APP_PRIVATE_KEY"); + expect(artifact.env?.GH_TOKEN).toBe("${{ github.token }}"); + expect(artifact.run).toContain("gh api --paginate"); + expect(artifact.run).toContain("candidate_attempt <= GITHUB_RUN_ATTEMPT"); + expect(artifact.run).toContain('echo "producer_attempt=$producer_attempt"'); + expect(paths.run).toContain('mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX"'); + expect(paths.run).toContain('mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX"'); + expect(download.uses).toBe( + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c", + ); + expect(download.with?.["artifact-ids"]).toBe("${{ steps.artifact.outputs.id }}"); + expect(download.with?.name).toBeUndefined(); + expect(download.with?.path).toBe("${{ steps.paths.outputs.input_root }}"); + expect(JSON.stringify(artifact.env ?? {})).not.toContain("clawgrit_app_token.outputs.token"); + expect(JSON.stringify(download.env ?? {})).not.toContain("clawgrit_app_token.outputs.token"); + expect(JSON.stringify(prepare.env ?? {})).not.toContain("clawgrit_app_token.outputs.token"); + expect(prepare.env?.TESTED_SHA).toBe("${{ needs.resolve_target.outputs.tested_sha }}"); + expect(prepare.env?.PRODUCER_ATTEMPT).toBe("${{ steps.artifact.outputs.producer_attempt }}"); + expect(prepare.run).toContain('run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"'); + expect(prepare.run).toContain('cat-file -e "HEAD:${dest_rel}/report.json"'); + expect(prepare.run).toContain('echo "already_published=true"'); + expect(prepare.run).toContain('git -C "$reports_root" diff --cached --quiet'); + expect(prepare.run).toContain('input_root="$(realpath "$INPUT_ROOT")"'); + expect(prepare.run).toContain('find "$input_root" -type f -path'); + expect(prepare.run).toContain("contains a symlink or special file"); + expect(prepare.run).toContain("config core.hooksPath /dev/null"); expect(prepare.run).toContain( - 'remote add origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"', + 'remote add origin "https://github.com/openclaw/clawgrit-reports.git"', ); - expect(publish.run).toContain( - 'remote set-url origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"', + expect(publish.env?.CLAWGRIT_REPORTS_APP_TOKEN).toBe( + "${{ steps.clawgrit_app_token.outputs.token }}", ); - expect(publish.run).toContain('git -C "$reports_root" push origin HEAD:main'); + expect(publish.if).toContain("steps.prepare.outputs.already_published != 'true'"); + expect(publish.run).not.toContain("${{ steps.kova.outputs."); + expect(publish.run).toContain("unset CLAWGRIT_REPORTS_APP_TOKEN"); + expect(publish.run).toContain("GIT_CONFIG_KEY_0=core.hooksPath"); + expect(publish.run).toContain("GIT_CONFIG_VALUE_0=/dev/null"); + expect(publish.run).toContain("GIT_CONFIG_KEY_1=http.https://github.com/.extraheader"); + expect(publish.run).toContain('GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}"'); + expect(publish.run).not.toContain("export GIT_CONFIG_"); + expect(readFileSync(WORKFLOW, "utf8")).not.toContain("https://x-access-token:"); }); - it("keeps optional clawgrit report publishing bounded", () => { - const prepare = findStep("Prepare clawgrit reports checkout"); - const publish = findStep("Publish to clawgrit reports"); + it("replays concurrent report commits on the current reports tip", () => { + const publish = findStep("Publish to clawgrit reports", "publish"); - expect(prepare.run).toContain('echo "ready=false" >> "$GITHUB_OUTPUT"'); - expect(prepare.run).toContain("timeout 60s git"); - expect(prepare.run).toContain("timeout 120s git"); - expect(prepare.run).toContain('echo "ready=true" >> "$GITHUB_OUTPUT"'); - expect(publish.if).toContain("steps.clawgrit_reports.outputs.ready == 'true'"); - expect(publish.run).toContain("timeout 120s git"); + expect(publish.run).toContain( + 'git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main', + ); + expect(publish.run).toContain('git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json"'); + expect(publish.run).toContain("git_local checkout --detach FETCH_HEAD"); + expect(publish.run).toContain('git_local cherry-pick -X theirs "$report_commit"'); + expect(publish.run).toContain('report_commit="$(git_local rev-parse HEAD)"'); + expect(publish.run).not.toContain("rebase FETCH_HEAD"); + }); + + it("reuses the producing artifact when only publisher jobs rerun", () => { + const artifact = findStep("Resolve Kova artifact", "publish"); + const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-artifact-resolver-")); + const bin = join(root, "bin"); + const output = join(root, "output"); + mkdirSync(bin); + writeFileSync( + join(bin, "gh"), + `#!/bin/sh +printf '%s\\n' \ + '101 openclaw-performance-mock-provider-9001-1' \ + '303 openclaw-performance-mock-provider-9001-3' +`, + ); + chmodSync(join(bin, "gh"), 0o755); + + try { + const result = spawnSync("bash", ["-c", artifact.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ""}`, + GITHUB_OUTPUT: output, + GITHUB_REPOSITORY: "openclaw/openclaw", + GITHUB_RUN_ATTEMPT: "2", + GITHUB_RUN_ID: "9001", + LANE_ID: "mock-provider", + REPORT_PUBLISH_REQUIRED: "true", + }, + }); + expect(result.status).toBe(0); + expect(readFileSync(output, "utf8")).toBe("id=101\nproducer_attempt=1\n"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("advertises a clawgrit URL only after an actual successful push", () => { + const publish = findStep("Publish to clawgrit reports", "publish"); + const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-publish-shell-")); + const bin = join(root, "bin"); + const reportsRoot = join(root, "reports"); + const reportUrl = + "https://github.com/openclaw/clawgrit-reports/tree/main/openclaw-performance/main/123-1/mock-provider"; + mkdirSync(bin); + mkdirSync(reportsRoot); + writeFileSync( + join(bin, "git"), + `#!/bin/bash +case "$*" in + *"config --local --get core.hooksPath"*) echo /dev/null ;; + *"remote get-url origin"*) echo https://github.com/openclaw/clawgrit-reports.git ;; + *" push origin HEAD:main"*) printf push > "$STUB_PUSH_MARKER"; exit "\${STUB_PUSH_STATUS:-0}" ;; + *" fetch --depth=1 origin main"*) exit 1 ;; + *) exit 0 ;; +esac +`, + ); + writeFileSync(join(bin, "sleep"), "#!/bin/sh\nexit 0\n"); + writeFileSync(join(bin, "timeout"), '#!/bin/sh\nshift\nexec "$@"\n'); + chmodSync(join(bin, "git"), 0o755); + chmodSync(join(bin, "sleep"), 0o755); + chmodSync(join(bin, "timeout"), 0o755); + + const execute = (pushStatus: string, appToken: string | null = "test-app-token") => { + const summary = join( + root, + `summary-${pushStatus}-${appToken === null ? "missing" : "token"}.md`, + ); + const pushMarker = join( + root, + `push-${pushStatus}-${appToken === null ? "missing" : "token"}.marker`, + ); + const result = spawnSync("bash", ["-c", publish.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ""}`, + ...(appToken === null ? {} : { CLAWGRIT_REPORTS_APP_TOKEN: appToken }), + DEST_REL: "openclaw-performance/main/123-1/mock-provider", + GITHUB_STEP_SUMMARY: summary, + REPORT_COMMIT: "a".repeat(40), + REPORT_PUBLISH_REQUIRED: "true", + REPORT_URL: reportUrl, + REPORTS_ROOT: reportsRoot, + RUNNER_TEMP: root, + STUB_PUSH_MARKER: pushMarker, + STUB_PUSH_STATUS: pushStatus, + }, + }); + return { + result, + pushMarker, + summary: readFileSync(summary, "utf8"), + }; + }; + + try { + const success = execute("0"); + expect(success.result.status).toBe(0); + expect(success.summary).toContain(`- Published report: ${reportUrl}`); + + const failure = execute("1"); + expect(failure.result.status).toBe(1); + expect(failure.summary).toContain("Clawgrit report publish failed"); + expect(failure.summary).toContain("ClawSweeper GitHub App installation"); + expect(failure.summary).not.toContain("Published report:"); + + const missing = execute("0", null); + expect(missing.result.status).toBe(1); + expect(missing.result.stdout).toContain("ClawSweeper GitHub App token is unavailable"); + expect(missing.summary).toContain("Clawgrit report publish unavailable"); + expect(missing.summary).not.toContain("Published report:"); + expect(existsSync(missing.pushMarker)).toBe(false); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("preserves both reports when concurrent writers update one latest pointer", () => { + const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-report-race-")); + const remote = join(root, "reports.git"); + const seed = join(root, "seed"); + const writerA = join(root, "writer-a"); + const writerB = join(root, "writer-b"); + const verify = join(root, "verify"); + const latest = "openclaw-performance/main/latest-mock-provider.json"; + const reportA = "openclaw-performance/main/100-1/mock-provider"; + const reportB = "openclaw-performance/main/200-1/mock-provider"; + + const configureWriter = (repo: string) => { + runGit(repo, ["config", "user.name", "publisher-test"]); + runGit(repo, ["config", "user.email", "publisher-test@example.com"]); + runGit(repo, ["config", "commit.gpgsign", "false"]); + runGit(repo, ["config", "core.hooksPath", "/dev/null"]); + }; + const commitReport = (repo: string, reportPath: string, marker: string) => { + mkdirSync(join(repo, reportPath), { recursive: true }); + writeFileSync(join(repo, reportPath, "report.json"), JSON.stringify({ marker })); + writeFileSync(join(repo, latest), JSON.stringify({ path: reportPath })); + runGit(repo, ["add", "--", "openclaw-performance"]); + runGit(repo, ["commit", "-m", `perf: add ${marker}`]); + }; + + try { + runGit(root, ["init", "--bare", "--initial-branch=main", remote]); + mkdirSync(seed); + runGit(seed, ["init", "--initial-branch=main"]); + configureWriter(seed); + writeFileSync(join(seed, "README.md"), "reports\n"); + runGit(seed, ["add", "README.md"]); + runGit(seed, ["commit", "-m", "chore: seed"]); + runGit(seed, ["remote", "add", "origin", remote]); + runGit(seed, ["push", "origin", "HEAD:main"]); + runGit(root, ["clone", remote, writerA]); + runGit(root, ["clone", remote, writerB]); + configureWriter(writerA); + configureWriter(writerB); + + commitReport(writerA, reportA, "writer-a"); + const reportCommit = runGit(writerA, ["rev-parse", "HEAD"]); + commitReport(writerB, reportB, "writer-b"); + runGit(writerB, ["push", "origin", "HEAD:main"]); + const rejectedPush = spawnSync("git", ["push", "origin", "HEAD:main"], { + cwd: writerA, + encoding: "utf8", + }); + expect(rejectedPush.status).not.toBe(0); + + runGit(writerA, ["fetch", "--depth=1", "origin", "main"]); + const remoteHasA = spawnSync("git", ["cat-file", "-e", `FETCH_HEAD:${reportA}/report.json`], { + cwd: writerA, + }); + expect(remoteHasA.status).not.toBe(0); + runGit(writerA, ["checkout", "--detach", "FETCH_HEAD"]); + runGit(writerA, ["cherry-pick", "-X", "theirs", reportCommit]); + runGit(writerA, ["push", "origin", "HEAD:main"]); + + runGit(root, ["clone", remote, verify]); + expect(JSON.parse(readFileSync(join(verify, reportA, "report.json"), "utf8"))).toEqual({ + marker: "writer-a", + }); + expect(JSON.parse(readFileSync(join(verify, reportB, "report.json"), "utf8"))).toEqual({ + marker: "writer-b", + }); + expect(JSON.parse(readFileSync(join(verify, latest), "utf8"))).toEqual({ + path: reportA, + }); + } finally { + rmSync(root, { force: true, recursive: true }); + } }); it("requires the shared Kova report gate before tolerating partial verdicts", () => {