vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-14 08:46:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fail release performance when reports do not publish (#103126)

Browse Source 
* fix(ci): harden performance report publishing

* ci: use scoped app token for reports

* ci: pin Kova readiness probe fix

* ci: pin latest Kova readiness checks

* ci: pin Kova startup sequencing fix

* ci: pin calibrated Kova release policy
This commit is contained in:
Peter Steinberger
2026-07-10 01:27:26 +01:00
committed by GitHub
parent e26a850fbd
commit d8624a4e1b
4 changed files with 789 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

515
.github/workflows/openclaw-performance.yml vendored
View File

@@ -45,7 +45,7 @@ on:
kova_ref:
description: openclaw/Kova Git ref to install
required: false
default: 886a0005269de56632491cfac89bf55256fff778
default: a18f4c018151f5885d980804d863643bc78933b3
type: string
dispatch_id:
description: Optional parent workflow dispatch identifier
@@ -69,8 +69,51 @@ env:
KOVA_SCENARIO_TIMEOUT_MS: "300000"
jobs:
resolve_target:
name: Resolve target ref
runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
tested_ref: ${{ steps.resolve.outputs.tested_ref }}
tested_sha: ${{ steps.resolve.outputs.tested_sha }}
steps:
- name: Resolve OpenClaw target ref
id: resolve
env:
GH_TOKEN: ${{ github.token }}
TARGET_REF_INPUT: ${{ inputs.target_ref }}
shell: bash
run: |
set -euo pipefail
requested="${TARGET_REF_INPUT:-}"
if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then
echo "::error::target_ref must be a single line."
exit 1
fi
if [[ -z "$requested" ]]; then
echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
echo "tested_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
exit 0
fi
encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")"
if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then
echo "::error::Unable to resolve OpenClaw target_ref '${requested}'."
exit 1
fi
if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'."
exit 1
fi
echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT"
echo "tested_ref=$requested" >> "$GITHUB_OUTPUT"
echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT"
kova:
name: ${{ matrix.title }}
needs: resolve_target
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 240
strategy:
@@ -105,7 +148,7 @@ jobs:
include_filters: "scenario:agent-cold-warm-message"
expected_release_entries: "agent-cold-warm-message:mock-openai-provider"
env:
KOVA_REF: ${{ inputs.kova_ref || '886a0005269de56632491cfac89bf55256fff778' }}
KOVA_REF: ${{ inputs.kova_ref || 'a18f4c018151f5885d980804d863643bc78933b3' }}
KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }}
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }}
@@ -113,7 +156,8 @@ jobs:
SUMMARY_DIR: ${{ github.workspace }}/.artifacts/kova/summaries
SOURCE_PERF_DIR: ${{ github.workspace }}/.artifacts/openclaw-performance/source/${{ matrix.lane }}
LANE_ID: ${{ matrix.lane }}
TARGET_REF: ${{ inputs.target_ref || github.ref_name }}
TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
PROFILE: ${{ inputs.profile || 'diagnostic' }}
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
FAIL_ON_REGRESSION: ${{ inputs.fail_on_regression || 'false' }}
@@ -144,53 +188,11 @@ jobs:
echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Detect clawgrit report token
id: clawgrit
if: steps.lane.outputs.run == 'true'
env:
CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }}
shell: bash
run: |
set -euo pipefail
if [[ -n "${CLAWGRIT_REPORTS_TOKEN:-}" ]]; then
echo "present=true" >> "$GITHUB_OUTPUT"
else
echo "present=false" >> "$GITHUB_OUTPUT"
fi
- name: Resolve OpenClaw target ref
id: target
if: steps.lane.outputs.run == 'true'
env:
GH_TOKEN: ${{ github.token }}
TARGET_REF_INPUT: ${{ inputs.target_ref }}
shell: bash
run: |
set -euo pipefail
requested="${TARGET_REF_INPUT:-}"
if [[ -z "$requested" ]]; then
echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
exit 0
fi
encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")"
if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then
echo "::error::Unable to resolve OpenClaw target_ref '${requested}'." >&2
exit 1
fi
if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'." >&2
exit 1
fi
echo "checkout_ref=${resolved_sha}" >> "$GITHUB_OUTPUT"
echo "tested_ref=${requested}" >> "$GITHUB_OUTPUT"
- name: Checkout OpenClaw
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ steps.target.outputs.checkout_ref }}
ref: ${{ needs.resolve_target.outputs.checkout_ref }}
fetch-depth: 1
persist-credentials: false
@@ -209,11 +211,15 @@ jobs:
run: |
set -euo pipefail
tested_sha="$(git rev-parse HEAD)"
if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
exit 1
fi
echo "TESTED_REF=${TARGET_REF}" >> "$GITHUB_ENV"
echo "TESTED_SHA=${tested_sha}" >> "$GITHUB_ENV"
echo "TESTED_SHA=${EXPECTED_TESTED_SHA}" >> "$GITHUB_ENV"
{
echo "Tested ref: ${TARGET_REF}"
echo "Tested SHA: ${tested_sha}"
echo "Tested SHA: ${EXPECTED_TESTED_SHA}"
echo "Workflow ref: ${GITHUB_REF_NAME}"
echo "Workflow SHA: ${GITHUB_SHA}"
} >> "$GITHUB_STEP_SUMMARY"
@@ -437,7 +443,6 @@ jobs:
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
CLAWGRIT_REPORTS_TOKEN_PRESENT: ${{ steps.clawgrit.outputs.present || 'false' }}
shell: bash
run: |
set -euo pipefail
@@ -505,17 +510,8 @@ jobs:
kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json"
ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')"
run_slug="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
report_url=""
if [[ "${CLAWGRIT_REPORTS_TOKEN_PRESENT:-false}" == "true" ]]; then
report_url="https://github.com/openclaw/clawgrit-reports/tree/main/openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
fi
summary_path="$SUMMARY_DIR/${LANE_ID}.md"
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID")
if [[ -n "$report_url" ]]; then
summary_args+=(--report-url "$report_url")
fi
"${summary_args[@]}"
cat >> "$summary_path" <<EOF
@@ -564,16 +560,14 @@ jobs:
fi
- name: Fetch previous source performance baseline
if: ${{ steps.lane.outputs.run == 'true' && matrix.lane == 'mock-provider' && steps.clawgrit.outputs.present == 'true' }}
env:
CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }}
if: ${{ steps.lane.outputs.run == 'true' && matrix.lane == 'mock-provider' }}
shell: bash
run: |
set -euo pipefail
reports_root=".artifacts/clawgrit-baseline"
mkdir -p "$reports_root"
git -C "$reports_root" init -b main
git -C "$reports_root" remote add origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
if ! git -C "$reports_root" fetch --depth=1 origin main; then
echo "No previous source performance baseline could be fetched." >> "$GITHUB_STEP_SUMMARY"
exit 0
@@ -743,56 +737,221 @@ jobs:
if-no-files-found: error
retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }}
- name: Prepare clawgrit reports checkout
id: clawgrit_reports
if: ${{ steps.kova.outputs.report_json != '' && steps.clawgrit.outputs.present == 'true' }}
publish:
name: Publish ${{ matrix.title }} report
needs: [resolve_target, kova]
if: ${{ always() && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' }}
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
actions: read
contents: read
strategy:
fail-fast: false
matrix:
include:
- lane: mock-provider
title: mock provider
- lane: mock-deep-profile
title: mock deep profile
- lane: live-openai-candidate
title: live OpenAI candidate
env:
LANE_ID: ${{ matrix.lane }}
REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }}
steps:
- name: Decide report publication lane
id: lane
shell: bash
run: |
set -euo pipefail
run_lane=true
if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
run_lane=false
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
run_lane=false
fi
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
- name: Resolve Kova artifact
id: artifact
if: steps.lane.outputs.run == 'true'
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }}
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
prefix="openclaw-performance-${LANE_ID}-${GITHUB_RUN_ID}-"
if ! artifacts="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" --jq '.artifacts[] | select(.expired == false) | [.id, .name] | @tsv')"; then
echo "::${annotation}::Unable to list Kova artifacts for this workflow run."
exit 1
fi
artifact_id=""
producer_attempt=0
while IFS=$'\t' read -r candidate_id candidate_name; do
[[ "$candidate_name" == "$prefix"* ]] || continue
candidate_attempt="${candidate_name#"$prefix"}"
[[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > producer_attempt )); then
artifact_id="$candidate_id"
producer_attempt="$candidate_attempt"
fi
done <<< "$artifacts"
if [[ -z "$artifact_id" ]]; then
echo "::${annotation}::No Kova artifact is available for ${LANE_ID} through attempt ${GITHUB_RUN_ATTEMPT}."
exit 1
fi
echo "id=$artifact_id" >> "$GITHUB_OUTPUT"
echo "producer_attempt=$producer_attempt" >> "$GITHUB_OUTPUT"
- name: Create isolated publisher paths
id: paths
if: steps.artifact.outputs.id != ''
shell: bash
run: |
set -euo pipefail
input_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX")"
reports_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX")"
chmod 0700 "$input_root" "$reports_root"
echo "input_root=$input_root" >> "$GITHUB_OUTPUT"
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
- name: Download Kova artifacts
id: download
if: steps.artifact.outputs.id != ''
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ steps.artifact.outputs.id }}
path: ${{ steps.paths.outputs.input_root }}
- name: Prepare clawgrit report commit
id: prepare
if: steps.download.outcome == 'success'
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
INPUT_ROOT: ${{ steps.paths.outputs.input_root }}
REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }}
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.producer_attempt }}
shell: bash
run: |
set -euo pipefail
echo "ready=false" >> "$GITHUB_OUTPUT"
reports_root=".artifacts/clawgrit-reports"
mkdir -p "$reports_root"
git -C "$reports_root" init -b main
git -C "$reports_root" remote add origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"
if timeout 60s git -C "$reports_root" ls-remote --exit-code --heads origin main >/dev/null 2>&1; then
if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then
echo "::warning::Skipping optional clawgrit report publish because the reports checkout fetch timed out or failed."
exit 0
fi
git -C "$reports_root" checkout -B main FETCH_HEAD
else
git -C "$reports_root" checkout -B main
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
input_root="$(realpath "$INPUT_ROOT")"
reports_root="$(realpath "$REPORTS_ROOT")"
if [[ "$input_root" != "$RUNNER_TEMP/"* || "$reports_root" != "$RUNNER_TEMP/"* ]]; then
echo "::${annotation}::Publisher paths escaped RUNNER_TEMP."
exit 1
fi
if [[ ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
echo "::${annotation}::Kova artifact attempt metadata is invalid."
exit 1
fi
echo "ready=true" >> "$GITHUB_OUTPUT"
- name: Publish to clawgrit reports
if: ${{ steps.kova.outputs.report_json != '' && steps.clawgrit.outputs.present == 'true' && steps.clawgrit_reports.outputs.ready == 'true' }}
env:
CLAWGRIT_REPORTS_TOKEN: ${{ secrets.CLAWGRIT_REPORTS_TOKEN }}
shell: bash
run: |
set -euo pipefail
reports_root=".artifacts/clawgrit-reports"
git -C "$reports_root" remote set-url origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"
ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')"
run_slug="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
dest="${reports_root}/openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
mapfile -d '' report_jsons < <(find "$input_root" -type f -path "*/kova/reports/${LANE_ID}/*.json" -print0)
if [[ "${#report_jsons[@]}" != "1" ]]; then
echo "::${annotation}::Expected exactly one Kova JSON report for ${LANE_ID}; found ${#report_jsons[@]}."
exit 1
fi
report_json="$(realpath "${report_jsons[0]}")"
if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then
echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty."
exit 1
fi
report_md_path="${report_json%.json}.md"
if [[ -e "$report_md_path" ]]; then
if [[ ! -f "$report_md_path" || -L "$report_md_path" ]]; then
echo "::${annotation}::Kova Markdown report is not a canonical artifact file."
exit 1
fi
report_md="$(realpath "$report_md_path")"
if [[ "$report_md" != "$input_root/"* ]]; then
echo "::${annotation}::Kova Markdown report escaped the downloaded artifact."
exit 1
fi
else
report_md=""
fi
mapfile -d '' summaries < <(find "$input_root" -type f -path "*/kova/summaries/${LANE_ID}.md" -print0)
mapfile -d '' bundles < <(find "$input_root" -type d -path "*/kova/bundles/${LANE_ID}" -print0)
if [[ "${#summaries[@]}" != "1" || "${#bundles[@]}" != "1" ]]; then
echo "::${annotation}::Kova summary or bundle is missing for ${LANE_ID}."
exit 1
fi
summary="$(realpath "${summaries[0]}")"
bundle="$(realpath "${bundles[0]}")"
if [[ "$summary" != "$input_root/"* || "$bundle" != "$input_root/"* || ! -s "$summary" ]]; then
echo "::${annotation}::Kova summary or bundle escaped the downloaded artifact."
exit 1
fi
if find "$bundle" -type l -print -quit | grep -q . || find "$bundle" ! -type d ! -type f -print -quit | grep -q .; then
echo "::${annotation}::Kova bundle contains a symlink or special file."
exit 1
fi
source_dir=""
mapfile -d '' sources < <(find "$input_root" -type d -path "*/openclaw-performance/source/${LANE_ID}" -print0)
if [[ "${#sources[@]}" == "1" ]]; then
source_dir="$(realpath "${sources[0]}")"
if [[ "$source_dir" != "$input_root/"* ]] || find "$source_dir" -type l -print -quit | grep -q . || find "$source_dir" ! -type d ! -type f -print -quit | grep -q .; then
echo "::${annotation}::Source probe artifact contains an unsafe path."
exit 1
fi
elif [[ "${#sources[@]}" != "0" ]]; then
echo "::${annotation}::Expected at most one source probe directory for ${LANE_ID}."
exit 1
fi
git -C "$reports_root" init -b main
git -C "$reports_root" config core.hooksPath /dev/null
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then
echo "::${annotation}::Unable to fetch openclaw/clawgrit-reports before publishing."
exit 1
fi
git -C "$reports_root" checkout -B main FETCH_HEAD
ref_slug="$(printf '%s' "$TESTED_REF" | tr -c 'A-Za-z0-9._-' '-')"
if [[ -z "$ref_slug" || "$ref_slug" == "." || "$ref_slug" == ".." ]]; then
echo "::${annotation}::Tested ref does not produce a safe report path."
exit 1
fi
run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"
dest_rel="openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
report_url="https://github.com/openclaw/clawgrit-reports/tree/main/${dest_rel}"
if git -C "$reports_root" cat-file -e "HEAD:${dest_rel}/report.json" 2>/dev/null; then
{
echo "### Clawgrit report already published"
echo
echo "- Published report: ${report_url}"
} >> "$GITHUB_STEP_SUMMARY"
echo "already_published=true" >> "$GITHUB_OUTPUT"
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
echo "ready=true" >> "$GITHUB_OUTPUT"
exit 0
fi
dest="${reports_root}/${dest_rel}"
mkdir -p "$dest"
cp "${{ steps.kova.outputs.report_json }}" "$dest/report.json"
if [[ -f "${{ steps.kova.outputs.report_md }}" ]]; then
cp "${{ steps.kova.outputs.report_md }}" "$dest/report.md"
install -m 0644 "$report_json" "$dest/report.json"
if [[ -n "$report_md" ]]; then
install -m 0644 "$report_md" "$dest/report.md"
fi
cp "$SUMMARY_DIR/${LANE_ID}.md" "$dest/index.md"
if [[ -d "$BUNDLE_DIR" ]]; then
mkdir -p "$dest/bundles"
cp -R "$BUNDLE_DIR"/. "$dest/bundles/"
fi
if [[ -d "$SOURCE_PERF_DIR" ]]; then
install -m 0644 "$summary" "$dest/index.md"
mkdir -p "$dest/bundles"
cp -R "$bundle"/. "$dest/bundles/"
if [[ -n "$source_dir" ]]; then
mkdir -p "$dest/source"
cp -R "$SOURCE_PERF_DIR"/. "$dest/source/"
if [[ -f "$SOURCE_PERF_DIR/index.md" ]]; then
cp -R "$source_dir"/. "$dest/source/"
if [[ -f "$source_dir/index.md" ]]; then
cat >> "$dest/index.md" <<'EOF'
## Source probes
@@ -801,49 +960,137 @@ jobs:
EOF
fi
fi
cat > "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json" <<EOF
{
"repository": "${GITHUB_REPOSITORY}",
"ref": "${TESTED_REF}",
"sha": "${TESTED_SHA}",
"tested_ref": "${TESTED_REF}",
"tested_sha": "${TESTED_SHA}",
"workflow_ref": "${GITHUB_REF_NAME}",
"workflow_sha": "${GITHUB_SHA}",
"workflow": "${GITHUB_WORKFLOW}",
"run_id": "${GITHUB_RUN_ID}",
"run_attempt": "${GITHUB_RUN_ATTEMPT}",
"lane": "${LANE_ID}",
"path": "openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
}
EOF
jq -n \
--arg repository "$GITHUB_REPOSITORY" \
--arg ref "$TESTED_REF" \
--arg sha "$TESTED_SHA" \
--arg workflow_ref "$GITHUB_REF_NAME" \
--arg workflow_sha "$GITHUB_SHA" \
--arg workflow "$GITHUB_WORKFLOW" \
--arg run_id "$GITHUB_RUN_ID" \
--arg run_attempt "$PRODUCER_ATTEMPT" \
--arg lane "$LANE_ID" \
--arg path "$dest_rel" \
'{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \
> "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json"
git -C "$reports_root" config user.name "openclaw-performance[bot]"
git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com"
git -C "$reports_root" add openclaw-performance
if git -C "$reports_root" diff --cached --quiet; then
echo "No clawgrit report changes to publish."
exit 0
git -C "$reports_root" add -- openclaw-performance
if ! git -C "$reports_root" diff --cached --quiet; then
git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}"
fi
git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}"
report_commit="$(git -C "$reports_root" rev-parse HEAD)"
echo "already_published=false" >> "$GITHUB_OUTPUT"
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
echo "report_commit=$report_commit" >> "$GITHUB_OUTPUT"
echo "dest_rel=$dest_rel" >> "$GITHUB_OUTPUT"
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
echo "ready=true" >> "$GITHUB_OUTPUT"
- name: Create clawgrit reports app token
id: clawgrit_app_token
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
with:
client-id: Iv23liOECG0slfuhz093
private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
owner: openclaw
repositories: clawgrit-reports
permission-contents: write
- name: Publish to clawgrit reports
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }}
REPORTS_ROOT: ${{ steps.prepare.outputs.reports_root }}
REPORT_COMMIT: ${{ steps.prepare.outputs.report_commit }}
DEST_REL: ${{ steps.prepare.outputs.dest_rel }}
REPORT_URL: ${{ steps.prepare.outputs.report_url }}
shell: bash
run: |
set -euo pipefail
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
if [[ -z "${CLAWGRIT_REPORTS_APP_TOKEN:-}" ]]; then
{
echo "### Clawgrit report publish unavailable"
echo
echo "The repository-scoped ClawSweeper GitHub App token could not be created."
} >> "$GITHUB_STEP_SUMMARY"
echo "::${annotation}::ClawSweeper GitHub App token is unavailable; Kova artifacts were uploaded, but the report cannot be published."
exit 1
fi
token="$CLAWGRIT_REPORTS_APP_TOKEN"
unset CLAWGRIT_REPORTS_APP_TOKEN
auth_header="$(printf 'x-access-token:%s' "$token" | base64 | tr -d '\n')"
unset token
echo "::add-mask::${auth_header}"
reports_root="$(realpath "$REPORTS_ROOT")"
if [[ "$reports_root" != "$RUNNER_TEMP/"* || ! "$REPORT_COMMIT" =~ ^[0-9a-f]{40}$ || ! "$DEST_REL" =~ ^openclaw-performance/[A-Za-z0-9._-]+/[0-9]+-[0-9]+/[A-Za-z0-9._-]+$ ]]; then
echo "::${annotation}::Prepared report metadata failed validation."
exit 1
fi
if [[ "$REPORT_URL" != "https://github.com/openclaw/clawgrit-reports/tree/main/${DEST_REL}" ]]; then
echo "::${annotation}::Prepared report URL failed validation."
exit 1
fi
git_local() {
git -C "$reports_root" -c core.hooksPath=/dev/null "$@"
}
push_report() {
GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=http.https://github.com/.extraheader GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}" timeout 120s git -C "$reports_root" push origin HEAD:main
}
if [[ "$(git_local config --local --get core.hooksPath)" != "/dev/null" || "$(git_local remote get-url origin)" != "https://github.com/openclaw/clawgrit-reports.git" ]]; then
echo "::${annotation}::Prepared reports checkout failed integrity validation."
exit 1
fi
git_local cat-file -e "${REPORT_COMMIT}^{commit}"
report_commit="$REPORT_COMMIT"
for attempt in 1 2 3 4 5; do
if git -C "$reports_root" push origin HEAD:main; then
if push_report; then
{
echo "### Clawgrit report published"
echo
echo "- Published report: ${REPORT_URL}"
} >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
if [[ "$attempt" == "5" ]]; then
{
echo "### Clawgrit report publish skipped"
echo "### Clawgrit report publish failed"
echo
echo "Kova artifacts were uploaded, but publishing the optional clawgrit report failed after ${attempt} attempts."
echo "Check the \`CLAWGRIT_REPORTS_TOKEN\` secret or the reports repository permissions."
echo "Kova artifacts were uploaded, but publishing the clawgrit report failed after ${attempt} attempts."
echo "Check the ClawSweeper GitHub App installation and contents permission for openclaw/clawgrit-reports."
} >> "$GITHUB_STEP_SUMMARY"
echo "::warning::Kova artifacts uploaded, but optional clawgrit report publish failed after ${attempt} attempts."
exit 0
echo "::${annotation}::Kova artifacts uploaded, but clawgrit report publish failed after ${attempt} attempts."
exit 1
fi
sleep $((attempt * 2))
timeout 120s git -C "$reports_root" fetch --depth=1 origin main || {
echo "::warning::Skipping optional clawgrit report rebase because the reports fetch timed out or failed."
exit 0
}
git -C "$reports_root" rebase FETCH_HEAD
if ! timeout 120s git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main; then
echo "::warning::Unable to refresh openclaw/clawgrit-reports after publish attempt ${attempt}; retrying."
continue
fi
if git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json" 2>/dev/null; then
# A timed-out push may have succeeded. Preserve any newer latest pointer.
git_local checkout --detach FETCH_HEAD
continue
fi
git_local checkout --detach FETCH_HEAD
# The run path is unique; only the shared latest pointer can conflict.
# Replaying with the current commit wins that pointer deterministically.
if ! git_local cherry-pick -X theirs "$report_commit"; then
git_local cherry-pick --abort || true
echo "::${annotation}::Unable to replay the clawgrit report after a concurrent publish."
exit 1
fi
report_commit="$(git_local rev-parse HEAD)"
done

2
docs/ci.md
View File

@@ -234,7 +234,7 @@ The workflow installs OCM from a pinned release and Kova from `openclaw/Kova` at
The mock-provider lane also runs OpenClaw-native source probes after the Kova pass: gateway boot timing and memory across default, skipped-channel, internal-hook, and fifty-plugin startup cases; bundled plugin import RSS, repeated mock-OpenAI `channel-chat-baseline` hello loops, CLI startup commands against the booted gateway, and the SQLite state smoke performance probe. When the previous published mock-provider source report is available for the tested ref, the source summary compares current RSS and heap values against that baseline and marks large RSS increases as `watch`. The source probe Markdown summary lives at `source/index.md` in the report bundle, with raw JSON beside it.
Every lane uploads GitHub artifacts. When `CLAWGRIT_REPORTS_TOKEN` is configured, the workflow also commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts into `openclaw/clawgrit-reports` under `openclaw-performance/<tested-ref>/<run-id>-<attempt>/<lane>/`. The current tested-ref pointer is written as `openclaw-performance/<tested-ref>/latest-<lane>.json`.
Every lane uploads GitHub artifacts. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts under `openclaw-performance/<tested-ref>/<run-id>-<attempt>/<lane>/`; the current tested-ref pointer is `openclaw-performance/<tested-ref>/latest-<lane>.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication.
## Full Release Validation

11
docs/help/testing.md
View File

@@ -86,10 +86,13 @@ When debugging real providers/models (requires real creds):
- Runtime performance reports: dispatch `OpenClaw Performance` with
`live_openai_candidate=true` for a real `openai/gpt-5.5` agent turn or
`deep_profile=true` for Kova CPU/heap/trace artifacts. Daily scheduled runs
publish mock-provider, deep-profile, and GPT 5.5 lane artifacts to
`openclaw/clawgrit-reports` when `CLAWGRIT_REPORTS_TOKEN` is configured. The
mock-provider report also includes source-level gateway boot, memory,
plugin-pressure, repeated fake-model hello-loop, and CLI startup numbers.
publish mock-provider, deep-profile, and GPT 5.5 lane reports to
`openclaw/clawgrit-reports` from a separate artifact-consuming publisher job;
missing or invalid publisher authentication fails scheduled and
`profile=release` runs. Manual non-release dispatches keep the GitHub artifacts
and treat report publication as advisory. The mock-provider report also
includes source-level gateway boot, memory, plugin-pressure, repeated
fake-model hello-loop, and CLI startup numbers.
- Docker live model sweep: `pnpm test:docker:live-models`
- Each selected model runs a text turn plus a small file-read-style probe.
Models whose metadata advertises `image` input also run a tiny image turn.

428
test/scripts/openclaw-performance-workflow.test.ts
View File

@@ -1,5 +1,17 @@
// Openclaw Performance Workflow tests cover openclaw performance workflow script behavior.
import { readFileSync } from "node:fs";
import { execFileSync, spawnSync } from "node:child_process";
import {
chmodSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
realpathSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { parse } from "yaml";
@@ -11,10 +23,17 @@ type WorkflowStep = {
if?: string;
run?: string;
env?: Record<string, string>;
uses?: string;
with?: Record<string, string>;
"continue-on-error"?: boolean | string;
};
type WorkflowJob = {
env?: Record<string, string>;
if?: string;
needs?: string | string[];
permissions?: Record<string, string>;
"runs-on"?: string;
steps?: WorkflowStep[];
strategy?: {
matrix?: {
@@ -31,13 +50,21 @@ function readWorkflow(): Workflow {
return parse(readFileSync(WORKFLOW, "utf8")) as Workflow;
}
function findStep(name: string): WorkflowStep {
const steps = readWorkflow().jobs?.kova?.steps ?? [];
function findStep(name: string, job = "kova"): WorkflowStep {
const steps = readWorkflow().jobs?.[job]?.steps ?? [];
const step = steps.find((candidate) => candidate.name === name);
expect(step).toBeDefined();
return step as WorkflowStep;
}
function runGit(cwd: string, args: string[]): string {
return execFileSync("git", args, {
cwd,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
}).trim();
}
function kovaMatrixEntries(): Array<Record<string, string>> {
return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? [];
}
@@ -55,53 +82,398 @@ describe("OpenClaw performance workflow", () => {
it("pins the Kova evaluator that reads agent payloads", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
const kovaRef = "886a0005269de56632491cfac89bf55256fff778";
const kovaRef = "a18f4c018151f5885d980804d863643bc78933b3";
expect(workflow).toContain(`default: ${kovaRef}`);
expect(workflow).toContain(`inputs.kova_ref || '${kovaRef}'`);
});
it("resolves dispatch target refs before checkout", () => {
const resolveTarget = findStep("Resolve OpenClaw target ref");
it("resolves each target once before benchmark and publication fan out", () => {
const workflow = readWorkflow();
const resolveTarget = findStep("Resolve OpenClaw target ref", "resolve_target");
const checkout = findStep("Checkout OpenClaw");
const record = findStep("Record tested revision");
expect(resolveTarget.id).toBe("target");
expect(resolveTarget.if).toBe("steps.lane.outputs.run == 'true'");
expect(workflow.jobs?.kova?.needs).toBe("resolve_target");
expect(resolveTarget.id).toBe("resolve");
expect(resolveTarget.env?.GH_TOKEN).toBe("${{ github.token }}");
expect(resolveTarget.env?.TARGET_REF_INPUT).toBe("${{ inputs.target_ref }}");
expect(resolveTarget.run).toContain("encodeURIComponent");
expect(resolveTarget.run).toContain(
'gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}"',
);
expect(resolveTarget.run).toContain("checkout_ref=${resolved_sha}");
expect(checkout.with?.ref).toBe("${{ steps.target.outputs.checkout_ref }}");
expect(resolveTarget.run).toContain("checkout_ref=$resolved_sha");
expect(resolveTarget.run).toContain("tested_sha=$resolved_sha");
expect(checkout.with?.ref).toBe("${{ needs.resolve_target.outputs.checkout_ref }}");
expect(record.run).toContain('[[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]');
expect(
Object.values(workflow.jobs ?? {})
.flatMap((job) => job.steps ?? [])
.filter((step) => step.name === "Resolve OpenClaw target ref"),
).toHaveLength(1);
});
it("uses the clawgrit reports token for every report repo push path", () => {
const prepare = findStep("Prepare clawgrit reports checkout");
const publish = findStep("Publish to clawgrit reports");
it("fetches the public clawgrit baseline without publisher credentials", () => {
const workflowText = readFileSync(WORKFLOW, "utf8");
const baseline = findStep("Fetch previous source performance baseline");
expect(prepare.env?.CLAWGRIT_REPORTS_TOKEN).toBe("${{ secrets.CLAWGRIT_REPORTS_TOKEN }}");
expect(publish.env?.CLAWGRIT_REPORTS_TOKEN).toBe("${{ secrets.CLAWGRIT_REPORTS_TOKEN }}");
expect(baseline.if).toBe(
"${{ steps.lane.outputs.run == 'true' && matrix.lane == 'mock-provider' }}",
);
expect(baseline.env?.CLAWGRIT_REPORTS_TOKEN).toBeUndefined();
expect(baseline.run).toContain(
'remote add origin "https://github.com/openclaw/clawgrit-reports.git"',
);
expect(workflowText).not.toContain("https://x-access-token:");
});
it("isolates required publication in a fresh artifact-consuming job", () => {
const workflow = readWorkflow();
const publisher = workflow.jobs?.publish;
const kovaSteps = workflow.jobs?.kova?.steps ?? [];
const publishSteps = publisher?.steps ?? [];
const appTokenIndex = publishSteps.findIndex(
(step) => step.name === "Create clawgrit reports app token",
);
const artifactIndex = publishSteps.findIndex((step) => step.name === "Resolve Kova artifact");
const downloadIndex = publishSteps.findIndex((step) => step.name === "Download Kova artifacts");
const prepareIndex = publishSteps.findIndex(
(step) => step.name === "Prepare clawgrit report commit",
);
const pushIndex = publishSteps.findIndex((step) => step.name === "Publish to clawgrit reports");
expect(publisher?.needs).toEqual(["resolve_target", "kova"]);
expect(publisher?.if).toBe(
"${{ always() && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' }}",
);
expect(publisher?.["runs-on"]).toBe("ubuntu-24.04");
expect(publisher?.permissions?.actions).toBe("read");
expect(publisher?.env?.REPORT_PUBLISH_REQUIRED).toBe(
"${{ github.event_name == 'schedule' || inputs.profile == 'release' }}",
);
expect(kovaSteps.some((step) => step.name === "Upload Kova artifacts")).toBe(true);
expect(JSON.stringify(kovaSteps)).not.toContain("CLAWSWEEPER_APP_PRIVATE_KEY");
expect(artifactIndex).toBeGreaterThanOrEqual(0);
expect(downloadIndex).toBeGreaterThan(artifactIndex);
expect(prepareIndex).toBeGreaterThan(downloadIndex);
expect(appTokenIndex).toBeGreaterThan(prepareIndex);
expect(pushIndex).toBeGreaterThan(appTokenIndex);
});
it("mints only a short-lived repo-scoped ClawSweeper app token", () => {
const workflowText = readFileSync(WORKFLOW, "utf8");
const publisher = readWorkflow().jobs?.publish;
const publishSteps = publisher?.steps ?? [];
const appToken = findStep("Create clawgrit reports app token", "publish");
const publish = findStep("Publish to clawgrit reports", "publish");
const appTokenOutput = "${{ steps.clawgrit_app_token.outputs.token }}";
const tokenConsumers = publishSteps.filter((step) =>
Object.values(step.env ?? {}).includes(appTokenOutput),
);
expect(appToken.id).toBe("clawgrit_app_token");
expect(appToken.if).toBe(
"${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
);
expect(appToken.uses).toBe(
"actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3",
);
expect(appToken.with).toEqual({
"client-id": "Iv23liOECG0slfuhz093",
"private-key": "${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}",
owner: "openclaw",
repositories: "clawgrit-reports",
"permission-contents": "write",
});
expect(appToken.with?.["skip-token-revoke"]).toBeUndefined();
expect(tokenConsumers.map((step) => step.name)).toEqual(["Publish to clawgrit reports"]);
expect(publish.env?.CLAWGRIT_REPORTS_APP_TOKEN).toBe(appTokenOutput);
expect(workflowText.split(appTokenOutput)).toHaveLength(2);
expect(workflowText.split("${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}")).toHaveLength(2);
expect(publish.if).toBe(
"${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
);
expect(workflowText).not.toContain("CLAWGRIT_REPORTS_TOKEN");
expect(workflowText).not.toContain("secrets.GH_APP_PRIVATE_KEY");
expect(workflowText).not.toContain('app-id: "2729701"');
});
it("keeps manual non-release publication advisory", () => {
const continuation = "${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}";
const steps = [
findStep("Create clawgrit reports app token", "publish"),
findStep("Resolve Kova artifact", "publish"),
findStep("Download Kova artifacts", "publish"),
findStep("Prepare clawgrit report commit", "publish"),
findStep("Publish to clawgrit reports", "publish"),
];
for (const step of steps) {
expect(step["continue-on-error"]).toBe(continuation);
}
for (const step of steps.filter((candidate) => candidate.run)) {
expect(step.run).toContain(
'annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"',
);
}
});
it("keeps app credentials out of artifact processing and scopes them to Git push", () => {
const workflow = readWorkflow();
const kovaJob = workflow.jobs?.kova;
const artifact = findStep("Resolve Kova artifact", "publish");
const paths = findStep("Create isolated publisher paths", "publish");
const download = findStep("Download Kova artifacts", "publish");
const prepare = findStep("Prepare clawgrit report commit", "publish");
const publish = findStep("Publish to clawgrit reports", "publish");
expect(JSON.stringify(kovaJob)).not.toContain("CLAWSWEEPER_APP_PRIVATE_KEY");
expect(artifact.env?.GH_TOKEN).toBe("${{ github.token }}");
expect(artifact.run).toContain("gh api --paginate");
expect(artifact.run).toContain("candidate_attempt <= GITHUB_RUN_ATTEMPT");
expect(artifact.run).toContain('echo "producer_attempt=$producer_attempt"');
expect(paths.run).toContain('mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX"');
expect(paths.run).toContain('mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX"');
expect(download.uses).toBe(
"actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c",
);
expect(download.with?.["artifact-ids"]).toBe("${{ steps.artifact.outputs.id }}");
expect(download.with?.name).toBeUndefined();
expect(download.with?.path).toBe("${{ steps.paths.outputs.input_root }}");
expect(JSON.stringify(artifact.env ?? {})).not.toContain("clawgrit_app_token.outputs.token");
expect(JSON.stringify(download.env ?? {})).not.toContain("clawgrit_app_token.outputs.token");
expect(JSON.stringify(prepare.env ?? {})).not.toContain("clawgrit_app_token.outputs.token");
expect(prepare.env?.TESTED_SHA).toBe("${{ needs.resolve_target.outputs.tested_sha }}");
expect(prepare.env?.PRODUCER_ATTEMPT).toBe("${{ steps.artifact.outputs.producer_attempt }}");
expect(prepare.run).toContain('run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"');
expect(prepare.run).toContain('cat-file -e "HEAD:${dest_rel}/report.json"');
expect(prepare.run).toContain('echo "already_published=true"');
expect(prepare.run).toContain('git -C "$reports_root" diff --cached --quiet');
expect(prepare.run).toContain('input_root="$(realpath "$INPUT_ROOT")"');
expect(prepare.run).toContain('find "$input_root" -type f -path');
expect(prepare.run).toContain("contains a symlink or special file");
expect(prepare.run).toContain("config core.hooksPath /dev/null");
expect(prepare.run).toContain(
'remote add origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"',
'remote add origin "https://github.com/openclaw/clawgrit-reports.git"',
);
expect(publish.run).toContain(
'remote set-url origin "https://x-access-token:${CLAWGRIT_REPORTS_TOKEN}@github.com/openclaw/clawgrit-reports.git"',
expect(publish.env?.CLAWGRIT_REPORTS_APP_TOKEN).toBe(
"${{ steps.clawgrit_app_token.outputs.token }}",
);
expect(publish.run).toContain('git -C "$reports_root" push origin HEAD:main');
expect(publish.if).toContain("steps.prepare.outputs.already_published != 'true'");
expect(publish.run).not.toContain("${{ steps.kova.outputs.");
expect(publish.run).toContain("unset CLAWGRIT_REPORTS_APP_TOKEN");
expect(publish.run).toContain("GIT_CONFIG_KEY_0=core.hooksPath");
expect(publish.run).toContain("GIT_CONFIG_VALUE_0=/dev/null");
expect(publish.run).toContain("GIT_CONFIG_KEY_1=http.https://github.com/.extraheader");
expect(publish.run).toContain('GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}"');
expect(publish.run).not.toContain("export GIT_CONFIG_");
expect(readFileSync(WORKFLOW, "utf8")).not.toContain("https://x-access-token:");
});
it("keeps optional clawgrit report publishing bounded", () => {
const prepare = findStep("Prepare clawgrit reports checkout");
const publish = findStep("Publish to clawgrit reports");
it("replays concurrent report commits on the current reports tip", () => {
const publish = findStep("Publish to clawgrit reports", "publish");
expect(prepare.run).toContain('echo "ready=false" >> "$GITHUB_OUTPUT"');
expect(prepare.run).toContain("timeout 60s git");
expect(prepare.run).toContain("timeout 120s git");
expect(prepare.run).toContain('echo "ready=true" >> "$GITHUB_OUTPUT"');
expect(publish.if).toContain("steps.clawgrit_reports.outputs.ready == 'true'");
expect(publish.run).toContain("timeout 120s git");
expect(publish.run).toContain(
'git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main',
);
expect(publish.run).toContain('git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json"');
expect(publish.run).toContain("git_local checkout --detach FETCH_HEAD");
expect(publish.run).toContain('git_local cherry-pick -X theirs "$report_commit"');
expect(publish.run).toContain('report_commit="$(git_local rev-parse HEAD)"');
expect(publish.run).not.toContain("rebase FETCH_HEAD");
});
it("reuses the producing artifact when only publisher jobs rerun", () => {
const artifact = findStep("Resolve Kova artifact", "publish");
const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-artifact-resolver-"));
const bin = join(root, "bin");
const output = join(root, "output");
mkdirSync(bin);
writeFileSync(
join(bin, "gh"),
`#!/bin/sh
printf '%s\\n' \
'101 openclaw-performance-mock-provider-9001-1' \
'303 openclaw-performance-mock-provider-9001-3'
`,
);
chmodSync(join(bin, "gh"), 0o755);
try {
const result = spawnSync("bash", ["-c", artifact.run ?? ""], {
encoding: "utf8",
env: {
...process.env,
PATH: `${bin}:${process.env.PATH ?? ""}`,
GITHUB_OUTPUT: output,
GITHUB_REPOSITORY: "openclaw/openclaw",
GITHUB_RUN_ATTEMPT: "2",
GITHUB_RUN_ID: "9001",
LANE_ID: "mock-provider",
REPORT_PUBLISH_REQUIRED: "true",
},
});
expect(result.status).toBe(0);
expect(readFileSync(output, "utf8")).toBe("id=101\nproducer_attempt=1\n");
} finally {
rmSync(root, { force: true, recursive: true });
}
});
it("advertises a clawgrit URL only after an actual successful push", () => {
const publish = findStep("Publish to clawgrit reports", "publish");
const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-publish-shell-"));
const bin = join(root, "bin");
const reportsRoot = join(root, "reports");
const reportUrl =
"https://github.com/openclaw/clawgrit-reports/tree/main/openclaw-performance/main/123-1/mock-provider";
mkdirSync(bin);
mkdirSync(reportsRoot);
writeFileSync(
join(bin, "git"),
`#!/bin/bash
case "$*" in
*"config --local --get core.hooksPath"*) echo /dev/null ;;
*"remote get-url origin"*) echo https://github.com/openclaw/clawgrit-reports.git ;;
*" push origin HEAD:main"*) printf push > "$STUB_PUSH_MARKER"; exit "\${STUB_PUSH_STATUS:-0}" ;;
*" fetch --depth=1 origin main"*) exit 1 ;;
*) exit 0 ;;
esac
`,
);
writeFileSync(join(bin, "sleep"), "#!/bin/sh\nexit 0\n");
writeFileSync(join(bin, "timeout"), '#!/bin/sh\nshift\nexec "$@"\n');
chmodSync(join(bin, "git"), 0o755);
chmodSync(join(bin, "sleep"), 0o755);
chmodSync(join(bin, "timeout"), 0o755);
const execute = (pushStatus: string, appToken: string | null = "test-app-token") => {
const summary = join(
root,
`summary-${pushStatus}-${appToken === null ? "missing" : "token"}.md`,
);
const pushMarker = join(
root,
`push-${pushStatus}-${appToken === null ? "missing" : "token"}.marker`,
);
const result = spawnSync("bash", ["-c", publish.run ?? ""], {
encoding: "utf8",
env: {
...process.env,
PATH: `${bin}:${process.env.PATH ?? ""}`,
...(appToken === null ? {} : { CLAWGRIT_REPORTS_APP_TOKEN: appToken }),
DEST_REL: "openclaw-performance/main/123-1/mock-provider",
GITHUB_STEP_SUMMARY: summary,
REPORT_COMMIT: "a".repeat(40),
REPORT_PUBLISH_REQUIRED: "true",
REPORT_URL: reportUrl,
REPORTS_ROOT: reportsRoot,
RUNNER_TEMP: root,
STUB_PUSH_MARKER: pushMarker,
STUB_PUSH_STATUS: pushStatus,
},
});
return {
result,
pushMarker,
summary: readFileSync(summary, "utf8"),
};
};
try {
const success = execute("0");
expect(success.result.status).toBe(0);
expect(success.summary).toContain(`- Published report: ${reportUrl}`);
const failure = execute("1");
expect(failure.result.status).toBe(1);
expect(failure.summary).toContain("Clawgrit report publish failed");
expect(failure.summary).toContain("ClawSweeper GitHub App installation");
expect(failure.summary).not.toContain("Published report:");
const missing = execute("0", null);
expect(missing.result.status).toBe(1);
expect(missing.result.stdout).toContain("ClawSweeper GitHub App token is unavailable");
expect(missing.summary).toContain("Clawgrit report publish unavailable");
expect(missing.summary).not.toContain("Published report:");
expect(existsSync(missing.pushMarker)).toBe(false);
} finally {
rmSync(root, { force: true, recursive: true });
}
});
it("preserves both reports when concurrent writers update one latest pointer", () => {
const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-report-race-"));
const remote = join(root, "reports.git");
const seed = join(root, "seed");
const writerA = join(root, "writer-a");
const writerB = join(root, "writer-b");
const verify = join(root, "verify");
const latest = "openclaw-performance/main/latest-mock-provider.json";
const reportA = "openclaw-performance/main/100-1/mock-provider";
const reportB = "openclaw-performance/main/200-1/mock-provider";
const configureWriter = (repo: string) => {
runGit(repo, ["config", "user.name", "publisher-test"]);
runGit(repo, ["config", "user.email", "publisher-test@example.com"]);
runGit(repo, ["config", "commit.gpgsign", "false"]);
runGit(repo, ["config", "core.hooksPath", "/dev/null"]);
};
const commitReport = (repo: string, reportPath: string, marker: string) => {
mkdirSync(join(repo, reportPath), { recursive: true });
writeFileSync(join(repo, reportPath, "report.json"), JSON.stringify({ marker }));
writeFileSync(join(repo, latest), JSON.stringify({ path: reportPath }));
runGit(repo, ["add", "--", "openclaw-performance"]);
runGit(repo, ["commit", "-m", `perf: add ${marker}`]);
};
try {
runGit(root, ["init", "--bare", "--initial-branch=main", remote]);
mkdirSync(seed);
runGit(seed, ["init", "--initial-branch=main"]);
configureWriter(seed);
writeFileSync(join(seed, "README.md"), "reports\n");
runGit(seed, ["add", "README.md"]);
runGit(seed, ["commit", "-m", "chore: seed"]);
runGit(seed, ["remote", "add", "origin", remote]);
runGit(seed, ["push", "origin", "HEAD:main"]);
runGit(root, ["clone", remote, writerA]);
runGit(root, ["clone", remote, writerB]);
configureWriter(writerA);
configureWriter(writerB);
commitReport(writerA, reportA, "writer-a");
const reportCommit = runGit(writerA, ["rev-parse", "HEAD"]);
commitReport(writerB, reportB, "writer-b");
runGit(writerB, ["push", "origin", "HEAD:main"]);
const rejectedPush = spawnSync("git", ["push", "origin", "HEAD:main"], {
cwd: writerA,
encoding: "utf8",
});
expect(rejectedPush.status).not.toBe(0);
runGit(writerA, ["fetch", "--depth=1", "origin", "main"]);
const remoteHasA = spawnSync("git", ["cat-file", "-e", `FETCH_HEAD:${reportA}/report.json`], {
cwd: writerA,
});
expect(remoteHasA.status).not.toBe(0);
runGit(writerA, ["checkout", "--detach", "FETCH_HEAD"]);
runGit(writerA, ["cherry-pick", "-X", "theirs", reportCommit]);
runGit(writerA, ["push", "origin", "HEAD:main"]);
runGit(root, ["clone", remote, verify]);
expect(JSON.parse(readFileSync(join(verify, reportA, "report.json"), "utf8"))).toEqual({
marker: "writer-a",
});
expect(JSON.parse(readFileSync(join(verify, reportB, "report.json"), "utf8"))).toEqual({
marker: "writer-b",
});
expect(JSON.parse(readFileSync(join(verify, latest), "utf8"))).toEqual({
path: reportA,
});
} finally {
rmSync(root, { force: true, recursive: true });
}
});
it("requires the shared Kova report gate before tolerating partial verdicts", () => {