mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-15 07:46:07 +00:00
f84b6f6091
* ci(perf): pin PID-aware Kova evaluator * test(ui): follow localized automation labels * ci(perf): pin compatible Kova runner * test(ui): bound chat hover browser wait * docs(agents): require release validation helper * ci(perf): retain redacted Kova reports * style(ui): format chat hover test * docs(agents): avoid dependency scripts in PR worktrees * docs(agents): guard shared checkout branches * docs(agents): quote gh api query endpoints
1258 lines
55 KiB
YAML
1258 lines
55 KiB
YAML
name: OpenClaw Performance
|
|
|
|
run-name: ${{ inputs.dispatch_id != '' && format('OpenClaw Performance {0}', inputs.dispatch_id) || 'OpenClaw Performance' }}
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "11 5 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
target_ref:
|
|
description: OpenClaw ref to benchmark; defaults to the workflow ref
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
profile:
|
|
description: Kova profile to run
|
|
required: false
|
|
default: diagnostic
|
|
type: choice
|
|
options:
|
|
- smoke
|
|
- diagnostic
|
|
- soak
|
|
- release
|
|
repeat:
|
|
description: Repeat count for non-profiled Kova runs
|
|
required: false
|
|
default: "3"
|
|
type: string
|
|
deep_profile:
|
|
description: Run the deep-profile lane with CPU/heap/trace artifacts
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
live_openai_candidate:
|
|
description: Run the live OpenAI GPT 5.6 agent-turn lane
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
fail_on_regression:
|
|
description: Fail the workflow when Kova exits non-zero
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
publish_reports:
|
|
description: Publish completed reports to openclaw/clawgrit-reports
|
|
required: false
|
|
default: true
|
|
type: boolean
|
|
kova_ref:
|
|
description: openclaw/Kova Git ref to install
|
|
required: false
|
|
default: 4dde6b6022d94058abced883343de7ee8ce69917
|
|
type: string
|
|
dispatch_id:
|
|
description: Optional parent workflow dispatch identifier
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-{1}', github.workflow, github.run_id) || format('{0}-{1}', github.workflow, github.ref) }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
OCM_VERSION: v0.2.25
|
|
OCM_LINUX_X64_SHA256: 57530199d21eb5bfa29695749928b40fd2869484c7edff69b7c65bfc84f2f1aa
|
|
KOVA_REPOSITORY: openclaw/Kova
|
|
PERFORMANCE_MODEL_ID: gpt-5.6-luna
|
|
KOVA_SCENARIO_TIMEOUT_MS: "300000"
|
|
|
|
jobs:
|
|
resolve_target:
|
|
name: Resolve target ref
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
outputs:
|
|
checkout_ref: ${{ steps.resolve.outputs.checkout_ref }}
|
|
tested_ref: ${{ steps.resolve.outputs.tested_ref }}
|
|
tested_sha: ${{ steps.resolve.outputs.tested_sha }}
|
|
steps:
|
|
- name: Resolve OpenClaw target ref
|
|
id: resolve
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
TARGET_REF_INPUT: ${{ inputs.target_ref }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
requested="${TARGET_REF_INPUT:-}"
|
|
if [[ "$requested" == *$'\n'* || "$requested" == *$'\r'* ]]; then
|
|
echo "::error::target_ref must be a single line."
|
|
exit 1
|
|
fi
|
|
if [[ -z "$requested" ]]; then
|
|
echo "checkout_ref=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
|
|
echo "tested_ref=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
|
|
echo "tested_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
|
|
encoded_ref="$(node -e 'process.stdout.write(encodeURIComponent(process.argv[1]))' "$requested")"
|
|
if ! resolved_sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${encoded_ref}" --jq '.sha')"; then
|
|
echo "::error::Unable to resolve OpenClaw target_ref '${requested}'."
|
|
exit 1
|
|
fi
|
|
if [[ ! "$resolved_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "::error::OpenClaw target_ref '${requested}' resolved to invalid SHA '${resolved_sha}'."
|
|
exit 1
|
|
fi
|
|
echo "checkout_ref=$resolved_sha" >> "$GITHUB_OUTPUT"
|
|
echo "tested_ref=$requested" >> "$GITHUB_OUTPUT"
|
|
echo "tested_sha=$resolved_sha" >> "$GITHUB_OUTPUT"
|
|
|
|
kova:
|
|
name: ${{ matrix.title }}
|
|
needs: resolve_target
|
|
runs-on: blacksmith-16vcpu-ubuntu-2404
|
|
timeout-minutes: 240
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- lane: mock-provider
|
|
title: Kova mock provider performance
|
|
auth: mock
|
|
repeat: input
|
|
deep_profile: "false"
|
|
live: "false"
|
|
managed_service: "true"
|
|
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:bundled-plugin-startup,scenario:bundled-runtime-deps,scenario:agent-cold-warm-message"
|
|
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,bundled-runtime-deps:missing-plugin-index,bundled-plugin-startup:fresh,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
|
|
- lane: mock-deep-profile
|
|
title: Kova mock provider deep profile
|
|
auth: mock
|
|
repeat: "1"
|
|
deep_profile: "true"
|
|
live: "false"
|
|
managed_service: "true"
|
|
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message"
|
|
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
|
|
- lane: live-openai-candidate
|
|
title: Kova live OpenAI GPT 5.6 agent turn
|
|
auth: live
|
|
repeat: "1"
|
|
deep_profile: "false"
|
|
live: "true"
|
|
managed_service: "false"
|
|
include_filters: "scenario:agent-cold-warm-message"
|
|
expected_release_entries: "agent-cold-warm-message:mock-openai-provider"
|
|
env:
|
|
KOVA_REF: ${{ inputs.kova_ref || '4dde6b6022d94058abced883343de7ee8ce69917' }}
|
|
KOVA_HOME: ${{ github.workspace }}/.artifacts/kova/home/${{ matrix.lane }}
|
|
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
|
|
REPORT_DIR: ${{ github.workspace }}/.artifacts/kova/reports/${{ matrix.lane }}
|
|
BUNDLE_DIR: ${{ github.workspace }}/.artifacts/kova/bundles/${{ matrix.lane }}
|
|
SUMMARY_DIR: ${{ github.workspace }}/.artifacts/kova/summaries
|
|
LANE_ID: ${{ matrix.lane }}
|
|
TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
|
|
EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
|
|
PROFILE: ${{ inputs.profile || 'diagnostic' }}
|
|
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
|
|
FAIL_ON_REGRESSION: ${{ inputs.fail_on_regression || 'false' }}
|
|
INCLUDE_FILTERS: ${{ matrix.include_filters }}
|
|
EXPECTED_RELEASE_ENTRIES: ${{ matrix.expected_release_entries }}
|
|
AUTH_MODE: ${{ matrix.auth }}
|
|
MATRIX_REPEAT: ${{ matrix.repeat }}
|
|
MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }}
|
|
MATRIX_LIVE: ${{ matrix.live }}
|
|
steps:
|
|
- name: Decide lane
|
|
id: lane
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
run_lane=true
|
|
reason=""
|
|
if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
|
|
run_lane=false
|
|
reason="deep_profile input is false"
|
|
fi
|
|
if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
|
|
run_lane=false
|
|
reason="live_openai_candidate input is false"
|
|
fi
|
|
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
|
|
if [[ "$run_lane" != "true" ]]; then
|
|
echo "Skipping ${LANE_ID}: ${reason}" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Checkout OpenClaw
|
|
if: steps.lane.outputs.run == 'true'
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ needs.resolve_target.outputs.checkout_ref }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Checkout performance workflow helpers
|
|
if: steps.lane.outputs.run == 'true'
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
path: .artifacts/performance-workflow
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Record tested revision
|
|
if: steps.lane.outputs.run == 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tested_sha="$(git rev-parse HEAD)"
|
|
if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
|
|
echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
|
|
exit 1
|
|
fi
|
|
echo "TESTED_REF=${TARGET_REF}" >> "$GITHUB_ENV"
|
|
echo "TESTED_SHA=${EXPECTED_TESTED_SHA}" >> "$GITHUB_ENV"
|
|
{
|
|
echo "Tested ref: ${TARGET_REF}"
|
|
echo "Tested SHA: ${EXPECTED_TESTED_SHA}"
|
|
echo "Workflow ref: ${GITHUB_REF_NAME}"
|
|
echo "Workflow SHA: ${GITHUB_SHA}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Set up Node environment
|
|
if: steps.lane.outputs.run == 'true'
|
|
uses: ./.github/actions/setup-node-env
|
|
with:
|
|
install-bun: "false"
|
|
|
|
- name: Prepare systemd user session
|
|
if: ${{ steps.lane.outputs.run == 'true' && matrix.managed_service == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
user="$(id -un)"
|
|
uid="$(id -u)"
|
|
|
|
test "$(ps -p 1 -o comm= | xargs)" = systemd
|
|
sudo systemctl is-active --quiet systemd-logind.service
|
|
sudo loginctl enable-linger "$user"
|
|
sudo systemctl start "user@${uid}.service"
|
|
|
|
runtime_dir="$(loginctl show-user "$user" --property=RuntimePath --value)"
|
|
test -n "$runtime_dir"
|
|
test -d "$runtime_dir"
|
|
test -O "$runtime_dir"
|
|
|
|
export XDG_RUNTIME_DIR="$runtime_dir"
|
|
test -S "$XDG_RUNTIME_DIR/systemd/private"
|
|
echo "XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" >> "$GITHUB_ENV"
|
|
|
|
if [[ -S "$runtime_dir/bus" ]]; then
|
|
export DBUS_SESSION_BUS_ADDRESS="unix:path=${runtime_dir}/bus"
|
|
echo "DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" >> "$GITHUB_ENV"
|
|
fi
|
|
|
|
systemctl --user show-environment >/dev/null
|
|
|
|
- name: Install OCM and Kova
|
|
if: steps.lane.outputs.run == 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
KOVA_SRC="${RUNNER_TEMP}/kova-src"
|
|
echo "KOVA_SRC=$KOVA_SRC" >> "$GITHUB_ENV"
|
|
mkdir -p "$HOME/.local/bin" "$(dirname "$KOVA_SRC")" "${RUNNER_TEMP}/ocm-install"
|
|
|
|
ocm_archive="${RUNNER_TEMP}/ocm-${OCM_VERSION}-x86_64-unknown-linux-gnu.tar.gz"
|
|
curl -fsSL --proto '=https' --tlsv1.2 \
|
|
--retry 8 --retry-max-time 180 --retry-all-errors --retry-connrefused \
|
|
-o "$ocm_archive" \
|
|
"https://github.com/shakkernerd/ocm/releases/download/${OCM_VERSION}/ocm-x86_64-unknown-linux-gnu.tar.gz"
|
|
echo "${OCM_LINUX_X64_SHA256} ${ocm_archive}" | sha256sum -c -
|
|
tar -xzf "$ocm_archive" -C "${RUNNER_TEMP}/ocm-install"
|
|
install -m 0755 "${RUNNER_TEMP}/ocm-install/ocm" "$HOME/.local/bin/ocm"
|
|
|
|
git init -b main "$KOVA_SRC"
|
|
git -C "$KOVA_SRC" remote add origin "https://github.com/${KOVA_REPOSITORY}.git"
|
|
git -C "$KOVA_SRC" fetch --filter=blob:none --depth 1 origin "$KOVA_REF"
|
|
git -C "$KOVA_SRC" checkout --detach FETCH_HEAD
|
|
npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund
|
|
node - "$KOVA_SRC" <<'NODE'
|
|
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
|
|
const root = process.argv[2];
|
|
const packageJsonPath = require.resolve("mock-ai-provider/package.json", {
|
|
paths: [root],
|
|
});
|
|
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
|
|
const bin =
|
|
typeof packageJson.bin === "string"
|
|
? packageJson.bin
|
|
: packageJson.bin?.["mock-ai-provider"];
|
|
if (typeof bin !== "string" || bin.length === 0) {
|
|
throw new Error("mock-ai-provider does not declare its CLI entrypoint");
|
|
}
|
|
fs.accessSync(path.resolve(path.dirname(packageJsonPath), bin), fs.constants.R_OK);
|
|
fs.accessSync(
|
|
path.join(root, "node_modules", ".bin", "mock-ai-provider"),
|
|
fs.constants.X_OK,
|
|
);
|
|
require.resolve("zod", { paths: [root] });
|
|
NODE
|
|
cat > "$HOME/.local/bin/kova" <<EOF
|
|
#!/usr/bin/env bash
|
|
export KOVA_HOME="${KOVA_HOME}"
|
|
exec node "${KOVA_SRC}/bin/kova.mjs" "\$@"
|
|
EOF
|
|
chmod 0755 "$HOME/.local/bin/kova"
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Configure OCM local workspace dependencies
|
|
if: steps.lane.outputs.run == 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
npm_wrapper="$PERFORMANCE_HELPER_DIR/scripts/ocm-npm-workspace-deps.mjs"
|
|
workspace_dependency_dirs=""
|
|
if [[ -f "${GITHUB_WORKSPACE}/packages/ai/package.json" ]]; then
|
|
workspace_dependency_dirs="${GITHUB_WORKSPACE}/packages/ai"
|
|
fi
|
|
chmod 0755 "$npm_wrapper"
|
|
{
|
|
echo "OCM_INTERNAL_NPM_BIN=$npm_wrapper"
|
|
echo "OPENCLAW_OCM_REAL_NPM_BIN=$(command -v npm)"
|
|
echo "OPENCLAW_OCM_WORKSPACE_DEPENDENCY_DIRS=$workspace_dependency_dirs"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Kova version and plan sanity
|
|
if: steps.lane.outputs.run == 'true'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
repeat="$REQUESTED_REPEAT"
|
|
if [[ "$MATRIX_REPEAT" != "input" ]]; then
|
|
repeat="$MATRIX_REPEAT"
|
|
fi
|
|
plan_dir="${RUNNER_TEMP}/kova-plans"
|
|
plan_json="${plan_dir}/${LANE_ID}.json"
|
|
mkdir -p "$plan_dir"
|
|
|
|
kova version --json
|
|
kova matrix plan \
|
|
--profile "$PROFILE" \
|
|
--target "local-build:${GITHUB_WORKSPACE}" \
|
|
--include "$INCLUDE_FILTERS" \
|
|
--parallel 1 \
|
|
--repeat "$repeat" \
|
|
--json >"$plan_json"
|
|
node --input-type=module - "$plan_json" <<'NODE'
|
|
import fs from "node:fs";
|
|
|
|
const plan = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
const expectedFilters = process.env.INCLUDE_FILTERS.split(",");
|
|
const actualFilters = plan.controls?.include;
|
|
if (
|
|
!Array.isArray(actualFilters) ||
|
|
actualFilters.length !== expectedFilters.length ||
|
|
actualFilters.some((filter, index) => filter !== expectedFilters[index])
|
|
) {
|
|
throw new Error("Kova plan did not preserve the requested include filters");
|
|
}
|
|
|
|
if (process.env.PROFILE === "release") {
|
|
const expectedEntries = process.env.EXPECTED_RELEASE_ENTRIES.split(",").sort();
|
|
if (!Array.isArray(plan.entries)) {
|
|
throw new Error("Kova release plan did not contain entries");
|
|
}
|
|
const actualEntries = plan.entries
|
|
.map((entry) => {
|
|
if (entry.status !== "SELECTED" || !entry.scenario?.id || !entry.state?.id) {
|
|
throw new Error("Kova release plan contained an invalid selected entry");
|
|
}
|
|
return `${entry.scenario.id}:${entry.state.id}`;
|
|
})
|
|
.sort();
|
|
if (
|
|
actualEntries.length !== expectedEntries.length ||
|
|
actualEntries.some((entry, index) => entry !== expectedEntries[index])
|
|
) {
|
|
throw new Error("Kova release plan entries did not match the required lane coverage");
|
|
}
|
|
}
|
|
NODE
|
|
echo "KOVA_PLAN_JSON=$plan_json" >> "$GITHUB_ENV"
|
|
echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"
|
|
|
|
- name: Configure live OpenAI auth
|
|
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
|
|
echo "OPENAI_API_KEY is not configured; live GPT 5.6 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY"
|
|
exit 1
|
|
fi
|
|
kova setup --ci --json
|
|
kova setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --json
|
|
|
|
- name: Run Kova
|
|
id: kova
|
|
if: steps.lane.outputs.run == 'true'
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$REPORT_DIR" "$BUNDLE_DIR" "$SUMMARY_DIR"
|
|
|
|
repeat="$KOVA_LANE_REPEAT"
|
|
|
|
args=(
|
|
matrix run
|
|
--profile "$PROFILE"
|
|
--target "local-build:${GITHUB_WORKSPACE}"
|
|
--include "$INCLUDE_FILTERS"
|
|
--auth "$AUTH_MODE"
|
|
--parallel 1
|
|
--repeat "$repeat"
|
|
--timeout-ms "$KOVA_SCENARIO_TIMEOUT_MS"
|
|
--report-dir "$REPORT_DIR"
|
|
--execute
|
|
--json
|
|
)
|
|
|
|
if [[ "$AUTH_MODE" == "live" ]]; then
|
|
args+=(--model "$PERFORMANCE_MODEL_ID")
|
|
fi
|
|
if [[ "$MATRIX_DEEP_PROFILE" == "true" ]]; then
|
|
args+=(--deep-profile)
|
|
fi
|
|
if [[ "$FAIL_ON_REGRESSION" == "true" ]]; then
|
|
args+=(--gate)
|
|
fi
|
|
|
|
log_path="$REPORT_DIR/${LANE_ID}.log"
|
|
set +e
|
|
kova "${args[@]}" 2>&1 | tee "$log_path"
|
|
status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
report_json="$(node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR")"
|
|
report_md="${report_json%.json}.md"
|
|
echo "status=$status" >> "$GITHUB_OUTPUT"
|
|
echo "report_json=$report_json" >> "$GITHUB_OUTPUT"
|
|
echo "report_md=$report_md" >> "$GITHUB_OUTPUT"
|
|
|
|
set +e
|
|
node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-workflow-evidence.mjs" \
|
|
--plan "$KOVA_PLAN_JSON" \
|
|
--report "$report_json" \
|
|
--profile "$PROFILE" \
|
|
--target "local-build:${GITHUB_WORKSPACE}" \
|
|
--repeat "$repeat" \
|
|
--include "$INCLUDE_FILTERS" \
|
|
--auth "$AUTH_MODE" \
|
|
--model "$PERFORMANCE_MODEL_ID"
|
|
evidence_status=$?
|
|
set -e
|
|
|
|
effective_status="$status"
|
|
if [[ "$evidence_status" == "0" && "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
|
|
if node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"
|
|
then
|
|
effective_status=0
|
|
{
|
|
echo "Kova returned a non-zero release-gate verdict, but the trusted report adapter found only filtered coverage or profiling-affected resource thresholds with no baseline regression."
|
|
echo
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
echo "effective_status=$effective_status" >> "$GITHUB_OUTPUT"
|
|
|
|
set +e
|
|
kova report bundle "$report_json" --output-dir "$BUNDLE_DIR" --json | tee "$BUNDLE_DIR/bundle.json"
|
|
bundle_status=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
summary_path="$SUMMARY_DIR/${LANE_ID}.md"
|
|
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/kova-ci-summary.mjs" --report "$report_json" --output "$summary_path" --lane "$LANE_ID")
|
|
set +e
|
|
"${summary_args[@]}"
|
|
summary_status=$?
|
|
set -e
|
|
if [[ "$summary_status" != "0" ]]; then
|
|
cat > "$summary_path" <<EOF
|
|
# OpenClaw Performance Report
|
|
|
|
- Lane: ${LANE_ID}
|
|
- Kova report: ${report_json}
|
|
- Summary generation failed with status ${summary_status}; inspect the Kova report and lane log.
|
|
EOF
|
|
fi
|
|
cat >> "$summary_path" <<EOF
|
|
|
|
## Test scope
|
|
|
|
- Repository: ${GITHUB_REPOSITORY}
|
|
- Tested ref: ${TESTED_REF}
|
|
- Tested SHA: ${TESTED_SHA}
|
|
- Workflow ref: ${GITHUB_REF_NAME}
|
|
- Workflow SHA: ${GITHUB_SHA}
|
|
- Kova repository: ${KOVA_REPOSITORY}
|
|
- Kova ref: ${KOVA_REF}
|
|
- Kova profile: ${PROFILE}
|
|
- Kova scenario timeout: ${KOVA_SCENARIO_TIMEOUT_MS}ms
|
|
- Lane auth: ${AUTH_MODE}
|
|
- Lane model: ${PERFORMANCE_MODEL_ID}
|
|
- Lane repeat: ${repeat}
|
|
- Include filters: ${INCLUDE_FILTERS}
|
|
EOF
|
|
cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
if [[ "$evidence_status" != "0" || "$bundle_status" != "0" || "$summary_status" != "0" ]]; then
|
|
exit 1
|
|
fi
|
|
if [[ "$FAIL_ON_REGRESSION" == "true" && "$effective_status" != "0" ]]; then
|
|
exit "$effective_status"
|
|
fi
|
|
|
|
- name: Validate Kova evidence
|
|
if: ${{ always() && steps.lane.outputs.run == 'true' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
missing=0
|
|
if ! node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-selector.mjs" --report-dir "$REPORT_DIR" >/dev/null; then
|
|
echo "::error::Exactly one full Kova JSON report is required for ${LANE_ID}."
|
|
missing=1
|
|
fi
|
|
if [[ ! -s "$BUNDLE_DIR/bundle.json" ]]; then
|
|
echo "::error::Kova bundle evidence is missing for ${LANE_ID}."
|
|
missing=1
|
|
fi
|
|
if [[ ! -s "$SUMMARY_DIR/${LANE_ID}.md" ]]; then
|
|
echo "::error::Kova summary evidence is missing for ${LANE_ID}."
|
|
missing=1
|
|
fi
|
|
if [[ "$missing" != "0" ]]; then
|
|
exit 1
|
|
fi
|
|
|
|
- name: Upload Kova artifacts
|
|
if: ${{ always() && steps.lane.outputs.run == 'true' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: openclaw-performance-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: |
|
|
.artifacts/kova/reports/${{ matrix.lane }}
|
|
.artifacts/kova/bundles/${{ matrix.lane }}
|
|
.artifacts/kova/summaries/${{ matrix.lane }}.md
|
|
if-no-files-found: error
|
|
retention-days: ${{ matrix.deep_profile == 'true' && 14 || 30 }}
|
|
|
|
source_performance:
|
|
name: OpenClaw source performance probes
|
|
needs: resolve_target
|
|
runs-on: blacksmith-16vcpu-ubuntu-2404
|
|
timeout-minutes: 120
|
|
env:
|
|
PERFORMANCE_HELPER_DIR: ${{ github.workspace }}/.artifacts/performance-workflow
|
|
SOURCE_PERF_DIR: ${{ github.workspace }}/.artifacts/openclaw-performance/source/mock-provider
|
|
TARGET_REF: ${{ needs.resolve_target.outputs.tested_ref }}
|
|
EXPECTED_TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
|
|
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
|
|
TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
|
|
REQUESTED_REPEAT: ${{ inputs.repeat || '3' }}
|
|
steps:
|
|
- name: Checkout OpenClaw source target
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ needs.resolve_target.outputs.checkout_ref }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Checkout source performance helpers
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
path: .artifacts/performance-workflow
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Record source performance revision
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
tested_sha="$(git rev-parse HEAD)"
|
|
if [[ "$tested_sha" != "$EXPECTED_TESTED_SHA" ]]; then
|
|
echo "::error::Checked out ${tested_sha}, expected resolved target ${EXPECTED_TESTED_SHA}."
|
|
exit 1
|
|
fi
|
|
{
|
|
echo "Tested ref: ${TESTED_REF}"
|
|
echo "Tested SHA: ${TESTED_SHA}"
|
|
echo "Workflow ref: ${GITHUB_REF_NAME}"
|
|
echo "Workflow SHA: ${GITHUB_SHA}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Set up source performance environment
|
|
uses: ./.github/actions/setup-node-env
|
|
with:
|
|
install-bun: "false"
|
|
|
|
- name: Fetch previous source performance baseline
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
reports_root=".artifacts/clawgrit-baseline"
|
|
mkdir -p "$reports_root"
|
|
git -C "$reports_root" init -b main
|
|
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
|
|
if ! git -C "$reports_root" fetch --filter=blob:none --depth=1 origin main; then
|
|
echo "No previous source performance baseline could be fetched." >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
ref_slug="$(printf '%s' "${TESTED_REF}" | tr -c 'A-Za-z0-9._-' '-')"
|
|
pointer="openclaw-performance/${ref_slug}/latest-mock-provider.json"
|
|
if ! git -C "$reports_root" cat-file -e "FETCH_HEAD:${pointer}"; then
|
|
echo "No previous source performance baseline exists for ${TESTED_REF}." >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
if ! latest_path="$(git -C "$reports_root" show "FETCH_HEAD:${pointer}" | node -e "const fs=require('node:fs'); const data=JSON.parse(fs.readFileSync(0,'utf8')); const value=String(data.path || ''); if (!/^openclaw-performance\\/[A-Za-z0-9._-]+\\/[0-9]+-[0-9]+\\/mock-provider$/u.test(value)) process.exit(1); process.stdout.write(value);")"; then
|
|
echo "Previous source performance baseline pointer is invalid." >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
git -C "$reports_root" sparse-checkout init --no-cone
|
|
printf '/%s/source/\n' "$latest_path" | git -C "$reports_root" sparse-checkout set --stdin
|
|
git -C "$reports_root" checkout --detach FETCH_HEAD
|
|
baseline_source="${reports_root}/${latest_path}/source"
|
|
if [[ -d "$baseline_source" ]]; then
|
|
baseline_source="$(realpath "$baseline_source")"
|
|
echo "SOURCE_PERF_BASELINE_DIR=$baseline_source" >> "$GITHUB_ENV"
|
|
echo "Using source performance baseline: ${latest_path}/source" >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "Previous source performance baseline has no source directory." >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Run OpenClaw source performance probes
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
source_runs="$REQUESTED_REPEAT"
|
|
if ! [[ "$source_runs" =~ ^[0-9]+$ ]] || [[ "$source_runs" -lt 1 ]]; then
|
|
source_runs=3
|
|
fi
|
|
|
|
mkdir -p "$SOURCE_PERF_DIR/mock-hello"
|
|
if ! node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:gateway:cpu-scenarios'] && scripts['test:extensions:memory'] && scripts.openclaw && fs.existsSync('scripts/bench-cli-startup.ts') && fs.existsSync('scripts/profile-extension-memory.mjs') ? 0 : 1)"; then
|
|
cat > "$SOURCE_PERF_DIR/index.md" <<EOF
|
|
# OpenClaw Source Performance
|
|
|
|
Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
|
|
Source probes skipped for this tested ref because one or more probe entry points are not present in the checked-out source tree.
|
|
|
|
## Test scope
|
|
|
|
- Tested ref: ${TESTED_REF}
|
|
- Tested SHA: ${TESTED_SHA}
|
|
- Required scripts: test:gateway:cpu-scenarios, test:extensions:memory, openclaw, scripts/bench-cli-startup.ts, scripts/profile-extension-memory.mjs
|
|
EOF
|
|
cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
|
|
# target_ref may predate the dedicated profile; preserve the historical full build there.
|
|
if node -e "import('./scripts/build-all.mjs').then((module) => process.exit(module.BUILD_ALL_PROFILES?.sourcePerformance ? 0 : 1)).catch(() => process.exit(1))"; then
|
|
OPENCLAW_BUILD_PRIVATE_QA=1 node scripts/build-all.mjs sourcePerformance
|
|
else
|
|
pnpm build
|
|
fi
|
|
|
|
pnpm test:gateway:cpu-scenarios \
|
|
--output-dir "$SOURCE_PERF_DIR/gateway-cpu" \
|
|
--runs "$source_runs" \
|
|
--warmup 1 \
|
|
--skip-qa \
|
|
--startup-case default \
|
|
--startup-case skipChannels \
|
|
--startup-case oneInternalHook \
|
|
--startup-case allInternalHooks \
|
|
--startup-case fiftyPlugins \
|
|
--startup-case fiftyStartupLazyPlugins
|
|
|
|
pnpm test:extensions:memory \
|
|
-- --json "$SOURCE_PERF_DIR/extension-memory.json"
|
|
|
|
for run_index in $(seq 1 "$source_runs"); do
|
|
run_dir="$SOURCE_PERF_DIR/mock-hello/run-$(printf '%03d' "$run_index")"
|
|
pnpm openclaw qa suite \
|
|
--provider-mode mock-openai \
|
|
--model "mock-openai/${PERFORMANCE_MODEL_ID}" \
|
|
--concurrency 1 \
|
|
--output-dir "$(realpath --relative-to="$GITHUB_WORKSPACE" "$run_dir")" \
|
|
--scenario channel-chat-baseline
|
|
done
|
|
|
|
gateway_home="$(mktemp -d)"
|
|
gateway_port="$(node -e "const net=require('node:net'); const s=net.createServer(); s.listen(0,'127.0.0.1',()=>{ console.log(s.address().port); s.close(); });")"
|
|
gateway_state="$gateway_home/.openclaw"
|
|
gateway_config="$gateway_state/openclaw.json"
|
|
gateway_log="$SOURCE_PERF_DIR/cli-gateway.log"
|
|
gateway_pid=""
|
|
mkdir -p "$gateway_state"
|
|
cat > "$gateway_config" <<EOF
|
|
{
|
|
"browser": { "enabled": false },
|
|
"gateway": {
|
|
"mode": "local",
|
|
"port": ${gateway_port},
|
|
"bind": "loopback",
|
|
"auth": { "mode": "none" },
|
|
"controlUi": { "enabled": false },
|
|
"tailscale": { "mode": "off" }
|
|
},
|
|
"plugins": {
|
|
"enabled": true,
|
|
"entries": { "browser": { "enabled": false } }
|
|
}
|
|
}
|
|
EOF
|
|
cleanup_gateway() {
|
|
if [[ -n "${gateway_pid:-}" ]] && kill -0 "$gateway_pid" 2>/dev/null; then
|
|
kill "$gateway_pid" 2>/dev/null || true
|
|
wait "$gateway_pid" 2>/dev/null || true
|
|
fi
|
|
rm -rf "$gateway_home"
|
|
}
|
|
trap cleanup_gateway EXIT
|
|
OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" OPENCLAW_SKIP_CHANNELS=1 \
|
|
node dist/entry.js gateway run --bind loopback --port "$gateway_port" --auth none --allow-unconfigured --force \
|
|
>"$gateway_log" 2>&1 &
|
|
gateway_pid="$!"
|
|
|
|
for _ in $(seq 1 120); do
|
|
if curl -fsS "http://127.0.0.1:${gateway_port}/healthz" >/dev/null; then
|
|
break
|
|
fi
|
|
if ! kill -0 "$gateway_pid" 2>/dev/null; then
|
|
cat "$gateway_log" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
curl -fsS "http://127.0.0.1:${gateway_port}/healthz" >/dev/null
|
|
|
|
OPENCLAW_HOME="$gateway_home" OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" OPENCLAW_GATEWAY_PORT="$gateway_port" \
|
|
node --import tsx scripts/bench-cli-startup.ts \
|
|
--case gatewayHealthJson \
|
|
--case configGetGatewayPort \
|
|
--runs "$source_runs" \
|
|
--warmup 1 \
|
|
--output "$SOURCE_PERF_DIR/cli-startup.json"
|
|
cleanup_gateway
|
|
trap - EXIT
|
|
|
|
if node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:sqlite:perf:smoke'] && fs.existsSync('scripts/bench-sqlite-state.ts') ? 0 : 1)"; then
|
|
pnpm test:sqlite:perf:smoke
|
|
cp .artifacts/sqlite-perf/smoke.json "$SOURCE_PERF_DIR/sqlite-perf-smoke.json"
|
|
else
|
|
echo "SQLite state smoke probe is not available in ${TESTED_REF}; continuing with the remaining source probes." >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
summary_args=(node "$PERFORMANCE_HELPER_DIR/scripts/openclaw-performance-source-summary.mjs" \
|
|
--source-dir "$SOURCE_PERF_DIR" \
|
|
--output "$SOURCE_PERF_DIR/index.md")
|
|
if [[ -n "${SOURCE_PERF_BASELINE_DIR:-}" && -d "$SOURCE_PERF_BASELINE_DIR" ]]; then
|
|
summary_args+=(--baseline-source-dir "$SOURCE_PERF_BASELINE_DIR")
|
|
fi
|
|
"${summary_args[@]}"
|
|
|
|
cat "$SOURCE_PERF_DIR/index.md" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload source performance artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: openclaw-performance-source-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: .artifacts/openclaw-performance/source
|
|
if-no-files-found: error
|
|
retention-days: 30
|
|
|
|
publish:
|
|
name: Publish ${{ matrix.title }} report
|
|
needs: [resolve_target, kova, source_performance]
|
|
if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- lane: mock-provider
|
|
title: mock provider
|
|
- lane: mock-deep-profile
|
|
title: mock deep profile
|
|
- lane: live-openai-candidate
|
|
title: live OpenAI candidate
|
|
env:
|
|
LANE_ID: ${{ matrix.lane }}
|
|
PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs
|
|
PERFORMANCE_REPORT_SELECTOR: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-selector.mjs
|
|
PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000"
|
|
REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }}
|
|
steps:
|
|
- name: Decide report publication lane
|
|
id: lane
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
run_lane=true
|
|
if [[ "$LANE_ID" == "mock-deep-profile" && "${{ github.event_name }}" != "schedule" && "${{ inputs.deep_profile || 'false' }}" != "true" ]]; then
|
|
run_lane=false
|
|
fi
|
|
if [[ "$LANE_ID" == "live-openai-candidate" && "${{ github.event_name }}" != "schedule" && "${{ inputs.live_openai_candidate || 'false' }}" != "true" ]]; then
|
|
run_lane=false
|
|
fi
|
|
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Checkout performance publisher helper
|
|
if: steps.lane.outputs.run == 'true'
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ github.sha }}
|
|
path: .artifacts/performance-publisher
|
|
sparse-checkout: |
|
|
scripts/lib/kova-report-publish-files.mjs
|
|
scripts/lib/kova-report-selector.mjs
|
|
sparse-checkout-cone-mode: false
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
- name: Resolve Kova artifact
|
|
id: artifact
|
|
if: steps.lane.outputs.run == 'true'
|
|
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
|
|
prefix="openclaw-performance-${LANE_ID}-${GITHUB_RUN_ID}-"
|
|
source_prefix="openclaw-performance-source-${GITHUB_RUN_ID}-"
|
|
if ! artifacts="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" --jq '.artifacts[] | select(.expired == false) | [.id, .name] | @tsv')"; then
|
|
echo "::${annotation}::Unable to list Kova artifacts for this workflow run."
|
|
exit 1
|
|
fi
|
|
|
|
artifact_id=""
|
|
producer_attempt=0
|
|
source_artifact_id=""
|
|
source_producer_attempt=0
|
|
while IFS=$'\t' read -r candidate_id candidate_name; do
|
|
if [[ "$candidate_name" == "$prefix"* ]]; then
|
|
candidate_attempt="${candidate_name#"$prefix"}"
|
|
[[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
|
|
if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > producer_attempt )); then
|
|
artifact_id="$candidate_id"
|
|
producer_attempt="$candidate_attempt"
|
|
fi
|
|
elif [[ "$LANE_ID" == "mock-provider" && "$candidate_name" == "$source_prefix"* ]]; then
|
|
candidate_attempt="${candidate_name#"$source_prefix"}"
|
|
[[ "$candidate_id" =~ ^[0-9]+$ && "$candidate_attempt" =~ ^[0-9]+$ ]] || continue
|
|
if (( candidate_attempt <= GITHUB_RUN_ATTEMPT && candidate_attempt > source_producer_attempt )); then
|
|
source_artifact_id="$candidate_id"
|
|
source_producer_attempt="$candidate_attempt"
|
|
fi
|
|
fi
|
|
done <<< "$artifacts"
|
|
|
|
if [[ -z "$artifact_id" ]]; then
|
|
echo "::${annotation}::No Kova artifact is available for ${LANE_ID} through attempt ${GITHUB_RUN_ATTEMPT}."
|
|
exit 1
|
|
fi
|
|
artifact_ids="$artifact_id"
|
|
if [[ "$LANE_ID" == "mock-provider" ]]; then
|
|
if [[ -z "$source_artifact_id" ]]; then
|
|
echo "::${annotation}::No source performance artifact is available through attempt ${GITHUB_RUN_ATTEMPT}."
|
|
exit 1
|
|
fi
|
|
artifact_ids="${artifact_ids},${source_artifact_id}"
|
|
if (( source_producer_attempt > producer_attempt )); then
|
|
producer_attempt="$source_producer_attempt"
|
|
fi
|
|
fi
|
|
echo "id=$artifact_id" >> "$GITHUB_OUTPUT"
|
|
echo "ids=$artifact_ids" >> "$GITHUB_OUTPUT"
|
|
echo "producer_attempt=$producer_attempt" >> "$GITHUB_OUTPUT"
|
|
echo "source_producer_attempt=$source_producer_attempt" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Create isolated publisher paths
|
|
id: paths
|
|
if: steps.artifact.outputs.id != ''
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
input_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-input.XXXXXX")"
|
|
reports_root="$(mktemp -d "${RUNNER_TEMP}/clawgrit-reports.XXXXXX")"
|
|
chmod 0700 "$input_root" "$reports_root"
|
|
echo "input_root=$input_root" >> "$GITHUB_OUTPUT"
|
|
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Download Kova artifacts
|
|
id: download
|
|
if: steps.artifact.outputs.id != ''
|
|
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
artifact-ids: ${{ steps.artifact.outputs.ids }}
|
|
path: ${{ steps.paths.outputs.input_root }}
|
|
|
|
- name: Prepare clawgrit report commit
|
|
id: prepare
|
|
if: steps.download.outcome == 'success'
|
|
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
|
env:
|
|
ARTIFACT_ID: ${{ steps.artifact.outputs.id }}
|
|
INPUT_ROOT: ${{ steps.paths.outputs.input_root }}
|
|
REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }}
|
|
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
|
|
TESTED_SHA: ${{ needs.resolve_target.outputs.tested_sha }}
|
|
PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.producer_attempt }}
|
|
SOURCE_PRODUCER_ATTEMPT: ${{ steps.artifact.outputs.source_producer_attempt }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
echo "ready=false" >> "$GITHUB_OUTPUT"
|
|
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
|
|
input_root="$(realpath "$INPUT_ROOT")"
|
|
reports_root="$(realpath "$REPORTS_ROOT")"
|
|
if [[ "$input_root" != "$RUNNER_TEMP/"* || "$reports_root" != "$RUNNER_TEMP/"* ]]; then
|
|
echo "::${annotation}::Publisher paths escaped RUNNER_TEMP."
|
|
exit 1
|
|
fi
|
|
if [[ ! "$ARTIFACT_ID" =~ ^[0-9]+$ || ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" || ! "$SOURCE_PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$SOURCE_PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
|
|
echo "::${annotation}::Kova artifact attempt metadata is invalid."
|
|
exit 1
|
|
fi
|
|
|
|
# download-artifact extracts one ID directly, but nests multiple IDs
|
|
# under artifact names. Recursive Kova lookup supports both layouts.
|
|
mapfile -d '' report_dirs < <(find "$input_root" -type d -path "*/reports/${LANE_ID}" -print0)
|
|
if [[ "${#report_dirs[@]}" != "1" ]]; then
|
|
echo "::${annotation}::Expected exactly one Kova report directory for ${LANE_ID}; found ${#report_dirs[@]}."
|
|
exit 1
|
|
fi
|
|
if ! report_json="$(node "$PERFORMANCE_REPORT_SELECTOR" --report-dir "${report_dirs[0]}")"; then
|
|
echo "::${annotation}::Exactly one full Kova JSON report is required for ${LANE_ID}."
|
|
exit 1
|
|
fi
|
|
report_json="$(realpath "$report_json")"
|
|
if [[ "$report_json" != "$input_root/"* || ! -s "$report_json" ]]; then
|
|
echo "::${annotation}::Kova JSON report is outside the downloaded artifact or empty."
|
|
exit 1
|
|
fi
|
|
report_md_path="${report_json%.json}.md"
|
|
if [[ -e "$report_md_path" ]]; then
|
|
if [[ ! -f "$report_md_path" || -L "$report_md_path" ]]; then
|
|
echo "::${annotation}::Kova Markdown report is not a canonical artifact file."
|
|
exit 1
|
|
fi
|
|
report_md="$(realpath "$report_md_path")"
|
|
if [[ "$report_md" != "$input_root/"* ]]; then
|
|
echo "::${annotation}::Kova Markdown report escaped the downloaded artifact."
|
|
exit 1
|
|
fi
|
|
else
|
|
report_md=""
|
|
fi
|
|
|
|
mapfile -d '' summaries < <(find "$input_root" -type f -path "*/summaries/${LANE_ID}.md" -print0)
|
|
mapfile -d '' bundles < <(find "$input_root" -type d -path "*/bundles/${LANE_ID}" -print0)
|
|
if [[ "${#summaries[@]}" != "1" || "${#bundles[@]}" != "1" ]]; then
|
|
echo "::${annotation}::Kova summary or bundle is missing for ${LANE_ID}."
|
|
exit 1
|
|
fi
|
|
summary="$(realpath "${summaries[0]}")"
|
|
bundle="$(realpath "${bundles[0]}")"
|
|
if [[ "$summary" != "$input_root/"* || "$bundle" != "$input_root/"* || ! -s "$summary" ]]; then
|
|
echo "::${annotation}::Kova summary or bundle escaped the downloaded artifact."
|
|
exit 1
|
|
fi
|
|
if find "$bundle" -type l -print -quit | grep -q . || find "$bundle" ! -type d ! -type f -print -quit | grep -q .; then
|
|
echo "::${annotation}::Kova bundle contains a symlink or special file."
|
|
exit 1
|
|
fi
|
|
|
|
source_dir=""
|
|
if [[ "$LANE_ID" == "mock-provider" ]]; then
|
|
if [[ "$SOURCE_PRODUCER_ATTEMPT" == "0" ]]; then
|
|
echo "::${annotation}::Source probe artifact attempt metadata is missing."
|
|
exit 1
|
|
fi
|
|
# The mock lane downloads two IDs, so its source artifact is nested
|
|
# under the immutable artifact name rather than extracted directly.
|
|
source_path="${input_root}/openclaw-performance-source-${GITHUB_RUN_ID}-${SOURCE_PRODUCER_ATTEMPT}/${LANE_ID}"
|
|
if [[ ! -d "$source_path" ]]; then
|
|
echo "::${annotation}::Source probe artifact is missing for ${LANE_ID}."
|
|
exit 1
|
|
fi
|
|
source_dir="$(realpath "$source_path")"
|
|
if [[ "$source_dir" != "$input_root/"* ]] || find "$source_dir" -type l -print -quit | grep -q . || find "$source_dir" ! -type d ! -type f -print -quit | grep -q .; then
|
|
echo "::${annotation}::Source probe artifact contains an unsafe path."
|
|
exit 1
|
|
fi
|
|
elif [[ "$SOURCE_PRODUCER_ATTEMPT" != "0" ]]; then
|
|
echo "::${annotation}::Unexpected source probe artifact attempt for ${LANE_ID}."
|
|
exit 1
|
|
fi
|
|
|
|
git -C "$reports_root" init -b main
|
|
git -C "$reports_root" config core.hooksPath /dev/null
|
|
git -C "$reports_root" remote add origin "https://github.com/openclaw/clawgrit-reports.git"
|
|
if ! timeout 120s git -C "$reports_root" fetch --depth=1 origin main; then
|
|
echo "::${annotation}::Unable to fetch openclaw/clawgrit-reports before publishing."
|
|
exit 1
|
|
fi
|
|
git -C "$reports_root" checkout -B main FETCH_HEAD
|
|
|
|
ref_slug="$(printf '%s' "$TESTED_REF" | tr -c 'A-Za-z0-9._-' '-')"
|
|
if [[ -z "$ref_slug" || "$ref_slug" == "." || "$ref_slug" == ".." ]]; then
|
|
echo "::${annotation}::Tested ref does not produce a safe report path."
|
|
exit 1
|
|
fi
|
|
run_slug="${GITHUB_RUN_ID}-${PRODUCER_ATTEMPT}"
|
|
dest_rel="openclaw-performance/${ref_slug}/${run_slug}/${LANE_ID}"
|
|
report_url="https://github.com/openclaw/clawgrit-reports/tree/main/${dest_rel}"
|
|
if git -C "$reports_root" cat-file -e "HEAD:${dest_rel}/report.json" 2>/dev/null; then
|
|
{
|
|
echo "### Clawgrit report already published"
|
|
echo
|
|
echo "- Published report: ${report_url}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
echo "already_published=true" >> "$GITHUB_OUTPUT"
|
|
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
|
|
echo "ready=true" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
dest="${reports_root}/${dest_rel}"
|
|
mkdir -p "$dest"
|
|
install -m 0644 "$report_json" "$dest/report.json"
|
|
if [[ -n "$report_md" ]]; then
|
|
install -m 0644 "$report_md" "$dest/report.md"
|
|
fi
|
|
install -m 0644 "$summary" "$dest/index.md"
|
|
if [[ -n "$source_dir" ]]; then
|
|
mkdir -p "$dest/source"
|
|
cp -R "$source_dir"/. "$dest/source/"
|
|
if [[ -f "$source_dir/index.md" ]]; then
|
|
cat >> "$dest/index.md" <<'EOF'
|
|
|
|
## Source probes
|
|
|
|
Additional gateway boot, memory, plugin pressure, mock hello-loop, CLI startup, and SQLite state smoke numbers are in [source/index.md](source/index.md).
|
|
EOF
|
|
fi
|
|
fi
|
|
|
|
artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}"
|
|
{
|
|
echo
|
|
echo "## Full diagnostic artifact"
|
|
echo
|
|
echo "The complete Kova bundle remains in [Actions artifact ${ARTIFACT_ID}](${artifact_url}); its checksum is published under the bundles directory."
|
|
} >> "$dest/index.md"
|
|
|
|
jq -n \
|
|
--arg repository "$GITHUB_REPOSITORY" \
|
|
--arg ref "$TESTED_REF" \
|
|
--arg sha "$TESTED_SHA" \
|
|
--arg workflow_ref "$GITHUB_REF_NAME" \
|
|
--arg workflow_sha "$GITHUB_SHA" \
|
|
--arg workflow "$GITHUB_WORKFLOW" \
|
|
--arg run_id "$GITHUB_RUN_ID" \
|
|
--arg run_attempt "$PRODUCER_ATTEMPT" \
|
|
--arg lane "$LANE_ID" \
|
|
--arg path "$dest_rel" \
|
|
'{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \
|
|
> "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json"
|
|
|
|
node "$PERFORMANCE_PUBLISHER_HELPER" \
|
|
--bundle-dir "$bundle" \
|
|
--bundle-destination "$dest/bundles" \
|
|
--publish-root "$dest" \
|
|
--max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"
|
|
|
|
git -C "$reports_root" config user.name "openclaw-performance[bot]"
|
|
git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com"
|
|
git -C "$reports_root" add -- openclaw-performance
|
|
if ! git -C "$reports_root" diff --cached --quiet; then
|
|
git -C "$reports_root" commit -m "perf: add OpenClaw ${LANE_ID} report ${TESTED_SHA::12}"
|
|
fi
|
|
report_commit="$(git -C "$reports_root" rev-parse HEAD)"
|
|
echo "already_published=false" >> "$GITHUB_OUTPUT"
|
|
echo "reports_root=$reports_root" >> "$GITHUB_OUTPUT"
|
|
echo "report_commit=$report_commit" >> "$GITHUB_OUTPUT"
|
|
echo "dest_rel=$dest_rel" >> "$GITHUB_OUTPUT"
|
|
echo "report_url=$report_url" >> "$GITHUB_OUTPUT"
|
|
echo "ready=true" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Create clawgrit reports app token
|
|
id: clawgrit_app_token
|
|
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
|
|
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
|
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
|
|
with:
|
|
client-id: Iv23liOECG0slfuhz093
|
|
private-key: ${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}
|
|
owner: openclaw
|
|
repositories: clawgrit-reports
|
|
permission-contents: write
|
|
|
|
- name: Publish to clawgrit reports
|
|
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
|
|
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
|
env:
|
|
CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }}
|
|
REPORTS_ROOT: ${{ steps.prepare.outputs.reports_root }}
|
|
REPORT_COMMIT: ${{ steps.prepare.outputs.report_commit }}
|
|
DEST_REL: ${{ steps.prepare.outputs.dest_rel }}
|
|
REPORT_URL: ${{ steps.prepare.outputs.report_url }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
annotation="$([[ "$REPORT_PUBLISH_REQUIRED" == "true" ]] && printf error || printf warning)"
|
|
if [[ -z "${CLAWGRIT_REPORTS_APP_TOKEN:-}" ]]; then
|
|
{
|
|
echo "### Clawgrit report publish unavailable"
|
|
echo
|
|
echo "The repository-scoped ClawSweeper GitHub App token could not be created."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
echo "::${annotation}::ClawSweeper GitHub App token is unavailable; Kova artifacts were uploaded, but the report cannot be published."
|
|
exit 1
|
|
fi
|
|
token="$CLAWGRIT_REPORTS_APP_TOKEN"
|
|
unset CLAWGRIT_REPORTS_APP_TOKEN
|
|
auth_header="$(printf 'x-access-token:%s' "$token" | base64 | tr -d '\n')"
|
|
unset token
|
|
echo "::add-mask::${auth_header}"
|
|
|
|
reports_root="$(realpath "$REPORTS_ROOT")"
|
|
if [[ "$reports_root" != "$RUNNER_TEMP/"* || ! "$REPORT_COMMIT" =~ ^[0-9a-f]{40}$ || ! "$DEST_REL" =~ ^openclaw-performance/[A-Za-z0-9._-]+/[0-9]+-[0-9]+/[A-Za-z0-9._-]+$ ]]; then
|
|
echo "::${annotation}::Prepared report metadata failed validation."
|
|
exit 1
|
|
fi
|
|
if [[ "$REPORT_URL" != "https://github.com/openclaw/clawgrit-reports/tree/main/${DEST_REL}" ]]; then
|
|
echo "::${annotation}::Prepared report URL failed validation."
|
|
exit 1
|
|
fi
|
|
|
|
git_local() {
|
|
git -C "$reports_root" -c core.hooksPath=/dev/null "$@"
|
|
}
|
|
push_report() {
|
|
GIT_TERMINAL_PROMPT=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=http.https://github.com/.extraheader GIT_CONFIG_VALUE_1="AUTHORIZATION: basic ${auth_header}" timeout 120s git -C "$reports_root" push origin HEAD:main
|
|
}
|
|
write_publish_success() {
|
|
{
|
|
echo "### Clawgrit report published"
|
|
echo
|
|
echo "- Published report: ${REPORT_URL}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
}
|
|
write_publish_failure() {
|
|
{
|
|
echo "### Clawgrit report publish failed"
|
|
echo
|
|
echo "Kova artifacts were uploaded, but publishing the clawgrit report failed after $1 attempts."
|
|
echo "Check the ClawSweeper GitHub App installation and contents permission for openclaw/clawgrit-reports."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
echo "::${annotation}::Kova artifacts uploaded, but clawgrit report publish failed after $1 attempts."
|
|
}
|
|
|
|
if [[ "$(git_local config --local --get core.hooksPath)" != "/dev/null" || "$(git_local remote get-url origin)" != "https://github.com/openclaw/clawgrit-reports.git" ]]; then
|
|
echo "::${annotation}::Prepared reports checkout failed integrity validation."
|
|
exit 1
|
|
fi
|
|
git_local cat-file -e "${REPORT_COMMIT}^{commit}"
|
|
report_commit="$REPORT_COMMIT"
|
|
|
|
for attempt in 1 2 3 4 5; do
|
|
if push_report; then
|
|
write_publish_success
|
|
exit 0
|
|
fi
|
|
|
|
sleep $((attempt * 2))
|
|
if ! timeout 120s git -C "$reports_root" -c core.hooksPath=/dev/null fetch --depth=1 origin main; then
|
|
if [[ "$attempt" != "5" ]]; then
|
|
echo "::warning::Unable to refresh openclaw/clawgrit-reports after publish attempt ${attempt}; retrying."
|
|
continue
|
|
fi
|
|
else
|
|
if git_local cat-file -e "FETCH_HEAD:${DEST_REL}/report.json" 2>/dev/null; then
|
|
# A timed-out push may have succeeded; verify its unique report before failing.
|
|
write_publish_success
|
|
exit 0
|
|
fi
|
|
if [[ "$attempt" != "5" ]]; then
|
|
git_local checkout --detach FETCH_HEAD
|
|
# The run path is unique; only the shared latest pointer can conflict.
|
|
# Replaying with the current commit wins that pointer deterministically.
|
|
if ! git_local cherry-pick -X theirs "$report_commit"; then
|
|
git_local cherry-pick --abort || true
|
|
echo "::${annotation}::Unable to replay the clawgrit report after a concurrent publish."
|
|
exit 1
|
|
fi
|
|
report_commit="$(git_local rev-parse HEAD)"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
write_publish_failure "$attempt"
|
|
exit 1
|
|
done
|
|
|
|
artifact_only_guard:
|
|
name: Verify artifact-only report mode
|
|
needs: [resolve_target, kova, publish]
|
|
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.publish_reports != true }}
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Verify report publisher stayed disabled
|
|
env:
|
|
PUBLISH_RESULT: ${{ needs.publish.result }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "$PUBLISH_RESULT" != "skipped" ]]; then
|
|
echo "::error::Artifact-only performance mode requires the report publisher to stay skipped; got ${PUBLISH_RESULT}."
|
|
exit 1
|
|
fi
|
|
{
|
|
echo "### Artifact-only report mode"
|
|
echo
|
|
echo "Clawgrit report publication stayed disabled; Kova evidence is available only as workflow artifacts."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|