mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-30 05:23:34 +00:00
465 lines
18 KiB
YAML
465 lines
18 KiB
YAML
name: Maturity scorecard
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
qa_evidence_run_id:
|
|
description: Optional workflow run id containing qa-evidence.json
|
|
required: false
|
|
type: string
|
|
ref:
|
|
description: OpenClaw branch, tag, or SHA containing the maturity score source
|
|
required: true
|
|
default: main
|
|
type: string
|
|
expected_sha:
|
|
description: Optional full SHA that ref must resolve to
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
workflow_call:
|
|
inputs:
|
|
qa_evidence_run_id:
|
|
description: Optional workflow run id containing qa-evidence.json
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
ref:
|
|
description: OpenClaw branch, tag, or SHA containing the maturity score source
|
|
required: true
|
|
type: string
|
|
expected_sha:
|
|
description: Optional full SHA that ref must resolve to
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
secrets:
|
|
OPENAI_API_KEY:
|
|
description: OpenAI API key used by live QA profile scenarios
|
|
required: true
|
|
OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY:
|
|
description: Optional OpenAI API key used by maturity scorecard agent steps
|
|
required: false
|
|
GH_APP_PRIVATE_KEY:
|
|
description: Optional GitHub App private key for generated docs PR creation
|
|
required: false
|
|
GH_APP_PRIVATE_KEY_FALLBACK:
|
|
description: Optional fallback GitHub App private key for generated docs PR creation
|
|
required: false
|
|
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ format('{0}-{1}-{2}', github.workflow, inputs.ref, inputs.qa_evidence_run_id || github.run_id) }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
NODE_VERSION: "24.x"
|
|
|
|
jobs:
|
|
validate_selected_ref:
|
|
name: Validate selected ref
|
|
runs-on: ubuntu-24.04
|
|
outputs:
|
|
selected_revision: ${{ steps.validate.outputs.selected_revision }}
|
|
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
|
|
steps:
|
|
- name: Checkout selected ref
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
persist-credentials: false
|
|
ref: ${{ inputs.ref }}
|
|
fetch-depth: 0
|
|
|
|
- name: Validate selected ref
|
|
id: validate
|
|
env:
|
|
EXPECTED_SHA: ${{ inputs.expected_sha }}
|
|
INPUT_REF: ${{ inputs.ref }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
selected_revision="$(git rev-parse HEAD)"
|
|
expected_sha="${EXPECTED_SHA,,}"
|
|
trusted_reason=""
|
|
|
|
if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "expected_sha must be a full 40-character SHA; got: ${EXPECTED_SHA}" >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${expected_sha// }" && "${selected_revision,,}" != "$expected_sha" ]]; then
|
|
echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
|
|
|
|
if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
|
|
trusted_reason="main-ancestor"
|
|
elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
|
|
trusted_reason="release-tag"
|
|
elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
|
|
git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}"
|
|
release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")"
|
|
if [[ "$selected_revision" == "$release_branch_sha" ]]; then
|
|
trusted_reason="release-branch-head"
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$trusted_reason" ]]; then
|
|
echo "Ref '${INPUT_REF}' resolved to $selected_revision, which is not trusted for this secret-bearing maturity scorecard run." >&2
|
|
echo "Allowed refs must be on main, point to a release tag, or match a release branch head." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT"
|
|
echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT"
|
|
{
|
|
echo "### Target"
|
|
echo
|
|
echo "- Requested ref: \`${INPUT_REF}\`"
|
|
echo "- Resolved SHA: \`$selected_revision\`"
|
|
echo "- Trust reason: \`$trusted_reason\`"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
generate_qa_evidence:
|
|
name: Generate full taxonomy QA evidence
|
|
needs: validate_selected_ref
|
|
if: ${{ inputs.qa_evidence_run_id == '' }}
|
|
uses: ./.github/workflows/qa-profile-evidence.yml
|
|
with:
|
|
ref: ${{ inputs.ref }}
|
|
expected_sha: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
|
qa_profile: all
|
|
secrets:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
|
|
publish:
|
|
name: Publish maturity docs PR
|
|
needs:
|
|
- validate_selected_ref
|
|
- generate_qa_evidence
|
|
if: ${{ always() && needs.validate_selected_ref.result == 'success' && (inputs.qa_evidence_run_id != '' || needs.generate_qa_evidence.result == 'success') }}
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
permissions:
|
|
actions: read
|
|
contents: read
|
|
steps:
|
|
- name: Checkout selected ref
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
|
with:
|
|
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
|
fetch-depth: 1
|
|
fetch-tags: false
|
|
persist-credentials: false
|
|
submodules: false
|
|
|
|
- name: Setup Node environment
|
|
uses: ./.github/actions/setup-node-env
|
|
with:
|
|
node-version: ${{ env.NODE_VERSION }}
|
|
install-bun: "false"
|
|
|
|
- name: Download provided QA evidence artifact
|
|
if: ${{ inputs.qa_evidence_run_id != '' }}
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
QA_EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p .artifacts/maturity-evidence
|
|
gh run download "$QA_EVIDENCE_RUN_ID" \
|
|
--repo "$GITHUB_REPOSITORY" \
|
|
--dir .artifacts/maturity-evidence
|
|
|
|
- name: Download generated QA evidence artifact
|
|
if: ${{ inputs.qa_evidence_run_id == '' }}
|
|
env:
|
|
GENERATED_ARTIFACT_NAME: ${{ needs.generate_qa_evidence.outputs.artifact_name }}
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -z "${GENERATED_ARTIFACT_NAME:-}" ]]; then
|
|
echo "Generated QA evidence workflow did not expose an artifact name." >&2
|
|
exit 1
|
|
fi
|
|
mkdir -p .artifacts/maturity-evidence
|
|
gh run download "$GITHUB_RUN_ID" \
|
|
--repo "$GITHUB_REPOSITORY" \
|
|
--name "$GENERATED_ARTIFACT_NAME" \
|
|
--dir .artifacts/maturity-evidence
|
|
|
|
- name: Require one QA evidence file
|
|
id: evidence
|
|
env:
|
|
QA_EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
mapfile -t evidence_paths < <(find .artifacts/maturity-evidence -type f -name qa-evidence.json | sort)
|
|
if [[ "${#evidence_paths[@]}" -eq 0 ]]; then
|
|
echo "Expected a qa-evidence.json file in the downloaded QA evidence artifact." >&2
|
|
exit 1
|
|
fi
|
|
if [[ "${#evidence_paths[@]}" -gt 1 ]]; then
|
|
echo "Expected exactly one qa-evidence.json file, found ${#evidence_paths[@]}:" >&2
|
|
printf '%s\n' "${evidence_paths[@]}" >&2
|
|
exit 1
|
|
fi
|
|
echo "qa_evidence_path=${evidence_paths[0]}" >> "$GITHUB_OUTPUT"
|
|
{
|
|
echo "### QA evidence"
|
|
echo
|
|
echo "- Evidence path: \`${evidence_paths[0]}\`"
|
|
echo "- Evidence source run: \`${QA_EVIDENCE_RUN_ID:-$GITHUB_RUN_ID}\`"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Validate QA evidence manifest
|
|
env:
|
|
QA_EVIDENCE_PATH: ${{ steps.evidence.outputs.qa_evidence_path }}
|
|
TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
|
run: |
|
|
set -euo pipefail
|
|
node --input-type=module <<'NODE'
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const evidencePath = process.env.QA_EVIDENCE_PATH;
|
|
const targetSha = process.env.TARGET_SHA;
|
|
if (!evidencePath) {
|
|
throw new Error("QA_EVIDENCE_PATH is required");
|
|
}
|
|
if (!targetSha) {
|
|
throw new Error("TARGET_SHA is required");
|
|
}
|
|
|
|
const evidence = JSON.parse(fs.readFileSync(evidencePath, "utf8"));
|
|
if (evidence.profile !== "all") {
|
|
throw new Error(`qa-evidence.json profile must be all, got ${JSON.stringify(evidence.profile)}`);
|
|
}
|
|
|
|
const artifactDir = path.dirname(evidencePath);
|
|
const manifestNames = fs
|
|
.readdirSync(artifactDir)
|
|
.filter((name) => name.endsWith("qa-profile-evidence-manifest.json"))
|
|
.sort();
|
|
if (manifestNames.length !== 1) {
|
|
throw new Error(
|
|
`Expected exactly one QA profile evidence manifest next to qa-evidence.json, found ${manifestNames.length}`,
|
|
);
|
|
}
|
|
|
|
const manifestPath = path.join(artifactDir, manifestNames[0]);
|
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
|
const manifestProfile = manifest.qaProfile ?? evidence.profile;
|
|
if (manifestProfile !== "all") {
|
|
throw new Error(`QA evidence manifest profile must be all, got ${JSON.stringify(manifestProfile)}`);
|
|
}
|
|
if (manifest.targetSha !== targetSha) {
|
|
throw new Error(`QA evidence manifest targetSha ${manifest.targetSha} does not match selected ref ${targetSha}`);
|
|
}
|
|
NODE
|
|
|
|
- name: Ensure maturity scorecard agent key exists
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
|
|
echo "Missing OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY or OPENAI_API_KEY secret." >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run Codex maturity scorecard agent
|
|
uses: openai/codex-action@e0fdf01220eb9a88167c4898839d273e3f2609d1
|
|
env:
|
|
MATURITY_EVIDENCE_DIR: .artifacts/maturity-evidence
|
|
MATURITY_SCORES_PATH: qa/maturity-scores.yaml
|
|
MATURITY_TAXONOMY_PATH: taxonomy.yaml
|
|
with:
|
|
openai-api-key: ${{ secrets.OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}
|
|
prompt-file: .github/codex/prompts/maturity-scorecard-agent.md
|
|
model: ${{ vars.OPENCLAW_CI_OPENAI_MODEL_BARE }}
|
|
effort: high
|
|
sandbox: workspace-write
|
|
safety-strategy: drop-sudo
|
|
|
|
- name: Enforce focused maturity score patch
|
|
run: |
|
|
set -euo pipefail
|
|
git restore --staged :/
|
|
|
|
allowed='^qa/maturity-scores\.yaml$'
|
|
bad_tracked="$(
|
|
git diff --name-only HEAD -- | while IFS= read -r path; do
|
|
if [[ ! "$path" =~ $allowed ]]; then
|
|
printf '%s\n' "$path"
|
|
fi
|
|
done
|
|
)"
|
|
if [[ -n "$bad_tracked" ]]; then
|
|
echo "Maturity scorecard agent touched forbidden tracked paths:"
|
|
printf '%s\n' "$bad_tracked"
|
|
exit 1
|
|
fi
|
|
|
|
bad_untracked="$(
|
|
git ls-files --others --exclude-standard | while IFS= read -r path; do
|
|
if [[ "$path" != "qa/maturity-scores.yaml" ]]; then
|
|
printf '%s\n' "$path"
|
|
fi
|
|
done
|
|
)"
|
|
if [[ -n "$bad_untracked" ]]; then
|
|
echo "Maturity scorecard agent created forbidden untracked paths:"
|
|
printf '%s\n' "$bad_untracked"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f qa/maturity-scores.yaml ]]; then
|
|
echo "Maturity scorecard agent must produce qa/maturity-scores.yaml." >&2
|
|
exit 1
|
|
fi
|
|
|
|
- name: Validate maturity score sources
|
|
run: |
|
|
node --import tsx --input-type=module <<'NODE'
|
|
import { readValidatedQaMaturityScoreSources } from "./extensions/qa-lab/src/scorecard-taxonomy.ts";
|
|
|
|
const { warnings } = readValidatedQaMaturityScoreSources({
|
|
scoresPath: "qa/maturity-scores.yaml",
|
|
taxonomyPath: "taxonomy.yaml",
|
|
});
|
|
for (const warning of warnings) {
|
|
console.error(`warning: ${warning}`);
|
|
}
|
|
NODE
|
|
|
|
- name: Render artifact docs
|
|
run: |
|
|
set -euo pipefail
|
|
pnpm maturity:render -- \
|
|
--output-dir .artifacts/maturity-docs \
|
|
--static-assets-dir .artifacts/maturity-docs/assets/maturity \
|
|
--scores qa/maturity-scores.yaml \
|
|
--evidence-dir .artifacts/maturity-evidence \
|
|
--strict-inputs
|
|
{
|
|
echo "### Maturity scorecard docs"
|
|
echo
|
|
echo "- Source validation: passed"
|
|
echo "- Artifact docs: \`.artifacts/maturity-docs\`"
|
|
echo "- Strict inputs: \`true\`"
|
|
echo "- QA evidence: included"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Render committed docs preview
|
|
run: |
|
|
set -euo pipefail
|
|
pnpm maturity:render -- \
|
|
--output-dir docs \
|
|
--scores qa/maturity-scores.yaml \
|
|
--evidence-dir .artifacts/maturity-evidence \
|
|
--strict-inputs
|
|
|
|
- name: Create generated docs PR app token
|
|
if: ${{ github.event_name == 'workflow_dispatch' }}
|
|
id: app-token
|
|
continue-on-error: true
|
|
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
|
|
with:
|
|
app-id: "2729701"
|
|
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
|
permission-contents: write
|
|
permission-pull-requests: write
|
|
|
|
- name: Create generated docs PR fallback app token
|
|
if: ${{ github.event_name == 'workflow_dispatch' && steps.app-token.outcome == 'failure' }}
|
|
id: app-token-fallback
|
|
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
|
|
with:
|
|
app-id: "2971289"
|
|
private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }}
|
|
permission-contents: write
|
|
permission-pull-requests: write
|
|
|
|
- name: Open generated docs PR
|
|
if: ${{ github.event_name == 'workflow_dispatch' }}
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }}
|
|
QA_EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id }}
|
|
REF_INPUT: ${{ inputs.ref }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -z "${GH_TOKEN:-}" ]]; then
|
|
echo "Maturity scorecard PR creation requires the OpenClaw GitHub App token secrets." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "$(git status --porcelain -- qa/maturity-scores.yaml docs/maturity/scorecard.md docs/maturity/taxonomy.md)" ]]; then
|
|
{
|
|
echo
|
|
echo "- Pull request: skipped; generated scorecard matches selected ref"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
|
|
evidence_run_id="${QA_EVIDENCE_RUN_ID:-$GITHUB_RUN_ID}"
|
|
branch="automation/maturity-scorecard-${evidence_run_id}"
|
|
base_branch="${REF_INPUT:-main}"
|
|
if ! git ls-remote --exit-code --heads origin "$base_branch" >/dev/null 2>&1; then
|
|
base_branch="main"
|
|
fi
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
gh auth setup-git
|
|
git fetch --no-tags --depth=1 origin "refs/heads/${branch}:refs/remotes/origin/${branch}" || true
|
|
git switch -C "$branch"
|
|
git add qa/maturity-scores.yaml docs/maturity/scorecard.md docs/maturity/taxonomy.md
|
|
git commit -m "docs: update maturity scorecard"
|
|
git push --force-with-lease origin "$branch"
|
|
|
|
body_file=".artifacts/maturity-scorecard-pr-body.md"
|
|
mkdir -p "$(dirname "$body_file")"
|
|
cat > "$body_file" <<BODY
|
|
## Summary
|
|
|
|
- render maturity scorecard docs from \`qa/maturity-scores.yaml\` and full taxonomy QA evidence
|
|
- maturity source ref: ${REF_INPUT}
|
|
- QA evidence run: ${evidence_run_id}
|
|
|
|
## Verification
|
|
|
|
- QA Lab maturity score validation passed
|
|
- Maturity scorecard workflow rendered docs from all profile qa-evidence.json artifacts with strict inputs
|
|
BODY
|
|
|
|
pr_url="$(gh pr list --head "$branch" --state open --json url --jq '.[0].url // ""')"
|
|
if [[ -n "$pr_url" ]]; then
|
|
gh pr edit "$pr_url" \
|
|
--title "docs: update maturity scorecard" \
|
|
--body-file "$body_file"
|
|
else
|
|
pr_url="$(gh pr create \
|
|
--base "$base_branch" \
|
|
--head "$branch" \
|
|
--title "docs: update maturity scorecard" \
|
|
--body-file "$body_file")"
|
|
fi
|
|
{
|
|
echo
|
|
echo "- Pull request: ${pr_url}"
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload maturity docs artifact
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: maturity-scorecard-docs-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: .artifacts/maturity-docs/
|
|
retention-days: 30
|
|
if-no-files-found: error
|