diff --git a/scripts/github/real-behavior-proof-policy.mjs b/scripts/github/real-behavior-proof-policy.mjs index 454b35364944..b81cb8b05e73 100644 --- a/scripts/github/real-behavior-proof-policy.mjs +++ b/scripts/github/real-behavior-proof-policy.mjs @@ -143,11 +143,10 @@ export async function isMaintainerTeamMember({ return body?.state === "active"; } -export function extractRealBehaviorProofSection(body = "") { +function extractMarkdownSection(headingRegex, body = "") { // Normalize CRLF → LF so regexes and section slicing see GitHub web-editor PR // bodies the same way as locally-authored Markdown. const normalizedBody = normalizeLineEndings(body); - const headingRegex = /^#{2,6}\s+real behavior proof\b[^\n]*$/gim; const match = headingRegex.exec(normalizedBody); if (!match) { return ""; @@ -158,6 +157,14 @@ export function extractRealBehaviorProofSection(body = "") { return (nextHeading ? rest.slice(0, nextHeading.index) : rest).trim(); } +export function extractRealBehaviorProofSection(body = "") { + return extractMarkdownSection(/^#{2,6}\s+real behavior proof\b[^\n]*$/im, body); +} + +function extractOutOfScopeFollowUpsSection(body = "") { + return extractMarkdownSection(/^#{2,6}\s+out-of-scope follow-ups\b[^\n]*$/im, body); +} + function fieldLineRegex(name) { return new RegExp( `^\\s*(?:[-*]\\s*)?(?:\\*\\*)?${escapeRegex(name)}(?:\\s*\\([^)]*\\))?(?:\\*\\*)?\\s*:\\s*(.*)$`, @@ -300,7 +307,8 @@ export function evaluateRealBehaviorProof({ pullRequest, labels } = {}) { return result("skipped", "Maintainer, collaborator, or bot PRs do not require this gate."); } - const section = extractRealBehaviorProofSection(pullRequest?.body ?? ""); + const body = pullRequest?.body ?? ""; + const section = extractRealBehaviorProofSection(body); if (!section) { return result( "missing", @@ -311,6 +319,9 @@ export function evaluateRealBehaviorProof({ pullRequest, labels } = {}) { const fields = Object.fromEntries( requiredProofFields.map((field) => [field.key, extractFieldValue(section, field)]), ); + if (!fields.notTested) { + fields.notTested = extractOutOfScopeFollowUpsSection(body); + } const missingFields = requiredProofFields .filter((field) => isMissingValue(fields[field.key] ?? "", field)) .map((field) => field.key); diff --git a/test/scripts/real-behavior-proof-policy.test.ts b/test/scripts/real-behavior-proof-policy.test.ts index d6f14c2828b9..f32f600c6c6f 100644 --- a/test/scripts/real-behavior-proof-policy.test.ts +++ b/test/scripts/real-behavior-proof-policy.test.ts @@ -85,6 +85,37 @@ describe("real-behavior-proof-policy", () => { expect(labelsForRealBehaviorProof(evaluation)).toEqual([PROOF_SUPPLIED_LABEL]); }); + it("accepts out-of-scope follow-ups as not-tested proof detail", () => { + const body = [ + "## Real behavior proof", + "", + "- Behavior addressed: Cron validation keeps Google Gemini 3 low thinking.", + "- Real environment tested: Local macOS source checkout, Node 24.", + "- Exact steps or command run after this patch:", + " 1. Built the local checkout with `node scripts/build-all.mjs`.", + " 2. Ran a redacted behavior probe for `provider=google`, `model=gemini-3-flash-preview`, and `catalogReasoning=false`.", + '- Evidence after fix: `.artifacts/behavior-85156/after-installed.json` recorded `lowSupported: true` and `fallbackFromLow: "low"`.', + "- Observed result after fix:", + " - `levels: off, minimal, low, medium, adaptive, high`", + " - `lowSupported: true`", + " - `fallbackFromLow: low`", + " - `local command version: OpenClaw 2026.5.21`", + "", + "## Out-of-scope Follow-ups", + "- No live systemd cron schedule was tested.", + "- No real Google provider request was sent.", + ].join("\n"); + const evaluation = evaluateRealBehaviorProof({ + pullRequest: externalPr(body), + }); + + expect(evaluation.status).toBe("passed"); + expect(evaluation.fields?.notTested).toBe( + "- No live systemd cron schedule was tested.\n- No real Google provider request was sent.", + ); + expect(labelsForRealBehaviorProof(evaluation)).toEqual([PROOF_SUPPLIED_LABEL]); + }); + it("fails external PRs without a real behavior proof section", () => { const evaluation = evaluateRealBehaviorProof({ pullRequest: externalPr("## Summary\n\n- Fixed startup."),