import { describe, expect, it } from "vitest"; import { candidateLabels, classifyPullRequestCandidateLabels, managedLabelSpecs, runBarnacleAutoResponse, } from "../../scripts/github/barnacle-auto-response.mjs"; import { PROOF_SUFFICIENT_LABEL, PROOF_SUPPLIED_LABEL, } from "../../scripts/github/real-behavior-proof-policy.mjs"; const blankTemplateBody = [ "## Summary", "", "Describe the problem and fix in 2–5 bullets:", "", "- Problem:", "- Why it matters:", "- What changed:", "- What did NOT change (scope boundary):", "", "## Linked Issue/PR", "", "- Closes #", "- Related #", "", "## Root Cause (if applicable)", "", "- Root cause:", "", "## Regression Test Plan (if applicable)", "", "- Target test or file:", ].join("\n"); function pr(title: string, body = blankTemplateBody) { return { title, body, }; } function realBehaviorProofBody(evidence: string, overrides: Record = {}) { const fields = { behavior: "Gateway status now reports the Discord channel as ready.", environment: "macOS 15.4, Node 24, local OpenClaw gateway, redacted Discord token.", steps: "pnpm openclaw gateway restart and pnpm openclaw gateway status", evidence, observedResult: "The gateway stayed connected and Discord reported ready.", notTested: "No known gaps.", ...overrides, }; return [ "## Real behavior proof", "", `- Behavior or issue addressed: ${fields.behavior}`, `- Real environment tested: ${fields.environment}`, `- Exact steps or command run after this patch: ${fields.steps}`, `- Evidence after fix: ${fields.evidence}`, `- Observed result after fix: ${fields.observedResult}`, `- What was not tested: ${fields.notTested}`, ].join("\n"); } function file(filename: string, status = "modified") { return { filename, status, }; } function barnacleContext( pullRequest: Record, labels: string[] = [], options: Record = {}, ) { return { repo: { owner: "openclaw", repo: "openclaw", }, payload: { action: options.action ?? "opened", label: options.label, sender: options.sender, pull_request: { number: 123, title: "Cleanup plugin docs", body: blankTemplateBody, author_association: "CONTRIBUTOR", user: { login: "contributor", }, labels: labels.map((name) => ({ name })), ...pullRequest, }, }, }; } function barnacleIssueContext( issue: Record, labels: string[] = [], options: Record = {}, ) { return { repo: { owner: "openclaw", repo: "openclaw", }, payload: { action: options.action ?? "opened", label: options.label, sender: options.sender, issue: { number: 456, title: "OpenClaw issue", body: "", author_association: "CONTRIBUTOR", user: { login: "contributor", }, labels: labels.map((name) => ({ name })), ...issue, }, comment: options.comment, }, }; } function barnacleGithub( files: ReturnType[], options: { maintainerLogins?: string[]; removeLabelNotFound?: string[]; repositoryRoles?: Record; } = {}, ) { const maintainerLogins = new Set( (options.maintainerLogins ?? []).map((login) => login.toLowerCase()), ); const removeLabelNotFound = new Set(options.removeLabelNotFound ?? []); const repositoryRoles = Object.fromEntries( Object.entries(options.repositoryRoles ?? {}).map(([login, role]) => [ login.toLowerCase(), role, ]), ); const calls = { addLabels: [] as Array<{ issue_number: number; labels: string[] }>, createComment: [] as Array<{ issue_number: number; body: string }>, lock: [] as Array<{ issue_number: number; lock_reason?: string }>, removeLabel: [] as Array<{ issue_number: number; name: string }>, update: [] as Array<{ issue_number: number; state?: string }>, }; const github = { paginate: async () => files, rest: { issues: { addLabels: async (params: { issue_number: number; labels: string[] }) => { calls.addLabels.push(params); }, createComment: async (params: { issue_number: number; body: string }) => { calls.createComment.push(params); }, createLabel: async () => undefined, getLabel: async (params: { name: string }) => ({ data: { color: managedLabelSpecs[params.name as keyof typeof managedLabelSpecs]?.color ?? "C5DEF5", description: managedLabelSpecs[params.name as keyof typeof managedLabelSpecs]?.description ?? "", }, }), lock: async (params: { issue_number: number; lock_reason?: string }) => { calls.lock.push(params); }, removeLabel: async (params: { issue_number: number; name: string }) => { calls.removeLabel.push(params); if (removeLabelNotFound.has(params.name)) { const error = new Error("not found") as Error & { status: number }; error.status = 404; throw error; } }, update: async (params: { issue_number: number; state?: string }) => { calls.update.push(params); }, updateLabel: async () => undefined, }, pulls: { listFiles: async () => files, }, repos: { getCollaboratorPermissionLevel: async ({ username }: { username: string }) => { const role = repositoryRoles[username.toLowerCase()] ?? "read"; return { data: { permission: role, role_name: role, }, }; }, }, teams: { getMembershipForUserInOrg: async ({ username }: { username: string }) => { if (maintainerLogins.has(username.toLowerCase())) { return { data: { state: "active", }, }; } const error = new Error("not found") as Error & { status: number }; error.status = 404; throw error; }, }, }, }; return { calls, github }; } describe("barnacle-auto-response", () => { it("keeps Barnacle-owned labels documented and ClawHub spelled correctly", () => { expect(managedLabelSpecs["r: skill"].description).toContain("ClawHub"); expect(managedLabelSpecs["r: skill"].description).not.toContain("Clawdhub"); expect(managedLabelSpecs.dirty.description).toContain("dirty/unrelated"); expect(managedLabelSpecs["r: support"].description).toContain("support requests"); expect(managedLabelSpecs["r: false-positive"].description).toContain("false positive"); expect(managedLabelSpecs["r: third-party-extension"].description).toContain("ClawHub"); expect(managedLabelSpecs["r: too-many-prs"].description).toContain("twenty active PRs"); expect(managedLabelSpecs[PROOF_SUPPLIED_LABEL].color).toBe("C2E0C6"); expect(managedLabelSpecs[PROOF_SUFFICIENT_LABEL].color).toBe("0E8A16"); for (const label of Object.values(candidateLabels)) { expect(managedLabelSpecs[label]).toBeDefined(); expect(managedLabelSpecs[label].description).toMatch(/^Candidate:/); } }); it("labels docs-only discoverability churn without closing it", () => { const labels = classifyPullRequestCandidateLabels(pr("Update README translation"), [ file("README.md"), ]); expect(labels).toEqual( expect.arrayContaining([ candidateLabels.blankTemplate, candidateLabels.lowSignalDocs, candidateLabels.docsDiscoverability, ]), ); }); it("does not treat template boilerplate as behavior evidence for test-only churn", () => { const labels = classifyPullRequestCandidateLabels(pr("Add test coverage"), [ file("src/gateway/foo.test.ts"), ]); expect(labels).toEqual( expect.arrayContaining([candidateLabels.blankTemplate, candidateLabels.testOnlyNoBug]), ); }); it("labels external PRs that are missing real behavior proof", () => { const labels = classifyPullRequestCandidateLabels(pr("Fix gateway startup"), [ file("src/gateway/server.ts"), ]); expect(labels).toContain(candidateLabels.needsRealBehaviorProof); expect(labels).not.toContain(candidateLabels.mockOnlyProof); }); it("labels external PRs whose proof is only tests or mocks", () => { const labels = classifyPullRequestCandidateLabels( pr( "Fix gateway startup", realBehaviorProofBody("pnpm test passed with Vitest mocks.", { steps: "pnpm test", observedResult: "CI passes.", }), ), [file("src/gateway/server.ts")], ); expect(labels).toContain(candidateLabels.mockOnlyProof); expect(labels).not.toContain(candidateLabels.needsRealBehaviorProof); }); it("labels external PRs that include real behavior proof as supplied", () => { const labels = classifyPullRequestCandidateLabels( pr( "Fix gateway startup", realBehaviorProofBody("![after](https://github.com/user-attachments/assets/gateway-ready)"), ), [file("src/gateway/server.ts")], ); expect(labels).toContain(PROOF_SUPPLIED_LABEL); expect(labels).not.toContain(candidateLabels.needsRealBehaviorProof); expect(labels).not.toContain(candidateLabels.mockOnlyProof); }); it("labels CRLF-formatted external PRs with screenshot proof as supplied", () => { const labels = classifyPullRequestCandidateLabels( pr( "Fix gateway startup", realBehaviorProofBody( "![after](https://github.com/user-attachments/assets/gateway-ready)", ).replace(/\n/g, "\r\n"), ), [file("src/gateway/server.ts")], ); expect(labels).toContain(PROOF_SUPPLIED_LABEL); expect(labels).not.toContain(candidateLabels.needsRealBehaviorProof); expect(labels).not.toContain(candidateLabels.mockOnlyProof); }); it("uses linked issues as context and suppresses low-signal docs labels", () => { const labels = classifyPullRequestCandidateLabels( pr("Update docs", `${blankTemplateBody}\n\nRelated #12345`), [file("docs/plugins/community.md")], ); expect(labels).not.toContain(candidateLabels.lowSignalDocs); expect(labels).not.toContain(candidateLabels.docsDiscoverability); }); it("warns on broad high-surface PRs instead of auto-closing them as dirty", () => { const labels = classifyPullRequestCandidateLabels(pr("Cleanup plugin docs"), [ file("ui/src/app.ts"), file("src/gateway/server.ts"), file("extensions/slack/src/index.ts"), file("docs/plugins/community.md"), ]); expect(labels).toContain(candidateLabels.dirtyCandidate); }); it("suppresses dirty-candidate when the PR has concrete behavior context", () => { const body = [ "- Problem: gateway crashes when plugin metadata is missing", "- Why it matters: users lose the running session", "- What changed: add a guard around metadata loading", ].join("\n"); const labels = classifyPullRequestCandidateLabels(pr("Fix gateway crash", body), [ file("ui/src/app.ts"), file("src/gateway/server.ts"), file("extensions/slack/src/index.ts"), file("docs/plugins/community.md"), ]); expect(labels).not.toContain(candidateLabels.dirtyCandidate); }); it("does not classify a linked core plugin auto-enable fix as an external plugin candidate", () => { const labels = classifyPullRequestCandidateLabels( pr( "Fix duplicate plugin auto-enable entries", [ "- Problem: openclaw doctor --fix adds duplicate installed plugin entries", "- Why it matters: users get noisy config churn", "- What changed: respect manifest-provided channel auto-loads", "", "Fixes #37548", "", "This touches external plugin install state but fixes core config repair behavior.", ].join("\n"), ), [ file("src/config/plugin-auto-enable.shared.ts"), file("src/config/plugin-auto-enable.channels.test.ts"), file("src/config/plugin-auto-enable.test-helpers.ts"), ], ); expect(labels).not.toContain(candidateLabels.externalPluginCandidate); }); it("does not mutate maintainer-authored PRs", async () => { const { calls, github } = barnacleGithub([ file("ui/src/app.ts"), file("src/gateway/server.ts"), file("extensions/slack/src/index.ts"), file("docs/plugins/community.md"), ]); await runBarnacleAutoResponse({ github, context: barnacleContext({ author_association: "OWNER", user: { login: "maintainer", }, }), core: { info: () => undefined, }, }); expect(calls.addLabels).toEqual([]); expect(calls.createComment).toEqual([]); expect(calls.removeLabel).toEqual([]); expect(calls.update).toEqual([]); }); it("leaves stale Barnacle labels alone on maintainer-authored PRs", async () => { const { calls, github } = barnacleGithub([ file("ui/src/app.ts"), file("src/gateway/server.ts"), file("extensions/slack/src/index.ts"), file("docs/plugins/community.md"), ]); await runBarnacleAutoResponse({ github, context: barnacleContext( { author_association: "OWNER", user: { login: "maintainer", }, }, [candidateLabels.dirtyCandidate, "r: too-many-prs"], ), core: { info: () => undefined, }, }); expect(calls.addLabels).toEqual([]); expect(calls.createComment).toEqual([]); expect(calls.removeLabel).toEqual([]); expect(calls.update).toEqual([]); }); it("does not mutate maintainer-authored issues", async () => { const { calls, github } = barnacleGithub([]); await runBarnacleAutoResponse({ github, context: barnacleIssueContext({ title: "TestFlight access", author_association: "OWNER", user: { login: "maintainer", }, }), core: { info: () => undefined, }, }); expect(calls.addLabels).toEqual([]); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("does not action close labels on maintainer-authored issues", async () => { const { calls, github } = barnacleGithub([]); await runBarnacleAutoResponse({ github, context: barnacleIssueContext( { title: "Need help with setup", author_association: "MEMBER", user: { login: "maintainer", }, }, ["r: support"], { action: "labeled", label: { name: "r: support" }, }, ), core: { info: () => undefined, }, }); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("closes issues tagged as false positives", async () => { const { calls, github } = barnacleGithub([]); await runBarnacleAutoResponse({ github, context: barnacleIssueContext({}, ["r: false-positive"], { action: "labeled", label: { name: "r: false-positive" }, sender: { login: "maintainer", type: "User" }, }), core: { info: () => undefined, }, }); expect(calls.createComment).toContainEqual( expect.objectContaining({ body: expect.stringContaining("false positive"), }), ); expect(calls.update).toContainEqual(expect.objectContaining({ state: "closed" })); }); it("does not respond to maintainer comments on contributor items", async () => { const { calls, github } = barnacleGithub([], { maintainerLogins: ["maintainer"] }); await runBarnacleAutoResponse({ github, context: barnacleIssueContext( { title: "Contributor issue", user: { login: "contributor", }, }, [], { action: "created", comment: { body: "testflight", user: { login: "maintainer", type: "User", }, }, }, ), core: { info: () => undefined, }, }); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("does not close automation PRs for the active PR limit", async () => { for (const automationPullRequest of [ { head: { ref: "clawsweeper/openclaw-openclaw-73880" }, login: "app/openclaw-clawsweeper" }, { headRefName: "clawsweeper/openclaw-openclaw-73880", login: "app/openclaw-clawsweeper" }, { head: { ref: "clownfish/ghcrawl-156993-autonomous-smoke" }, login: "app/openclaw-clownfish", }, { headRefName: "clownfish/ghcrawl-156993-autonomous-smoke", login: "app/openclaw-clownfish" }, ]) { const { calls, github } = barnacleGithub([]); const { login, ...pullRequest } = automationPullRequest; await runBarnacleAutoResponse({ github, context: barnacleContext( { ...pullRequest, user: { login, }, }, ["r: too-many-prs"], ), core: { info: () => undefined, }, }); expect(calls.removeLabel).toContainEqual( expect.objectContaining({ name: "r: too-many-prs" }), ); expect(calls.createComment).not.toContainEqual( expect.objectContaining({ body: expect.stringContaining("more than 20 active PRs"), }), ); expect(calls.update).not.toContainEqual(expect.objectContaining({ state: "closed" })); } }); it("removes stale PR-limit labels from GitHub App-authored PRs", async () => { const { calls, github } = barnacleGithub([file("README.md")]); await runBarnacleAutoResponse({ github, context: barnacleContext( { user: { login: "renovate[bot]", type: "Bot", }, }, ["r: too-many-prs"], ), core: { info: () => undefined, }, }); expect(calls.removeLabel).toContainEqual(expect.objectContaining({ name: "r: too-many-prs" })); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("does not close GitHub App-authored PRs when stale PR-limit label removal returns 404", async () => { const { calls, github } = barnacleGithub([file("README.md")], { removeLabelNotFound: ["r: too-many-prs"], }); await runBarnacleAutoResponse({ github, context: barnacleContext( { user: { login: "renovate[bot]", type: "Bot", }, }, ["r: too-many-prs"], ), core: { info: () => undefined, }, }); expect(calls.removeLabel).toContainEqual(expect.objectContaining({ name: "r: too-many-prs" })); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("still adds candidate labels to broad contributor PRs", async () => { const { calls, github } = barnacleGithub([ file("ui/src/app.ts"), file("src/gateway/server.ts"), file("extensions/slack/src/index.ts"), file("docs/plugins/community.md"), ]); await runBarnacleAutoResponse({ github, context: barnacleContext({}), core: { info: () => undefined, }, }); expect(calls.addLabels).toContainEqual( expect.objectContaining({ labels: expect.arrayContaining([candidateLabels.dirtyCandidate]), }), ); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("adds proof labels to external PRs without auto-closing by default", async () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext({}), core: { info: () => undefined, }, }); expect(calls.addLabels).toContainEqual( expect.objectContaining({ labels: expect.arrayContaining([candidateLabels.needsRealBehaviorProof]), }), ); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("removes stale proof labels when override is present", async () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext({}, [ candidateLabels.needsRealBehaviorProof, candidateLabels.mockOnlyProof, PROOF_SUPPLIED_LABEL, PROOF_SUFFICIENT_LABEL, "proof: override", ]), core: { info: () => undefined, }, }); expect(calls.removeLabel.map((call) => call.name)).toEqual( expect.arrayContaining([ candidateLabels.needsRealBehaviorProof, candidateLabels.mockOnlyProof, PROOF_SUPPLIED_LABEL, PROOF_SUFFICIENT_LABEL, ]), ); expect(calls.update).toEqual([]); }); it("removes stale negative proof labels and adds supplied when proof is present", async () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext( { body: realBehaviorProofBody( "![after](https://github.com/user-attachments/assets/gateway-ready)", ), }, [candidateLabels.needsRealBehaviorProof, candidateLabels.mockOnlyProof], ), core: { info: () => undefined, }, }); expect(calls.removeLabel.map((call) => call.name)).toEqual( expect.arrayContaining([ candidateLabels.needsRealBehaviorProof, candidateLabels.mockOnlyProof, ]), ); expect(calls.addLabels).toContainEqual( expect.objectContaining({ labels: expect.arrayContaining([PROOF_SUPPLIED_LABEL]), }), ); }); it.each(["edited", "synchronize"])( "removes stale sufficient proof label after PR %s events", async (action) => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext( { body: realBehaviorProofBody( "![after](https://github.com/user-attachments/assets/gateway-ready)", ), }, [PROOF_SUPPLIED_LABEL, PROOF_SUFFICIENT_LABEL], { action }, ), core: { info: () => undefined, }, }); expect(calls.removeLabel).toContainEqual( expect.objectContaining({ name: PROOF_SUFFICIENT_LABEL }), ); }, ); it("preserves ClawSweeper's sufficient proof label on ordinary label events", async () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext( { body: realBehaviorProofBody( "![after](https://github.com/user-attachments/assets/gateway-ready)", ), }, [PROOF_SUPPLIED_LABEL, PROOF_SUFFICIENT_LABEL], { action: "labeled", label: { name: PROOF_SUFFICIENT_LABEL }, sender: { login: "openclaw-clawsweeper[bot]", type: "Bot" }, }, ), core: { info: () => undefined, }, }); expect(calls.removeLabel).not.toContainEqual( expect.objectContaining({ name: PROOF_SUFFICIENT_LABEL }), ); }); it("actions manually applied candidate labels", async () => { const { calls, github } = barnacleGithub([file("extensions/example/openclaw.plugin.json")]); await runBarnacleAutoResponse({ github, context: barnacleContext({}, [candidateLabels.externalPluginCandidate], { action: "labeled", label: { name: candidateLabels.externalPluginCandidate }, sender: { login: "maintainer", type: "User" }, }), core: { info: () => undefined, }, }); expect(calls.createComment).toContainEqual( expect.objectContaining({ body: expect.stringContaining("ClawHub"), }), ); expect(calls.update).toContainEqual(expect.objectContaining({ state: "closed" })); }); it("keeps bot-applied candidate labels passive", async () => { const { calls, github } = barnacleGithub([file("extensions/example/openclaw.plugin.json")]); await runBarnacleAutoResponse({ github, context: barnacleContext({}, [candidateLabels.externalPluginCandidate], { action: "labeled", label: { name: candidateLabels.externalPluginCandidate }, sender: { login: "openclaw-bot[bot]", type: "Bot" }, }), core: { info: () => undefined, }, }); expect(calls.createComment).toEqual([]); expect(calls.update).toEqual([]); }); it("actions existing candidate labels when a maintainer adds trigger-response", async () => { const { calls, github } = barnacleGithub([file("src/gateway/foo.test.ts")]); await runBarnacleAutoResponse({ github, context: barnacleContext({}, [candidateLabels.testOnlyNoBug, "trigger-response"], { action: "labeled", label: { name: "trigger-response" }, sender: { login: "maintainer", type: "User" }, }), core: { info: () => undefined, }, }); expect(calls.removeLabel).toContainEqual(expect.objectContaining({ name: "trigger-response" })); expect(calls.createComment).toContainEqual( expect.objectContaining({ body: expect.stringContaining("does not include real behavior proof"), }), ); expect(calls.update).toContainEqual(expect.objectContaining({ state: "closed" })); }); });