From ff98dd93f589392cc983ffd7fbf4b73a7035a01f Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Mon, 20 Jul 2026 17:08:41 -0700 Subject: [PATCH] ci: scan pull requests with TruffleHog (#111935) * ci: scan pull requests with TruffleHog * ci: scan staged changes with TruffleHog * ci: fetch pull request base for secret scan * ci: fetch complete pull request scan history --- .github/workflows/ci.yml | 27 +++++++ git-hooks/pre-commit | 30 ++++++++ test/git-hooks-pre-commit.test.ts | 97 ++++++++++++++++++++++++- test/scripts/ci-workflow-guards.test.ts | 35 +++++++++ 4 files changed, 187 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b83b5faa1fe9..064bc1b9dbd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -878,6 +878,33 @@ jobs: base-sha: ${{ steps.diff_base.outputs.sha }} fetch-ref: ${{ github.event_name == 'push' && github.ref_name || github.event.pull_request.base.ref }} + - name: Fetch pull request scan history + if: github.event_name == 'pull_request' + env: + PR_COMMIT_COUNT: ${{ github.event.pull_request.commits }} + PR_MERGE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if ! [[ "$PR_COMMIT_COUNT" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid pull request commit count: $PR_COMMIT_COUNT" + exit 2 + fi + # Include the synthetic merge, every pull request commit, and one + # ancestor so TruffleHog can clone and resolve the bounded range. + fetch_depth=$((PR_COMMIT_COUNT + 2)) + timeout --signal=TERM --kill-after=10s 120s git \ + -c protocol.version=2 \ + fetch --no-tags --no-recurse-submodules --depth="$fetch_depth" origin "$PR_MERGE_SHA" + + - name: Scan pull request for leaked credentials + if: github.event_name == 'pull_request' + uses: trufflesecurity/trufflehog@27b0417c16317ca9a472a9a8092acce143b49c55 # v3.95.9 + with: + base: ${{ steps.diff_base.outputs.sha }} + head: ${{ github.sha }} + version: "3.95.9@sha256:59b244249d1a1aef4baa24fe73d3c931616264482580d806d77f6c74d26b3e42" + extra_args: --results=verified,unknown --fail-on-scan-errors + - name: Prepare trusted pre-commit config if: github.event_name == 'pull_request' env: diff --git a/git-hooks/pre-commit b/git-hooks/pre-commit index 76fee645db11..d6304932d3ca 100755 --- a/git-hooks/pre-commit +++ b/git-hooks/pre-commit @@ -40,6 +40,17 @@ if [ "${#files[@]}" -eq 0 ]; then exit 0 fi +if ! command -v trufflehog >/dev/null 2>&1; then + cat >&2 <<'EOF' +OpenClaw requires TruffleHog for pre-commit secret scanning. + +Install it, then retry the commit: + macOS: brew install trufflehog + Other platforms: https://github.com/trufflesecurity/trufflehog#installation +EOF + exit 1 +fi + restage_files=() for file in "${files[@]}"; do if ! git check-ignore --no-index -q -- "$file"; then @@ -59,3 +70,22 @@ fi if [ "${#restage_files[@]}" -gt 0 ]; then git add -- "${restage_files[@]}" fi + +staged_snapshot="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-trufflehog.XXXXXX")" +trap 'rm -rf "$staged_snapshot"' EXIT +for file in "${files[@]}"; do + if [[ "$(git cat-file -t ":0:$file")" != "blob" ]]; then + continue + fi + snapshot_path="$staged_snapshot/$file" + mkdir -p "${snapshot_path%/*}" + git cat-file blob ":0:$file" > "$snapshot_path" +done + +trufflehog \ + --no-update \ + --no-color \ + --results=verified,unknown \ + --fail \ + --fail-on-scan-errors \ + filesystem "$staged_snapshot" diff --git a/test/git-hooks-pre-commit.test.ts b/test/git-hooks-pre-commit.test.ts index 229ae8013f8f..f786c111370e 100644 --- a/test/git-hooks-pre-commit.test.ts +++ b/test/git-hooks-pre-commit.test.ts @@ -80,6 +80,7 @@ function installPreCommitFixture(dir: string): string { const fakeBinDir = path.join(dir, "bin"); mkdirSync(fakeBinDir, { recursive: true }); writeExecutable(fakeBinDir, "node", "#!/usr/bin/env bash\nexit 0\n"); + writeExecutable(fakeBinDir, "trufflehog", "#!/usr/bin/env bash\nexit 0\n"); return fakeBinDir; } @@ -244,19 +245,111 @@ describe("git-hooks/pre-commit (integration)", () => { it("still formats staged files during a normal commit", () => { const dir = makeTempRepoRoot(tempDirs, "openclaw-pre-commit-normal-"); run(dir, "git", ["init", "-q", "--initial-branch=main"]); - installPreCommitFixture(dir); + const fakeBinDir = installPreCommitFixture(dir); + run(dir, "rm", ["-f", path.join(fakeBinDir, "node")]); const logPath = installFormattingRecorder(dir); writeFileSync(path.join(dir, "changed.ts"), "export const value = 1;\n", "utf8"); run(dir, "git", ["add", "--", "changed.ts"]); - run(dir, "bash", ["git-hooks/pre-commit"]); + run(dir, "bash", ["git-hooks/pre-commit"], { + PATH: `${fakeBinDir}:${process.env.PATH ?? ""}`, + }); expect(readFormatterLog(logPath)).toEqual([ "oxfmt --write --no-error-on-unmatched-pattern changed.ts", ]); }); + it("scans only staged versions of changed files with TruffleHog", () => { + const dir = makeTempRepoRoot(tempDirs, "openclaw-pre-commit-trufflehog-"); + run(dir, "git", ["init", "-q", "--initial-branch=main"]); + const fakeBinDir = installPreCommitFixture(dir); + const logPath = path.join(dir, "trufflehog.log"); + writeExecutable( + fakeBinDir, + "trufflehog", + `#!/usr/bin/env bash +printf 'env=%s\n' "\${TRUFFLEHOG_PRE_COMMIT:-}" > ${JSON.stringify(logPath)} +printf 'args=%s\n' "$*" >> ${JSON.stringify(logPath)} +`, + ); + + writeFileSync(path.join(dir, "base.txt"), "base\n", "utf8"); + run(dir, "git", ["add", "--", "base.txt"]); + run(dir, "git", [ + "-c", + "user.name=Test User", + "-c", + "user.email=test@example.invalid", + "commit", + "-q", + "-m", + "base", + ]); + writeFileSync(path.join(dir, "changed.txt"), "safe staged content\n", "utf8"); + run(dir, "git", ["add", "--", "changed.txt"]); + + run(dir, "bash", ["git-hooks/pre-commit"], { + PATH: `${fakeBinDir}:${process.env.PATH ?? ""}`, + }); + + const log = readFileSync(logPath, "utf8"); + expect(log).toContain("env=\n"); + expect(log).toContain( + "args=--no-update --no-color --results=verified,unknown --fail --fail-on-scan-errors filesystem ", + ); + }); + + it("scans the staged index snapshot before the first commit", () => { + const dir = makeTempRepoRoot(tempDirs, "openclaw-pre-commit-first-"); + run(dir, "git", ["init", "-q", "--initial-branch=main"]); + const fakeBinDir = installPreCommitFixture(dir); + const logPath = path.join(dir, "trufflehog.log"); + writeExecutable( + fakeBinDir, + "trufflehog", + `#!/usr/bin/env bash +printf 'env=%s\n' "\${TRUFFLEHOG_PRE_COMMIT:-}" > ${JSON.stringify(logPath)} +printf 'args=%s\n' "$*" >> ${JSON.stringify(logPath)} +`, + ); + + writeFileSync(path.join(dir, "changed.txt"), "safe staged content\n", "utf8"); + run(dir, "git", ["add", "--", "changed.txt"]); + + run(dir, "bash", ["git-hooks/pre-commit"], { + PATH: `${fakeBinDir}:${process.env.PATH ?? ""}`, + }); + + const log = readFileSync(logPath, "utf8"); + expect(log).toContain("env=\n"); + expect(log).toContain( + "args=--no-update --no-color --results=verified,unknown --fail --fail-on-scan-errors filesystem ", + ); + }); + + it("blocks with install guidance when TruffleHog is missing", () => { + const dir = makeTempRepoRoot(tempDirs, "openclaw-pre-commit-no-trufflehog-"); + run(dir, "git", ["init", "-q", "--initial-branch=main"]); + const fakeBinDir = installPreCommitFixture(dir); + run(dir, "rm", ["-f", path.join(fakeBinDir, "trufflehog")]); + + writeFileSync(path.join(dir, "changed.txt"), "safe staged content\n", "utf8"); + run(dir, "git", ["add", "--", "changed.txt"]); + + const failure = runFailure(dir, "bash", ["git-hooks/pre-commit"], { + PATH: `${fakeBinDir}:/usr/bin:/bin`, + }); + + expect(failure.status).toBe(1); + expect(failure.stderr).toContain( + "OpenClaw requires TruffleHog for pre-commit secret scanning.", + ); + expect(failure.stderr).toContain("brew install trufflehog"); + expect(failure.stderr).toContain("https://github.com/trufflesecurity/trufflehog#installation"); + }); + it("does not run the changed-scope check for non-doc staged changes", () => { const dir = makeTempRepoRoot(tempDirs, "openclaw-pre-commit-no-check-changed-"); run(dir, "git", ["init", "-q", "--initial-branch=main"]); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index d22044e048fc..5f6d532d91dd 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -27,6 +27,7 @@ const UPLOAD_ARTIFACT_V7 = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64 const DOWNLOAD_ARTIFACT_V8 = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; const CREATE_GITHUB_APP_TOKEN_V3 = "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1"; +const TRUFFLEHOG_V3_95_9 = "trufflesecurity/trufflehog@27b0417c16317ca9a472a9a8092acce143b49c55"; const MANTIS_GITHUB_APP_CLIENT_ID = "Iv23liPJCozR0uHm6P7G"; const OPENGREP_PR_DIFF_WORKFLOW = ".github/workflows/opengrep-precise.yml"; const OPENGREP_FULL_WORKFLOW = ".github/workflows/opengrep-precise-full.yml"; @@ -1899,6 +1900,40 @@ describe("ci workflow guards", () => { expect(workflow.jobs["pnpm-store-warmup"]["runs-on"]).toContain("blacksmith-4vcpu-ubuntu-2404"); }); + it("scans only the pull request commit range for leaked credentials", () => { + const securitySteps = readCiWorkflow().jobs["security-fast"].steps as WorkflowStep[]; + const fetchScanHistoryIndex = securitySteps.findIndex( + (step) => step.name === "Fetch pull request scan history", + ); + const scanIndex = securitySteps.findIndex( + (step) => step.name === "Scan pull request for leaked credentials", + ); + const fetchScanHistoryStep = expectDefined( + securitySteps[fetchScanHistoryIndex], + "TruffleHog history fetch step", + ); + const scanStep = expectDefined(securitySteps[scanIndex], "TruffleHog pull request scan step"); + + expect(scanIndex).toBeGreaterThan(fetchScanHistoryIndex); + expect(fetchScanHistoryStep.if).toBe("github.event_name == 'pull_request'"); + expect(fetchScanHistoryStep.env).toEqual({ + PR_COMMIT_COUNT: "${{ github.event.pull_request.commits }}", + PR_MERGE_SHA: "${{ github.sha }}", + }); + expect(fetchScanHistoryStep.run).toContain("fetch_depth=$((PR_COMMIT_COUNT + 2))"); + expect(fetchScanHistoryStep.run).toContain( + 'fetch --no-tags --no-recurse-submodules --depth="$fetch_depth" origin "$PR_MERGE_SHA"', + ); + expect(scanStep.if).toBe("github.event_name == 'pull_request'"); + expect(scanStep.uses).toBe(TRUFFLEHOG_V3_95_9); + expect(scanStep.with).toEqual({ + base: "${{ steps.diff_base.outputs.sha }}", + head: "${{ github.sha }}", + version: "3.95.9@sha256:59b244249d1a1aef4baa24fe73d3c931616264482580d806d77f6c74d26b3e42", + extra_args: "--results=verified,unknown --fail-on-scan-errors", + }); + }); + it("keeps sticky dependency snapshots on trusted Blacksmith Node shards", () => { const workflow = readCiWorkflow(); const blacksmithJobs = Object.entries(workflow.jobs).filter(([, job]) => {