From 427d5d4f69dbab55a2352871592b5f613dc2ef75 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 10:40:35 +0100 Subject: [PATCH] ci: guard unused dead-code files --- .github/workflows/ci.yml | 1 + docs/ci.md | 2 +- package.json | 1 + scripts/check-deadcode-unused-files.mjs | 144 ++++++++++++++++++ scripts/deadcode-unused-files.allowlist.mjs | 77 ++++++++++ scripts/test-projects.test-support.mjs | 9 ++ .../check-deadcode-unused-files.test.ts | 57 +++++++ 7 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 scripts/check-deadcode-unused-files.mjs create mode 100644 scripts/deadcode-unused-files.allowlist.mjs create mode 100644 test/scripts/check-deadcode-unused-files.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a5a2c2a459..ea2f58fd49f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1286,6 +1286,7 @@ jobs: ;; dependencies) pnpm deadcode:dependencies + pnpm deadcode:unused-files ;; policy-guards) pnpm lint:webhook:no-low-level-body-read diff --git a/docs/ci.md b/docs/ci.md index 239a64ba431..23cbc05bfa0 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -8,7 +8,7 @@ read_when: The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. Manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out the full normal CI graph for release candidates or broad validation, with Android lanes opt-in through `include_android` for standalone manual runs. Release-only plugin prerelease lanes live in the separate `Plugin Prerelease` workflow and run only from `Full Release Validation` or an explicit manual dispatch. -The `check-dependencies` shard runs `pnpm deadcode:dependencies`, a production Knip dependency-only pass pinned to the latest Knip version used by that script, with pnpm's minimum release age disabled for the `dlx` install. It gates newly unused, unlisted, unresolved, binary, or catalog dependencies without enabling Knip's full unused-file mode, which remains a manual audit because OpenClaw intentionally loads many plugin and runtime surfaces through manifests and string specifiers. +The `check-dependencies` shard runs `pnpm deadcode:dependencies`, a production Knip dependency-only pass pinned to the latest Knip version used by that script, with pnpm's minimum release age disabled for the `dlx` install. It also runs `pnpm deadcode:unused-files`, which compares Knip's production unused-file findings against `scripts/deadcode-unused-files.allowlist.mjs`. That guard fails when a PR adds a new unreviewed unused file or leaves a stale allowlist entry after cleanup, while preserving intentional dynamic plugin, generated, build, live-test, and package bridge surfaces that Knip cannot resolve statically. `Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the diff --git a/package.json b/package.json index fec2c2d5904..1b6848aaecb 100644 --- a/package.json +++ b/package.json @@ -1308,6 +1308,7 @@ "deadcode:report:ci:ts-unused": "mkdir -p .artifacts/deadcode && pnpm deadcode:ts-unused > .artifacts/deadcode/ts-unused-exports.txt 2>&1 || true", "deadcode:ts-prune": "pnpm dlx ts-prune src extensions scripts", "deadcode:ts-unused": "pnpm dlx ts-unused-exports tsconfig.json --ignoreTestFiles --exitWithCount", + "deadcode:unused-files": "node scripts/check-deadcode-unused-files.mjs", "deps:root-ownership": "node scripts/root-dependency-ownership-audit.mjs", "deps:root-ownership:check": "node scripts/root-dependency-ownership-audit.mjs --check", "deps:sbom-risk": "node scripts/sbom-risk-report.mjs", diff --git a/scripts/check-deadcode-unused-files.mjs b/scripts/check-deadcode-unused-files.mjs new file mode 100644 index 00000000000..184c4ea691c --- /dev/null +++ b/scripts/check-deadcode-unused-files.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { KNIP_UNUSED_FILE_ALLOWLIST } from "./deadcode-unused-files.allowlist.mjs"; + +const KNIP_VERSION = "6.8.0"; +const KNIP_ARGS = [ + "--config", + "knip.config.ts", + "--production", + "--no-progress", + "--reporter", + "compact", + "--files", + "--no-config-hints", +]; + +function normalizeRepoPath(value) { + return value.replaceAll("\\", "/").replace(/^\.\//u, ""); +} + +function uniqueSorted(values) { + return [...new Set(values.map(normalizeRepoPath))].toSorted((left, right) => + left.localeCompare(right), + ); +} + +export function parseKnipCompactUnusedFiles(output) { + const files = []; + let inUnusedFilesSection = false; + let sawUnusedFilesSection = false; + + for (const line of output.split(/\r?\n/u)) { + if (/^Unused files \(\d+\)$/u.test(line)) { + inUnusedFilesSection = true; + sawUnusedFilesSection = true; + continue; + } + if (inUnusedFilesSection && line.trim() === "") { + break; + } + + const separatorIndex = line.lastIndexOf(": "); + if (separatorIndex === -1) { + continue; + } + if (sawUnusedFilesSection && !inUnusedFilesSection) { + continue; + } + files.push(line.slice(separatorIndex + 2).trim()); + } + + return uniqueSorted(files); +} + +export function compareUnusedFilesToAllowlist(actualFiles, allowlistFiles) { + const actual = uniqueSorted(actualFiles); + const allowed = uniqueSorted(allowlistFiles); + const allowedSet = new Set(allowed); + const actualSet = new Set(actual); + + return { + actual, + allowed, + unexpected: actual.filter((file) => !allowedSet.has(file)), + stale: allowed.filter((file) => !actualSet.has(file)), + duplicateAllowedCount: allowlistFiles.length - new Set(allowlistFiles).size, + allowlistIsSorted: + JSON.stringify(allowlistFiles.map(normalizeRepoPath)) === JSON.stringify(allowed), + }; +} + +export function formatUnusedFileComparison(comparison) { + const lines = []; + if (!comparison.allowlistIsSorted) { + lines.push("deadcode unused-file allowlist is not sorted."); + } + if (comparison.duplicateAllowedCount > 0) { + lines.push( + `deadcode unused-file allowlist contains ${comparison.duplicateAllowedCount} duplicate entr${ + comparison.duplicateAllowedCount === 1 ? "y" : "ies" + }.`, + ); + } + if (comparison.unexpected.length > 0) { + lines.push("Unexpected unused files:"); + lines.push(...comparison.unexpected.map((file) => ` ${file}`)); + } + if (comparison.stale.length > 0) { + lines.push("Stale allowlist entries:"); + lines.push(...comparison.stale.map((file) => ` ${file}`)); + } + return lines.join("\n"); +} + +export function runKnipUnusedFiles() { + const result = spawnSync( + "pnpm", + ["--config.minimum-release-age=0", "dlx", `knip@${KNIP_VERSION}`, ...KNIP_ARGS], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return { + status: result.status, + signal: result.signal, + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + }; +} + +export function checkUnusedFiles(output, allowlistFiles = KNIP_UNUSED_FILE_ALLOWLIST) { + const actual = parseKnipCompactUnusedFiles(output); + const comparison = compareUnusedFilesToAllowlist(actual, allowlistFiles); + return { + ok: + comparison.allowlistIsSorted && + comparison.duplicateAllowedCount === 0 && + comparison.unexpected.length === 0 && + comparison.stale.length === 0, + comparison, + message: formatUnusedFileComparison(comparison), + }; +} + +function main() { + const result = runKnipUnusedFiles(); + const check = checkUnusedFiles(result.output); + if (!check.ok) { + if (check.message) { + console.error(check.message); + } + process.exitCode = 1; + return; + } + + console.log( + `[deadcode] Knip unused-file allowlist matched ${check.comparison.actual.length} intentional entries.`, + ); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/deadcode-unused-files.allowlist.mjs b/scripts/deadcode-unused-files.allowlist.mjs new file mode 100644 index 00000000000..1d4baec7e1b --- /dev/null +++ b/scripts/deadcode-unused-files.allowlist.mjs @@ -0,0 +1,77 @@ +// Intentional Knip unused-file findings. These are dynamic entrypoints, +// generated/build inputs, manifest-discovered plugin surfaces, live-test +// helpers, or package bridge files that static production scanning cannot see. +export const KNIP_UNUSED_FILE_ALLOWLIST = [ + "extensions/diffs/src/viewer-client.ts", + "extensions/diffs/src/viewer-payload.ts", + "extensions/mattermost/src/config-schema.ts", + "extensions/memory-core/src/memory-tool-manager-mock.ts", + "src/agents/subagent-registry.runtime.ts", + "src/auto-reply/inbound.group-require-mention-test-plugins.ts", + "src/auto-reply/reply/get-reply.test-loader.ts", + "src/cli/daemon-cli-compat.ts", + "src/cli/debug-timing.ts", + "src/commands/doctor/shared/deprecation-compat.ts", + "src/config/doc-baseline.runtime.ts", + "src/config/doc-baseline.ts", + "src/gateway/gateway-cli-backend.live-helpers.ts", + "src/gateway/gateway-cli-backend.live-probe-helpers.ts", + "src/gateway/gateway-codex-harness.live-helpers.ts", + "src/infra/changelog-unreleased.ts", + "src/mcp/openclaw-tools-serve.ts", + "src/mcp/plugin-tools-handlers.ts", + "src/mcp/plugin-tools-serve.ts", + "src/mcp/tools-stdio-server.ts", + "src/memory-host-sdk/engine-embeddings.ts", + "src/memory-host-sdk/engine-foundation.ts", + "src/memory-host-sdk/engine.ts", + "src/memory-host-sdk/host/batch-error-utils.ts", + "src/memory-host-sdk/host/batch-http.ts", + "src/memory-host-sdk/host/batch-output.ts", + "src/memory-host-sdk/host/batch-provider-common.ts", + "src/memory-host-sdk/host/batch-runner.ts", + "src/memory-host-sdk/host/batch-status.ts", + "src/memory-host-sdk/host/batch-upload.ts", + "src/memory-host-sdk/host/batch-utils.ts", + "src/memory-host-sdk/host/embedding-chunk-limits.ts", + "src/memory-host-sdk/host/embedding-input-limits.ts", + "src/memory-host-sdk/host/embedding-model-limits.ts", + "src/memory-host-sdk/host/embedding-provider-adapter-utils.ts", + "src/memory-host-sdk/host/embedding-vectors.ts", + "src/memory-host-sdk/host/embeddings-debug.ts", + "src/memory-host-sdk/host/embeddings-model-normalize.ts", + "src/memory-host-sdk/host/embeddings-remote-client.ts", + "src/memory-host-sdk/host/embeddings-remote-fetch.ts", + "src/memory-host-sdk/host/embeddings-remote-provider.ts", + "src/memory-host-sdk/host/embeddings.ts", + "src/memory-host-sdk/host/embeddings.types.ts", + "src/memory-host-sdk/host/fs-utils.ts", + "src/memory-host-sdk/host/hash.ts", + "src/memory-host-sdk/host/internal.ts", + "src/memory-host-sdk/host/memory-schema.ts", + "src/memory-host-sdk/host/multimodal.ts", + "src/memory-host-sdk/host/node-llama.ts", + "src/memory-host-sdk/host/post-json.ts", + "src/memory-host-sdk/host/qmd-process.ts", + "src/memory-host-sdk/host/qmd-query-parser.ts", + "src/memory-host-sdk/host/qmd-scope.ts", + "src/memory-host-sdk/host/query-expansion.ts", + "src/memory-host-sdk/host/read-file-shared.ts", + "src/memory-host-sdk/host/read-file.ts", + "src/memory-host-sdk/host/remote-http.ts", + "src/memory-host-sdk/host/secret-input.ts", + "src/memory-host-sdk/host/session-files.ts", + "src/memory-host-sdk/host/sqlite-vec.ts", + "src/memory-host-sdk/host/sqlite.ts", + "src/memory-host-sdk/host/status-format.ts", + "src/memory-host-sdk/runtime-cli.ts", + "src/memory-host-sdk/runtime-core.ts", + "src/memory-host-sdk/runtime-files.ts", + "src/memory-host-sdk/runtime.ts", + "src/plugins/build-smoke-entry.ts", + "src/plugins/contracts/host-hook-fixture.ts", + "src/plugins/contracts/rootdir-boundary-canary.ts", + "src/plugins/contracts/tts-contract-suites.ts", + "src/plugins/runtime-sidecar-paths-baseline.ts", + "src/tasks/task-registry-control.runtime.ts", +]; diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index a74674cc3e6..0730ccc19bd 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -230,6 +230,11 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/github/barnacle-auto-response.mjs", ["test/scripts/barnacle-auto-response.test.ts"]], ["scripts/changed-lanes.mjs", ["test/scripts/changed-lanes.test.ts"]], ["scripts/check-changed.mjs", ["test/scripts/changed-lanes.test.ts"]], + ["scripts/check-deadcode-unused-files.mjs", ["test/scripts/check-deadcode-unused-files.test.ts"]], + [ + "scripts/deadcode-unused-files.allowlist.mjs", + ["test/scripts/check-deadcode-unused-files.test.ts"], + ], ["scripts/lib/live-docker-stage.sh", ["test/scripts/live-docker-stage.test.ts"]], ["scripts/lib/openclaw-test-state.mjs", ["test/scripts/openclaw-test-state.test.ts"]], ["scripts/lib/vitest-local-scheduling.mjs", ["test/scripts/vitest-local-scheduling.test.ts"]], @@ -274,6 +279,10 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ const TOOLING_TEST_TARGETS = new Map([ ["test/scripts/barnacle-auto-response.test.ts", ["test/scripts/barnacle-auto-response.test.ts"]], ["test/scripts/changed-lanes.test.ts", ["test/scripts/changed-lanes.test.ts"]], + [ + "test/scripts/check-deadcode-unused-files.test.ts", + ["test/scripts/check-deadcode-unused-files.test.ts"], + ], ["test/scripts/live-docker-stage.test.ts", ["test/scripts/live-docker-stage.test.ts"]], ["test/scripts/openclaw-test-state.test.ts", ["test/scripts/openclaw-test-state.test.ts"]], [ diff --git a/test/scripts/check-deadcode-unused-files.test.ts b/test/scripts/check-deadcode-unused-files.test.ts new file mode 100644 index 00000000000..67adf292ee9 --- /dev/null +++ b/test/scripts/check-deadcode-unused-files.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { + checkUnusedFiles, + compareUnusedFilesToAllowlist, + parseKnipCompactUnusedFiles, +} from "../../scripts/check-deadcode-unused-files.mjs"; + +describe("check-deadcode-unused-files", () => { + it("parses the compact Knip unused-file section", () => { + expect( + parseKnipCompactUnusedFiles(` +> openclaw@2026.4.27 deadcode:knip /repo +> pnpm dlx knip --reporter compact --files + +Unused files (2) +src/b.ts: src/b.ts +src/a.ts: src/a.ts + +Unused dependencies (1) +left-pad: package.json +`), + ).toEqual(["src/a.ts", "src/b.ts"]); + }); + + it("parses Knip's files-only compact output", () => { + expect(parseKnipCompactUnusedFiles("src/b.ts: src/b.ts\nsrc/a.ts: src/a.ts\n")).toEqual([ + "src/a.ts", + "src/b.ts", + ]); + }); + + it("reports unexpected and stale allowlist entries", () => { + expect( + compareUnusedFilesToAllowlist(["src/a.ts", "src/new.ts"], ["src/a.ts", "src/old.ts"]), + ).toMatchObject({ + unexpected: ["src/new.ts"], + stale: ["src/old.ts"], + duplicateAllowedCount: 0, + allowlistIsSorted: true, + }); + }); + + it("accepts exactly allowlisted unused files", () => { + expect(checkUnusedFiles("Unused files (1)\nsrc/a.ts: src/a.ts\n", ["src/a.ts"])).toMatchObject({ + ok: true, + message: "", + }); + }); + + it("rejects unsorted allowlists", () => { + expect( + compareUnusedFilesToAllowlist(["src/a.ts", "src/b.ts"], ["src/b.ts", "src/a.ts"]), + ).toMatchObject({ + allowlistIsSorted: false, + }); + }); +});