From 39bcd1e08834af9b5b568d2b4bd563d27927ab1a Mon Sep 17 00:00:00 2001 From: Pavan Kumar Gondhi Date: Wed, 13 May 2026 10:26:24 +0530 Subject: [PATCH] fix(plugins): scan installed dependency runtime code [AI] (#81066) * fix: scan installed plugin dependency code * addressing review-skill * addressing review-skill * addressing codex review * addressing codex review * addressing codex review * addressing codex review * addressing codex review * addressing codex review * addressing codex review * addressing codex review * addressing ci * addressing ci * docs: add changelog entry for PR merge --- CHANGELOG.md | 1 + .../session-tool-result-guard-wrapper.ts | 2 +- ...ult-guard.tool-result-persist-hook.test.ts | 2 +- src/agents/session-tool-result-guard.ts | 2 +- src/logging/redact.test.ts | 2 +- src/logging/redact.ts | 2 +- src/plugins/install-security-scan.runtime.ts | 294 ++++++++++- src/plugins/install-security-scan.ts | 4 + src/plugins/install.test.ts | 497 ++++++++++++++++++ src/plugins/install.ts | 85 ++- src/security/audit-extra.async.test.ts | 2 + src/security/skill-scanner.test.ts | 18 + src/security/skill-scanner.ts | 87 ++- 13 files changed, 960 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92623e63099..48b3b140b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- fix(plugins): scan installed dependency runtime code [AI]. (#81066) Thanks @pgondhi987. - Inherit tool restrictions for delegated sessions [AI]. (#80979) Thanks @pgondhi987. - Telegram: discard legacy long-poll update offsets that cannot be tied to the current bot token, so token rotation no longer leaves bots silently skipping new messages. (#80671) Thanks @sxxtony. - browser: enforce navigation checks for act interactions [AI]. (#81070) Thanks @pgondhi987. diff --git a/src/agents/session-tool-result-guard-wrapper.ts b/src/agents/session-tool-result-guard-wrapper.ts index 618f2f5b102..b7f16234b09 100644 --- a/src/agents/session-tool-result-guard-wrapper.ts +++ b/src/agents/session-tool-result-guard-wrapper.ts @@ -181,4 +181,4 @@ export function guardSessionManager( (sessionManager as GuardedSessionManager).flushPendingToolResults = guard.flushPendingToolResults; (sessionManager as GuardedSessionManager).clearPendingToolResults = guard.clearPendingToolResults; return sessionManager as GuardedSessionManager; -} \ No newline at end of file +} diff --git a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts index 42689852b37..3265cf4f462 100644 --- a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts +++ b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts @@ -821,4 +821,4 @@ describe("before_message_write hook", () => { appendToolCallAndResult(sm); expectPersistedToolResultTextCapped(sm); }); -}); \ No newline at end of file +}); diff --git a/src/agents/session-tool-result-guard.ts b/src/agents/session-tool-result-guard.ts index 695a6d5dd4b..ad1ec2858bb 100644 --- a/src/agents/session-tool-result-guard.ts +++ b/src/agents/session-tool-result-guard.ts @@ -707,4 +707,4 @@ export function installSessionToolResultGuard( clearPendingToolResults, getPendingIds: pendingState.getPendingIds, }; -} \ No newline at end of file +} diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index 7872b21377a..c7a44341509 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -567,4 +567,4 @@ describe("redactSensitiveLines", () => { expect(joined).toContain("…redacted…"); expect(joined).not.toContain("ABCDEF1234567890"); }); -}); \ No newline at end of file +}); diff --git a/src/logging/redact.ts b/src/logging/redact.ts index b52b4db5841..44698030d31 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -360,4 +360,4 @@ export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOp return lines; } return redactText(lines.join("\n"), resolved.patterns).split("\n"); -} \ No newline at end of file +} diff --git a/src/plugins/install-security-scan.runtime.ts b/src/plugins/install-security-scan.runtime.ts index be8cec5dd1e..b2b543c3de4 100644 --- a/src/plugins/install-security-scan.runtime.ts +++ b/src/plugins/install-security-scan.runtime.ts @@ -75,6 +75,11 @@ type PackageManifestTraversalResult = { packageManifestPaths: string[]; }; +type InstalledPackageScanRoot = { + packageDir: string; + realPath: string; +}; + type PluginInstallRequestKind = | "skill-install" | "plugin-dir" @@ -321,6 +326,7 @@ function buildBuiltinScanFromSummary(summary: { critical: number; warn: number; info: number; + truncated: boolean; findings: InstallScanFinding[]; }): BuiltinInstallScan { return { @@ -338,6 +344,7 @@ const DEFAULT_PACKAGE_MANIFEST_TRAVERSAL_LIMITS: PackageManifestTraversalLimits maxDirectories: 10_000, maxManifests: 10_000, }; +const DEFAULT_INSTALLED_PACKAGE_CODE_SCAN_MAX_FILES = 10_000; function readPositiveIntegerEnv(name: string, fallback: number): number { const rawValue = process.env[name]; @@ -368,6 +375,171 @@ function resolvePackageManifestTraversalLimits(): PackageManifestTraversalLimits }; } +function resolveInstalledPackageCodeScanMaxFiles(): number { + return readPositiveIntegerEnv( + "OPENCLAW_INSTALL_SCAN_MAX_CODE_FILES", + DEFAULT_INSTALLED_PACKAGE_CODE_SCAN_MAX_FILES, + ); +} + +function isSamePathOrInside(parentPath: string, candidatePath: string): boolean { + return parentPath === candidatePath || isPathInside(parentPath, candidatePath); +} + +function getErrnoCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) { + return undefined; + } + const code = (error as { code?: unknown }).code; + return typeof code === "string" ? code : undefined; +} + +function isInstallScannableDependencyName(name: string): boolean { + if (name.startsWith("@")) { + const parts = name.split("/"); + return ( + parts.length === 2 && parts.every((part) => part.length > 0 && part !== "." && part !== "..") + ); + } + return ( + name.length > 0 && !name.includes("/") && !name.includes("\\") && name !== "." && name !== ".." + ); +} + +function collectManifestRuntimeDependencyNames(manifest: PackageManifest): string[] { + const dependencyNames = new Set(); + for (const dependencies of [manifest.dependencies, manifest.optionalDependencies]) { + for (const dependencyName of Object.keys(dependencies ?? {})) { + if (isInstallScannableDependencyName(dependencyName)) { + dependencyNames.add(dependencyName); + } + } + } + for (const dependencyName of Object.keys(manifest.peerDependencies ?? {})) { + if (dependencyName !== "openclaw" && isInstallScannableDependencyName(dependencyName)) { + dependencyNames.add(dependencyName); + } + } + return [...dependencyNames].toSorted((left, right) => left.localeCompare(right)); +} + +async function resolveInstalledPackageScanRoot(params: { + boundaryRealPath: string; + dependencyName: string; + packageDir: string; +}): Promise { + const packageDir = path.join(params.packageDir, "node_modules", params.dependencyName); + let stats: Awaited>; + try { + stats = await fs.stat(packageDir); + } catch (error) { + if (getErrnoCode(error) === "ENOENT") { + return undefined; + } + throw error; + } + if (!stats.isDirectory()) { + return undefined; + } + + const realPath = await fs.realpath(packageDir).catch(() => path.resolve(packageDir)); + if (!isSamePathOrInside(params.boundaryRealPath, realPath)) { + throw new Error( + `installed dependency scan found package outside install root at ${packageDir}`, + ); + } + return { packageDir, realPath }; +} + +async function collectInstalledPackageScanRoots(params: { + additionalPackageDirs?: string[]; + dependencyScanRootDir?: string; + packageDir: string; +}): Promise { + const limits = resolvePackageManifestTraversalLimits(); + const boundaryDir = params.dependencyScanRootDir ?? params.packageDir; + const boundaryRealPath = await fs.realpath(boundaryDir).catch(() => path.resolve(boundaryDir)); + const packageRealPath = await fs + .realpath(params.packageDir) + .catch(() => path.resolve(params.packageDir)); + if (!isSamePathOrInside(boundaryRealPath, packageRealPath)) { + throw new Error( + `installed dependency scan found package outside install root at ${params.packageDir}`, + ); + } + + const queue: InstalledPackageScanRoot[] = [ + { packageDir: params.packageDir, realPath: packageRealPath }, + ]; + for (const packageDir of params.additionalPackageDirs ?? []) { + const realPath = await fs.realpath(packageDir).catch(() => path.resolve(packageDir)); + if (!isSamePathOrInside(boundaryRealPath, realPath)) { + throw new Error( + `installed dependency scan found package outside install root at ${packageDir}`, + ); + } + queue.push({ packageDir, realPath }); + } + const visitedRealPaths = new Set(); + const scanRoots: string[] = []; + let queueIndex = 0; + + while (queueIndex < queue.length) { + const current = queue[queueIndex]; + queueIndex += 1; + if (!current || visitedRealPaths.has(current.realPath)) { + continue; + } + visitedRealPaths.add(current.realPath); + if (visitedRealPaths.size > limits.maxDirectories) { + throw new Error( + `installed dependency scan exceeded max packages (${limits.maxDirectories}) under ${boundaryDir}`, + ); + } + scanRoots.push(current.packageDir); + + const manifest = await tryReadJson( + path.join(current.packageDir, "package.json"), + ); + if (!manifest) { + continue; + } + for (const dependencyName of collectManifestRuntimeDependencyNames(manifest)) { + const nestedCandidate = await resolveInstalledPackageScanRoot({ + boundaryRealPath, + dependencyName, + packageDir: current.packageDir, + }); + const candidate = + nestedCandidate ?? + (params.dependencyScanRootDir + ? await resolveInstalledPackageScanRoot({ + boundaryRealPath, + dependencyName, + packageDir: params.dependencyScanRootDir, + }) + : undefined); + if (candidate && !visitedRealPaths.has(candidate.realPath)) { + queue.push(candidate); + } + } + } + + return scanRoots; +} + +async function collectNonOverlappingPackageScanRoots(packageDirs: string[]): Promise { + const selectedRoots: InstalledPackageScanRoot[] = []; + for (const packageDir of packageDirs) { + const realPath = await fs.realpath(packageDir).catch(() => path.resolve(packageDir)); + if (selectedRoots.some((selectedRoot) => isSamePathOrInside(selectedRoot.realPath, realPath))) { + continue; + } + selectedRoots.push({ packageDir, realPath }); + } + return selectedRoots.map((selectedRoot) => selectedRoot.packageDir); +} + async function collectPackageManifestPaths(params: { allowManagedNpmRootPackagePeerSymlinks?: boolean; rootDir: string; @@ -493,10 +665,25 @@ async function collectPackageManifestPaths(params: { }; } +function formatPackageScanRelativePath(params: { + packageDir: string; + relativePath: string; + relativeRootDir?: string; +}): string { + if (!params.relativeRootDir) { + return params.relativePath; + } + const packageRelativePath = path.relative(params.relativeRootDir, params.packageDir); + return packageRelativePath + ? path.join(packageRelativePath, params.relativePath) + : params.relativePath; +} + async function scanManifestDependencyDenylist(params: { allowManagedNpmRootPackagePeerSymlinks?: boolean; logger: InstallScanLogger; packageDir: string; + relativeRootDir?: string; targetLabel: string; }): Promise { const traversalResult = await collectPackageManifestPaths({ @@ -515,7 +702,11 @@ async function scanManifestDependencyDenylist(params: { continue; } - const manifestRelativePath = path.relative(params.packageDir, manifestPath) || "package.json"; + const manifestRelativePath = formatPackageScanRelativePath({ + packageDir: params.packageDir, + relativePath: path.relative(params.packageDir, manifestPath) || "package.json", + relativeRootDir: params.relativeRootDir, + }); const reason = buildBlockedDependencyReason({ findings: blockedDependencies, manifestPackageName: manifest.name, @@ -536,7 +727,11 @@ async function scanManifestDependencyDenylist(params: { if (traversalResult.blockedDirectoryFinding) { const reason = buildBlockedDependencyDirectoryReason({ dependencyName: traversalResult.blockedDirectoryFinding.dependencyName, - directoryRelativePath: traversalResult.blockedDirectoryFinding.directoryRelativePath, + directoryRelativePath: formatPackageScanRelativePath({ + packageDir: params.packageDir, + relativePath: traversalResult.blockedDirectoryFinding.directoryRelativePath, + relativeRootDir: params.relativeRootDir, + }), targetLabel: params.targetLabel, }); params.logger.warn?.(`WARNING: ${reason}`); @@ -550,7 +745,11 @@ async function scanManifestDependencyDenylist(params: { if (traversalResult.blockedFileFinding) { const reason = buildBlockedDependencyFileReason({ dependencyName: traversalResult.blockedFileFinding.dependencyName, - fileRelativePath: traversalResult.blockedFileFinding.fileRelativePath, + fileRelativePath: formatPackageScanRelativePath({ + packageDir: params.packageDir, + relativePath: traversalResult.blockedFileFinding.fileRelativePath, + relativeRootDir: params.relativeRootDir, + }), targetLabel: params.targetLabel, }); params.logger.warn?.(`WARNING: ${reason}`); @@ -565,8 +764,14 @@ async function scanManifestDependencyDenylist(params: { } async function scanDirectoryTarget(params: { + excludeTestFiles?: boolean; + failOnTruncated?: boolean; + includeHiddenDirectories?: boolean; + includeNestedNodeModulesTestFiles?: boolean; + includeNodeModules?: boolean; includeFiles?: string[]; logger: InstallScanLogger; + maxFiles?: number; path: string; suppressBuiltinWarnings?: boolean; suspiciousMessage: string; @@ -575,9 +780,18 @@ async function scanDirectoryTarget(params: { }): Promise { try { const scanSummary = await scanDirectoryWithSummary(params.path, { - excludeTestFiles: true, + excludeTestFiles: params.excludeTestFiles ?? true, + includeHiddenDirectories: params.includeHiddenDirectories, + includeNestedNodeModulesTestFiles: params.includeNestedNodeModulesTestFiles, + includeNodeModules: params.includeNodeModules, includeFiles: params.includeFiles, + maxFiles: params.maxFiles, }); + if (params.failOnTruncated && scanSummary.truncated) { + return buildBuiltinScanFromError( + `code safety scan reached file limit (${params.maxFiles ?? "configured limit"})`, + ); + } const builtinScan = buildBuiltinScanFromSummary(scanSummary); if (params.suppressBuiltinWarnings) { return builtinScan; @@ -934,17 +1148,81 @@ export async function scanPackageInstallSourceRuntime( } export async function scanInstalledPackageDependencyTreeRuntime(params: { + additionalPackageDirs?: string[]; allowManagedNpmRootPackagePeerSymlinks?: boolean; + dangerouslyForceUnsafeInstall?: boolean; + dependencyScanRootDir?: string; logger: InstallScanLogger; packageDir: string; pluginId: string; + trustedSourceLinkedOfficialInstall?: boolean; }): Promise { - return await scanManifestDependencyDenylist({ - logger: params.logger, + const scanRoots = await collectInstalledPackageScanRoots({ + ...(params.additionalPackageDirs + ? { additionalPackageDirs: params.additionalPackageDirs } + : {}), + dependencyScanRootDir: params.dependencyScanRootDir, packageDir: params.packageDir, - allowManagedNpmRootPackagePeerSymlinks: params.allowManagedNpmRootPackagePeerSymlinks, - targetLabel: `Plugin "${params.pluginId}" installation`, }); + const directoryScanRoots = await collectNonOverlappingPackageScanRoots(scanRoots); + for (const packageDir of directoryScanRoots) { + const dependencyBlocked = await scanManifestDependencyDenylist({ + logger: params.logger, + packageDir, + allowManagedNpmRootPackagePeerSymlinks: params.allowManagedNpmRootPackagePeerSymlinks, + relativeRootDir: params.dependencyScanRootDir ?? params.packageDir, + targetLabel: `Plugin "${params.pluginId}" installation`, + }); + if (dependencyBlocked) { + return dependencyBlocked; + } + } + + let remainingMaxFiles = resolveInstalledPackageCodeScanMaxFiles(); + const pluginRootRealPath = await fs + .realpath(params.packageDir) + .catch(() => path.resolve(params.packageDir)); + for (const packageDir of directoryScanRoots) { + if (remainingMaxFiles <= 0) { + return resolveBuiltinScanDecision({ + builtinScan: buildBuiltinScanFromError( + "code safety scan reached file limit (configured limit)", + ), + logger: params.logger, + dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + trustedSourceLinkedOfficialInstall: params.trustedSourceLinkedOfficialInstall, + targetLabel: `Plugin "${params.pluginId}" installation`, + }); + } + const packageRealPath = await fs.realpath(packageDir).catch(() => path.resolve(packageDir)); + const isPluginRoot = packageRealPath === pluginRootRealPath; + const builtinScan = await scanDirectoryTarget({ + excludeTestFiles: isPluginRoot, + failOnTruncated: true, + includeHiddenDirectories: true, + includeNestedNodeModulesTestFiles: isPluginRoot, + includeNodeModules: true, + logger: params.logger, + maxFiles: remainingMaxFiles, + path: packageDir, + suppressBuiltinWarnings: params.trustedSourceLinkedOfficialInstall === true, + suspiciousMessage: `Plugin "{target}" installed tree has {count} suspicious code pattern(s). Run "openclaw security audit --deep" for details.`, + targetName: params.pluginId, + warningMessage: `WARNING: Plugin "${params.pluginId}" installed tree contains dangerous code patterns`, + }); + const builtinBlocked = resolveBuiltinScanDecision({ + builtinScan, + logger: params.logger, + dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + trustedSourceLinkedOfficialInstall: params.trustedSourceLinkedOfficialInstall, + targetLabel: `Plugin "${params.pluginId}" installation`, + }); + if (builtinBlocked) { + return builtinBlocked; + } + remainingMaxFiles -= builtinScan.scannedFiles; + } + return undefined; } export async function scanFileInstallSourceRuntime( diff --git a/src/plugins/install-security-scan.ts b/src/plugins/install-security-scan.ts index 14ea1c4d13f..c7172ebdd3a 100644 --- a/src/plugins/install-security-scan.ts +++ b/src/plugins/install-security-scan.ts @@ -80,10 +80,14 @@ export async function scanPackageInstallSource( } export async function scanInstalledPackageDependencyTree(params: { + additionalPackageDirs?: string[]; allowManagedNpmRootPackagePeerSymlinks?: boolean; + dangerouslyForceUnsafeInstall?: boolean; + dependencyScanRootDir?: string; logger: InstallScanLogger; packageDir: string; pluginId: string; + trustedSourceLinkedOfficialInstall?: boolean; }): Promise { const { scanInstalledPackageDependencyTreeRuntime } = await loadInstallSecurityScanRuntime(); return await scanInstalledPackageDependencyTreeRuntime(params); diff --git a/src/plugins/install.test.ts b/src/plugins/install.test.ts index 844efc638a3..11c813214c6 100644 --- a/src/plugins/install.test.ts +++ b/src/plugins/install.test.ts @@ -11,6 +11,7 @@ import * as installSecurityScan from "./install-security-scan.js"; import { installPluginFromArchive, installPluginFromDir, + installPluginFromInstalledPackageDir, PLUGIN_INSTALL_ERROR_CODE, resolvePluginInstallDir, } from "./install.js"; @@ -811,6 +812,211 @@ describe("installPluginFromArchive", () => { expect(warnings).toStrictEqual([]); }); + it("blocks archive installs when dependency install materializes dangerous runtime code", async () => { + const stateDir = suiteTempRootTracker.makeTempDir(); + const extensionsDir = path.join(stateDir, "extensions"); + fs.mkdirSync(extensionsDir, { recursive: true }); + + const archivePath = await ensureDynamicArchiveTemplate({ + outName: "dependency-runtime-code-plugin.tgz", + packageJson: { + name: "dependency-runtime-code-plugin", + version: "1.0.0", + openclaw: { extensions: ["./dist/index.js"] }, + dependencies: { + "telemetry-helper": "1.0.0", + }, + }, + withDistIndex: true, + distIndexJsContent: `const telemetry = require("telemetry-helper");\nmodule.exports = telemetry;\n`, + }); + + const run = vi.mocked(runCommandWithTimeout); + run.mockImplementationOnce(async (_cmd, options) => { + if (!options || typeof options === "number" || !options.cwd) { + throw new Error("expected npm install cwd"); + } + const dependencyDir = path.join(options.cwd, "node_modules", "telemetry-helper"); + fs.mkdirSync(dependencyDir, { recursive: true }); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ name: "telemetry-helper", version: "1.0.0", main: "index.cjs" }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "index.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + return { + code: 0, + stdout: "", + stderr: "", + signal: null, + killed: false, + termination: "exit" as const, + }; + }); + + const { result, warnings } = await installFromArchiveWithWarnings({ + archivePath, + extensionsDir, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_BLOCKED); + expect(result.error).toContain( + 'Plugin "dependency-runtime-code-plugin" installation blocked', + ); + expect(result.error).toContain("dangerous code patterns detected"); + expect(result.error).toContain("node_modules/telemetry-helper/index.cjs"); + } + expectWarningIncludes(warnings, "installed tree contains dangerous code patterns"); + }); + + it("blocks archive installs when dependency runtime code is loaded from a hidden directory", async () => { + const stateDir = suiteTempRootTracker.makeTempDir(); + const extensionsDir = path.join(stateDir, "extensions"); + fs.mkdirSync(extensionsDir, { recursive: true }); + + const archivePath = await ensureDynamicArchiveTemplate({ + outName: "hidden-dependency-runtime-code-plugin.tgz", + packageJson: { + name: "hidden-dependency-runtime-code-plugin", + version: "1.0.0", + openclaw: { extensions: ["./dist/index.js"] }, + dependencies: { + "hidden-telemetry-helper": "1.0.0", + }, + }, + withDistIndex: true, + distIndexJsContent: `const telemetry = require("hidden-telemetry-helper");\nmodule.exports = telemetry;\n`, + }); + + const run = vi.mocked(runCommandWithTimeout); + run.mockImplementationOnce(async (_cmd, options) => { + if (!options || typeof options === "number" || !options.cwd) { + throw new Error("expected npm install cwd"); + } + const dependencyDir = path.join(options.cwd, "node_modules", "hidden-telemetry-helper"); + const hiddenPayloadDir = path.join(dependencyDir, ".payload"); + fs.mkdirSync(hiddenPayloadDir, { recursive: true }); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ + name: "hidden-telemetry-helper", + version: "1.0.0", + main: "index.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "index.cjs"), + `module.exports = require("./.payload/runtime.cjs");\n`, + "utf-8", + ); + fs.writeFileSync( + path.join(hiddenPayloadDir, "runtime.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + return { + code: 0, + stdout: "", + stderr: "", + signal: null, + killed: false, + termination: "exit" as const, + }; + }); + + const { result, warnings } = await installFromArchiveWithWarnings({ + archivePath, + extensionsDir, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_BLOCKED); + expect(result.error).toContain( + 'Plugin "hidden-dependency-runtime-code-plugin" installation blocked', + ); + expect(result.error).toContain("dangerous code patterns detected"); + expect(result.error).toContain("node_modules/hidden-telemetry-helper/.payload/runtime.cjs"); + } + expectWarningIncludes(warnings, "installed tree contains dangerous code patterns"); + }); + + it("fails archive installs when installed runtime code scan reaches its file cap", async () => { + vi.stubEnv("OPENCLAW_INSTALL_SCAN_MAX_CODE_FILES", "1"); + const stateDir = suiteTempRootTracker.makeTempDir(); + const extensionsDir = path.join(stateDir, "extensions"); + fs.mkdirSync(extensionsDir, { recursive: true }); + + const archivePath = await ensureDynamicArchiveTemplate({ + outName: "capped-dependency-runtime-code-plugin.tgz", + packageJson: { + name: "capped-dependency-runtime-code-plugin", + version: "1.0.0", + openclaw: { extensions: ["./dist/index.js"] }, + dependencies: { + "capped-telemetry-helper": "1.0.0", + }, + }, + withDistIndex: true, + distIndexJsContent: `const telemetry = require("capped-telemetry-helper");\nmodule.exports = telemetry;\n`, + }); + + const run = vi.mocked(runCommandWithTimeout); + run.mockImplementationOnce(async (_cmd, options) => { + if (!options || typeof options === "number" || !options.cwd) { + throw new Error("expected npm install cwd"); + } + const dependencyDir = path.join(options.cwd, "node_modules", "capped-telemetry-helper"); + fs.mkdirSync(dependencyDir, { recursive: true }); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ + name: "capped-telemetry-helper", + version: "1.0.0", + main: "index.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "index.cjs"), + `module.exports = require("./runtime.cjs");\n`, + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "runtime.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + return { + code: 0, + stdout: "", + stderr: "", + signal: null, + killed: false, + termination: "exit" as const, + }; + }); + + const { result } = await installFromArchiveWithWarnings({ + archivePath, + extensionsDir, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_FAILED); + expect(result.error).toContain("code safety scan failed"); + expect(result.error).toContain("code safety scan reached file limit (1)"); + } + }); + it("installs flat-root plugin archives from ClawHub-style downloads", async () => { const result = await installArchivePackageAndReturnResult({ packageJson: { @@ -2856,6 +3062,297 @@ describe("installPluginFromDir", () => { expect(vi.mocked(runCommandWithTimeout)).not.toHaveBeenCalled(); }); + it("does not scan pre-existing sibling packages from a managed npm root", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const npmRoot = path.join(caseDir, "npm-root"); + const newPluginDir = path.join(npmRoot, "node_modules", "new-managed-plugin"); + const existingPluginDir = path.join(npmRoot, "node_modules", "existing-official-plugin"); + fs.mkdirSync(newPluginDir, { recursive: true }); + fs.mkdirSync(existingPluginDir, { recursive: true }); + writeMinimalPackagePlugin(newPluginDir, "new-managed-plugin"); + writeMinimalPackagePlugin(existingPluginDir, "existing-official-plugin"); + fs.writeFileSync( + path.join(existingPluginDir, "index.js"), + `const childProcess = require("node:child_process");\nchildProcess.spawn("node", ["-v"]);\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: newPluginDir, + dependencyScanRootDir: npmRoot, + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.pluginId).toBe("new-managed-plugin"); + } + }); + + it("scans flattened managed npm dependencies reachable from the installed package", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const npmRoot = path.join(caseDir, "npm-root"); + const pluginDir = path.join(npmRoot, "node_modules", "managed-plugin-with-dep"); + const dependencyDir = path.join(npmRoot, "node_modules", "flattened-runtime-helper"); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(dependencyDir, { recursive: true }); + writeMinimalPackagePlugin(pluginDir, "managed-plugin-with-dep"); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "managed-plugin-with-dep", + version: "1.0.0", + dependencies: { + "flattened-runtime-helper": "1.0.0", + }, + openclaw: { extensions: ["index.js"] }, + }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ + name: "flattened-runtime-helper", + version: "1.0.0", + main: "index.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyDir, "index.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + dependencyScanRootDir: npmRoot, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_BLOCKED); + expect(result.error).toContain("flattened-runtime-helper/index.cjs"); + } + }); + + it("scans installed managed npm peer dependencies reachable from the installed package", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const npmRoot = path.join(caseDir, "npm-root"); + const pluginDir = path.join(npmRoot, "node_modules", "managed-plugin-with-peer"); + const peerDependencyDir = path.join(npmRoot, "node_modules", "peer-runtime-helper"); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(peerDependencyDir, { recursive: true }); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "managed-plugin-with-peer", + version: "1.0.0", + peerDependencies: { + "peer-runtime-helper": "^1.0.0", + }, + openclaw: { extensions: ["index.js"] }, + }), + "utf-8", + ); + fs.writeFileSync(path.join(pluginDir, "index.js"), "export {};\n", "utf-8"); + fs.writeFileSync( + path.join(peerDependencyDir, "package.json"), + JSON.stringify({ + name: "peer-runtime-helper", + version: "1.0.0", + main: "index.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(peerDependencyDir, "index.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + dependencyScanRootDir: npmRoot, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_BLOCKED); + expect(result.error).toContain("peer-runtime-helper/index.cjs"); + } + }); + + it("scans installed dependency runtime entrypoints with test-like paths", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const npmRoot = path.join(caseDir, "npm-root"); + const pluginDir = path.join(npmRoot, "node_modules", "managed-plugin-with-test-entry-dep"); + const dependencyDir = path.join(npmRoot, "node_modules", "test-entry-helper"); + const dependencyTestsDir = path.join(dependencyDir, "tests"); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(dependencyTestsDir, { recursive: true }); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "managed-plugin-with-test-entry-dep", + version: "1.0.0", + dependencies: { + "test-entry-helper": "1.0.0", + }, + openclaw: { extensions: ["index.js"] }, + }), + "utf-8", + ); + fs.writeFileSync(path.join(pluginDir, "index.js"), "export {};\n", "utf-8"); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ + name: "test-entry-helper", + version: "1.0.0", + main: "tests/runtime.test.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(dependencyTestsDir, "runtime.test.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + dependencyScanRootDir: npmRoot, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe(PLUGIN_INSTALL_ERROR_CODE.SECURITY_SCAN_BLOCKED); + expect(result.error).toContain("test-entry-helper/tests/runtime.test.cjs"); + } + }); + + it("keeps plugin-root test files excluded during installed tree scans", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const pluginDir = path.join(caseDir, "plugin-with-test-files"); + const testsDir = path.join(pluginDir, "tests"); + fs.mkdirSync(testsDir, { recursive: true }); + writeMinimalPackagePlugin(pluginDir, "plugin-with-test-files"); + fs.writeFileSync( + path.join(testsDir, "dangerous.test.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.pluginId).toBe("plugin-with-test-files"); + } + }); + + it("prefers nested managed npm dependencies over pre-existing root fallbacks", async () => { + const caseDir = suiteTempRootTracker.makeTempDir(); + const npmRoot = path.join(caseDir, "npm-root"); + const pluginDir = path.join(npmRoot, "node_modules", "managed-plugin-with-nested-dep"); + const nestedDependencyDir = path.join(pluginDir, "node_modules", "shared-runtime-helper"); + const rootFallbackDir = path.join(npmRoot, "node_modules", "shared-runtime-helper"); + fs.mkdirSync(nestedDependencyDir, { recursive: true }); + fs.mkdirSync(rootFallbackDir, { recursive: true }); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "managed-plugin-with-nested-dep", + version: "1.0.0", + dependencies: { + "shared-runtime-helper": "2.0.0", + }, + openclaw: { extensions: ["index.js"] }, + }), + "utf-8", + ); + fs.writeFileSync(path.join(pluginDir, "index.js"), "export {};\n", "utf-8"); + fs.writeFileSync( + path.join(nestedDependencyDir, "package.json"), + JSON.stringify({ + name: "shared-runtime-helper", + version: "2.0.0", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(nestedDependencyDir, "index.cjs"), + "module.exports = {};\n", + "utf-8", + ); + fs.writeFileSync( + path.join(rootFallbackDir, "package.json"), + JSON.stringify({ + name: "shared-runtime-helper", + version: "1.0.0", + main: "index.cjs", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(rootFallbackDir, "index.cjs"), + `const childProcess = require("node:child_process");\nchildProcess.execSync("node -v", { encoding: "utf8" });\nmodule.exports = {};\n`, + "utf-8", + ); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + dependencyScanRootDir: npmRoot, + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.pluginId).toBe("managed-plugin-with-nested-dep"); + } + }); + + it("does not double-count nested dependency files against the installed tree scan cap", async () => { + vi.stubEnv("OPENCLAW_INSTALL_SCAN_MAX_CODE_FILES", "4"); + const caseDir = suiteTempRootTracker.makeTempDir(); + const pluginDir = path.join(caseDir, "isolated-plugin"); + const dependencyDir = path.join(pluginDir, "node_modules", "nested-runtime-helper"); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(dependencyDir, { recursive: true }); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "isolated-plugin", + version: "1.0.0", + dependencies: { + "nested-runtime-helper": "1.0.0", + }, + openclaw: { extensions: ["index.js"] }, + }), + "utf-8", + ); + fs.writeFileSync(path.join(pluginDir, "index.js"), "export {};\n", "utf-8"); + fs.writeFileSync( + path.join(dependencyDir, "package.json"), + JSON.stringify({ + name: "nested-runtime-helper", + version: "1.0.0", + }), + "utf-8", + ); + fs.writeFileSync(path.join(dependencyDir, "first.cjs"), "module.exports = 1;\n", "utf-8"); + fs.writeFileSync(path.join(dependencyDir, "second.cjs"), "module.exports = 2;\n", "utf-8"); + + const result = await installPluginFromInstalledPackageDir({ + packageDir: pluginDir, + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.pluginId).toBe("isolated-plugin"); + } + }); + it.each([ { name: "rejects plugins whose minHostVersion is newer than the current host", diff --git a/src/plugins/install.ts b/src/plugins/install.ts index 7d67c7cca2d..5286de76ef3 100644 --- a/src/plugins/install.ts +++ b/src/plugins/install.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import type { Dirent } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { packageNameMatchesId } from "../infra/install-safe-path.js"; @@ -400,6 +401,65 @@ function resolveInstalledNpmResolutionMismatch(params: { return null; } +async function listManagedNpmRootPackageNames(npmRoot: string): Promise> { + const nodeModulesDir = path.join(npmRoot, "node_modules"); + let entries: Dirent[]; + try { + entries = await fs.readdir(nodeModulesDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return new Set(); + } + throw error; + } + + const packageNames = new Set(); + for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) { + if (entry.name === ".bin" || entry.name === "openclaw") { + continue; + } + if (entry.name.startsWith("@")) { + const scopeDir = path.join(nodeModulesDir, entry.name); + let scopedEntries: Dirent[]; + try { + scopedEntries = await fs.readdir(scopeDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw error; + } + for (const scopedEntry of scopedEntries.toSorted((left, right) => + left.name.localeCompare(right.name), + )) { + if (scopedEntry.isDirectory() || scopedEntry.isSymbolicLink()) { + packageNames.add(`${entry.name}/${scopedEntry.name}`); + } + } + continue; + } + if (entry.isDirectory() || entry.isSymbolicLink()) { + packageNames.add(entry.name); + } + } + return packageNames; +} + +function resolveManagedNpmRootPackageDir(npmRoot: string, packageName: string): string { + return path.join(npmRoot, "node_modules", ...packageName.split("/")); +} + +async function listNewManagedNpmRootPackageDirs(params: { + beforeInstallPackageNames: Set; + npmRoot: string; +}): Promise { + const afterInstallPackageNames = await listManagedNpmRootPackageNames(params.npmRoot); + return [...afterInstallPackageNames] + .filter((packageName) => !params.beforeInstallPackageNames.has(packageName)) + .map((packageName) => resolveManagedNpmRootPackageDir(params.npmRoot, packageName)) + .toSorted((left, right) => left.localeCompare(right)); +} + function resolveTrustedNpmPackPackageName(packageName: string | undefined): | { ok: true; @@ -489,6 +549,7 @@ async function installPluginFromManagedNpmRoot( logger.info?.(`Repaired stale openclaw peer dependency in ${npmRoot}`); } } + const preInstallRootPackageNames = await listManagedNpmRootPackageNames(npmRoot); const managedOverrides = await readOpenClawManagedNpmRootOverrides(); await upsertManagedNpmRootDependency({ npmRoot, @@ -623,8 +684,13 @@ async function installPluginFromManagedNpmRoot( }; } + const newRootPackageDirs = await listNewManagedNpmRootPackageDirs({ + beforeInstallPackageNames: preInstallRootPackageNames, + npmRoot, + }); const result = await installPluginFromInstalledPackageDir({ dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + additionalDependencyPackageDirs: newRootPackageDirs, packageDir: installRoot, dependencyScanRootDir: npmRoot, logger, @@ -1214,21 +1280,30 @@ async function validatePackagePluginInstallSource(params: { async function scanAndLinkInstalledPackage(params: { runtime: Awaited>; installedDir: string; + additionalDependencyPackageDirs?: string[]; dependencyScanRootDir?: string; pluginId: string; peerDependencies: Record; + dangerouslyForceUnsafeInstall?: boolean; + trustedSourceLinkedOfficialInstall?: boolean; logger: PluginInstallLogger; }): Promise | null> { const scanResult = await runInstallSourceScan({ subject: `Plugin "${params.pluginId}"`, scan: async () => await params.runtime.scanInstalledPackageDependencyTree({ + ...(params.additionalDependencyPackageDirs + ? { additionalPackageDirs: params.additionalDependencyPackageDirs } + : {}), allowManagedNpmRootPackagePeerSymlinks: params.dependencyScanRootDir !== undefined && path.resolve(params.dependencyScanRootDir) !== path.resolve(params.installedDir), + dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + dependencyScanRootDir: params.dependencyScanRootDir, logger: params.logger, - packageDir: params.dependencyScanRootDir ?? params.installedDir, + packageDir: params.installedDir, pluginId: params.pluginId, + trustedSourceLinkedOfficialInstall: params.trustedSourceLinkedOfficialInstall, }), }); if (scanResult) { @@ -1250,6 +1325,7 @@ async function scanAndLinkInstalledPackage(params: { export async function installPluginFromInstalledPackageDir( params: { + additionalDependencyPackageDirs?: string[]; packageDir: string; dependencyScanRootDir?: string; } & PackageInstallCommonParams, @@ -1273,9 +1349,14 @@ export async function installPluginFromInstalledPackageDir( const postInstallError = await scanAndLinkInstalledPackage({ runtime, installedDir: params.packageDir, + ...(params.additionalDependencyPackageDirs + ? { additionalDependencyPackageDirs: params.additionalDependencyPackageDirs } + : {}), dependencyScanRootDir: params.dependencyScanRootDir, pluginId: validated.plugin.pluginId, peerDependencies: validated.plugin.peerDependencies, + dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + trustedSourceLinkedOfficialInstall: params.trustedSourceLinkedOfficialInstall, logger, }); if (postInstallError) { @@ -1364,6 +1445,8 @@ async function installPluginFromPackageDir( installedDir, pluginId: plugin.pluginId, peerDependencies: plugin.peerDependencies, + dangerouslyForceUnsafeInstall: params.dangerouslyForceUnsafeInstall, + trustedSourceLinkedOfficialInstall: params.trustedSourceLinkedOfficialInstall, logger, }); }, diff --git a/src/security/audit-extra.async.test.ts b/src/security/audit-extra.async.test.ts index b855a18e059..362e31b7cc7 100644 --- a/src/security/audit-extra.async.test.ts +++ b/src/security/audit-extra.async.test.ts @@ -108,6 +108,7 @@ description: test skill critical: 1, warn: 0, info: 0, + truncated: false, findings: [ { ruleId: "dangerous-exec", @@ -173,6 +174,7 @@ description: test skill critical: dirPath.includes(`${path.sep}demo`) ? 1 : 0, warn: 0, info: 0, + truncated: false, findings: dirPath.includes(`${path.sep}demo`) ? [ { diff --git a/src/security/skill-scanner.test.ts b/src/security/skill-scanner.test.ts index a67c5c7b1fc..180ced64126 100644 --- a/src/security/skill-scanner.test.ts +++ b/src/security/skill-scanner.test.ts @@ -138,6 +138,7 @@ type SummaryCase = { critical?: number; warn?: number; info?: number; + truncated?: boolean; findingCount?: number; maxFindings?: number; expectedRuleId?: string; @@ -529,9 +530,23 @@ describe("scanDirectoryWithSummary", () => { options: { maxFiles: 2 }, expected: { scannedFiles: 2, + truncated: true, maxFindings: 2, }, }, + { + name: "does not mark scans truncated when file count exactly matches maxFiles", + files: { + "a.js": `const x = eval("a");`, + "b.js": `const x = eval("b");`, + }, + options: { maxFiles: 2 }, + expected: { + scannedFiles: 2, + truncated: false, + findingCount: 2, + }, + }, { name: "skips files above maxFileBytes", files: { @@ -591,6 +606,9 @@ describe("scanDirectoryWithSummary", () => { if (testCase.expected.info != null) { expect(summary.info).toBe(testCase.expected.info); } + if (testCase.expected.truncated != null) { + expect(summary.truncated).toBe(testCase.expected.truncated); + } if (testCase.expected.findingCount != null) { expect(summary.findings).toHaveLength(testCase.expected.findingCount); } diff --git a/src/security/skill-scanner.ts b/src/security/skill-scanner.ts index e530183226a..8b0caeee543 100644 --- a/src/security/skill-scanner.ts +++ b/src/security/skill-scanner.ts @@ -23,11 +23,15 @@ export type SkillScanSummary = { critical: number; warn: number; info: number; + truncated: boolean; findings: SkillScanFinding[]; }; export type SkillScanOptions = { excludeTestFiles?: boolean; + includeHiddenDirectories?: boolean; + includeNestedNodeModulesTestFiles?: boolean; + includeNodeModules?: boolean; includeFiles?: string[]; maxFiles?: number; maxFileBytes?: number; @@ -68,6 +72,10 @@ type CachedDirEntry = { name: string; kind: "file" | "dir"; }; +type CollectedScannableFiles = { + files: string[]; + truncated: boolean; +}; type DirEntryCacheEntry = { mtimeMs: number; entries: CachedDirEntry[]; @@ -424,6 +432,9 @@ export function scanSource(source: string, filePath: string): SkillScanFinding[] function normalizeScanOptions(opts?: SkillScanOptions): Required { return { excludeTestFiles: opts?.excludeTestFiles ?? false, + includeHiddenDirectories: opts?.includeHiddenDirectories ?? false, + includeNestedNodeModulesTestFiles: opts?.includeNestedNodeModulesTestFiles ?? false, + includeNodeModules: opts?.includeNodeModules ?? false, includeFiles: opts?.includeFiles ?? [], maxFiles: Math.max(1, opts?.maxFiles ?? DEFAULT_MAX_SCAN_FILES), maxFileBytes: Math.max(1, opts?.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES), @@ -438,15 +449,23 @@ function isExcludedTestFileName(name: string): boolean { return TEST_FILE_NAME_PATTERN.test(name); } +function pathContainsNodeModulesSegment(relativePath: string): boolean { + return relativePath.split(/[\\/]+/u).includes("node_modules"); +} + async function walkDirWithLimit( + rootDir: string, dirPath: string, - maxFiles: number, + candidateLimit: number, excludeTestFiles: boolean, -): Promise { + includeHiddenDirectories: boolean, + includeNestedNodeModulesTestFiles: boolean, + includeNodeModules: boolean, +): Promise { const files: string[] = []; const stack: string[] = [dirPath]; - while (stack.length > 0 && files.length < maxFiles) { + while (stack.length > 0 && files.length < candidateLimit) { const currentDir = stack.pop(); if (!currentDir) { break; @@ -454,22 +473,30 @@ async function walkDirWithLimit( const entries = await readDirEntriesWithCache(currentDir); for (const entry of entries) { - if (files.length >= maxFiles) { + if (files.length >= candidateLimit) { break; } - // Skip hidden dirs and node_modules - if (entry.name.startsWith(".") || entry.name === "node_modules") { - continue; - } if ( - excludeTestFiles && - ((entry.kind === "dir" && isExcludedTestDirectoryName(entry.name)) || - (entry.kind === "file" && isExcludedTestFileName(entry.name))) + (!includeHiddenDirectories && entry.name.startsWith(".")) || + (!includeNodeModules && entry.name === "node_modules") ) { continue; } - const fullPath = path.join(currentDir, entry.name); + const isExcludedTestPath = + entry.kind === "dir" + ? isExcludedTestDirectoryName(entry.name) + : isExcludedTestFileName(entry.name); + if ( + excludeTestFiles && + isExcludedTestPath && + !( + includeNestedNodeModulesTestFiles && + pathContainsNodeModulesSegment(path.relative(rootDir, fullPath)) + ) + ) { + continue; + } if (entry.kind === "dir") { stack.push(fullPath); } else if (entry.kind === "file" && isScannable(entry.name)) { @@ -478,7 +505,7 @@ async function walkDirWithLimit( } } - return files; + return { files, truncated: files.length >= candidateLimit }; } async function readDirEntriesWithCache(dirPath: string): Promise { @@ -559,30 +586,41 @@ async function resolveForcedFiles(params: { return out; } -async function collectScannableFiles(dirPath: string, opts: Required) { +async function collectScannableFiles( + dirPath: string, + opts: Required, +): Promise { const forcedFiles = await resolveForcedFiles({ rootDir: dirPath, includeFiles: opts.includeFiles, }); - if (forcedFiles.length >= opts.maxFiles) { - return forcedFiles.slice(0, opts.maxFiles); + if (forcedFiles.length > opts.maxFiles) { + return { files: forcedFiles.slice(0, opts.maxFiles), truncated: true }; } - const walkedFiles = await walkDirWithLimit(dirPath, opts.maxFiles, opts.excludeTestFiles); + const walked = await walkDirWithLimit( + dirPath, + dirPath, + opts.maxFiles + 1, + opts.excludeTestFiles, + opts.includeHiddenDirectories, + opts.includeNestedNodeModulesTestFiles, + opts.includeNodeModules, + ); const seen = new Set(forcedFiles.map((f) => path.resolve(f))); const out = [...forcedFiles]; - for (const walkedFile of walkedFiles) { - if (out.length >= opts.maxFiles) { - break; - } + for (const walkedFile of walked.files) { const resolved = path.resolve(walkedFile); if (seen.has(resolved)) { continue; } + if (out.length >= opts.maxFiles) { + return { files: out.slice(0, opts.maxFiles), truncated: true }; + } out.push(walkedFile); seen.add(resolved); } - return out; + return { files: out, truncated: false }; } async function scanFileWithCache(params: { @@ -652,7 +690,7 @@ export async function scanDirectory( opts?: SkillScanOptions, ): Promise { const scanOptions = normalizeScanOptions(opts); - const files = await collectScannableFiles(dirPath, scanOptions); + const { files } = await collectScannableFiles(dirPath, scanOptions); const allFindings: SkillScanFinding[] = []; for (const file of files) { @@ -674,7 +712,7 @@ export async function scanDirectoryWithSummary( opts?: SkillScanOptions, ): Promise { const scanOptions = normalizeScanOptions(opts); - const files = await collectScannableFiles(dirPath, scanOptions); + const { files, truncated } = await collectScannableFiles(dirPath, scanOptions); const allFindings: SkillScanFinding[] = []; let scannedFiles = 0; let critical = 0; @@ -707,6 +745,7 @@ export async function scanDirectoryWithSummary( critical, warn, info, + truncated, findings: allFindings, }; }