diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6c40bd995..365d41df729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai - Plugins/source metadata: warn when `openclaw.install.defaultChoice` is invalid or points at a missing source, keeping catalog diagnostics explicit without breaking existing plugins. Thanks @vincentkoc. - Plugins/source metadata: warn when `openclaw.install.expectedIntegrity` is present without a valid npm source, keeping orphaned integrity metadata visible without rejecting existing plugins. Thanks @vincentkoc. - Diagnostics/OTEL: add a lightweight diagnostic trace-context carrier for future span correlation without adding OTEL SDK state to core. Thanks @vincentkoc. +- Dependencies/SBOM: add an ownership-backed dependency risk report for root closure size, native/build-risk packages, and missing owner records. Thanks @vincentkoc. - Diagnostics/OTEL: attach diagnostic trace context to exported OTEL logs so log records can correlate with future spans without adding retained process state. Thanks @vincentkoc. - Diagnostics/OTEL: pass immutable per-run diagnostic trace context through agent and tool hook contexts, and parent exported diagnostic spans from validated context without retaining global trace state. Thanks @vincentkoc. - Diagnostics/OTEL: make exporter startup restart-safe so config reloads do not retain stale SDKs, log transports, or diagnostic event listeners. Thanks @vincentkoc. diff --git a/package.json b/package.json index 07b36043b21..13f6dee469c 100644 --- a/package.json +++ b/package.json @@ -1326,6 +1326,8 @@ "deadcode:ts-unused": "pnpm dlx ts-unused-exports tsconfig.json --ignoreTestFiles --exitWithCount", "deps:root-ownership": "node scripts/root-dependency-ownership-audit.mjs", "deps:root-ownership:check": "node scripts/root-dependency-ownership-audit.mjs --check", + "deps:sbom-risk": "node scripts/sbom-risk-report.mjs", + "deps:sbom-risk:check": "node scripts/sbom-risk-report.mjs --check", "dev": "node scripts/run-node.mjs", "docs:bin": "node scripts/build-docs-list.mjs", "docs:check-i18n-glossary": "node scripts/check-docs-i18n-glossary.mjs", diff --git a/scripts/lib/dependency-ownership.json b/scripts/lib/dependency-ownership.json new file mode 100644 index 00000000000..285970d1abd --- /dev/null +++ b/scripts/lib/dependency-ownership.json @@ -0,0 +1,220 @@ +{ + "schemaVersion": 1, + "dependencies": { + "@agentclientprotocol/sdk": { + "owner": "core:mcp-acp", + "class": "core-runtime", + "risk": ["protocol-client"] + }, + "@anthropic-ai/vertex-sdk": { + "owner": "provider:anthropic-vertex", + "class": "default-runtime-initially", + "risk": ["provider-sdk"] + }, + "@clack/prompts": { + "owner": "core:cli", + "class": "core-runtime", + "risk": ["interactive-cli"] + }, + "@lydell/node-pty": { + "owner": "core:tui-terminal", + "class": "core-runtime", + "risk": ["native", "terminal"] + }, + "@mariozechner/pi-agent-core": { + "owner": "capability:agent-runtime-pi", + "class": "default-runtime-initially", + "risk": ["large-transitive-cone", "agent-runtime"] + }, + "@mariozechner/pi-ai": { + "owner": "capability:agent-runtime-pi", + "class": "default-runtime-initially", + "risk": ["large-transitive-cone", "provider-sdk-fanout"] + }, + "@mariozechner/pi-coding-agent": { + "owner": "capability:agent-runtime-pi", + "class": "default-runtime-initially", + "risk": ["large-transitive-cone", "agent-runtime"] + }, + "@mariozechner/pi-tui": { + "owner": "capability:tui-pi", + "class": "default-runtime-initially", + "risk": ["tui-runtime"] + }, + "@modelcontextprotocol/sdk": { + "owner": "core:mcp", + "class": "core-runtime", + "risk": ["protocol-client", "network"] + }, + "@mozilla/readability": { + "owner": "capability:web-extract-local", + "class": "default-runtime-initially", + "risk": ["parser", "untrusted-html"] + }, + "@napi-rs/canvas": { + "owner": "capability:document-and-image-rendering", + "class": "default-runtime-initially", + "risk": ["native", "parser", "untrusted-files"] + }, + "@vincentkoc/qrcode-tui": { + "owner": "core:qr-setup", + "class": "default-runtime-initially", + "risk": ["terminal-rendering"] + }, + "ajv": { + "owner": "core:json-schema-validation", + "class": "core-runtime", + "risk": ["schema-validation"] + }, + "chalk": { + "owner": "core:cli", + "class": "core-runtime", + "risk": ["formatting"] + }, + "chokidar": { + "owner": "core:watch-mode", + "class": "core-runtime", + "risk": ["filesystem-watch"] + }, + "cli-highlight": { + "owner": "capability:tui", + "class": "default-runtime-initially", + "risk": ["syntax-highlighting", "large-transitive-cone"] + }, + "commander": { + "owner": "core:cli", + "class": "core-runtime", + "risk": ["cli-parser"] + }, + "croner": { + "owner": "core:scheduler", + "class": "core-runtime", + "risk": ["scheduler"] + }, + "dotenv": { + "owner": "core:config", + "class": "core-runtime", + "risk": ["env-loading"] + }, + "express": { + "owner": "capability:http-route-host", + "class": "default-runtime-initially", + "risk": ["http-server", "large-transitive-cone"] + }, + "file-type": { + "owner": "core:media-admission", + "class": "core-runtime", + "risk": ["file-sniffing", "untrusted-files"] + }, + "https-proxy-agent": { + "owner": "core:proxy", + "class": "core-runtime", + "risk": ["network", "proxy"] + }, + "ipaddr.js": { + "owner": "core:ssrf-guard", + "class": "core-runtime", + "risk": ["network-policy"] + }, + "jiti": { + "owner": "core:plugin-loader", + "class": "core-runtime", + "risk": ["dynamic-code-loading"] + }, + "json5": { + "owner": "core:config", + "class": "core-runtime", + "risk": ["config-parser"] + }, + "jszip": { + "owner": "core:archive-handling", + "class": "core-runtime", + "risk": ["archive-parser", "untrusted-files"] + }, + "linkedom": { + "owner": "capability:web-extract-local", + "class": "default-runtime-initially", + "risk": ["parser", "untrusted-html"] + }, + "markdown-it": { + "owner": "core:markdown-rendering", + "class": "core-runtime", + "risk": ["parser", "markdown"] + }, + "node-llama-cpp": { + "owner": "capability:memory-local-embeddings", + "class": "optional-peer-runtime", + "risk": ["native", "local-model-runtime", "large-transitive-cone"] + }, + "openai": { + "owner": "provider:openai", + "class": "default-runtime-initially", + "risk": ["provider-sdk", "network"] + }, + "osc-progress": { + "owner": "core:terminal-progress", + "class": "core-runtime", + "risk": ["terminal-rendering"] + }, + "pdfjs-dist": { + "owner": "capability:document-extract", + "class": "default-runtime-initially", + "risk": ["parser", "untrusted-files"] + }, + "proxy-agent": { + "owner": "core:proxy", + "class": "core-runtime", + "risk": ["network", "proxy"] + }, + "semver": { + "owner": "core:package-versioning", + "class": "core-runtime", + "risk": ["version-parser"] + }, + "sharp": { + "owner": "capability:image-ops", + "class": "default-runtime-initially", + "risk": ["native", "parser", "untrusted-files"] + }, + "sqlite-vec": { + "owner": "capability:memory-sqlite-vec", + "class": "default-runtime-initially", + "risk": ["native", "database-extension"] + }, + "tar": { + "owner": "core:archive-handling", + "class": "core-runtime", + "risk": ["archive-parser", "untrusted-files"] + }, + "tslog": { + "owner": "core:logging", + "class": "core-runtime", + "risk": ["logging"] + }, + "typebox": { + "owner": "core:json-schema-contracts", + "class": "core-runtime", + "risk": ["schema-generation"] + }, + "undici": { + "owner": "core:http-client", + "class": "core-runtime", + "risk": ["network"] + }, + "ws": { + "owner": "core:gateway-websocket", + "class": "core-runtime", + "risk": ["network", "websocket"] + }, + "yaml": { + "owner": "core:config-and-tooling", + "class": "core-runtime", + "risk": ["parser"] + }, + "zod": { + "owner": "core:config-and-plugin-sdk-validation", + "class": "core-runtime", + "risk": ["schema-validation"] + } + } +} diff --git a/scripts/sbom-risk-report.mjs b/scripts/sbom-risk-report.mjs new file mode 100644 index 00000000000..5b3760ac926 --- /dev/null +++ b/scripts/sbom-risk-report.mjs @@ -0,0 +1,310 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { collectRootDependencyOwnershipAudit } from "./root-dependency-ownership-audit.mjs"; + +const DEFAULT_OWNERSHIP_PATH = "scripts/lib/dependency-ownership.json"; +const PROD_IMPORTER_SECTIONS = ["dependencies", "optionalDependencies"]; +const TRANSITIVE_SECTIONS = ["dependencies", "optionalDependencies"]; +const compareStrings = (left, right) => left.localeCompare(right); + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function readLockfile(filePath) { + return parseYaml(fs.readFileSync(filePath, "utf8")); +} + +function normalizeDependencies(record = {}) { + const entries = []; + for (const section of PROD_IMPORTER_SECTIONS) { + for (const [name, value] of Object.entries(record[section] ?? {})) { + const version = + value && typeof value === "object" && "version" in value ? value.version : value; + const specifier = + value && typeof value === "object" && "specifier" in value ? value.specifier : undefined; + if (typeof version === "string") { + entries.push({ name, section, specifier, version }); + } + } + } + return entries.toSorted((left, right) => left.name.localeCompare(right.name)); +} + +export function packageNameFromLockKey(lockKey) { + const peerSuffixIndex = lockKey.indexOf("("); + const baseKey = peerSuffixIndex >= 0 ? lockKey.slice(0, peerSuffixIndex) : lockKey; + if (baseKey.startsWith("@")) { + const secondAt = baseKey.indexOf("@", 1); + return secondAt >= 0 ? baseKey.slice(0, secondAt) : baseKey; + } + const firstAt = baseKey.indexOf("@"); + return firstAt >= 0 ? baseKey.slice(0, firstAt) : baseKey; +} + +function lockKeyForDependency(name, version) { + if (!version || version.startsWith("link:") || version.startsWith("workspace:")) { + return undefined; + } + if (version.startsWith("file:")) { + return undefined; + } + if (version.startsWith("npm:")) { + return version.slice("npm:".length); + } + if (version.startsWith("@")) { + return version; + } + return `${name}@${version}`; +} + +function dependencyEntriesFromSnapshot(snapshot = {}) { + const entries = []; + for (const section of TRANSITIVE_SECTIONS) { + for (const [name, version] of Object.entries(snapshot[section] ?? {})) { + if (typeof version === "string") { + entries.push({ name, version }); + } + } + } + return entries; +} + +function collectClosure(lockfile, rootKeys) { + const seen = new Set(); + const missing = new Set(); + const queue = [...rootKeys].filter(Boolean); + while (queue.length > 0) { + const key = queue.shift(); + if (seen.has(key)) { + continue; + } + seen.add(key); + const snapshot = lockfile.snapshots?.[key]; + if (!snapshot) { + missing.add(key); + continue; + } + for (const dependency of dependencyEntriesFromSnapshot(snapshot)) { + const dependencyKey = lockKeyForDependency(dependency.name, dependency.version); + if (dependencyKey && !seen.has(dependencyKey)) { + queue.push(dependencyKey); + } + } + } + return { + missing: [...missing].toSorted(compareStrings), + packageKeys: [...seen].toSorted(compareStrings), + }; +} + +function collectBuildRiskPackages(lockfile) { + return Object.entries(lockfile.packages ?? {}) + .filter(([, record]) => record.requiresBuild || record.hasBin || record.os || record.cpu) + .map(([lockKey, record]) => ({ + name: packageNameFromLockKey(lockKey), + lockKey, + requiresBuild: record.requiresBuild === true, + hasBin: Boolean(record.hasBin), + platformRestricted: Boolean(record.os || record.cpu || record.libc), + })) + .toSorted((left, right) => left.lockKey.localeCompare(right.lockKey)); +} + +function ownershipFor(dependencyOwnership, name) { + return dependencyOwnership.dependencies?.[name]; +} + +export function collectSbomRiskReport(params = {}) { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const packageJson = readJson(path.join(repoRoot, "package.json")); + const lockfile = readLockfile(path.join(repoRoot, "pnpm-lock.yaml")); + const ownershipPath = path.resolve(repoRoot, params.ownershipPath ?? DEFAULT_OWNERSHIP_PATH); + const dependencyOwnership = readJson(ownershipPath); + const rootImporter = lockfile.importers?.["."] ?? {}; + const rootDependencies = normalizeDependencies(rootImporter); + const sourceAudit = new Map( + collectRootDependencyOwnershipAudit({ repoRoot }).map((record) => [record.depName, record]), + ); + + const rootDependencyRows = rootDependencies.map((dependency) => { + const rootKey = lockKeyForDependency(dependency.name, dependency.version); + const closure = collectClosure(lockfile, rootKey ? [rootKey] : []); + const ownership = ownershipFor(dependencyOwnership, dependency.name); + const sourceRecord = sourceAudit.get(dependency.name); + return { + name: dependency.name, + specifier: + dependency.specifier ?? + packageJson.dependencies?.[dependency.name] ?? + packageJson.optionalDependencies?.[dependency.name] ?? + null, + section: dependency.section, + resolved: dependency.version, + owner: ownership?.owner ?? null, + class: ownership?.class ?? null, + risk: ownership?.risk ?? [], + sourceCategory: sourceRecord?.category ?? null, + sourceSections: sourceRecord?.sections ?? [], + sourceFileCount: sourceRecord?.fileCount ?? 0, + closureSize: closure.packageKeys.length, + missingSnapshotKeys: closure.missing, + }; + }); + + const rootClosure = collectClosure( + lockfile, + rootDependencies + .map((dependency) => lockKeyForDependency(dependency.name, dependency.version)) + .filter(Boolean), + ); + const importerClosures = Object.entries(lockfile.importers ?? {}) + .map(([importer, record]) => { + const dependencies = normalizeDependencies(record); + const closure = collectClosure( + lockfile, + dependencies + .map((dependency) => lockKeyForDependency(dependency.name, dependency.version)) + .filter(Boolean), + ); + return { + importer, + directDependencyCount: dependencies.length, + closureSize: closure.packageKeys.length, + }; + }) + .toSorted((left, right) => { + if (right.closureSize !== left.closureSize) { + return right.closureSize - left.closureSize; + } + return left.importer.localeCompare(right.importer); + }); + + const rootDependencyNames = new Set(rootDependencies.map((dependency) => dependency.name)); + const ownershipGaps = rootDependencies + .filter((dependency) => !ownershipFor(dependencyOwnership, dependency.name)) + .map((dependency) => dependency.name) + .toSorted(compareStrings); + const staleOwnershipRecords = Object.keys(dependencyOwnership.dependencies ?? {}) + .filter((name) => !rootDependencyNames.has(name)) + .toSorted(compareStrings); + const ownershipWarnings = rootDependencyRows + .filter( + (dependency) => + dependency.owner?.startsWith("plugin:") && + (dependency.sourceSections.includes("src") || + dependency.sourceSections.includes("packages") || + dependency.sourceSections.includes("ui")), + ) + .map((dependency) => ({ + name: dependency.name, + owner: dependency.owner, + sourceSections: dependency.sourceSections, + message: "plugin-owned dependency is still imported by core-owned source", + })); + + return { + schemaVersion: 1, + summary: { + importerCount: Object.keys(lockfile.importers ?? {}).length, + lockfilePackageCount: Object.keys(lockfile.packages ?? {}).length, + rootDirectDependencyCount: rootDependencies.length, + rootClosurePackageCount: rootClosure.packageKeys.length, + rootOwnershipRecordCount: Object.keys(dependencyOwnership.dependencies ?? {}).length, + buildRiskPackageCount: collectBuildRiskPackages(lockfile).length, + }, + ownershipGaps, + staleOwnershipRecords, + ownershipWarnings, + buildRiskPackages: collectBuildRiskPackages(lockfile).slice(0, 50), + topRootDependencyCones: rootDependencyRows + .toSorted((left, right) => { + if (right.closureSize !== left.closureSize) { + return right.closureSize - left.closureSize; + } + return left.name.localeCompare(right.name); + }) + .slice(0, 20), + rootDependencies: rootDependencyRows, + importerClosures: importerClosures.slice(0, 30), + }; +} + +export function collectSbomRiskCheckErrors(report) { + return report.ownershipGaps.map( + (name) => `root dependency '${name}' is missing from ${DEFAULT_OWNERSHIP_PATH}`, + ); +} + +function printTextReport(report) { + console.log("# SBOM dependency risk report"); + console.log(""); + console.log(`importers: ${report.summary.importerCount}`); + console.log(`lockfile packages: ${report.summary.lockfilePackageCount}`); + console.log(`root direct dependencies: ${report.summary.rootDirectDependencyCount}`); + console.log(`root closure packages: ${report.summary.rootClosurePackageCount}`); + console.log(`build/native/bin risk packages: ${report.summary.buildRiskPackageCount}`); + console.log(`ownership records: ${report.summary.rootOwnershipRecordCount}`); + if (report.ownershipGaps.length > 0) { + console.log(""); + console.log("## Ownership gaps"); + for (const name of report.ownershipGaps) { + console.log(`- ${name}`); + } + } + if (report.ownershipWarnings.length > 0) { + console.log(""); + console.log("## Ownership warnings"); + for (const warning of report.ownershipWarnings) { + console.log(`- ${warning.name}: ${warning.message} (${warning.sourceSections.join(",")})`); + } + } + console.log(""); + console.log("## Largest root dependency cones"); + for (const dependency of report.topRootDependencyCones) { + const owner = dependency.owner ?? "unowned"; + console.log( + `- ${dependency.name}: closure=${dependency.closureSize} owner=${owner} class=${dependency.class ?? "-"}`, + ); + } + console.log(""); + console.log("## Largest importer closures"); + for (const importer of report.importerClosures.slice(0, 15)) { + console.log( + `- ${importer.importer}: closure=${importer.closureSize} direct=${importer.directDependencyCount}`, + ); + } +} + +function main(argv = process.argv.slice(2)) { + const asJson = argv.includes("--json"); + const check = argv.includes("--check"); + const report = collectSbomRiskReport(); + if (check) { + const errors = collectSbomRiskCheckErrors(report); + if (errors.length > 0) { + for (const error of errors) { + console.error(`[sbom-risk] ${error}`); + } + process.exitCode = 1; + return; + } + if (!asJson) { + console.error("[sbom-risk] ok"); + return; + } + } + if (asJson) { + console.log(JSON.stringify(report, null, 2)); + return; + } + printTextReport(report); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(); +} diff --git a/test/scripts/sbom-risk-report.test.ts b/test/scripts/sbom-risk-report.test.ts new file mode 100644 index 00000000000..7925254dc30 --- /dev/null +++ b/test/scripts/sbom-risk-report.test.ts @@ -0,0 +1,121 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + collectSbomRiskCheckErrors, + collectSbomRiskReport, + packageNameFromLockKey, +} from "../../scripts/sbom-risk-report.mjs"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { force: true, recursive: true }); + } +}); + +function makeTempRepo() { + const dir = mkdtempSync(path.join(tmpdir(), "openclaw-sbom-risk-")); + tempDirs.push(dir); + return dir; +} + +function writeRepoFile(repoRoot: string, relativePath: string, value: string) { + const filePath = path.join(repoRoot, relativePath); + mkdirSync(path.dirname(filePath), { recursive: true }); + writeFileSync(filePath, value, "utf8"); +} + +describe("packageNameFromLockKey", () => { + it("extracts scoped and unscoped names from pnpm snapshot keys", () => { + expect(packageNameFromLockKey("@scope/pkg@1.2.3(peer@1.0.0)")).toBe("@scope/pkg"); + expect(packageNameFromLockKey("left-pad@1.3.0")).toBe("left-pad"); + }); +}); + +describe("collectSbomRiskReport", () => { + it("reports root closure sizes, build-risk packages, and ownership gaps", () => { + const repoRoot = makeTempRepo(); + writeRepoFile( + repoRoot, + "package.json", + JSON.stringify({ + dependencies: { + "core-lib": "1.0.0", + "missing-owner": "2.0.0", + }, + }), + ); + writeRepoFile( + repoRoot, + "pnpm-lock.yaml", + ` +lockfileVersion: '9.0' +importers: + .: + dependencies: + core-lib: + specifier: 1.0.0 + version: 1.0.0 + missing-owner: + specifier: 2.0.0 + version: 2.0.0 + alias-domexception: + specifier: npm:@nolyfill/domexception@1.0.0 + version: npm:@nolyfill/domexception@1.0.0 +packages: + core-lib@1.0.0: {} + transitive-native@1.0.0: + requiresBuild: true + missing-owner@2.0.0: {} + '@nolyfill/domexception@1.0.0': {} +snapshots: + core-lib@1.0.0: + dependencies: + transitive-native: 1.0.0 + alias-domexception: '@nolyfill/domexception@1.0.0' + transitive-native@1.0.0: {} + missing-owner@2.0.0: {} + '@nolyfill/domexception@1.0.0': {} +`, + ); + writeRepoFile( + repoRoot, + "scripts/lib/dependency-ownership.json", + JSON.stringify({ + schemaVersion: 1, + dependencies: { + "alias-domexception": { + owner: "core:test", + class: "core-runtime", + risk: ["compat"], + }, + "core-lib": { owner: "core:test", class: "core-runtime", risk: ["network"] }, + }, + }), + ); + writeRepoFile(repoRoot, "src/index.ts", 'import "core-lib";\n'); + + const report = collectSbomRiskReport({ repoRoot }); + + expect(report.summary).toMatchObject({ + buildRiskPackageCount: 1, + importerCount: 1, + lockfilePackageCount: 4, + rootClosurePackageCount: 4, + rootDirectDependencyCount: 3, + rootOwnershipRecordCount: 2, + }); + expect(report.ownershipGaps).toEqual(["missing-owner"]); + expect(report.topRootDependencyCones[0]).toMatchObject({ + closureSize: 3, + name: "core-lib", + owner: "core:test", + }); + expect(collectSbomRiskCheckErrors(report)).toEqual([ + "root dependency 'missing-owner' is missing from scripts/lib/dependency-ownership.json", + ]); + }); +});