From 4aedffd37aedd5afa339cefcfe755986df9efaaa Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 20:22:41 +0100 Subject: [PATCH] refactor(plugins): split loader runtime helpers (#74545) * refactor(plugins): split loader runtime helpers * test(scripts): include discord api barrel lane * test(ci): align built artifact guard expectations * fix(plugins): avoid redundant cache key assertion --- .../plugins/bundled.shape-guard.test.ts | 3 +- .../bundled-runtime-deps-jiti-aliases.test.ts | 107 +++ src/plugins/bundled-runtime-deps-json.ts | 95 +++ src/plugins/bundled-runtime-deps-specs.ts | 104 +++ src/plugins/bundled-runtime-deps.ts | 217 +----- src/plugins/loader-channel-setup.ts | 224 ++++++ src/plugins/loader-provenance.ts | 268 +++++++ src/plugins/loader-records.ts | 195 +++++ src/plugins/loader.ts | 694 +----------------- 9 files changed, 1039 insertions(+), 868 deletions(-) create mode 100644 src/plugins/bundled-runtime-deps-jiti-aliases.test.ts create mode 100644 src/plugins/bundled-runtime-deps-json.ts create mode 100644 src/plugins/bundled-runtime-deps-specs.ts create mode 100644 src/plugins/loader-channel-setup.ts create mode 100644 src/plugins/loader-provenance.ts create mode 100644 src/plugins/loader-records.ts diff --git a/src/channels/plugins/bundled.shape-guard.test.ts b/src/channels/plugins/bundled.shape-guard.test.ts index b76e8fa8efd..009d34c3645 100644 --- a/src/channels/plugins/bundled.shape-guard.test.ts +++ b/src/channels/plugins/bundled.shape-guard.test.ts @@ -698,6 +698,7 @@ describe("bundled channel entry shape guards", () => { " name: 'Alpha',", " description: 'Alpha',", " importMetaUrl: import.meta.url,", + " features: { accountInspect: true },", " plugin: { specifier: './plugin.js' },", "});", "", @@ -733,7 +734,7 @@ describe("bundled channel entry shape guards", () => { "./bundled.js?scope=bundled-runtime-deps", ); - expect(bundled.getBundledChannelPlugin("alpha")).toBeUndefined(); + expect(bundled.hasBundledChannelEntryFeature("alpha", "accountInspect")).toBe(true); expect(testGlobal.__bundledRuntimeDepMarker).toBeUndefined(); } finally { restoreBundledPluginsDir(previousBundledPluginsDir); diff --git a/src/plugins/bundled-runtime-deps-jiti-aliases.test.ts b/src/plugins/bundled-runtime-deps-jiti-aliases.test.ts new file mode 100644 index 00000000000..5e1ef201c39 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-jiti-aliases.test.ts @@ -0,0 +1,107 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + clearBundledRuntimeDependencyJitiAliases, + registerBundledRuntimeDependencyJitiAliases, + resolveBundledRuntimeDependencyJitiAliasMap, +} from "./bundled-runtime-deps-jiti-aliases.js"; + +const tempDirs: string[] = []; + +function makeTempRoot(): string { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-runtime-aliases-")); + tempDirs.push(tempDir); + return tempDir; +} + +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function writeFile(filePath: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, "export default null;\n", "utf8"); +} + +function packageRoot(rootDir: string, packageName: string): string { + return path.join(rootDir, "node_modules", ...packageName.split("/")); +} + +afterEach(() => { + clearBundledRuntimeDependencyJitiAliases(); + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +describe("bundled runtime dependency Jiti aliases", () => { + it("registers root, subpath, wildcard, and scoped package aliases", () => { + const rootDir = makeTempRoot(); + writeJson(path.join(rootDir, "package.json"), { + dependencies: { + plain: "1.0.0", + wild: "1.0.0", + "@scope/pkg": "1.0.0", + }, + }); + + const plainRoot = packageRoot(rootDir, "plain"); + writeJson(path.join(plainRoot, "package.json"), { + exports: { + ".": { import: "./esm/index.js", default: "./cjs/index.js" }, + "./feature": "./features/feature.js", + }, + }); + writeFile(path.join(plainRoot, "esm/index.js")); + writeFile(path.join(plainRoot, "features/feature.js")); + + const wildRoot = packageRoot(rootDir, "wild"); + writeJson(path.join(wildRoot, "package.json"), { + exports: { + "./sub/*": "./dist/*.js", + }, + }); + writeFile(path.join(wildRoot, "dist/a.js")); + writeFile(path.join(wildRoot, "dist/nested/b.js")); + + const scopedRoot = packageRoot(rootDir, "@scope/pkg"); + writeJson(path.join(scopedRoot, "package.json"), { + module: "./index.mjs", + }); + writeFile(path.join(scopedRoot, "index.mjs")); + + registerBundledRuntimeDependencyJitiAliases(rootDir); + + expect(resolveBundledRuntimeDependencyJitiAliasMap()).toEqual({ + "wild/sub/nested/b": path.join(wildRoot, "dist/nested/b.js"), + "plain/feature": path.join(plainRoot, "features/feature.js"), + "@scope/pkg": path.join(scopedRoot, "index.mjs"), + "wild/sub/a": path.join(wildRoot, "dist/a.js"), + plain: path.join(plainRoot, "esm/index.js"), + }); + }); + + it("ignores missing, private, and escaping export targets", () => { + const rootDir = makeTempRoot(); + writeJson(path.join(rootDir, "package.json"), { + dependencies: { + unsafe: "1.0.0", + }, + }); + const unsafeRoot = packageRoot(rootDir, "unsafe"); + writeJson(path.join(unsafeRoot, "package.json"), { + exports: { + ".": "../outside.js", + "./private": "#internal", + "./missing": "./missing.js", + }, + }); + + registerBundledRuntimeDependencyJitiAliases(rootDir); + + expect(resolveBundledRuntimeDependencyJitiAliasMap()).toBeUndefined(); + }); +}); diff --git a/src/plugins/bundled-runtime-deps-json.ts b/src/plugins/bundled-runtime-deps-json.ts new file mode 100644 index 00000000000..030a7650360 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-json.ts @@ -0,0 +1,95 @@ +import fs from "node:fs"; + +export type JsonObject = Record; + +const MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES = 2048; + +const runtimeDepsTextFileCache = new Map(); +const runtimeDepsJsonObjectCache = new Map< + string, + { signature: string; value: JsonObject | null } +>(); + +export function readRuntimeDepsJsonObject(filePath: string): JsonObject | null { + const signature = getRuntimeDepsFileSignature(filePath); + const cached = signature ? runtimeDepsJsonObjectCache.get(filePath) : undefined; + if (cached?.signature === signature) { + return cached.value; + } + const source = readRuntimeDepsTextFile(filePath, signature); + if (source === null) { + cacheRuntimeDepsJsonObject(filePath, signature, null); + return null; + } + try { + const parsed = JSON.parse(source) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + cacheRuntimeDepsJsonObject(filePath, signature, null); + return null; + } + const value = parsed as JsonObject; + cacheRuntimeDepsJsonObject(filePath, signature, value); + return value; + } catch { + cacheRuntimeDepsJsonObject(filePath, signature, null); + return null; + } +} + +function readRuntimeDepsTextFile(filePath: string, signature?: string | null): string | null { + const fileSignature = signature ?? getRuntimeDepsFileSignature(filePath); + const cached = fileSignature ? runtimeDepsTextFileCache.get(filePath) : undefined; + if (cached?.signature === fileSignature) { + return cached.value; + } + try { + const value = fs.readFileSync(filePath, "utf8"); + if (fileSignature) { + rememberRuntimeDepsCacheEntry(runtimeDepsTextFileCache, filePath, { + signature: fileSignature, + value, + }); + } + return value; + } catch { + return null; + } +} + +function getRuntimeDepsFileSignature(filePath: string): string | null { + try { + const stat = fs.statSync(filePath, { bigint: true }); + if (!stat.isFile()) { + return null; + } + return [ + stat.dev.toString(), + stat.ino.toString(), + stat.size.toString(), + stat.mtimeNs.toString(), + ].join(":"); + } catch { + return null; + } +} + +function cacheRuntimeDepsJsonObject( + filePath: string, + signature: string | null, + value: JsonObject | null, +): void { + if (!signature) { + return; + } + rememberRuntimeDepsCacheEntry(runtimeDepsJsonObjectCache, filePath, { signature, value }); +} + +function rememberRuntimeDepsCacheEntry(cache: Map, key: string, value: T): void { + if (cache.size >= MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES && !cache.has(key)) { + const oldestKey = cache.keys().next().value; + if (oldestKey !== undefined) { + cache.delete(oldestKey); + } + } + cache.set(key, value); +} diff --git a/src/plugins/bundled-runtime-deps-specs.ts b/src/plugins/bundled-runtime-deps-specs.ts new file mode 100644 index 00000000000..d3fd3632128 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-specs.ts @@ -0,0 +1,104 @@ +import path from "node:path"; +import { validSemver } from "./semver.runtime.js"; + +export type RuntimeDepEntry = { + name: string; + version: string; + pluginIds: string[]; +}; + +const BUNDLED_RUNTIME_DEP_SEGMENT_RE = /^[a-z0-9][a-z0-9._-]*$/; + +export function normalizeInstallableRuntimeDepName(rawName: string): string | null { + const depName = rawName.trim(); + if (depName === "") { + return null; + } + const segments = depName.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + return null; + } + if (segments.length === 1) { + return BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(segments[0] ?? "") ? depName : null; + } + if (segments.length !== 2 || !segments[0]?.startsWith("@")) { + return null; + } + const scope = segments[0].slice(1); + const packageName = segments[1]; + return BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(scope) && + BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(packageName ?? "") + ? depName + : null; +} + +function normalizeInstallableRuntimeDepVersion(rawVersion: unknown): string | null { + if (typeof rawVersion !== "string") { + return null; + } + const version = rawVersion.trim(); + if (version === "" || version.toLowerCase().startsWith("workspace:")) { + return null; + } + if (validSemver(version)) { + return version; + } + const rangePrefix = version[0]; + if ((rangePrefix === "^" || rangePrefix === "~") && validSemver(version.slice(1))) { + return version; + } + return null; +} + +export function parseInstallableRuntimeDep( + name: string, + rawVersion: unknown, +): { name: string; version: string } | null { + if (typeof rawVersion !== "string") { + return null; + } + const version = rawVersion.trim(); + if (version === "" || version.toLowerCase().startsWith("workspace:")) { + return null; + } + const normalizedName = normalizeInstallableRuntimeDepName(name); + if (!normalizedName) { + throw new Error(`Invalid bundled runtime dependency name: ${name}`); + } + const normalizedVersion = normalizeInstallableRuntimeDepVersion(version); + if (!normalizedVersion) { + throw new Error( + `Unsupported bundled runtime dependency spec for ${normalizedName}: ${version}`, + ); + } + return { name: normalizedName, version: normalizedVersion }; +} + +export function parseInstallableRuntimeDepSpec(spec: string): { name: string; version: string } { + const atIndex = spec.lastIndexOf("@"); + if (atIndex <= 0 || atIndex === spec.length - 1) { + throw new Error(`Invalid bundled runtime dependency install spec: ${spec}`); + } + const parsed = parseInstallableRuntimeDep(spec.slice(0, atIndex), spec.slice(atIndex + 1)); + if (!parsed) { + throw new Error(`Invalid bundled runtime dependency install spec: ${spec}`); + } + return parsed; +} + +function dependencySentinelPath(depName: string): string { + const normalizedDepName = normalizeInstallableRuntimeDepName(depName); + if (!normalizedDepName) { + throw new Error(`Invalid bundled runtime dependency name: ${depName}`); + } + return path.join("node_modules", ...normalizedDepName.split("/"), "package.json"); +} + +export function resolveDependencySentinelAbsolutePath(rootDir: string, depName: string): string { + const nodeModulesDir = path.resolve(rootDir, "node_modules"); + const sentinelPath = path.resolve(rootDir, dependencySentinelPath(depName)); + if (sentinelPath !== nodeModulesDir && !sentinelPath.startsWith(`${nodeModulesDir}${path.sep}`)) { + throw new Error(`Blocked runtime dependency path escape for ${depName}`); + } + return sentinelPath; +} diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 3042c6eda92..0d384dd2bd7 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -11,6 +11,7 @@ import { resolveHomeRelativePath } from "../infra/home-dir.js"; import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; import { sanitizeTerminalText } from "../terminal/safe-text.js"; import { beginBundledRuntimeDepsInstall } from "./bundled-runtime-deps-activity.js"; +import { readRuntimeDepsJsonObject, type JsonObject } from "./bundled-runtime-deps-json.js"; import { BUNDLED_RUNTIME_DEPS_LOCK_DIR, formatRuntimeDepsLockTimeoutMessage, @@ -29,12 +30,19 @@ import { type BundledRuntimeDepsPackageManager, type BundledRuntimeDepsPackageManagerRunner, } from "./bundled-runtime-deps-package-manager.js"; +import { + normalizeInstallableRuntimeDepName, + parseInstallableRuntimeDep, + parseInstallableRuntimeDepSpec, + resolveDependencySentinelAbsolutePath, + type RuntimeDepEntry, +} from "./bundled-runtime-deps-specs.js"; import { normalizePluginsConfigWithResolver, type NormalizedPluginsConfig, type NormalizePluginId, } from "./config-normalization-shared.js"; -import { satisfies, validSemver } from "./semver.runtime.js"; +import { satisfies } from "./semver.runtime.js"; export { createBundledRuntimeDepsInstallArgs, @@ -43,6 +51,7 @@ export { withBundledRuntimeDepsFilesystemLock, }; export type { BundledRuntimeDepsNpmRunner }; +export type { RuntimeDepEntry } from "./bundled-runtime-deps-specs.js"; export const __testing = { formatRuntimeDepsLockTimeoutMessage, @@ -50,12 +59,6 @@ export const __testing = { shouldRemoveRuntimeDepsLock, }; -export type RuntimeDepEntry = { - name: string; - version: string; - pluginIds: string[]; -}; - export type RuntimeDepConflict = { name: string; versions: string[]; @@ -91,7 +94,6 @@ export type BundledRuntimeDepsPlan = { installRootPlan: BundledRuntimeDepsInstallRootPlan; }; -type JsonObject = Record; const LEGACY_RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json"; // Packaged bundled plugins (Docker image, npm global install) keep their // `package.json` next to their entry point; running `npm install ` with @@ -104,14 +106,8 @@ const DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP = 20; const DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS = 10 * 60_000; const BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS = 5_000; const MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID = "openclaw-core"; -const MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES = 2048; const registeredBundledRuntimeDepNodePaths = new Set(); -const runtimeDepsTextFileCache = new Map(); -const runtimeDepsJsonObjectCache = new Map< - string, - { signature: string; value: JsonObject | null } ->(); function createBundledRuntimeDepsEnsureResult( installedSpecs: string[], @@ -119,183 +115,6 @@ function createBundledRuntimeDepsEnsureResult( return { installedSpecs }; } -const BUNDLED_RUNTIME_DEP_SEGMENT_RE = /^[a-z0-9][a-z0-9._-]*$/; - -function normalizeInstallableRuntimeDepName(rawName: string): string | null { - const depName = rawName.trim(); - if (depName === "") { - return null; - } - const segments = depName.split("/"); - if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { - return null; - } - if (segments.length === 1) { - return BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(segments[0] ?? "") ? depName : null; - } - if (segments.length !== 2 || !segments[0]?.startsWith("@")) { - return null; - } - const scope = segments[0].slice(1); - const packageName = segments[1]; - return BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(scope) && - BUNDLED_RUNTIME_DEP_SEGMENT_RE.test(packageName ?? "") - ? depName - : null; -} - -function normalizeInstallableRuntimeDepVersion(rawVersion: unknown): string | null { - if (typeof rawVersion !== "string") { - return null; - } - const version = rawVersion.trim(); - if (version === "" || version.toLowerCase().startsWith("workspace:")) { - return null; - } - if (validSemver(version)) { - return version; - } - const rangePrefix = version[0]; - if ((rangePrefix === "^" || rangePrefix === "~") && validSemver(version.slice(1))) { - return version; - } - return null; -} - -function parseInstallableRuntimeDep( - name: string, - rawVersion: unknown, -): { name: string; version: string } | null { - if (typeof rawVersion !== "string") { - return null; - } - const version = rawVersion.trim(); - if (version === "" || version.toLowerCase().startsWith("workspace:")) { - return null; - } - const normalizedName = normalizeInstallableRuntimeDepName(name); - if (!normalizedName) { - throw new Error(`Invalid bundled runtime dependency name: ${name}`); - } - const normalizedVersion = normalizeInstallableRuntimeDepVersion(version); - if (!normalizedVersion) { - throw new Error( - `Unsupported bundled runtime dependency spec for ${normalizedName}: ${version}`, - ); - } - return { name: normalizedName, version: normalizedVersion }; -} - -function parseInstallableRuntimeDepSpec(spec: string): { name: string; version: string } { - const atIndex = spec.lastIndexOf("@"); - if (atIndex <= 0 || atIndex === spec.length - 1) { - throw new Error(`Invalid bundled runtime dependency install spec: ${spec}`); - } - const parsed = parseInstallableRuntimeDep(spec.slice(0, atIndex), spec.slice(atIndex + 1)); - if (!parsed) { - throw new Error(`Invalid bundled runtime dependency install spec: ${spec}`); - } - return parsed; -} - -function dependencySentinelPath(depName: string): string { - const normalizedDepName = normalizeInstallableRuntimeDepName(depName); - if (!normalizedDepName) { - throw new Error(`Invalid bundled runtime dependency name: ${depName}`); - } - return path.join("node_modules", ...normalizedDepName.split("/"), "package.json"); -} - -function resolveDependencySentinelAbsolutePath(rootDir: string, depName: string): string { - const nodeModulesDir = path.resolve(rootDir, "node_modules"); - const sentinelPath = path.resolve(rootDir, dependencySentinelPath(depName)); - if (sentinelPath !== nodeModulesDir && !sentinelPath.startsWith(`${nodeModulesDir}${path.sep}`)) { - throw new Error(`Blocked runtime dependency path escape for ${depName}`); - } - return sentinelPath; -} - -function readJsonObject(filePath: string): JsonObject | null { - const signature = getRuntimeDepsFileSignature(filePath); - const cached = signature ? runtimeDepsJsonObjectCache.get(filePath) : undefined; - if (cached?.signature === signature) { - return cached.value; - } - const source = readRuntimeDepsTextFile(filePath, signature); - if (source === null) { - cacheRuntimeDepsJsonObject(filePath, signature, null); - return null; - } - try { - const parsed = JSON.parse(source) as unknown; - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - cacheRuntimeDepsJsonObject(filePath, signature, null); - return null; - } - const value = parsed as JsonObject; - cacheRuntimeDepsJsonObject(filePath, signature, value); - return value; - } catch { - cacheRuntimeDepsJsonObject(filePath, signature, null); - return null; - } -} - -function readRuntimeDepsTextFile(filePath: string, signature?: string | null): string | null { - const fileSignature = signature ?? getRuntimeDepsFileSignature(filePath); - const cached = fileSignature ? runtimeDepsTextFileCache.get(filePath) : undefined; - if (cached?.signature === fileSignature) { - return cached.value; - } - try { - const value = fs.readFileSync(filePath, "utf8"); - if (fileSignature) { - rememberRuntimeDepsCacheEntry(runtimeDepsTextFileCache, filePath, { - signature: fileSignature, - value, - }); - } - return value; - } catch { - return null; - } -} - -function getRuntimeDepsFileSignature(filePath: string): string | null { - try { - const stat = fs.statSync(filePath, { bigint: true }); - if (!stat.isFile()) { - return null; - } - return [ - stat.dev.toString(), - stat.ino.toString(), - stat.size.toString(), - stat.mtimeNs.toString(), - ].join(":"); - } catch { - return null; - } -} - -function cacheRuntimeDepsJsonObject( - filePath: string, - signature: string | null, - value: JsonObject | null, -): void { - if (!signature) { - return; - } - rememberRuntimeDepsCacheEntry(runtimeDepsJsonObjectCache, filePath, { signature, value }); -} - -function rememberRuntimeDepsCacheEntry(cache: Map, key: string, value: T): void { - if (cache.size >= MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES && !cache.has(key)) { - cache.delete(cache.keys().next().value as string); - } - cache.set(key, value); -} - function withBundledRuntimeDepsInstallRootLock(installRoot: string, run: () => T): T { return withBundledRuntimeDepsFilesystemLock(installRoot, BUNDLED_RUNTIME_DEPS_LOCK_DIR, run); } @@ -356,7 +175,7 @@ function collectMirroredPackageRuntimeDeps(packageRoot: string | null): { if (!packageRoot) { return []; } - const packageJson = readJsonObject(path.join(packageRoot, "package.json")); + const packageJson = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); if (!packageJson) { return []; } @@ -471,7 +290,7 @@ function sanitizePathSegment(value: string): string { } function readPackageVersion(packageRoot: string): string { - const parsed = readJsonObject(path.join(packageRoot, "package.json")); + const parsed = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); const version = parsed && typeof parsed.version === "string" ? parsed.version.trim() : ""; return version || "unknown"; } @@ -484,7 +303,7 @@ function normalizeRuntimeDepSpecs(specs: readonly string[]): string[] { } function readGeneratedInstallManifestSpecs(installRoot: string): string[] | null { - const parsed = readJsonObject(path.join(installRoot, "package.json")); + const parsed = readRuntimeDepsJsonObject(path.join(installRoot, "package.json")); if (parsed?.name !== "openclaw-runtime-deps-install") { return null; } @@ -503,7 +322,7 @@ function readGeneratedInstallManifestSpecs(installRoot: string): string[] | null } function readPackageRuntimeDepSpecs(packageRoot: string): string[] | null { - const parsed = readJsonObject(path.join(packageRoot, "package.json")); + const parsed = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); if (!parsed || parsed.name === "openclaw-runtime-deps-install") { return null; } @@ -850,7 +669,7 @@ function readBundledPluginRuntimeDepsManifest( if (cached) { return cached; } - const manifest = readJsonObject(path.join(pluginDir, "openclaw.plugin.json")); + const manifest = readRuntimeDepsJsonObject(path.join(pluginDir, "openclaw.plugin.json")); const channels = manifest?.channels; const legacyPluginIds = manifest?.legacyPluginIds; const providers = manifest?.providers; @@ -1173,7 +992,7 @@ function collectBundledPluginRuntimeDeps(params: { continue; } includedPluginIds.add(pluginId); - const packageJson = readJsonObject(path.join(pluginDir, "package.json")); + const packageJson = readRuntimeDepsJsonObject(path.join(pluginDir, "package.json")); if (!packageJson) { continue; } @@ -1413,7 +1232,7 @@ export function createBundledRuntimeDependencyAliasMap(params: { if (path.resolve(params.installRoot) === path.resolve(params.pluginRoot)) { return {}; } - const packageJson = readJsonObject(path.join(params.pluginRoot, "package.json")); + const packageJson = readRuntimeDepsJsonObject(path.join(params.pluginRoot, "package.json")); if (!packageJson) { return {}; } @@ -1869,7 +1688,7 @@ export function ensureBundledPluginRuntimeDeps(params: { ) { return createBundledRuntimeDepsEnsureResult([]); } - const packageJson = readJsonObject(path.join(params.pluginRoot, "package.json")); + const packageJson = readRuntimeDepsJsonObject(path.join(params.pluginRoot, "package.json")); if (!packageJson) { return createBundledRuntimeDepsEnsureResult([]); } diff --git a/src/plugins/loader-channel-setup.ts b/src/plugins/loader-channel-setup.ts new file mode 100644 index 00000000000..f86ece78e69 --- /dev/null +++ b/src/plugins/loader-channel-setup.ts @@ -0,0 +1,224 @@ +import type { ChannelPlugin } from "../channels/plugins/types.plugin.js"; +import { isChannelConfigured } from "../config/channel-configured.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { unwrapDefaultModuleExport } from "./module-export.js"; +import type { PluginRuntime } from "./runtime/types.js"; + +function mergeChannelPluginSection( + baseValue: T | undefined, + overrideValue: T | undefined, +): T | undefined { + if ( + baseValue && + overrideValue && + typeof baseValue === "object" && + typeof overrideValue === "object" + ) { + const merged = { + ...(baseValue as Record), + }; + for (const [key, value] of Object.entries(overrideValue as Record)) { + if (value !== undefined) { + merged[key] = value; + } + } + return { + ...merged, + } as T; + } + return overrideValue ?? baseValue; +} + +export function mergeSetupRuntimeChannelPlugin( + runtimePlugin: ChannelPlugin, + setupPlugin: ChannelPlugin, +): ChannelPlugin { + return { + ...runtimePlugin, + ...setupPlugin, + meta: mergeChannelPluginSection(runtimePlugin.meta, setupPlugin.meta), + capabilities: mergeChannelPluginSection(runtimePlugin.capabilities, setupPlugin.capabilities), + commands: mergeChannelPluginSection(runtimePlugin.commands, setupPlugin.commands), + doctor: mergeChannelPluginSection(runtimePlugin.doctor, setupPlugin.doctor), + reload: mergeChannelPluginSection(runtimePlugin.reload, setupPlugin.reload), + config: mergeChannelPluginSection(runtimePlugin.config, setupPlugin.config), + setup: mergeChannelPluginSection(runtimePlugin.setup, setupPlugin.setup), + messaging: mergeChannelPluginSection(runtimePlugin.messaging, setupPlugin.messaging), + actions: mergeChannelPluginSection(runtimePlugin.actions, setupPlugin.actions), + secrets: mergeChannelPluginSection(runtimePlugin.secrets, setupPlugin.secrets), + } as ChannelPlugin; +} + +export type BundledRuntimeChannelRegistration = { + id?: string; + loadChannelPlugin?: () => ChannelPlugin; + loadChannelSecrets?: () => ChannelPlugin["secrets"] | undefined; + setChannelRuntime?: (runtime: PluginRuntime) => void; +}; + +export function resolveBundledRuntimeChannelRegistration( + moduleExport: unknown, +): BundledRuntimeChannelRegistration { + const resolved = unwrapDefaultModuleExport(moduleExport); + if (!resolved || typeof resolved !== "object") { + return {}; + } + const entryRecord = resolved as { + kind?: unknown; + id?: unknown; + loadChannelPlugin?: unknown; + loadChannelSecrets?: unknown; + setChannelRuntime?: unknown; + }; + if ( + entryRecord.kind !== "bundled-channel-entry" || + typeof entryRecord.id !== "string" || + typeof entryRecord.loadChannelPlugin !== "function" + ) { + return {}; + } + return { + id: entryRecord.id, + loadChannelPlugin: entryRecord.loadChannelPlugin as () => ChannelPlugin, + ...(typeof entryRecord.loadChannelSecrets === "function" + ? { + loadChannelSecrets: entryRecord.loadChannelSecrets as () => + | ChannelPlugin["secrets"] + | undefined, + } + : {}), + ...(typeof entryRecord.setChannelRuntime === "function" + ? { + setChannelRuntime: entryRecord.setChannelRuntime as (runtime: PluginRuntime) => void, + } + : {}), + }; +} + +export function loadBundledRuntimeChannelPlugin(params: { + registration: BundledRuntimeChannelRegistration; +}): { + plugin?: ChannelPlugin; + loadError?: unknown; +} { + if (typeof params.registration.loadChannelPlugin !== "function") { + return {}; + } + try { + const loadedPlugin = params.registration.loadChannelPlugin(); + const loadedSecrets = params.registration.loadChannelSecrets?.(); + if (!loadedPlugin || typeof loadedPlugin !== "object") { + return {}; + } + const mergedSecrets = mergeChannelPluginSection(loadedPlugin.secrets, loadedSecrets); + return { + plugin: { + ...loadedPlugin, + ...(mergedSecrets !== undefined ? { secrets: mergedSecrets } : {}), + }, + }; + } catch (err) { + return { loadError: err }; + } +} + +export function resolveSetupChannelRegistration( + moduleExport: unknown, + params: { installRuntimeDeps?: boolean } = {}, +): { + plugin?: ChannelPlugin; + setChannelRuntime?: (runtime: PluginRuntime) => void; + usesBundledSetupContract?: boolean; + loadError?: unknown; +} { + const resolved = unwrapDefaultModuleExport(moduleExport); + if (!resolved || typeof resolved !== "object") { + return {}; + } + const setupEntryRecord = resolved as { + kind?: unknown; + loadSetupPlugin?: unknown; + loadSetupSecrets?: unknown; + setChannelRuntime?: unknown; + }; + if ( + setupEntryRecord.kind === "bundled-channel-setup-entry" && + typeof setupEntryRecord.loadSetupPlugin === "function" + ) { + try { + const setupLoadOptions = + params.installRuntimeDeps === false ? { installRuntimeDeps: false } : undefined; + const loadedPlugin = setupEntryRecord.loadSetupPlugin(setupLoadOptions); + const loadedSecrets = + typeof setupEntryRecord.loadSetupSecrets === "function" + ? (setupEntryRecord.loadSetupSecrets(setupLoadOptions) as + | ChannelPlugin["secrets"] + | undefined) + : undefined; + if (loadedPlugin && typeof loadedPlugin === "object") { + const mergedSecrets = mergeChannelPluginSection( + (loadedPlugin as ChannelPlugin).secrets, + loadedSecrets, + ); + return { + plugin: { + ...(loadedPlugin as ChannelPlugin), + ...(mergedSecrets !== undefined ? { secrets: mergedSecrets } : {}), + }, + usesBundledSetupContract: true, + ...(typeof setupEntryRecord.setChannelRuntime === "function" + ? { + setChannelRuntime: setupEntryRecord.setChannelRuntime as ( + runtime: PluginRuntime, + ) => void, + } + : {}), + }; + } + } catch (err) { + return { loadError: err }; + } + } + const setup = resolved as { + plugin?: unknown; + }; + if (!setup.plugin || typeof setup.plugin !== "object") { + return {}; + } + return { + plugin: setup.plugin as ChannelPlugin, + }; +} + +export function shouldLoadChannelPluginInSetupRuntime(params: { + manifestChannels: string[]; + setupSource?: string; + startupDeferConfiguredChannelFullLoadUntilAfterListen?: boolean; + cfg: OpenClawConfig; + env: NodeJS.ProcessEnv; + preferSetupRuntimeForChannelPlugins?: boolean; +}): boolean { + if (!params.setupSource || params.manifestChannels.length === 0) { + return false; + } + if ( + params.preferSetupRuntimeForChannelPlugins && + params.startupDeferConfiguredChannelFullLoadUntilAfterListen === true + ) { + return true; + } + return !params.manifestChannels.some((channelId) => + isChannelConfigured(params.cfg, channelId, params.env), + ); +} + +export function channelPluginIdBelongsToManifest(params: { + channelId: string | undefined; + pluginId: string; + manifestChannels: readonly string[]; +}): boolean { + if (!params.channelId) { + return true; + } + return params.channelId === params.pluginId || params.manifestChannels.includes(params.channelId); +} diff --git a/src/plugins/loader-provenance.ts b/src/plugins/loader-provenance.ts new file mode 100644 index 00000000000..cddf50c7d85 --- /dev/null +++ b/src/plugins/loader-provenance.ts @@ -0,0 +1,268 @@ +import { normalizeOptionalString } from "../shared/string-coerce.js"; +import { resolveUserPath } from "../utils.js"; +import type { PluginCandidate } from "./discovery.js"; +import { loadInstalledPluginIndexInstallRecordsSync } from "./installed-plugin-index-records.js"; +import type { PluginManifestRecord } from "./manifest-registry.js"; +import { isPathInside, safeStatSync } from "./path-safety.js"; +import type { PluginRecord, PluginRegistry } from "./registry.js"; +import type { PluginLogger } from "./types.js"; + +type PathMatcher = { + exact: Set; + dirs: string[]; +}; + +type InstallTrackingRule = { + trackedWithoutPaths: boolean; + matcher: PathMatcher; +}; + +export type PluginProvenanceIndex = { + loadPathMatcher: PathMatcher; + installRules: Map; +}; + +type OpenAllowlistWarningCache = { + hasOpenAllowlistWarning(cacheKey: string): boolean; + recordOpenAllowlistWarning(cacheKey: string): void; +}; + +function createPathMatcher(): PathMatcher { + return { exact: new Set(), dirs: [] }; +} + +function addPathToMatcher( + matcher: PathMatcher, + rawPath: string, + env: NodeJS.ProcessEnv = process.env, +): void { + const trimmed = rawPath.trim(); + if (!trimmed) { + return; + } + const resolved = resolveUserPath(trimmed, env); + if (!resolved) { + return; + } + if (matcher.exact.has(resolved) || matcher.dirs.includes(resolved)) { + return; + } + const stat = safeStatSync(resolved); + if (stat?.isDirectory()) { + matcher.dirs.push(resolved); + return; + } + matcher.exact.add(resolved); +} + +function matchesPathMatcher(matcher: PathMatcher, sourcePath: string): boolean { + if (matcher.exact.has(sourcePath)) { + return true; + } + return matcher.dirs.some((dirPath) => isPathInside(dirPath, sourcePath)); +} + +export function buildProvenanceIndex(params: { + normalizedLoadPaths: string[]; + env: NodeJS.ProcessEnv; +}): PluginProvenanceIndex { + const loadPathMatcher = createPathMatcher(); + for (const loadPath of params.normalizedLoadPaths) { + addPathToMatcher(loadPathMatcher, loadPath, params.env); + } + + const installRules = new Map(); + const installs = loadInstalledPluginIndexInstallRecordsSync({ env: params.env }); + for (const [pluginId, install] of Object.entries(installs)) { + const rule: InstallTrackingRule = { + trackedWithoutPaths: false, + matcher: createPathMatcher(), + }; + const trackedPaths = [install.installPath, install.sourcePath] + .map((entry) => normalizeOptionalString(entry)) + .filter((entry): entry is string => Boolean(entry)); + if (trackedPaths.length === 0) { + rule.trackedWithoutPaths = true; + } else { + for (const trackedPath of trackedPaths) { + addPathToMatcher(rule.matcher, trackedPath, params.env); + } + } + installRules.set(pluginId, rule); + } + + return { loadPathMatcher, installRules }; +} + +function isTrackedByProvenance(params: { + pluginId: string; + source: string; + index: PluginProvenanceIndex; + env: NodeJS.ProcessEnv; +}): boolean { + const sourcePath = resolveUserPath(params.source, params.env); + const installRule = params.index.installRules.get(params.pluginId); + if (installRule) { + if (installRule.trackedWithoutPaths) { + return true; + } + if (matchesPathMatcher(installRule.matcher, sourcePath)) { + return true; + } + } + return matchesPathMatcher(params.index.loadPathMatcher, sourcePath); +} + +function matchesExplicitInstallRule(params: { + pluginId: string; + source: string; + index: PluginProvenanceIndex; + env: NodeJS.ProcessEnv; +}): boolean { + const sourcePath = resolveUserPath(params.source, params.env); + const installRule = params.index.installRules.get(params.pluginId); + if (!installRule || installRule.trackedWithoutPaths) { + return false; + } + return matchesPathMatcher(installRule.matcher, sourcePath); +} + +function resolveCandidateDuplicateRank(params: { + candidate: PluginCandidate; + manifestByRoot: Map; + provenance: PluginProvenanceIndex; + env: NodeJS.ProcessEnv; +}): number { + const manifestRecord = params.manifestByRoot.get(params.candidate.rootDir); + const pluginId = manifestRecord?.id; + const isExplicitInstall = + params.candidate.origin === "global" && + pluginId !== undefined && + matchesExplicitInstallRule({ + pluginId, + source: params.candidate.source, + index: params.provenance, + env: params.env, + }); + + if (params.candidate.origin === "config") { + return 0; + } + if (params.candidate.origin === "global" && isExplicitInstall) { + return 1; + } + if (params.candidate.origin === "bundled") { + // Bundled plugin ids stay reserved unless the operator configured an override. + return 2; + } + if (params.candidate.origin === "workspace") { + return 3; + } + return 4; +} + +export function compareDuplicateCandidateOrder(params: { + left: PluginCandidate; + right: PluginCandidate; + manifestByRoot: Map; + provenance: PluginProvenanceIndex; + env: NodeJS.ProcessEnv; +}): number { + const leftPluginId = params.manifestByRoot.get(params.left.rootDir)?.id; + const rightPluginId = params.manifestByRoot.get(params.right.rootDir)?.id; + if (!leftPluginId || leftPluginId !== rightPluginId) { + return 0; + } + return ( + resolveCandidateDuplicateRank({ + candidate: params.left, + manifestByRoot: params.manifestByRoot, + provenance: params.provenance, + env: params.env, + }) - + resolveCandidateDuplicateRank({ + candidate: params.right, + manifestByRoot: params.manifestByRoot, + provenance: params.provenance, + env: params.env, + }) + ); +} + +export function warnWhenAllowlistIsOpen(params: { + emitWarning: boolean; + logger: PluginLogger; + pluginsEnabled: boolean; + allow: string[]; + warningCacheKey: string; + warningCache: OpenAllowlistWarningCache; + discoverablePlugins: Array<{ id: string; source: string; origin: PluginRecord["origin"] }>; +}) { + if (!params.emitWarning) { + return; + } + if (!params.pluginsEnabled) { + return; + } + if (params.allow.length > 0) { + return; + } + const autoDiscoverable = params.discoverablePlugins.filter( + (entry) => entry.origin === "workspace" || entry.origin === "global", + ); + if (autoDiscoverable.length === 0) { + return; + } + if (params.warningCache.hasOpenAllowlistWarning(params.warningCacheKey)) { + return; + } + const preview = autoDiscoverable + .slice(0, 6) + .map((entry) => `${entry.id} (${entry.source})`) + .join(", "); + const extra = autoDiscoverable.length > 6 ? ` (+${autoDiscoverable.length - 6} more)` : ""; + params.warningCache.recordOpenAllowlistWarning(params.warningCacheKey); + params.logger.warn( + `[plugins] plugins.allow is empty; discovered non-bundled plugins may auto-load: ${preview}${extra}. Set plugins.allow to explicit trusted ids.`, + ); +} + +export function warnAboutUntrackedLoadedPlugins(params: { + registry: PluginRegistry; + provenance: PluginProvenanceIndex; + allowlist: string[]; + emitWarning: boolean; + logger: PluginLogger; + env: NodeJS.ProcessEnv; +}) { + const allowSet = new Set(params.allowlist); + for (const plugin of params.registry.plugins) { + if (plugin.status !== "loaded" || plugin.origin === "bundled") { + continue; + } + if (allowSet.has(plugin.id)) { + continue; + } + if ( + isTrackedByProvenance({ + pluginId: plugin.id, + source: plugin.source, + index: params.provenance, + env: params.env, + }) + ) { + continue; + } + const message = + "loaded without install/load-path provenance; treat as untracked local code and pin trust via plugins.allow or install records"; + params.registry.diagnostics.push({ + level: "warn", + pluginId: plugin.id, + source: plugin.source, + message, + }); + if (params.emitWarning) { + params.logger.warn(`[plugins] ${plugin.id}: ${message} (${plugin.source})`); + } + } +} diff --git a/src/plugins/loader-records.ts b/src/plugins/loader-records.ts new file mode 100644 index 00000000000..167873c84df --- /dev/null +++ b/src/plugins/loader-records.ts @@ -0,0 +1,195 @@ +import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; +import type { PluginCompatCode } from "./compat/registry.js"; +import type { PluginActivationState } from "./config-state.js"; +import type { PluginBundleFormat, PluginFormat } from "./manifest-types.js"; +import type { PluginManifestContracts } from "./manifest.js"; +import type { PluginRecord, PluginRegistry } from "./registry.js"; +import type { PluginLogger } from "./types.js"; + +export function createPluginRecord(params: { + id: string; + name?: string; + description?: string; + version?: string; + format?: PluginFormat; + bundleFormat?: PluginBundleFormat; + bundleCapabilities?: string[]; + source: string; + rootDir?: string; + origin: PluginRecord["origin"]; + workspaceDir?: string; + enabled: boolean; + compat?: readonly PluginCompatCode[]; + activationState?: PluginActivationState; + syntheticAuthRefs?: string[]; + configSchema: boolean; + contracts?: PluginManifestContracts; +}): PluginRecord { + return { + id: params.id, + name: params.name ?? params.id, + description: params.description, + version: params.version, + format: params.format ?? "openclaw", + bundleFormat: params.bundleFormat, + bundleCapabilities: params.bundleCapabilities, + source: params.source, + rootDir: params.rootDir, + origin: params.origin, + workspaceDir: params.workspaceDir, + enabled: params.enabled, + compat: params.compat, + explicitlyEnabled: params.activationState?.explicitlyEnabled, + activated: params.activationState?.activated, + activationSource: params.activationState?.source, + activationReason: params.activationState?.reason, + syntheticAuthRefs: params.syntheticAuthRefs ?? [], + status: params.enabled ? "loaded" : "disabled", + toolNames: [], + hookNames: [], + channelIds: [], + cliBackendIds: [], + providerIds: [], + speechProviderIds: [], + realtimeTranscriptionProviderIds: [], + realtimeVoiceProviderIds: [], + mediaUnderstandingProviderIds: [], + imageGenerationProviderIds: [], + videoGenerationProviderIds: [], + musicGenerationProviderIds: [], + webFetchProviderIds: [], + webSearchProviderIds: [], + migrationProviderIds: [], + contextEngineIds: [], + memoryEmbeddingProviderIds: [], + agentHarnessIds: [], + gatewayMethods: [], + cliCommands: [], + services: [], + gatewayDiscoveryServiceIds: [], + commands: [], + httpRoutes: 0, + hookCount: 0, + configSchema: params.configSchema, + configUiHints: undefined, + configJsonSchema: undefined, + contracts: params.contracts, + }; +} + +export function markPluginActivationDisabled(record: PluginRecord, reason?: string): void { + record.activated = false; + record.activationSource = "disabled"; + record.activationReason = reason; +} + +export function formatAutoEnabledActivationReason( + reasons: readonly string[] | undefined, +): string | undefined { + if (!reasons || reasons.length === 0) { + return undefined; + } + return reasons.join("; "); +} + +export function recordPluginError(params: { + logger: PluginLogger; + registry: PluginRegistry; + record: PluginRecord; + seenIds: Map; + pluginId: string; + origin: PluginRecord["origin"]; + phase: PluginRecord["failurePhase"]; + error: unknown; + logPrefix: string; + diagnosticMessagePrefix: string; +}) { + const errorText = + process.env.OPENCLAW_PLUGIN_LOADER_DEBUG_STACKS === "1" && + params.error instanceof Error && + typeof params.error.stack === "string" + ? params.error.stack + : String(params.error); + const deprecatedApiHint = + errorText.includes("api.registerHttpHandler") && errorText.includes("is not a function") + ? "deprecated api.registerHttpHandler(...) was removed; use api.registerHttpRoute(...) for plugin-owned routes or registerPluginHttpRoute(...) for dynamic lifecycle routes" + : null; + const displayError = deprecatedApiHint ? `${deprecatedApiHint} (${errorText})` : errorText; + params.logger.error(`${params.logPrefix}${displayError}`); + params.record.status = "error"; + params.record.error = displayError; + params.record.failedAt = new Date(); + params.record.failurePhase = params.phase; + params.registry.plugins.push(params.record); + params.seenIds.set(params.pluginId, params.origin); + params.registry.diagnostics.push({ + level: "error", + pluginId: params.record.id, + source: params.record.source, + message: `${params.diagnosticMessagePrefix}${displayError}`, + }); +} + +export function formatPluginFailureSummary(failedPlugins: PluginRecord[]): string { + const grouped = new Map, string[]>(); + for (const plugin of failedPlugins) { + const phase = plugin.failurePhase ?? "load"; + const ids = grouped.get(phase); + if (ids) { + ids.push(plugin.id); + continue; + } + grouped.set(phase, [plugin.id]); + } + return [...grouped.entries()].map(([phase, ids]) => `${phase}: ${ids.join(", ")}`).join("; "); +} + +function isPluginLoadDebugEnabled(env: NodeJS.ProcessEnv): boolean { + const normalized = normalizeLowercaseStringOrEmpty(env.OPENCLAW_PLUGIN_LOAD_DEBUG); + return normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on"; +} + +function describePluginModuleExportShape( + value: unknown, + label = "export", + seen: Set = new Set(), +): string[] { + if (value === null) { + return [`${label}:null`]; + } + if (typeof value !== "object") { + return [`${label}:${typeof value}`]; + } + if (seen.has(value)) { + return [`${label}:circular`]; + } + seen.add(value); + + const record = value as Record; + const keys = Object.keys(record).toSorted(); + const visibleKeys = keys.slice(0, 8); + const extraCount = keys.length - visibleKeys.length; + const keySummary = + visibleKeys.length > 0 + ? `${visibleKeys.join(",")}${extraCount > 0 ? `,+${extraCount}` : ""}` + : "none"; + const details = [`${label}:object keys=${keySummary}`]; + + for (const key of ["default", "module", "register", "activate"]) { + if (Object.prototype.hasOwnProperty.call(record, key)) { + details.push(...describePluginModuleExportShape(record[key], `${label}.${key}`, seen)); + } + } + return details; +} + +export function formatMissingPluginRegisterError( + moduleExport: unknown, + env: NodeJS.ProcessEnv, +): string { + const message = "plugin export missing register/activate"; + if (!isPluginLoadDebugEnabled(env)) { + return message; + } + return `${message} (module shape: ${describePluginModuleExportShape(moduleExport).join("; ")})`; +} diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 701022bb31c..15aa273a57b 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -6,8 +6,6 @@ import { listRegisteredAgentHarnesses, restoreRegisteredAgentHarnesses, } from "../agents/harness/registry.js"; -import type { ChannelPlugin } from "../channels/plugins/types.plugin.js"; -import { isChannelConfigured } from "../config/channel-configured.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginInstallRecord } from "../config/types.plugins.js"; import type { GatewayRequestHandler } from "../gateway/server-methods/types.js"; @@ -19,10 +17,7 @@ import { resolveMemoryDreamingConfig, resolveMemoryDreamingPluginConfig, } from "../memory-host-sdk/dreaming.js"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "../shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; import { clearDetachedTaskLifecycleRuntimeRegistration, getDetachedTaskLifecycleRuntimeRegistration, @@ -57,7 +52,6 @@ import { listRegisteredCompactionProviders, restoreRegisteredCompactionProviders, } from "./compaction-provider.js"; -import type { PluginCompatCode } from "./compat/registry.js"; import { applyTestPluginDefaults, createPluginActivationSource, @@ -67,7 +61,6 @@ import { resolveMemorySlotDecision, type PluginActivationConfigSource, type NormalizedPluginsConfig, - type PluginActivationState, } from "./config-state.js"; import { discoverOpenClawPlugins, type PluginCandidate } from "./discovery.js"; import { getGlobalHookRunner, initializeGlobalHookRunner } from "./hook-runner-global.js"; @@ -81,13 +74,34 @@ import { } from "./interactive-registry.js"; import { getCachedPluginJitiLoader, type PluginJitiLoaderCache } from "./jiti-loader-cache.js"; import { PluginLoaderCacheState } from "./loader-cache-state.js"; +import { + channelPluginIdBelongsToManifest, + loadBundledRuntimeChannelPlugin, + mergeSetupRuntimeChannelPlugin, + resolveBundledRuntimeChannelRegistration, + resolveSetupChannelRegistration, + shouldLoadChannelPluginInSetupRuntime, +} from "./loader-channel-setup.js"; +import { + buildProvenanceIndex, + compareDuplicateCandidateOrder, + warnAboutUntrackedLoadedPlugins, + warnWhenAllowlistIsOpen, +} from "./loader-provenance.js"; +import { + createPluginRecord, + formatAutoEnabledActivationReason, + formatMissingPluginRegisterError, + formatPluginFailureSummary, + markPluginActivationDisabled, + recordPluginError, +} from "./loader-records.js"; import { loadPluginManifestRegistry, type PluginManifestRecord, type PluginManifestRegistry, } from "./manifest-registry.js"; -import type { PluginBundleFormat, PluginDiagnostic, PluginFormat } from "./manifest-types.js"; -import type { PluginManifestContracts } from "./manifest.js"; +import type { PluginDiagnostic } from "./manifest-types.js"; import { clearMemoryEmbeddingProviders, listRegisteredMemoryEmbeddingProviders, @@ -104,7 +118,6 @@ import { restoreMemoryPluginState, } from "./memory-state.js"; import { unwrapDefaultModuleExport } from "./module-export.js"; -import { isPathInside, safeStatSync } from "./path-safety.js"; import { withProfile } from "./plugin-load-profile.js"; import { createPluginIdScopeSet, @@ -1027,406 +1040,6 @@ function resolvePluginModuleExport(moduleExport: unknown): { return {}; } -function isPluginLoadDebugEnabled(env: NodeJS.ProcessEnv): boolean { - const normalized = normalizeLowercaseStringOrEmpty(env.OPENCLAW_PLUGIN_LOAD_DEBUG); - return normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on"; -} - -function describePluginModuleExportShape( - value: unknown, - label = "export", - seen: Set = new Set(), -): string[] { - if (value === null) { - return [`${label}:null`]; - } - if (typeof value !== "object") { - return [`${label}:${typeof value}`]; - } - if (seen.has(value)) { - return [`${label}:circular`]; - } - seen.add(value); - - const record = value as Record; - const keys = Object.keys(record).toSorted(); - const visibleKeys = keys.slice(0, 8); - const extraCount = keys.length - visibleKeys.length; - const keySummary = - visibleKeys.length > 0 - ? `${visibleKeys.join(",")}${extraCount > 0 ? `,+${extraCount}` : ""}` - : "none"; - const details = [`${label}:object keys=${keySummary}`]; - - for (const key of ["default", "module", "register", "activate"]) { - if (Object.prototype.hasOwnProperty.call(record, key)) { - details.push(...describePluginModuleExportShape(record[key], `${label}.${key}`, seen)); - } - } - return details; -} - -function formatMissingPluginRegisterError(moduleExport: unknown, env: NodeJS.ProcessEnv): string { - const message = "plugin export missing register/activate"; - if (!isPluginLoadDebugEnabled(env)) { - return message; - } - return `${message} (module shape: ${describePluginModuleExportShape(moduleExport).join("; ")})`; -} - -function mergeChannelPluginSection( - baseValue: T | undefined, - overrideValue: T | undefined, -): T | undefined { - if ( - baseValue && - overrideValue && - typeof baseValue === "object" && - typeof overrideValue === "object" - ) { - const merged = { - ...(baseValue as Record), - }; - for (const [key, value] of Object.entries(overrideValue as Record)) { - if (value !== undefined) { - merged[key] = value; - } - } - return { - ...merged, - } as T; - } - return overrideValue ?? baseValue; -} - -function mergeSetupRuntimeChannelPlugin( - runtimePlugin: ChannelPlugin, - setupPlugin: ChannelPlugin, -): ChannelPlugin { - return { - ...runtimePlugin, - ...setupPlugin, - meta: mergeChannelPluginSection(runtimePlugin.meta, setupPlugin.meta), - capabilities: mergeChannelPluginSection(runtimePlugin.capabilities, setupPlugin.capabilities), - commands: mergeChannelPluginSection(runtimePlugin.commands, setupPlugin.commands), - doctor: mergeChannelPluginSection(runtimePlugin.doctor, setupPlugin.doctor), - reload: mergeChannelPluginSection(runtimePlugin.reload, setupPlugin.reload), - config: mergeChannelPluginSection(runtimePlugin.config, setupPlugin.config), - setup: mergeChannelPluginSection(runtimePlugin.setup, setupPlugin.setup), - messaging: mergeChannelPluginSection(runtimePlugin.messaging, setupPlugin.messaging), - actions: mergeChannelPluginSection(runtimePlugin.actions, setupPlugin.actions), - secrets: mergeChannelPluginSection(runtimePlugin.secrets, setupPlugin.secrets), - } as ChannelPlugin; -} - -function resolveBundledRuntimeChannelRegistration(moduleExport: unknown): { - id?: string; - loadChannelPlugin?: () => ChannelPlugin; - loadChannelSecrets?: () => ChannelPlugin["secrets"] | undefined; - setChannelRuntime?: (runtime: PluginRuntime) => void; -} { - const resolved = unwrapDefaultModuleExport(moduleExport); - if (!resolved || typeof resolved !== "object") { - return {}; - } - const entryRecord = resolved as { - kind?: unknown; - id?: unknown; - loadChannelPlugin?: unknown; - loadChannelSecrets?: unknown; - setChannelRuntime?: unknown; - }; - if ( - entryRecord.kind !== "bundled-channel-entry" || - typeof entryRecord.id !== "string" || - typeof entryRecord.loadChannelPlugin !== "function" - ) { - return {}; - } - return { - id: entryRecord.id, - loadChannelPlugin: entryRecord.loadChannelPlugin as () => ChannelPlugin, - ...(typeof entryRecord.loadChannelSecrets === "function" - ? { - loadChannelSecrets: entryRecord.loadChannelSecrets as () => - | ChannelPlugin["secrets"] - | undefined, - } - : {}), - ...(typeof entryRecord.setChannelRuntime === "function" - ? { - setChannelRuntime: entryRecord.setChannelRuntime as (runtime: PluginRuntime) => void, - } - : {}), - }; -} - -function loadBundledRuntimeChannelPlugin(params: { - registration: ReturnType; -}): { - plugin?: ChannelPlugin; - loadError?: unknown; -} { - if (typeof params.registration.loadChannelPlugin !== "function") { - return {}; - } - try { - const loadedPlugin = params.registration.loadChannelPlugin(); - const loadedSecrets = params.registration.loadChannelSecrets?.(); - if (!loadedPlugin || typeof loadedPlugin !== "object") { - return {}; - } - const mergedSecrets = mergeChannelPluginSection(loadedPlugin.secrets, loadedSecrets); - return { - plugin: { - ...loadedPlugin, - ...(mergedSecrets !== undefined ? { secrets: mergedSecrets } : {}), - }, - }; - } catch (err) { - return { loadError: err }; - } -} - -function resolveSetupChannelRegistration( - moduleExport: unknown, - params: { installRuntimeDeps?: boolean } = {}, -): { - plugin?: ChannelPlugin; - setChannelRuntime?: (runtime: PluginRuntime) => void; - usesBundledSetupContract?: boolean; - loadError?: unknown; -} { - const resolved = unwrapDefaultModuleExport(moduleExport); - if (!resolved || typeof resolved !== "object") { - return {}; - } - const setupEntryRecord = resolved as { - kind?: unknown; - loadSetupPlugin?: unknown; - loadSetupSecrets?: unknown; - setChannelRuntime?: unknown; - }; - if ( - setupEntryRecord.kind === "bundled-channel-setup-entry" && - typeof setupEntryRecord.loadSetupPlugin === "function" - ) { - try { - const setupLoadOptions = - params.installRuntimeDeps === false ? { installRuntimeDeps: false } : undefined; - const loadedPlugin = setupEntryRecord.loadSetupPlugin(setupLoadOptions); - const loadedSecrets = - typeof setupEntryRecord.loadSetupSecrets === "function" - ? (setupEntryRecord.loadSetupSecrets(setupLoadOptions) as - | ChannelPlugin["secrets"] - | undefined) - : undefined; - if (loadedPlugin && typeof loadedPlugin === "object") { - const mergedSecrets = mergeChannelPluginSection( - (loadedPlugin as ChannelPlugin).secrets, - loadedSecrets, - ); - return { - plugin: { - ...(loadedPlugin as ChannelPlugin), - ...(mergedSecrets !== undefined ? { secrets: mergedSecrets } : {}), - }, - usesBundledSetupContract: true, - ...(typeof setupEntryRecord.setChannelRuntime === "function" - ? { - setChannelRuntime: setupEntryRecord.setChannelRuntime as ( - runtime: PluginRuntime, - ) => void, - } - : {}), - }; - } - } catch (err) { - return { loadError: err }; - } - } - const setup = resolved as { - plugin?: unknown; - }; - if (!setup.plugin || typeof setup.plugin !== "object") { - return {}; - } - return { - plugin: setup.plugin as ChannelPlugin, - }; -} - -function shouldLoadChannelPluginInSetupRuntime(params: { - manifestChannels: string[]; - setupSource?: string; - startupDeferConfiguredChannelFullLoadUntilAfterListen?: boolean; - cfg: OpenClawConfig; - env: NodeJS.ProcessEnv; - preferSetupRuntimeForChannelPlugins?: boolean; -}): boolean { - if (!params.setupSource || params.manifestChannels.length === 0) { - return false; - } - if ( - params.preferSetupRuntimeForChannelPlugins && - params.startupDeferConfiguredChannelFullLoadUntilAfterListen === true - ) { - return true; - } - return !params.manifestChannels.some((channelId) => - isChannelConfigured(params.cfg, channelId, params.env), - ); -} - -function channelPluginIdBelongsToManifest(params: { - channelId: string | undefined; - pluginId: string; - manifestChannels: readonly string[]; -}): boolean { - if (!params.channelId) { - return true; - } - return params.channelId === params.pluginId || params.manifestChannels.includes(params.channelId); -} - -function createPluginRecord(params: { - id: string; - name?: string; - description?: string; - version?: string; - format?: PluginFormat; - bundleFormat?: PluginBundleFormat; - bundleCapabilities?: string[]; - source: string; - rootDir?: string; - origin: PluginRecord["origin"]; - workspaceDir?: string; - enabled: boolean; - compat?: readonly PluginCompatCode[]; - activationState?: PluginActivationState; - syntheticAuthRefs?: string[]; - configSchema: boolean; - contracts?: PluginManifestContracts; -}): PluginRecord { - return { - id: params.id, - name: params.name ?? params.id, - description: params.description, - version: params.version, - format: params.format ?? "openclaw", - bundleFormat: params.bundleFormat, - bundleCapabilities: params.bundleCapabilities, - source: params.source, - rootDir: params.rootDir, - origin: params.origin, - workspaceDir: params.workspaceDir, - enabled: params.enabled, - compat: params.compat, - explicitlyEnabled: params.activationState?.explicitlyEnabled, - activated: params.activationState?.activated, - activationSource: params.activationState?.source, - activationReason: params.activationState?.reason, - syntheticAuthRefs: params.syntheticAuthRefs ?? [], - status: params.enabled ? "loaded" : "disabled", - toolNames: [], - hookNames: [], - channelIds: [], - cliBackendIds: [], - providerIds: [], - speechProviderIds: [], - realtimeTranscriptionProviderIds: [], - realtimeVoiceProviderIds: [], - mediaUnderstandingProviderIds: [], - imageGenerationProviderIds: [], - videoGenerationProviderIds: [], - musicGenerationProviderIds: [], - webFetchProviderIds: [], - webSearchProviderIds: [], - migrationProviderIds: [], - contextEngineIds: [], - memoryEmbeddingProviderIds: [], - agentHarnessIds: [], - gatewayMethods: [], - cliCommands: [], - services: [], - gatewayDiscoveryServiceIds: [], - commands: [], - httpRoutes: 0, - hookCount: 0, - configSchema: params.configSchema, - configUiHints: undefined, - configJsonSchema: undefined, - contracts: params.contracts, - }; -} - -function markPluginActivationDisabled(record: PluginRecord, reason?: string): void { - record.activated = false; - record.activationSource = "disabled"; - record.activationReason = reason; -} - -function formatAutoEnabledActivationReason( - reasons: readonly string[] | undefined, -): string | undefined { - if (!reasons || reasons.length === 0) { - return undefined; - } - return reasons.join("; "); -} - -function recordPluginError(params: { - logger: PluginLogger; - registry: PluginRegistry; - record: PluginRecord; - seenIds: Map; - pluginId: string; - origin: PluginRecord["origin"]; - phase: PluginRecord["failurePhase"]; - error: unknown; - logPrefix: string; - diagnosticMessagePrefix: string; -}) { - const errorText = - process.env.OPENCLAW_PLUGIN_LOADER_DEBUG_STACKS === "1" && - params.error instanceof Error && - typeof params.error.stack === "string" - ? params.error.stack - : String(params.error); - const deprecatedApiHint = - errorText.includes("api.registerHttpHandler") && errorText.includes("is not a function") - ? "deprecated api.registerHttpHandler(...) was removed; use api.registerHttpRoute(...) for plugin-owned routes or registerPluginHttpRoute(...) for dynamic lifecycle routes" - : null; - const displayError = deprecatedApiHint ? `${deprecatedApiHint} (${errorText})` : errorText; - params.logger.error(`${params.logPrefix}${displayError}`); - params.record.status = "error"; - params.record.error = displayError; - params.record.failedAt = new Date(); - params.record.failurePhase = params.phase; - params.registry.plugins.push(params.record); - params.seenIds.set(params.pluginId, params.origin); - params.registry.diagnostics.push({ - level: "error", - pluginId: params.record.id, - source: params.record.source, - message: `${params.diagnosticMessagePrefix}${displayError}`, - }); -} - -function formatPluginFailureSummary(failedPlugins: PluginRecord[]): string { - const grouped = new Map, string[]>(); - for (const plugin of failedPlugins) { - const phase = plugin.failurePhase ?? "load"; - const ids = grouped.get(phase); - if (ids) { - ids.push(plugin.id); - continue; - } - grouped.set(phase, [plugin.id]); - } - return [...grouped.entries()].map(([phase, ids]) => `${phase}: ${ids.join(", ")}`).join("; "); -} - function pushDiagnostics(diagnostics: PluginDiagnostic[], append: PluginDiagnostic[]) { diagnostics.push(...append); } @@ -1444,261 +1057,6 @@ function maybeThrowOnPluginLoadError( throw new PluginLoadFailureError(registry); } -type PathMatcher = { - exact: Set; - dirs: string[]; -}; - -type InstallTrackingRule = { - trackedWithoutPaths: boolean; - matcher: PathMatcher; -}; - -type PluginProvenanceIndex = { - loadPathMatcher: PathMatcher; - installRules: Map; -}; - -function createPathMatcher(): PathMatcher { - return { exact: new Set(), dirs: [] }; -} - -function addPathToMatcher( - matcher: PathMatcher, - rawPath: string, - env: NodeJS.ProcessEnv = process.env, -): void { - const trimmed = rawPath.trim(); - if (!trimmed) { - return; - } - const resolved = resolveUserPath(trimmed, env); - if (!resolved) { - return; - } - if (matcher.exact.has(resolved) || matcher.dirs.includes(resolved)) { - return; - } - const stat = safeStatSync(resolved); - if (stat?.isDirectory()) { - matcher.dirs.push(resolved); - return; - } - matcher.exact.add(resolved); -} - -function matchesPathMatcher(matcher: PathMatcher, sourcePath: string): boolean { - if (matcher.exact.has(sourcePath)) { - return true; - } - return matcher.dirs.some((dirPath) => isPathInside(dirPath, sourcePath)); -} - -function buildProvenanceIndex(params: { - config: OpenClawConfig; - normalizedLoadPaths: string[]; - env: NodeJS.ProcessEnv; -}): PluginProvenanceIndex { - const loadPathMatcher = createPathMatcher(); - for (const loadPath of params.normalizedLoadPaths) { - addPathToMatcher(loadPathMatcher, loadPath, params.env); - } - - const installRules = new Map(); - const installs = loadInstalledPluginIndexInstallRecordsSync({ env: params.env }); - for (const [pluginId, install] of Object.entries(installs)) { - const rule: InstallTrackingRule = { - trackedWithoutPaths: false, - matcher: createPathMatcher(), - }; - const trackedPaths = [install.installPath, install.sourcePath] - .map((entry) => normalizeOptionalString(entry)) - .filter((entry): entry is string => Boolean(entry)); - if (trackedPaths.length === 0) { - rule.trackedWithoutPaths = true; - } else { - for (const trackedPath of trackedPaths) { - addPathToMatcher(rule.matcher, trackedPath, params.env); - } - } - installRules.set(pluginId, rule); - } - - return { loadPathMatcher, installRules }; -} - -function isTrackedByProvenance(params: { - pluginId: string; - source: string; - index: PluginProvenanceIndex; - env: NodeJS.ProcessEnv; -}): boolean { - const sourcePath = resolveUserPath(params.source, params.env); - const installRule = params.index.installRules.get(params.pluginId); - if (installRule) { - if (installRule.trackedWithoutPaths) { - return true; - } - if (matchesPathMatcher(installRule.matcher, sourcePath)) { - return true; - } - } - return matchesPathMatcher(params.index.loadPathMatcher, sourcePath); -} - -function matchesExplicitInstallRule(params: { - pluginId: string; - source: string; - index: PluginProvenanceIndex; - env: NodeJS.ProcessEnv; -}): boolean { - const sourcePath = resolveUserPath(params.source, params.env); - const installRule = params.index.installRules.get(params.pluginId); - if (!installRule || installRule.trackedWithoutPaths) { - return false; - } - return matchesPathMatcher(installRule.matcher, sourcePath); -} - -function resolveCandidateDuplicateRank(params: { - candidate: ReturnType["candidates"][number]; - manifestByRoot: Map["plugins"][number]>; - provenance: PluginProvenanceIndex; - env: NodeJS.ProcessEnv; -}): number { - const manifestRecord = params.manifestByRoot.get(params.candidate.rootDir); - const pluginId = manifestRecord?.id; - const isExplicitInstall = - params.candidate.origin === "global" && - pluginId !== undefined && - matchesExplicitInstallRule({ - pluginId, - source: params.candidate.source, - index: params.provenance, - env: params.env, - }); - - if (params.candidate.origin === "config") { - return 0; - } - if (params.candidate.origin === "global" && isExplicitInstall) { - return 1; - } - if (params.candidate.origin === "bundled") { - // Bundled plugin ids stay reserved unless the operator configured an override. - return 2; - } - if (params.candidate.origin === "workspace") { - return 3; - } - return 4; -} - -function compareDuplicateCandidateOrder(params: { - left: ReturnType["candidates"][number]; - right: ReturnType["candidates"][number]; - manifestByRoot: Map["plugins"][number]>; - provenance: PluginProvenanceIndex; - env: NodeJS.ProcessEnv; -}): number { - const leftPluginId = params.manifestByRoot.get(params.left.rootDir)?.id; - const rightPluginId = params.manifestByRoot.get(params.right.rootDir)?.id; - if (!leftPluginId || leftPluginId !== rightPluginId) { - return 0; - } - return ( - resolveCandidateDuplicateRank({ - candidate: params.left, - manifestByRoot: params.manifestByRoot, - provenance: params.provenance, - env: params.env, - }) - - resolveCandidateDuplicateRank({ - candidate: params.right, - manifestByRoot: params.manifestByRoot, - provenance: params.provenance, - env: params.env, - }) - ); -} - -function warnWhenAllowlistIsOpen(params: { - emitWarning: boolean; - logger: PluginLogger; - pluginsEnabled: boolean; - allow: string[]; - warningCacheKey: string; - discoverablePlugins: Array<{ id: string; source: string; origin: PluginRecord["origin"] }>; -}) { - if (!params.emitWarning) { - return; - } - if (!params.pluginsEnabled) { - return; - } - if (params.allow.length > 0) { - return; - } - const autoDiscoverable = params.discoverablePlugins.filter( - (entry) => entry.origin === "workspace" || entry.origin === "global", - ); - if (autoDiscoverable.length === 0) { - return; - } - if (pluginLoaderCacheState.hasOpenAllowlistWarning(params.warningCacheKey)) { - return; - } - const preview = autoDiscoverable - .slice(0, 6) - .map((entry) => `${entry.id} (${entry.source})`) - .join(", "); - const extra = autoDiscoverable.length > 6 ? ` (+${autoDiscoverable.length - 6} more)` : ""; - pluginLoaderCacheState.recordOpenAllowlistWarning(params.warningCacheKey); - params.logger.warn( - `[plugins] plugins.allow is empty; discovered non-bundled plugins may auto-load: ${preview}${extra}. Set plugins.allow to explicit trusted ids.`, - ); -} - -function warnAboutUntrackedLoadedPlugins(params: { - registry: PluginRegistry; - provenance: PluginProvenanceIndex; - allowlist: string[]; - emitWarning: boolean; - logger: PluginLogger; - env: NodeJS.ProcessEnv; -}) { - const allowSet = new Set(params.allowlist); - for (const plugin of params.registry.plugins) { - if (plugin.status !== "loaded" || plugin.origin === "bundled") { - continue; - } - if (allowSet.has(plugin.id)) { - continue; - } - if ( - isTrackedByProvenance({ - pluginId: plugin.id, - source: plugin.source, - index: params.provenance, - env: params.env, - }) - ) { - continue; - } - const message = - "loaded without install/load-path provenance; treat as untracked local code and pin trust via plugins.allow or install records"; - params.registry.diagnostics.push({ - level: "warn", - pluginId: plugin.id, - source: plugin.source, - message, - }); - if (params.emitWarning) { - params.logger.warn(`[plugins] ${plugin.id}: ${message} (${plugin.source})`); - } - } -} - function activatePluginRegistry( registry: PluginRegistry, cacheKey: string, @@ -1921,6 +1279,7 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi pluginsEnabled: normalized.enabled, allow: normalized.allow, warningCacheKey: cacheKey, + warningCache: pluginLoaderCacheState, // Keep warning input scoped as well so partial snapshot loads only mention the // plugins that were intentionally requested for this registry. discoverablePlugins: manifestRegistry.plugins @@ -1932,7 +1291,6 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi })), }); const provenance = buildProvenanceIndex({ - config: cfg, normalizedLoadPaths: normalized.loadPaths, env, }); @@ -2810,6 +2168,7 @@ export async function loadOpenClawPluginCliRegistry( pluginsEnabled: normalized.enabled, allow: normalized.allow, warningCacheKey: `${cacheKey}::cli-metadata`, + warningCache: pluginLoaderCacheState, discoverablePlugins: manifestRegistry.plugins .filter((plugin) => !onlyPluginIdSet || onlyPluginIdSet.has(plugin.id)) .map((plugin) => ({ @@ -2819,7 +2178,6 @@ export async function loadOpenClawPluginCliRegistry( })), }); const provenance = buildProvenanceIndex({ - config: cfg, normalizedLoadPaths: normalized.loadPaths, env, });