diff --git a/CHANGELOG.md b/CHANGELOG.md index f51f5a2fb2d..4efcbad0776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Docs: https://docs.openclaw.ai - Channels/sessions: prevent guarded inbound session recording from creating route-only phantom sessions while still allowing last-route updates for sessions that already exist. Carries forward #73009. Thanks @jzakirov. - Cron: accept `delivery.threadId` in Gateway cron add/update schemas so scheduled announce delivery can target Telegram forum topics and other threaded channel destinations through the documented delivery path. Fixes #73017. Thanks @coachsootz. - Plugins/runtime deps: stage bundled plugin dependencies imported by mirrored root dist chunks, so packaged memory and status commands do not miss `chokidar` or similar root-chunk dependencies after update. Fixes #72882 and #72970; carries forward #72992. Thanks @shrimpy8, @colin-chang, and @Schnup03. +- Plugins/runtime deps: reuse unchanged bundled plugin runtime mirrors instead of rebuilding plugin trees on every load, cutting avoidable writes and restart/reconnect I/O on slow storage. Fixes #72933. Thanks @jasonftl. - Agents/runtime context: deliver hidden runtime context through prompt-local system context while keeping the transcript-only custom entry out of provider user turns, and strip stale copied runtime-context prefaces from user-facing replies. Fixes #72386; carries forward #72969. Thanks @jhsmith409. - Channels/Telegram: skip the optional webhook-info API call during polling-mode status checks and startup bot-label probes so long-polling setups avoid an unnecessary Telegram round trip. Carries forward #72990. Thanks @danielgruneberg. - CLI/message: resolve targeted `openclaw message` channels to their owning plugin before loading the registry, and fall back to configured channel plugins when the channel must be inferred, so scripted sends avoid full bundled plugin registry scans without assuming channel ids match plugin ids. Fixes #73006. Thanks @jasonftl. diff --git a/src/plugins/bundled-runtime-mirror.ts b/src/plugins/bundled-runtime-mirror.ts new file mode 100644 index 00000000000..fa7fc29e96c --- /dev/null +++ b/src/plugins/bundled-runtime-mirror.ts @@ -0,0 +1,219 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const BUNDLED_RUNTIME_MIRROR_METADATA_FILE = ".openclaw-runtime-mirror.json"; +const BUNDLED_RUNTIME_MIRROR_METADATA_VERSION = 1; + +type BundledRuntimeMirrorMetadata = { + version: number; + pluginId: string; + sourceRoot: string; + sourceFingerprint: string; +}; + +export function refreshBundledPluginRuntimeMirrorRoot(params: { + pluginId: string; + sourceRoot: string; + targetRoot: string; + tempDirParent?: string; +}): boolean { + if (path.resolve(params.sourceRoot) === path.resolve(params.targetRoot)) { + return false; + } + const metadata = createBundledRuntimeMirrorMetadata(params); + if (isBundledRuntimeMirrorRootFresh(params.targetRoot, metadata)) { + return false; + } + const tempDir = fs.mkdtempSync( + path.join( + params.tempDirParent ?? path.dirname(params.targetRoot), + `.plugin-${sanitizeBundledRuntimeMirrorTempId(params.pluginId)}-`, + ), + ); + const stagedRoot = path.join(tempDir, "plugin"); + try { + copyBundledPluginRuntimeRoot(params.sourceRoot, stagedRoot); + writeBundledRuntimeMirrorMetadata(stagedRoot, metadata); + fs.rmSync(params.targetRoot, { recursive: true, force: true }); + fs.renameSync(stagedRoot, params.targetRoot); + return true; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +export function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void { + if (path.resolve(sourceRoot) === path.resolve(targetRoot)) { + return; + } + fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) { + if (shouldIgnoreBundledRuntimeMirrorEntry(entry.name)) { + continue; + } + const sourcePath = path.join(sourceRoot, entry.name); + const targetPath = path.join(targetRoot, entry.name); + if (entry.isDirectory()) { + copyBundledPluginRuntimeRoot(sourcePath, targetPath); + continue; + } + if (entry.isSymbolicLink()) { + fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath); + continue; + } + if (!entry.isFile()) { + continue; + } + fs.copyFileSync(sourcePath, targetPath); + try { + const sourceMode = fs.statSync(sourcePath).mode; + fs.chmodSync(targetPath, sourceMode | 0o600); + } catch { + // Readable copied files are enough for plugin loading. + } + } +} + +function createBundledRuntimeMirrorMetadata(params: { + pluginId: string; + sourceRoot: string; +}): BundledRuntimeMirrorMetadata { + return { + version: BUNDLED_RUNTIME_MIRROR_METADATA_VERSION, + pluginId: params.pluginId, + sourceRoot: resolveBundledRuntimeMirrorSourceRootId(params.sourceRoot), + sourceFingerprint: fingerprintBundledRuntimeMirrorSourceRoot(params.sourceRoot), + }; +} + +function isBundledRuntimeMirrorRootFresh( + targetRoot: string, + expected: BundledRuntimeMirrorMetadata, +): boolean { + try { + if (!fs.lstatSync(targetRoot).isDirectory()) { + return false; + } + } catch { + return false; + } + const actual = readBundledRuntimeMirrorMetadata(targetRoot); + return ( + actual?.version === expected.version && + actual.pluginId === expected.pluginId && + actual.sourceRoot === expected.sourceRoot && + actual.sourceFingerprint === expected.sourceFingerprint + ); +} + +function readBundledRuntimeMirrorMetadata(targetRoot: string): BundledRuntimeMirrorMetadata | null { + try { + const parsed = JSON.parse( + fs.readFileSync(path.join(targetRoot, BUNDLED_RUNTIME_MIRROR_METADATA_FILE), "utf8"), + ) as Partial; + if ( + parsed.version !== BUNDLED_RUNTIME_MIRROR_METADATA_VERSION || + typeof parsed.pluginId !== "string" || + typeof parsed.sourceRoot !== "string" || + typeof parsed.sourceFingerprint !== "string" + ) { + return null; + } + return parsed as BundledRuntimeMirrorMetadata; + } catch { + return null; + } +} + +function writeBundledRuntimeMirrorMetadata( + targetRoot: string, + metadata: BundledRuntimeMirrorMetadata, +): void { + fs.writeFileSync( + path.join(targetRoot, BUNDLED_RUNTIME_MIRROR_METADATA_FILE), + `${JSON.stringify(metadata, null, 2)}\n`, + "utf8", + ); +} + +function fingerprintBundledRuntimeMirrorSourceRoot(sourceRoot: string): string { + const hash = createHash("sha256"); + hashBundledRuntimeMirrorDirectory(hash, sourceRoot, sourceRoot); + return hash.digest("hex"); +} + +function hashBundledRuntimeMirrorDirectory( + hash: ReturnType, + sourceRoot: string, + directory: string, +): void { + const entries = fs + .readdirSync(directory, { withFileTypes: true }) + .filter((entry) => !shouldIgnoreBundledRuntimeMirrorEntry(entry.name)) + .toSorted((left, right) => left.name.localeCompare(right.name)); + + for (const entry of entries) { + const sourcePath = path.join(directory, entry.name); + const relativePath = path.relative(sourceRoot, sourcePath).replaceAll(path.sep, "/"); + const stat = fs.lstatSync(sourcePath, { bigint: true }); + if (entry.isDirectory()) { + updateBundledRuntimeMirrorHash(hash, [ + "dir", + relativePath, + formatBundledRuntimeMirrorMode(stat.mode), + ]); + hashBundledRuntimeMirrorDirectory(hash, sourceRoot, sourcePath); + continue; + } + if (entry.isSymbolicLink()) { + updateBundledRuntimeMirrorHash(hash, [ + "symlink", + relativePath, + formatBundledRuntimeMirrorMode(stat.mode), + stat.ctimeNs.toString(), + fs.readlinkSync(sourcePath), + ]); + continue; + } + if (!entry.isFile()) { + continue; + } + updateBundledRuntimeMirrorHash(hash, [ + "file", + relativePath, + formatBundledRuntimeMirrorMode(stat.mode), + stat.size.toString(), + stat.mtimeNs.toString(), + stat.ctimeNs.toString(), + ]); + } +} + +function updateBundledRuntimeMirrorHash( + hash: ReturnType, + fields: readonly string[], +): void { + hash.update(JSON.stringify(fields)); + hash.update("\n"); +} + +function formatBundledRuntimeMirrorMode(mode: bigint): string { + return (mode & 0o7777n).toString(8); +} + +function resolveBundledRuntimeMirrorSourceRootId(sourceRoot: string): string { + try { + return fs.realpathSync.native(sourceRoot); + } catch { + return path.resolve(sourceRoot); + } +} + +function shouldIgnoreBundledRuntimeMirrorEntry(name: string): boolean { + return name === "node_modules" || name === BUNDLED_RUNTIME_MIRROR_METADATA_FILE; +} + +function sanitizeBundledRuntimeMirrorTempId(pluginId: string): string { + return pluginId.replaceAll(/[^a-zA-Z0-9._-]/g, "_"); +} diff --git a/src/plugins/bundled-runtime-root.test.ts b/src/plugins/bundled-runtime-root.test.ts index ac9c67867d4..b39f332483a 100644 --- a/src/plugins/bundled-runtime-root.test.ts +++ b/src/plugins/bundled-runtime-root.test.ts @@ -19,6 +19,10 @@ afterEach(() => { } }); +async function waitForFilesystemTimestampTick(): Promise { + await new Promise((resolve) => setTimeout(resolve, 50)); +} + describe("prepareBundledPluginRuntimeRoot", () => { it("materializes root JavaScript chunks in external mirrors", () => { const packageRoot = makeTempRoot(); @@ -167,4 +171,122 @@ describe("prepareBundledPluginRuntimeRoot", () => { expect(prepared.modulePath).toBe(path.join(pluginRoot, "index.js")); expect(fs.readFileSync(distChunk, "utf8")).toContain("same-root"); }); + + it("reuses unchanged external runtime mirrors from the original plugin root", async () => { + const packageRoot = makeTempRoot(); + const stageDir = makeTempRoot(); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "whatsapp"); + const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27", type: "module" }), + "utf8", + ); + fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v1';\n", "utf8"); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: "@openclaw/whatsapp", + version: "1.0.0", + type: "module", + dependencies: { "whatsapp-runtime": "1.0.0" }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf8", + ); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + fs.mkdirSync(path.join(installRoot, "node_modules", "whatsapp-runtime"), { recursive: true }); + fs.writeFileSync( + path.join(installRoot, "node_modules", "whatsapp-runtime", "package.json"), + JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }), + "utf8", + ); + + const prepared = prepareBundledPluginRuntimeRoot({ + pluginId: "whatsapp", + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + const mirrorEntry = path.join(prepared.pluginRoot, "index.js"); + const initialStat = fs.statSync(mirrorEntry); + + await waitForFilesystemTimestampTick(); + + const preparedAgain = prepareBundledPluginRuntimeRoot({ + pluginId: "whatsapp", + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + const reusedStat = fs.statSync(mirrorEntry); + + expect(preparedAgain).toEqual(prepared); + expect(reusedStat.mtimeMs).toBe(initialStat.mtimeMs); + expect(fs.readFileSync(mirrorEntry, "utf8")).toContain("v1"); + }); + + it("refreshes external runtime mirrors when source files change", async () => { + const packageRoot = makeTempRoot(); + const stageDir = makeTempRoot(); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "whatsapp"); + const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27", type: "module" }), + "utf8", + ); + fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v1';\n", "utf8"); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: "@openclaw/whatsapp", + version: "1.0.0", + type: "module", + dependencies: { "whatsapp-runtime": "1.0.0" }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf8", + ); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + fs.mkdirSync(path.join(installRoot, "node_modules", "whatsapp-runtime"), { recursive: true }); + fs.writeFileSync( + path.join(installRoot, "node_modules", "whatsapp-runtime", "package.json"), + JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }), + "utf8", + ); + + const prepared = prepareBundledPluginRuntimeRoot({ + pluginId: "whatsapp", + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + const mirrorEntry = path.join(prepared.pluginRoot, "index.js"); + const initialStat = fs.statSync(mirrorEntry); + + await waitForFilesystemTimestampTick(); + fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v2';\n", "utf8"); + + prepareBundledPluginRuntimeRoot({ + pluginId: "whatsapp", + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + const refreshedStat = fs.statSync(mirrorEntry); + + expect(refreshedStat.mtimeMs).toBeGreaterThan(initialStat.mtimeMs); + expect(fs.readFileSync(mirrorEntry, "utf8")).toContain("v2"); + }); }); diff --git a/src/plugins/bundled-runtime-root.ts b/src/plugins/bundled-runtime-root.ts index b76651f8e45..895b08c932d 100644 --- a/src/plugins/bundled-runtime-root.ts +++ b/src/plugins/bundled-runtime-root.ts @@ -9,6 +9,10 @@ import { shouldMaterializeBundledRuntimeMirrorDistFile, withBundledRuntimeDepsFilesystemLock, } from "./bundled-runtime-deps.js"; +import { + copyBundledPluginRuntimeRoot, + refreshBundledPluginRuntimeMirrorRoot, +} from "./bundled-runtime-mirror.js"; const bundledRuntimeDepsRetainSpecsByInstallRoot = new Map(); const BUNDLED_RUNTIME_MIRROR_LOCK_DIR = ".openclaw-runtime-mirror.lock"; @@ -117,15 +121,12 @@ function mirrorBundledPluginRuntimeRoot(params: { if (path.resolve(mirrorRoot) === path.resolve(params.pluginRoot)) { return mirrorRoot; } - const tempDir = fs.mkdtempSync(path.join(mirrorParent, `.plugin-${params.pluginId}-`)); - const stagedRoot = path.join(tempDir, "plugin"); - try { - copyBundledPluginRuntimeRoot(params.pluginRoot, stagedRoot); - fs.rmSync(mirrorRoot, { recursive: true, force: true }); - fs.renameSync(stagedRoot, mirrorRoot); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } + refreshBundledPluginRuntimeMirrorRoot({ + pluginId: params.pluginId, + sourceRoot: params.pluginRoot, + targetRoot: mirrorRoot, + tempDirParent: mirrorParent, + }); return mirrorRoot; }, ); @@ -182,38 +183,6 @@ function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void { writeRuntimeJsonFile(packageJsonPath, { type: "module" }); } -function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void { - if (path.resolve(sourceRoot) === path.resolve(targetRoot)) { - return; - } - fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); - for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) { - if (entry.name === "node_modules") { - continue; - } - const sourcePath = path.join(sourceRoot, entry.name); - const targetPath = path.join(targetRoot, entry.name); - if (entry.isDirectory()) { - copyBundledPluginRuntimeRoot(sourcePath, targetPath); - continue; - } - if (entry.isSymbolicLink()) { - fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath); - continue; - } - if (!entry.isFile()) { - continue; - } - fs.copyFileSync(sourcePath, targetPath); - try { - const sourceMode = fs.statSync(sourcePath).mode; - fs.chmodSync(targetPath, sourceMode | 0o600); - } catch { - // Readable copied files are enough for plugin loading. - } - } -} - function writeRuntimeJsonFile(targetPath: string, value: unknown): void { fs.mkdirSync(path.dirname(targetPath), { recursive: true }); fs.writeFileSync(targetPath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 5c711fd8e7c..47071154581 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -149,6 +149,10 @@ const RESERVED_ADMIN_PLUGIN_METHOD = "config.plugin.inspect"; const RESERVED_ADMIN_SCOPE_WARNING = "gateway method scope coerced to operator.admin for reserved core namespace"; +async function waitForFilesystemTimestampTick(): Promise { + await new Promise((resolve) => setTimeout(resolve, 50)); +} + function writeBundledPlugin(params: { id: string; body?: string; @@ -2021,7 +2025,7 @@ module.exports = { expect(registry.plugins.find((entry) => entry.id === "discord")?.status).toBe("loaded"); }); - it("loads dist-runtime wrappers from an external stage dir", () => { + it("loads dist-runtime wrappers from an external stage dir", async () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); const bundledDir = path.join(packageRoot, "dist-runtime", "extensions"); @@ -2143,6 +2147,40 @@ module.exports = { expect(fs.lstatSync(path.join(actualInstallRoot, "dist", "pw-ai.js")).isSymbolicLink()).toBe( false, ); + + const runtimeMirrorEntry = path.join( + actualInstallRoot, + "dist-runtime", + "extensions", + "acpx", + "index.js", + ); + const canonicalMirrorEntry = path.join( + actualInstallRoot, + "dist", + "extensions", + "acpx", + "index.js", + ); + const runtimeMirrorStat = fs.statSync(runtimeMirrorEntry); + const canonicalMirrorStat = fs.statSync(canonicalMirrorEntry); + + await waitForFilesystemTimestampTick(); + + const reloadedRegistry = loadOpenClawPlugins({ + cache: false, + config: { + plugins: { + enabled: true, + }, + }, + }); + + const reloadedRecord = reloadedRegistry.plugins.find((entry) => entry.id === "acpx"); + expect(reloadedRecord?.error).toBeUndefined(); + expect(reloadedRecord?.status).toBe("loaded"); + expect(fs.statSync(runtimeMirrorEntry).mtimeMs).toBe(runtimeMirrorStat.mtimeMs); + expect(fs.statSync(canonicalMirrorEntry).mtimeMs).toBe(canonicalMirrorStat.mtimeMs); }); it("loads native ESM deps from a layered baseline stage dir", () => { diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 514059aab26..5e96aaa7476 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -43,6 +43,10 @@ import { withBundledRuntimeDepsFilesystemLock, type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; +import { + copyBundledPluginRuntimeRoot, + refreshBundledPluginRuntimeMirrorRoot, +} from "./bundled-runtime-mirror.js"; import { clearPluginCommands, listRegisteredPluginCommands, @@ -703,15 +707,12 @@ function mirrorBundledPluginRuntimeRoot(params: { if (path.resolve(mirrorRoot) === path.resolve(params.pluginRoot)) { return mirrorRoot; } - const tempDir = fs.mkdtempSync(path.join(mirrorParent, `.plugin-${params.pluginId}-`)); - const stagedRoot = path.join(tempDir, "plugin"); - try { - copyBundledPluginRuntimeRoot(params.pluginRoot, stagedRoot); - fs.rmSync(mirrorRoot, { recursive: true, force: true }); - fs.renameSync(stagedRoot, mirrorRoot); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } + refreshBundledPluginRuntimeMirrorRoot({ + pluginId: params.pluginId, + sourceRoot: params.pluginRoot, + targetRoot: mirrorRoot, + tempDirParent: mirrorParent, + }); return mirrorRoot; }, ); @@ -819,17 +820,12 @@ function mirrorCanonicalBundledRuntimeDistRoot(params: { return; } const targetCanonicalPluginRoot = path.join(targetCanonicalDistRoot, "extensions", pluginId); - const tempDir = fs.mkdtempSync( - path.join(path.dirname(targetCanonicalPluginRoot), `.plugin-${pluginId}-`), - ); - const stagedRoot = path.join(tempDir, "plugin"); - try { - copyBundledPluginRuntimeRoot(sourceCanonicalPluginRoot, stagedRoot); - fs.rmSync(targetCanonicalPluginRoot, { recursive: true, force: true }); - fs.renameSync(stagedRoot, targetCanonicalPluginRoot); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } + refreshBundledPluginRuntimeMirrorRoot({ + pluginId, + sourceRoot: sourceCanonicalPluginRoot, + targetRoot: targetCanonicalPluginRoot, + tempDirParent: path.dirname(targetCanonicalPluginRoot), + }); } function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void { @@ -840,38 +836,6 @@ function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void { writeRuntimeJsonFile(packageJsonPath, { type: "module" }); } -function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void { - if (path.resolve(sourceRoot) === path.resolve(targetRoot)) { - return; - } - fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); - for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) { - if (entry.name === "node_modules") { - continue; - } - const sourcePath = path.join(sourceRoot, entry.name); - const targetPath = path.join(targetRoot, entry.name); - if (entry.isDirectory()) { - copyBundledPluginRuntimeRoot(sourcePath, targetPath); - continue; - } - if (entry.isSymbolicLink()) { - fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath); - continue; - } - if (!entry.isFile()) { - continue; - } - fs.copyFileSync(sourcePath, targetPath); - try { - const sourceMode = fs.statSync(sourcePath).mode; - fs.chmodSync(targetPath, sourceMode | 0o600); - } catch { - // Readable copied files are enough for plugin loading. - } - } -} - function writeRuntimeJsonFile(targetPath: string, value: unknown): void { fs.mkdirSync(path.dirname(targetPath), { recursive: true }); fs.writeFileSync(targetPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");