diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0a1a64c9e..5edd3c6c24e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ Docs: https://docs.openclaw.ai - Plugins: cache unchanged plugin manifest loads by file signature, reducing repeated JSON/JSON5 parsing and manifest normalization in bursty startup and runtime registry paths. Refs #73532 and #73647; carries forward #73678. Thanks @TheDutchRuler. - Plugins/runtime-deps: cache unchanged bundled runtime mirror dist-file materialization decisions and close file-lock handles on owner-write failures, reducing repeated startup chunk scans and avoiding FileHandle-GC recovery stalls. Refs #73532. Thanks @oadiazp and @bstanbury. - Plugins/runtime-deps: retry and defer transient cleanup failures for owned runtime staging directories so CLI startup no longer aborts after a successful bundled dependency swap. Refs #73903. Thanks @bobfreeman1989. +- Plugins/runtime-deps: cache bundled runtime-deps JSON/package files and root chunk import scans by file signature, reducing repeated staged-runtime scanning during bundled channel startup. Refs #73647 and #73705. Thanks @mattmcintyre and @bmilne1981. - CLI/TUI: keep `chat.history` off model-catalog discovery so initial Gateway-backed TUI history loads cannot block behind slow provider/plugin model scans on low-core hosts. Refs #73524. Thanks @harshcatsystems-collab. - Channels/WhatsApp: flag recently reconnected linked accounts in channel status even when the socket is currently healthy, so flapping WhatsApp Web sessions no longer look clean after a brief reconnect. Refs #73602. Thanks @Vksh07. - Gateway: expose `gateway.handshakeTimeoutMs` in config, schema, and docs while preserving `OPENCLAW_HANDSHAKE_TIMEOUT_MS` precedence, so loaded or low-powered hosts can tune local WebSocket pre-auth handshakes without patching dist files. Supersedes #51282; refs #73592 and #73652. Thanks @henry-the-frog. diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index b22c3144b24..e6124adc0b9 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -76,12 +76,26 @@ const BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE = /(?:^|\n)\/\/#region extensions\ const BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE = /(?:^|[;\n])\s*(?:import|export)\s+(?:[^'"()]+?\s+from\s+)?["']([^"']+)["']|\bimport\(\s*["']([^"']+)["']\s*\)|\brequire\(\s*["']([^"']+)["']\s*\)/g; const NPM_EXECPATH_ENV_KEY = "npm_execpath"; +const MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES = 2048; const registeredBundledRuntimeDepNodePaths = new Set(); const bundledRuntimeMirrorMaterializeCache = new Map< string, { signature: string; materialize: boolean } >(); +const runtimeDepsTextFileCache = new Map(); +const runtimeDepsJsonObjectCache = new Map< + string, + { signature: string; value: JsonObject | null } +>(); +const runtimeDepsImportSpecifierCache = new Map< + string, + { signature: string; value: readonly string[] } +>(); +const runtimeMirrorMaterializeImportSpecifierCache = new Map< + string, + { signature: string; value: readonly string[] } +>(); export type BundledRuntimeDepsNpmRunner = { command: string; @@ -98,17 +112,19 @@ function statSignature(stat: Pick) } function computeBundledRuntimeMirrorDistFileMaterialization(sourcePath: string): boolean { - let source: string; - try { - source = fs.readFileSync(sourcePath, "utf8"); - } catch { + const signature = getRuntimeDepsFileSignature(sourcePath); + const source = readRuntimeDepsTextFile(sourcePath, signature); + if (source === null) { return false; } if (BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE.test(source)) { return true; } - for (const match of source.matchAll(BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE)) { - const specifier = match[1] ?? match[2] ?? match[3] ?? ""; + for (const specifier of readRuntimeMirrorMaterializeImportSpecifiers( + sourcePath, + signature, + source, + )) { if ( specifier !== "" && !specifier.startsWith(".") && @@ -279,17 +295,132 @@ function readInstalledDependencyVersion(rootDir: string, depName: string): strin } function readJsonObject(filePath: string): JsonObject | null { + const signature = getRuntimeDepsFileSignature(filePath); + const cached = signature ? runtimeDepsJsonObjectCache.get(filePath) : undefined; + if (cached?.signature === signature) { + return cached.value; + } + const source = readRuntimeDepsTextFile(filePath, signature); + if (source === null) { + cacheRuntimeDepsJsonObject(filePath, signature, null); + return null; + } try { - const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown; + const parsed = JSON.parse(source) as unknown; if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + cacheRuntimeDepsJsonObject(filePath, signature, null); return null; } - return parsed as JsonObject; + const value = parsed as JsonObject; + cacheRuntimeDepsJsonObject(filePath, signature, value); + return value; + } catch { + cacheRuntimeDepsJsonObject(filePath, signature, null); + return null; + } +} + +function readRuntimeDepsTextFile(filePath: string, signature?: string | null): string | null { + const fileSignature = signature ?? getRuntimeDepsFileSignature(filePath); + const cached = fileSignature ? runtimeDepsTextFileCache.get(filePath) : undefined; + if (cached?.signature === fileSignature) { + return cached.value; + } + try { + const value = fs.readFileSync(filePath, "utf8"); + if (fileSignature) { + rememberRuntimeDepsCacheEntry(runtimeDepsTextFileCache, filePath, { + signature: fileSignature, + value, + }); + } + return value; } catch { return null; } } +function readRuntimeDepsImportSpecifiers( + filePath: string, + signature: string | null, + source: string, +): readonly string[] { + const cached = signature ? runtimeDepsImportSpecifierCache.get(filePath) : undefined; + if (cached?.signature === signature) { + return cached.value; + } + const value = extractStaticRuntimeImportSpecifiers(source); + if (signature) { + rememberRuntimeDepsCacheEntry(runtimeDepsImportSpecifierCache, filePath, { signature, value }); + } + return value; +} + +function readRuntimeMirrorMaterializeImportSpecifiers( + filePath: string, + signature: string | null, + source: string, +): readonly string[] { + const cached = signature ? runtimeMirrorMaterializeImportSpecifierCache.get(filePath) : undefined; + if (cached?.signature === signature) { + return cached.value; + } + const value = extractRuntimeMirrorMaterializeImportSpecifiers(source); + if (signature) { + rememberRuntimeDepsCacheEntry(runtimeMirrorMaterializeImportSpecifierCache, filePath, { + signature, + value, + }); + } + return value; +} + +function extractRuntimeMirrorMaterializeImportSpecifiers(source: string): string[] { + const specifiers = new Set(); + for (const match of source.matchAll(BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE)) { + const specifier = match[1] ?? match[2] ?? match[3]; + if (specifier) { + specifiers.add(specifier); + } + } + return [...specifiers]; +} + +function getRuntimeDepsFileSignature(filePath: string): string | null { + try { + const stat = fs.statSync(filePath, { bigint: true }); + if (!stat.isFile()) { + return null; + } + return [ + stat.dev.toString(), + stat.ino.toString(), + stat.size.toString(), + stat.mtimeNs.toString(), + ].join(":"); + } catch { + return null; + } +} + +function cacheRuntimeDepsJsonObject( + filePath: string, + signature: string | null, + value: JsonObject | null, +): void { + if (!signature) { + return; + } + rememberRuntimeDepsCacheEntry(runtimeDepsJsonObjectCache, filePath, { signature, value }); +} + +function rememberRuntimeDepsCacheEntry(cache: Map, key: string, value: T): void { + if (cache.size >= MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES && !cache.has(key)) { + cache.delete(cache.keys().next().value as string); + } + cache.set(key, value); +} + function sleepSync(ms: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } @@ -686,9 +817,13 @@ function collectRootDistMirroredRuntimeDeps(params: { rootDir: distDir, skipTopLevelDirs: new Set(["extensions"]), })) { - const source = fs.readFileSync(filePath, "utf8"); + const signature = getRuntimeDepsFileSignature(filePath); + const source = readRuntimeDepsTextFile(filePath, signature); + if (source === null) { + continue; + } const relativePath = path.relative(distDir, filePath).replaceAll(path.sep, "/"); - for (const specifier of extractStaticRuntimeImportSpecifiers(source)) { + for (const specifier of readRuntimeDepsImportSpecifiers(filePath, signature, source)) { const dependencyName = packageNameFromSpecifier(specifier); if (!dependencyName) { continue; diff --git a/src/plugins/bundled-runtime-root.test.ts b/src/plugins/bundled-runtime-root.test.ts index 8ee368f5993..d71202455b2 100644 --- a/src/plugins/bundled-runtime-root.test.ts +++ b/src/plugins/bundled-runtime-root.test.ts @@ -68,6 +68,11 @@ describe("prepareBundledPluginRuntimeRoot", () => { "import JSON5 from 'json5'; export const parse = JSON5.parse;\n", "utf8", ); + fs.writeFileSync( + path.join(packageRoot, "dist", "string-runtime.js"), + `const text = 'not an import: from "zod"'; export const marker = text;\n`, + "utf8", + ); fs.writeFileSync( path.join(pluginRoot, "index.js"), `import { marker } from "../../pw-ai.js"; export default { id: "browser", marker };\n`, @@ -138,6 +143,82 @@ describe("prepareBundledPluginRuntimeRoot", () => { expect(fs.lstatSync(path.join(installRoot, "dist", "config-runtime.js")).isSymbolicLink()).toBe( true, ); + expect(fs.lstatSync(path.join(installRoot, "dist", "string-runtime.js")).isSymbolicLink()).toBe( + false, + ); + }); + + it("reuses root chunk materialization decisions across bundled plugin mirrors", () => { + const packageRoot = makeTempRoot(); + const stageDir = makeTempRoot(); + const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const rootChunk = path.join(packageRoot, "dist", "shared-runtime.js"); + const externalChunk = path.join(packageRoot, "dist", "external-runtime.js"); + fs.mkdirSync(path.join(packageRoot, "dist", "extensions"), { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27", type: "module" }), + "utf8", + ); + fs.writeFileSync(rootChunk, "export const shared = 'root';\n", "utf8"); + fs.writeFileSync(externalChunk, "import zod from 'zod'; export const schema = zod;\n", "utf8"); + + for (const pluginId of ["alpha", "beta"]) { + const pluginRoot = path.join(packageRoot, "dist", "extensions", pluginId); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(pluginRoot, "index.js"), + `import { shared } from "../../shared-runtime.js"; export default { id: ${JSON.stringify(pluginId)}, shared };\n`, + "utf8", + ); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: `@openclaw/${pluginId}`, + version: "1.0.0", + type: "module", + dependencies: { [`${pluginId}-runtime`]: "1.0.0" }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf8", + ); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + fs.mkdirSync(path.join(installRoot, "node_modules", `${pluginId}-runtime`), { + recursive: true, + }); + fs.writeFileSync( + path.join(installRoot, "node_modules", `${pluginId}-runtime`, "package.json"), + JSON.stringify({ name: `${pluginId}-runtime`, version: "1.0.0", type: "module" }), + "utf8", + ); + } + + const realReadFileSync = fs.readFileSync.bind(fs); + const readPaths: string[] = []; + vi.spyOn(fs, "readFileSync").mockImplementation(((target, options) => { + const targetPath = target.toString(); + if (targetPath === rootChunk || targetPath === externalChunk) { + readPaths.push(targetPath); + } + return realReadFileSync(target, options as never); + }) as typeof fs.readFileSync); + + for (const pluginId of ["alpha", "beta"]) { + const pluginRoot = path.join(packageRoot, "dist", "extensions", pluginId); + prepareBundledPluginRuntimeRoot({ + pluginId, + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + } + + expect(readPaths.filter((entry) => entry === rootChunk)).toHaveLength(1); + expect(readPaths.filter((entry) => entry === externalChunk)).toHaveLength(1); }); it("does not copy staged runtime mirror dist files onto themselves", () => {