diff --git a/CHANGELOG.md b/CHANGELOG.md index 35fec2275da..c5bb1a2f7bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Docs: https://docs.openclaw.ai - Discord/native commands: return the real status card for native `/status` interactions instead of falling through to the synthetic `✅ Done.` ack when the generic dispatcher produces no visible reply. (#54629) Thanks @tkozzer and @vincentkoc. - Hooks/Ollama: let LLM-backed session-memory slug generation honor an explicit `agents.defaults.timeoutSeconds` override instead of always aborting after 15 seconds, so slow local Ollama runs stop silently dropping back to generic filenames. (#66237) Thanks @dmak and @vincentkoc. - Media/transcription: remap `.aac` filenames to `.m4a` for OpenAI-compatible audio uploads so AAC voice notes stop failing MIME-sensitive transcription endpoints. (#66446) Thanks @ben-z. +- WhatsApp/Baileys media upload: keep encrypted upload POSTs streaming while still guarding generic-agent dispatcher wiring, so large outbound media sends avoid full-buffer RSS spikes and OOM regressions. (#65966) Thanks @frankekn. - UI/chat: replace marked.js with markdown-it so maliciously crafted markdown can no longer freeze the Control UI via ReDoS. (#46707) Thanks @zhangfnf. - Auto-reply/send policy: keep `sendPolicy: "deny"` from blocking inbound message processing, so the agent still runs its turn while all outbound delivery is suppressed for observer-style setups. (#65461, #53328) Thanks @omarshahine. - BlueBubbles: lazy-refresh the Private API server-info cache on send when reply threading or message effects are requested but status is unknown, so sends no longer silently degrade to plain messages when the 10-minute cache expires. (#65447, #43764) Thanks @omarshahine. diff --git a/extensions/whatsapp/package.json b/extensions/whatsapp/package.json index 4a37a7360b8..c7db4b453af 100644 --- a/extensions/whatsapp/package.json +++ b/extensions/whatsapp/package.json @@ -47,12 +47,12 @@ "compat": { "pluginApi": ">=2026.4.12" }, - "build": { - "openclawVersion": "2026.4.12" - }, "bundle": { "stageRuntimeDependencies": true }, + "build": { + "openclawVersion": "2026.4.12" + }, "release": { "publishToClawHub": true, "publishToNpm": true diff --git a/extensions/whatsapp/src/session.test.ts b/extensions/whatsapp/src/session.test.ts index d281b6afde5..ad140f21517 100644 --- a/extensions/whatsapp/src/session.test.ts +++ b/extensions/whatsapp/src/session.test.ts @@ -127,7 +127,11 @@ describe("web session", () => { fetchAgent?: unknown; }; expect(passed.agent).toBeDefined(); - expect(passed.fetchAgent).toBe(passed.agent); + expect(passed.fetchAgent).toBeDefined(); + expect(passed.fetchAgent).not.toBe(passed.agent); + expect(typeof (passed.fetchAgent as { dispatch?: unknown } | undefined)?.dispatch).toBe( + "function", + ); }); it("does not create a proxy agent when no env proxy is configured", async () => { diff --git a/extensions/whatsapp/src/session.ts b/extensions/whatsapp/src/session.ts index 306bddfc780..34d50414ab2 100644 --- a/extensions/whatsapp/src/session.ts +++ b/extensions/whatsapp/src/session.ts @@ -125,6 +125,7 @@ export async function createWaSocket( const { state, saveCreds } = await useMultiFileAuthState(authDir); const { version } = await fetchLatestBaileysVersion(); const agent = await resolveEnvProxyAgent(sessionLogger); + const fetchAgent = await resolveEnvFetchDispatcher(sessionLogger, agent); const sock = makeWASocket({ auth: { creds: state.creds, @@ -137,7 +138,9 @@ export async function createWaSocket( syncFullHistory: false, markOnlineOnConnect: false, agent, - fetchAgent: agent, + // Baileys types still model `fetchAgent` as a Node agent even though the + // runtime path accepts an undici dispatcher for upload fetches. + fetchAgent: fetchAgent as Agent | undefined, }); sock.ev.on("creds.update", () => enqueueSaveCreds(authDir, saveCreds, sessionLogger)); @@ -199,6 +202,58 @@ async function resolveEnvProxyAgent( }); } +async function resolveEnvFetchDispatcher( + logger: ReturnType, + agent?: unknown, +): Promise { + const proxyUrl = resolveProxyUrlFromAgent(agent); + const envProxyUrl = resolveEnvHttpsProxyUrl(); + if (!proxyUrl && !envProxyUrl) { + return undefined; + } + try { + const { EnvHttpProxyAgent, ProxyAgent } = await import("undici"); + return proxyUrl + ? new ProxyAgent({ allowH2: false, uri: proxyUrl }) + : new EnvHttpProxyAgent({ allowH2: false }); + } catch (error) { + logger.warn( + { error: String(error) }, + "Failed to initialize env proxy dispatcher for WhatsApp media uploads", + ); + return undefined; + } +} + +function resolveProxyUrlFromAgent(agent: unknown): string | undefined { + if (typeof agent !== "object" || agent === null || !("proxy" in agent)) { + return undefined; + } + const proxy = (agent as { proxy?: unknown }).proxy; + if (proxy instanceof URL) { + return proxy.toString(); + } + return typeof proxy === "string" && proxy.length > 0 ? proxy : undefined; +} + +function resolveEnvHttpsProxyUrl(env: NodeJS.ProcessEnv = process.env): string | undefined { + const lowerHttpsProxy = normalizeEnvProxyValue(env.https_proxy); + const lowerHttpProxy = normalizeEnvProxyValue(env.http_proxy); + const httpsProxy = + lowerHttpsProxy !== undefined ? lowerHttpsProxy : normalizeEnvProxyValue(env.HTTPS_PROXY); + const httpProxy = + lowerHttpProxy !== undefined ? lowerHttpProxy : normalizeEnvProxyValue(env.HTTP_PROXY); + return httpsProxy ?? httpProxy ?? undefined; +} + +function normalizeEnvProxyValue(value: string | undefined): string | null | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + export async function waitForWaConnection(sock: ReturnType) { return new Promise((resolve, reject) => { type OffCapable = { diff --git a/package.json b/package.json index 401403a709a..71015e17168 100644 --- a/package.json +++ b/package.json @@ -1523,6 +1523,9 @@ "strip-ansi": "^7.2.0" } } + }, + "patchedDependencies": { + "@whiskeysockets/baileys@7.0.0-rc.9": "patches/@whiskeysockets__baileys@7.0.0-rc.9.patch" } } } diff --git a/patches/@whiskeysockets__baileys@7.0.0-rc.9.patch b/patches/@whiskeysockets__baileys@7.0.0-rc.9.patch new file mode 100644 index 00000000000..c47805f1b3d --- /dev/null +++ b/patches/@whiskeysockets__baileys@7.0.0-rc.9.patch @@ -0,0 +1,46 @@ +diff --git a/lib/Utils/messages-media.js b/lib/Utils/messages-media.js +index 0d32dfb4882dfe029ba8804772d7d89404b08e76..73809fcd1d52362aef0c35cb7416c29d86482df0 100644 +--- a/lib/Utils/messages-media.js ++++ b/lib/Utils/messages-media.js +@@ -353,9 +353,17 @@ + const fileSha256 = sha256Plain.digest(); + const fileEncSha256 = sha256Enc.digest(); + encFileWriteStream.write(mac); ++ // Create finish promises before calling end() to avoid missing the event ++ const encFinishPromise = once(encFileWriteStream, 'finish'); ++ const originalFinishPromise = originalFileStream ? once(originalFileStream, 'finish') : Promise.resolve(); + encFileWriteStream.end(); + originalFileStream?.end?.(); + stream.destroy(); ++ // Wait for write streams to fully flush to disk before returning encFilePath. ++ // Without this await, the caller may open a read stream on the file before ++ // the OS has created it, causing a race-condition ENOENT crash. ++ await encFinishPromise; ++ await originalFinishPromise; + logger?.debug('encrypted data successfully'); + return { + mediaKey, +@@ -520,11 +528,10 @@ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let result; + try { + const stream = createReadStream(filePath); + const response = await fetch(url, { +- dispatcher: fetchAgent, + method: 'POST', + body: stream, + headers: { + ...(() => { + const hdrs = options?.headers; +@@ -535,6 +542,11 @@ + 'Content-Type': 'application/octet-stream', + Origin: DEFAULT_ORIGIN + }, ++ // Baileys passes a generic agent here in some runtimes. Undici's ++ // `dispatcher` only works with Dispatcher-compatible implementations, ++ // so only wire it through when the object actually implements ++ // `dispatch`. ++ ...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}), + duplex: 'half', + // Note: custom agents/proxy require undici Agent; omitted here. + signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b7fefafb71f..758c6fc249b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,11 @@ overrides: packageExtensionsChecksum: sha256-n+P/SQo4Pf+dHYpYn1Y6wL4cJEVoVzZ835N0OEp4TM8= +patchedDependencies: + '@whiskeysockets/baileys@7.0.0-rc.9': + hash: 23ec8efe1484afa57c51b96955ba331d1467521a8e676a18c2690da7e70a6201 + path: patches/@whiskeysockets__baileys@7.0.0-rc.9.patch + importers: .: @@ -114,7 +119,7 @@ importers: version: 7.15.0 '@whiskeysockets/baileys': specifier: 7.0.0-rc.9 - version: 7.0.0-rc.9(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5) + version: 7.0.0-rc.9(patch_hash=23ec8efe1484afa57c51b96955ba331d1467521a8e676a18c2690da7e70a6201)(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5) ajv: specifier: ^8.18.0 version: 8.18.0 @@ -1221,7 +1226,7 @@ importers: dependencies: '@whiskeysockets/baileys': specifier: 7.0.0-rc.9 - version: 7.0.0-rc.9(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5) + version: 7.0.0-rc.9(patch_hash=23ec8efe1484afa57c51b96955ba331d1467521a8e676a18c2690da7e70a6201)(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5) jimp: specifier: ^1.6.1 version: 1.6.1 @@ -11423,7 +11428,7 @@ snapshots: '@wasm-audio-decoders/common': 9.0.7 optional: true - '@whiskeysockets/baileys@7.0.0-rc.9(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5)': + '@whiskeysockets/baileys@7.0.0-rc.9(patch_hash=23ec8efe1484afa57c51b96955ba331d1467521a8e676a18c2690da7e70a6201)(audio-decode@2.2.3)(jimp@1.6.1)(sharp@0.34.5)': dependencies: '@cacheable/node-cache': 1.7.6 '@hapi/boom': 9.1.4 diff --git a/scripts/postinstall-bundled-plugins.mjs b/scripts/postinstall-bundled-plugins.mjs index 0c2e002d72b..06e1a301c7a 100644 --- a/scripts/postinstall-bundled-plugins.mjs +++ b/scripts/postinstall-bundled-plugins.mjs @@ -55,6 +55,47 @@ const BAILEYS_MEDIA_HOTFIX_REPLACEMENT = [ " await Promise.all([encFinishPromise, originalFinishPromise]);", " logger?.debug('encrypted data successfully');", ].join("\n"); +const BAILEYS_MEDIA_HOTFIX_SEQUENTIAL_REPLACEMENT = [ + " encFileWriteStream.write(mac);", + " const encFinishPromise = once(encFileWriteStream, 'finish');", + " const originalFinishPromise = originalFileStream ? once(originalFileStream, 'finish') : Promise.resolve();", + " encFileWriteStream.end();", + " originalFileStream?.end?.();", + " stream.destroy();", + " await encFinishPromise;", + " await originalFinishPromise;", + " logger?.debug('encrypted data successfully');", +].join("\n"); +const BAILEYS_MEDIA_HOTFIX_FINISH_PROMISES_RE = + /const\s+encFinishPromise\s*=\s*once\(encFileWriteStream,\s*'finish'\);\s*\n[\s\S]*const\s+originalFinishPromise\s*=\s*originalFileStream\s*\?\s*once\(originalFileStream,\s*'finish'\)\s*:\s*Promise\.resolve\(\);/u; +const BAILEYS_MEDIA_HOTFIX_PROMISE_ALL_RE = + /await\s+Promise\.all\(\[\s*encFinishPromise\s*,\s*originalFinishPromise\s*\]\);/u; +const BAILEYS_MEDIA_HOTFIX_SEQUENTIAL_AWAITS_RE = + /await\s+encFinishPromise;\s*(?:\/\/[^\n]*\n|\s)*await\s+originalFinishPromise;/u; +const BAILEYS_MEDIA_DISPATCHER_NEEDLE = [ + " const response = await fetch(url, {", + " dispatcher: fetchAgent,", + " method: 'POST',", +].join("\n"); +const BAILEYS_MEDIA_DISPATCHER_REPLACEMENT = [ + " const response = await fetch(url, {", + " method: 'POST',", +].join("\n"); +const BAILEYS_MEDIA_DISPATCHER_HEADER_NEEDLE = [ + " 'Content-Type': 'application/octet-stream',", + " Origin: DEFAULT_ORIGIN", + " },", +].join("\n"); +const BAILEYS_MEDIA_DISPATCHER_HEADER_REPLACEMENT = [ + " 'Content-Type': 'application/octet-stream',", + " Origin: DEFAULT_ORIGIN", + " },", + " // Baileys passes a generic agent here in some runtimes. Undici's", + " // `dispatcher` only works with Dispatcher-compatible implementations,", + " // so only wire it through when the object actually implements", + " // `dispatch`.", + " ...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", +].join("\n"); const BAILEYS_MEDIA_ONCE_IMPORT_RE = /import\s+\{\s*once\s*\}\s+from\s+['"]events['"]/u; const BAILEYS_MEDIA_ASYNC_CONTEXT_RE = /async\s+function\s+encryptedStream|encryptedStream\s*=\s*async/u; @@ -243,23 +284,59 @@ export function applyBaileysEncryptedStreamFinishHotfix(params = {}) { } const currentText = readFile(targetPath, "utf8"); - if (currentText.includes(BAILEYS_MEDIA_HOTFIX_REPLACEMENT)) { - return { applied: false, reason: "already_patched" }; - } - if (!currentText.includes(BAILEYS_MEDIA_HOTFIX_NEEDLE)) { - return { applied: false, reason: "unexpected_content" }; - } - if (!BAILEYS_MEDIA_ONCE_IMPORT_RE.test(currentText)) { - return { applied: false, reason: "missing_once_import", targetPath }; - } - if (!BAILEYS_MEDIA_ASYNC_CONTEXT_RE.test(currentText)) { - return { applied: false, reason: "not_async_context", targetPath }; + let patchedText = currentText; + let applied = false; + + const encryptedStreamAlreadyPatched = + patchedText.includes(BAILEYS_MEDIA_HOTFIX_REPLACEMENT) || + patchedText.includes(BAILEYS_MEDIA_HOTFIX_SEQUENTIAL_REPLACEMENT) || + (BAILEYS_MEDIA_HOTFIX_FINISH_PROMISES_RE.test(patchedText) && + (BAILEYS_MEDIA_HOTFIX_PROMISE_ALL_RE.test(patchedText) || + BAILEYS_MEDIA_HOTFIX_SEQUENTIAL_AWAITS_RE.test(patchedText))); + const encryptedStreamPatchable = patchedText.includes(BAILEYS_MEDIA_HOTFIX_NEEDLE); + + let encryptedStreamResolved = encryptedStreamAlreadyPatched; + if (!encryptedStreamResolved && encryptedStreamPatchable) { + if (!BAILEYS_MEDIA_ONCE_IMPORT_RE.test(patchedText)) { + return { applied: false, reason: "missing_once_import", targetPath }; + } + if (!BAILEYS_MEDIA_ASYNC_CONTEXT_RE.test(patchedText)) { + return { applied: false, reason: "not_async_context", targetPath }; + } + patchedText = patchedText.replace( + BAILEYS_MEDIA_HOTFIX_NEEDLE, + BAILEYS_MEDIA_HOTFIX_REPLACEMENT, + ); + applied = true; + encryptedStreamResolved = true; } - const patchedText = currentText.replace( - BAILEYS_MEDIA_HOTFIX_NEEDLE, - BAILEYS_MEDIA_HOTFIX_REPLACEMENT, + const dispatcherAlreadyPatched = patchedText.includes( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", ); + const dispatcherPatchable = + patchedText.includes(BAILEYS_MEDIA_DISPATCHER_NEEDLE) && + patchedText.includes(BAILEYS_MEDIA_DISPATCHER_HEADER_NEEDLE); + let dispatcherResolved = dispatcherAlreadyPatched; + + if (!dispatcherResolved && dispatcherPatchable) { + patchedText = patchedText + .replace(BAILEYS_MEDIA_DISPATCHER_NEEDLE, BAILEYS_MEDIA_DISPATCHER_REPLACEMENT) + .replace( + BAILEYS_MEDIA_DISPATCHER_HEADER_NEEDLE, + BAILEYS_MEDIA_DISPATCHER_HEADER_REPLACEMENT, + ); + applied = true; + dispatcherResolved = true; + } + + if (!dispatcherResolved) { + return { applied: false, reason: "unexpected_content", targetPath }; + } + + if (!applied) { + return { applied: false, reason: "already_patched" }; + } const tempPath = createTempPath(targetPath); const tempFd = openFile(tempPath, "wx", initialTargetValidation.mode); let tempFdClosed = false; @@ -298,12 +375,12 @@ function applyBundledPluginRuntimeHotfixes(params = {}) { const log = params.log ?? console; const baileysResult = applyBaileysEncryptedStreamFinishHotfix(params); if (baileysResult.applied) { - log.log("[postinstall] patched @whiskeysockets/baileys encryptedStream flush ordering"); + log.log("[postinstall] patched @whiskeysockets/baileys runtime hotfixes"); return; } if (baileysResult.reason !== "missing" && baileysResult.reason !== "already_patched") { log.warn( - `[postinstall] could not patch @whiskeysockets/baileys encryptedStream: ${baileysResult.reason}`, + `[postinstall] could not patch @whiskeysockets/baileys runtime hotfixes: ${baileysResult.reason}`, ); } } diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index 68b1abc4fbf..f0c7a1e9237 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -15,6 +15,13 @@ function writeJson(filePath, value) { fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } +function readOptionalUtf8(filePath) { + if (!fs.existsSync(filePath)) { + return null; + } + return fs.readFileSync(filePath, "utf8"); +} + function removePathIfExists(targetPath) { fs.rmSync(targetPath, { recursive: true, force: true }); } @@ -42,15 +49,43 @@ function replaceDir(targetPath, sourcePath) { removePathIfExists(sourcePath); } +function dependencyPathSegments(depName) { + if (typeof depName !== "string" || depName.length === 0) { + return null; + } + const segments = depName.split("/"); + if (depName.startsWith("@")) { + if (segments.length !== 2) { + return null; + } + const [scope, name] = segments; + if ( + !/^@[A-Za-z0-9._-]+$/.test(scope) || + !/^[A-Za-z0-9._-]+$/.test(name) || + scope === "@." || + scope === "@.." + ) { + return null; + } + return [scope, name]; + } + if (segments.length !== 1 || !/^[A-Za-z0-9._-]+$/.test(segments[0])) { + return null; + } + return segments; +} + function dependencyNodeModulesPath(nodeModulesDir, depName) { - return path.join(nodeModulesDir, ...depName.split("/")); + const segments = dependencyPathSegments(depName); + return segments ? path.join(nodeModulesDir, ...segments) : null; } function readInstalledDependencyVersion(nodeModulesDir, depName) { - const packageJsonPath = path.join( - dependencyNodeModulesPath(nodeModulesDir, depName), - "package.json", - ); + const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName); + if (depRoot === null) { + return null; + } + const packageJsonPath = path.join(depRoot, "package.json"); if (!fs.existsSync(packageJsonPath)) { return null; } @@ -62,6 +97,15 @@ function dependencyVersionSatisfied(spec, installedVersion) { return semverSatisfies(installedVersion, spec, { includePrerelease: false }); } +function readInstalledDependencyVersionFromRoot(depRoot) { + const packageJsonPath = path.join(depRoot, "package.json"); + if (!fs.existsSync(packageJsonPath)) { + return null; + } + const version = readJson(packageJsonPath).version; + return typeof version === "string" ? version : null; +} + const defaultStagedRuntimeDepGlobalPruneSuffixes = [".d.ts", ".map"]; const defaultStagedRuntimeDepPruneRules = new Map([ // Type declarations only; runtime resolves through lib/es entrypoints. @@ -103,7 +147,7 @@ const defaultStagedRuntimeDepPruneRules = new Map([ ["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }], ["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }], ]); -const runtimeDepsStagingVersion = 2; +const runtimeDepsStagingVersion = 3; function resolveRuntimeDepPruneConfig(params = {}) { return { @@ -113,38 +157,247 @@ function resolveRuntimeDepPruneConfig(params = {}) { }; } -function collectInstalledRuntimeClosure(rootNodeModulesDir, dependencySpecs) { +function resolveInstalledDependencyRoot(params) { + const candidates = []; + if (params.parentPackageRoot) { + const nestedDepRoot = dependencyNodeModulesPath( + path.join(params.parentPackageRoot, "node_modules"), + params.depName, + ); + if (nestedDepRoot !== null) { + candidates.push(nestedDepRoot); + } + } + const rootDepRoot = dependencyNodeModulesPath(params.rootNodeModulesDir, params.depName); + if (rootDepRoot !== null) { + candidates.push(rootDepRoot); + } + + for (const depRoot of candidates) { + const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); + if (installedVersion !== null && dependencyVersionSatisfied(params.spec, installedVersion)) { + return depRoot; + } + } + + return null; +} + +function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs) { const packageCache = new Map(); - const closure = new Set(); - const queue = Object.entries(dependencySpecs); + const directRoots = []; + const allRoots = []; + const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({ + depName, + spec, + parentPackageRoot: null, + direct: true, + })); + const seen = new Set(); while (queue.length > 0) { - const [depName, spec] = queue.shift(); + const current = queue.shift(); + const depRoot = resolveInstalledDependencyRoot({ + depName: current.depName, + spec: current.spec, + parentPackageRoot: current.parentPackageRoot, + rootNodeModulesDir, + }); + if (depRoot === null) { + return null; + } + const canonicalDepRoot = fs.realpathSync(depRoot); + + const seenKey = `${current.depName}\0${canonicalDepRoot}`; + if (seen.has(seenKey)) { + continue; + } + seen.add(seenKey); + + const record = { name: current.depName, root: depRoot, realRoot: canonicalDepRoot }; + allRoots.push(record); + if (current.direct) { + directRoots.push(record); + } + + const packageJson = + packageCache.get(canonicalDepRoot) ?? readJson(path.join(depRoot, "package.json")); + packageCache.set(canonicalDepRoot, packageJson); + for (const [childName, childSpec] of Object.entries(packageJson.dependencies ?? {})) { + queue.push({ + depName: childName, + spec: childSpec, + parentPackageRoot: depRoot, + direct: false, + }); + } + for (const [childName, childSpec] of Object.entries(packageJson.optionalDependencies ?? {})) { + queue.push({ + depName: childName, + spec: childSpec, + parentPackageRoot: depRoot, + direct: false, + }); + } + } + + return { allRoots, directRoots }; +} + +function pathIsInsideCopiedRoot(candidateRoot, copiedRoot) { + return candidateRoot === copiedRoot || candidateRoot.startsWith(`${copiedRoot}${path.sep}`); +} + +function findContainingRealRoot(candidatePath, allowedRealRoots) { + return ( + allowedRealRoots.find((rootPath) => pathIsInsideCopiedRoot(candidatePath, rootPath)) ?? null + ); +} + +function copyMaterializedDependencyTree(params) { + const { activeRoots, allowedRealRoots, sourcePath, targetPath } = params; + const sourceStats = fs.lstatSync(sourcePath); + + if (sourceStats.isSymbolicLink()) { + let resolvedPath; + try { + resolvedPath = fs.realpathSync(sourcePath); + } catch { + return false; + } + const containingRoot = findContainingRealRoot(resolvedPath, allowedRealRoots); + if (containingRoot === null) { + return false; + } + if (activeRoots.has(containingRoot)) { + return true; + } + const nextActiveRoots = new Set(activeRoots); + nextActiveRoots.add(containingRoot); + return copyMaterializedDependencyTree({ + activeRoots: nextActiveRoots, + allowedRealRoots, + sourcePath: resolvedPath, + targetPath, + }); + } + + if (sourceStats.isDirectory()) { + fs.mkdirSync(targetPath, { recursive: true }); + for (const entry of fs + .readdirSync(sourcePath, { withFileTypes: true }) + .toSorted((left, right) => left.name.localeCompare(right.name))) { + if ( + !copyMaterializedDependencyTree({ + activeRoots, + allowedRealRoots, + sourcePath: path.join(sourcePath, entry.name), + targetPath: path.join(targetPath, entry.name), + }) + ) { + return false; + } + } + return true; + } + + if (sourceStats.isFile()) { + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.copyFileSync(sourcePath, targetPath); + fs.chmodSync(targetPath, sourceStats.mode); + return true; + } + + return true; +} + +function selectRuntimeDependencyRootsToCopy(resolution) { + const rootsToCopy = []; + + for (const record of resolution.directRoots) { + rootsToCopy.push(record); + } + + for (const record of resolution.allRoots) { + if (rootsToCopy.some((entry) => pathIsInsideCopiedRoot(record.realRoot, entry.realRoot))) { + continue; + } + rootsToCopy.push(record); + } + + return rootsToCopy; +} + +function resolveInstalledDirectDependencyNames(rootNodeModulesDir, dependencySpecs) { + const directDependencyNames = []; + for (const [depName, spec] of Object.entries(dependencySpecs)) { const installedVersion = readInstalledDependencyVersion(rootNodeModulesDir, depName); if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) { return null; } - if (closure.has(depName)) { + directDependencyNames.push(depName); + } + return directDependencyNames; +} + +function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) { + const entries = fs + .readdirSync(currentDir, { withFileTypes: true }) + .toSorted((left, right) => left.name.localeCompare(right.name)); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry.name); + const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/"); + if (entry.isSymbolicLink()) { + hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`); continue; } - - const packageJsonPath = path.join( - dependencyNodeModulesPath(rootNodeModulesDir, depName), - "package.json", - ); - const packageJson = packageCache.get(depName) ?? readJson(packageJsonPath); - packageCache.set(depName, packageJson); - closure.add(depName); - - for (const [childName, childSpec] of Object.entries(packageJson.dependencies ?? {})) { - queue.push([childName, childSpec]); + if (entry.isDirectory()) { + hash.update(`dir:${relativePath}\n`); + appendDirectoryFingerprint(hash, rootDir, fullPath); + continue; } - for (const [childName, childSpec] of Object.entries(packageJson.optionalDependencies ?? {})) { - queue.push([childName, childSpec]); + if (!entry.isFile()) { + continue; } + const stat = fs.statSync(fullPath); + hash.update(`file:${relativePath}:${stat.size}\n`); + hash.update(fs.readFileSync(fullPath)); } +} - return [...closure]; +function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) { + const hash = createHash("sha256"); + for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) { + const depRoot = dependencyNodeModulesPath(rootNodeModulesDir, depName); + if (depRoot === null || !fs.existsSync(depRoot)) { + return null; + } + hash.update(`package:${depName}\n`); + appendDirectoryFingerprint(hash, depRoot); + } + return hash.digest("hex"); +} + +function resolveInstalledRuntimeClosureFingerprint(params) { + const dependencySpecs = { + ...params.packageJson.dependencies, + ...params.packageJson.optionalDependencies, + }; + if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(params.rootNodeModulesDir)) { + return null; + } + const resolution = collectInstalledRuntimeDependencyRoots( + params.rootNodeModulesDir, + dependencySpecs, + ); + if (resolution === null) { + return null; + } + return createInstalledRuntimeClosureFingerprint( + params.rootNodeModulesDir, + selectRuntimeDependencyRootsToCopy(resolution).map((record) => record.name), + ); } function walkFiles(rootDir, visitFile) { @@ -180,6 +433,9 @@ function pruneDependencyFilesBySuffixes(depRoot, suffixes) { function pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig) { const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName); + if (depRoot === null) { + return; + } const pruneRule = pruneConfig.pruneRules.get(depName); for (const relativePath of pruneRule?.paths ?? []) { removePathIfExists(path.join(depRoot, relativePath)); @@ -272,13 +528,21 @@ function resolveRuntimeDepsStampPath(pluginDir) { return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"); } -function createRuntimeDepsFingerprint(packageJson, pruneConfig) { +function createRuntimeDepsFingerprint(packageJson, pruneConfig, params = {}) { + const repoRoot = params.repoRoot; + const lockfilePath = + typeof repoRoot === "string" && repoRoot.length > 0 + ? path.join(repoRoot, "pnpm-lock.yaml") + : null; + const rootLockfile = lockfilePath ? readOptionalUtf8(lockfilePath) : null; return createHash("sha256") .update( JSON.stringify({ globalPruneSuffixes: pruneConfig.globalPruneSuffixes, packageJson, pruneRules: [...pruneConfig.pruneRules.entries()], + rootInstalledRuntimeFingerprint: params.rootInstalledRuntimeFingerprint ?? null, + rootLockfile, version: runtimeDepsStagingVersion, }), ) @@ -307,10 +571,19 @@ function stageInstalledRootRuntimeDeps(params) { return false; } - const dependencyNames = collectInstalledRuntimeClosure(rootNodeModulesDir, dependencySpecs); - if (dependencyNames === null) { + const directDependencyNames = resolveInstalledDirectDependencyNames( + rootNodeModulesDir, + dependencySpecs, + ); + if (directDependencyNames === null) { return false; } + const resolution = collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs); + if (resolution === null) { + return false; + } + const rootsToCopy = selectRuntimeDependencyRootsToCopy(resolution); + const allowedRealRoots = rootsToCopy.map((record) => record.realRoot); const nodeModulesDir = path.join(pluginDir, "node_modules"); const stampPath = resolveRuntimeDepsStampPath(pluginDir); @@ -323,11 +596,27 @@ function stageInstalledRootRuntimeDeps(params) { ); try { - for (const depName of dependencyNames) { - const sourcePath = dependencyNodeModulesPath(rootNodeModulesDir, depName); - const targetPath = dependencyNodeModulesPath(stagedNodeModulesDir, depName); + for (const record of rootsToCopy.toSorted((left, right) => + left.name.localeCompare(right.name), + )) { + const sourcePath = record.realRoot; + const targetPath = dependencyNodeModulesPath(stagedNodeModulesDir, record.name); + if (targetPath === null) { + return false; + } fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - fs.cpSync(sourcePath, targetPath, { recursive: true, force: true, dereference: true }); + const sourceRootReal = findContainingRealRoot(sourcePath, allowedRealRoots); + if ( + sourceRootReal === null || + !copyMaterializedDependencyTree({ + activeRoots: new Set([sourceRootReal]), + allowedRealRoots, + sourcePath, + targetPath, + }) + ) { + return false; + } } pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); @@ -435,7 +724,14 @@ export function stageBundledPluginRuntimeDeps(params = {}) { removePathIfExists(stampPath); continue; } - const fingerprint = createRuntimeDepsFingerprint(packageJson, pruneConfig); + const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({ + packageJson, + rootNodeModulesDir: path.join(repoRoot, "node_modules"), + }); + const fingerprint = createRuntimeDepsFingerprint(packageJson, pruneConfig, { + repoRoot, + rootInstalledRuntimeFingerprint, + }); const stamp = readRuntimeDepsStamp(stampPath); if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) { continue; diff --git a/src/plugins/stage-bundled-plugin-runtime-deps.test.ts b/src/plugins/stage-bundled-plugin-runtime-deps.test.ts index d056bdc15cc..f25e287fbb1 100644 --- a/src/plugins/stage-bundled-plugin-runtime-deps.test.ts +++ b/src/plugins/stage-bundled-plugin-runtime-deps.test.ts @@ -63,6 +63,117 @@ function writeRepoFile(repoRoot: string, relativePath: string, value: string) { fs.writeFileSync(fullPath, value, "utf8"); } +function createBaileysMessagesMediaSource(params?: { + dispatcherPatched?: boolean; + dispatcherHeaderDrifted?: boolean; + encryptedStreamPatched?: boolean; + encryptedStreamPatchedSequentially?: boolean; + encryptedStreamPatchedSequentiallyWithComments?: boolean; + encryptedStreamUnrecognized?: boolean; +}) { + const encryptedLines = params?.encryptedStreamUnrecognized + ? [ + " encFileWriteStream.write(mac);", + " logger?.debug('encrypted data changed upstream');", + ] + : params?.encryptedStreamPatchedSequentiallyWithComments + ? [ + " encFileWriteStream.write(mac);", + " const encFinishPromise = once(encFileWriteStream, 'finish');", + " const originalFinishPromise = originalFileStream ? once(originalFileStream, 'finish') : Promise.resolve();", + " encFileWriteStream.end();", + " originalFileStream?.end?.();", + " stream.destroy();", + " // Wait for write streams to fully flush to disk before returning encFilePath.", + " // Without this await, the caller may open a read stream on the file before", + " // the OS has created it, causing a race-condition ENOENT crash.", + " await encFinishPromise;", + " await originalFinishPromise;", + " logger?.debug('encrypted data successfully');", + ] + : params?.encryptedStreamPatchedSequentially + ? [ + " encFileWriteStream.write(mac);", + " const encFinishPromise = once(encFileWriteStream, 'finish');", + " const originalFinishPromise = originalFileStream ? once(originalFileStream, 'finish') : Promise.resolve();", + " encFileWriteStream.end();", + " originalFileStream?.end?.();", + " stream.destroy();", + " await encFinishPromise;", + " await originalFinishPromise;", + " logger?.debug('encrypted data successfully');", + ] + : params?.encryptedStreamPatched + ? [ + " encFileWriteStream.write(mac);", + " const encFinishPromise = once(encFileWriteStream, 'finish');", + " const originalFinishPromise = originalFileStream ? once(originalFileStream, 'finish') : Promise.resolve();", + " encFileWriteStream.end();", + " originalFileStream?.end?.();", + " stream.destroy();", + " await Promise.all([encFinishPromise, originalFinishPromise]);", + " logger?.debug('encrypted data successfully');", + ] + : [ + " encFileWriteStream.write(mac);", + " encFileWriteStream.end();", + " originalFileStream?.end?.();", + " stream.destroy();", + " logger?.debug('encrypted data successfully');", + ]; + const dispatcherLines = params?.dispatcherPatched + ? [ + " const response = await fetch(url, {", + " method: 'POST',", + " body: stream,", + " headers: {", + " 'Content-Type': 'application/octet-stream',", + " Origin: DEFAULT_ORIGIN", + " },", + " // Baileys passes a generic agent here in some runtimes. Undici's", + " // `dispatcher` only works with Dispatcher-compatible implementations,", + " // so only wire it through when the object actually implements", + " // `dispatch`.", + " ...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + " duplex: 'half',", + " });", + ] + : params?.dispatcherHeaderDrifted + ? [ + " const response = await fetch(url, {", + " dispatcher: fetchAgent,", + " method: 'POST',", + " body: stream,", + " headers: {", + " Origin: DEFAULT_ORIGIN,", + " 'Content-Type': 'application/octet-stream'", + " },", + " duplex: 'half',", + " });", + ] + : [ + " const response = await fetch(url, {", + " dispatcher: fetchAgent,", + " method: 'POST',", + " body: stream,", + " headers: {", + " 'Content-Type': 'application/octet-stream',", + " Origin: DEFAULT_ORIGIN", + " },", + " duplex: 'half',", + " });", + ]; + return [ + "import { once } from 'events';", + "const encryptedStream = async () => {", + ...encryptedLines, + "};", + "const upload = async () => {", + ...dispatcherLines, + "};", + ].join("\n"); +} + afterEach(() => { cleanupTrackedTempDirs(tempDirs); }); @@ -208,16 +319,7 @@ describe("stageBundledPluginRuntimeDeps", () => { writeRepoFile( repoRoot, "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", - [ - "import { once } from 'events';", - "const encryptedStream = async () => {", - " encFileWriteStream.write(mac);", - " encFileWriteStream.end();", - " originalFileStream?.end?.();", - " stream.destroy();", - " logger?.debug('encrypted data successfully');", - "};", - ].join("\n"), + createBaileysMessagesMediaSource(), ); const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); @@ -234,6 +336,171 @@ describe("stageBundledPluginRuntimeDeps", () => { expect(fs.readFileSync(targetPath, "utf8")).toContain( "await Promise.all([encFinishPromise, originalFinishPromise]);", ); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + ); + expect(fs.readFileSync(targetPath, "utf8")).not.toContain("dispatcher: fetchAgent,"); + }); + + it("patches the Baileys dispatcher guard when the flush hotfix is already present", async () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-hotfix-dispatcher-"); + const targetPath = path.join( + repoRoot, + "node_modules", + "@whiskeysockets", + "baileys", + "lib", + "Utils", + "messages-media.js", + ); + writeRepoFile( + repoRoot, + "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", + createBaileysMessagesMediaSource({ encryptedStreamPatched: true }), + ); + + const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); + const result = applyBaileysEncryptedStreamFinishHotfix({ packageRoot: repoRoot }); + + expect(result).toEqual({ + applied: true, + reason: "patched", + targetPath, + }); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "await Promise.all([encFinishPromise, originalFinishPromise]);", + ); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + ); + }); + + it("patches the Baileys dispatcher guard even when the encryptedStream block changed", async () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-hotfix-dispatcher-only-"); + const targetPath = path.join( + repoRoot, + "node_modules", + "@whiskeysockets", + "baileys", + "lib", + "Utils", + "messages-media.js", + ); + writeRepoFile( + repoRoot, + "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", + createBaileysMessagesMediaSource({ encryptedStreamUnrecognized: true }), + ); + + const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); + const result = applyBaileysEncryptedStreamFinishHotfix({ packageRoot: repoRoot }); + + expect(result).toEqual({ + applied: true, + reason: "patched", + targetPath, + }); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "logger?.debug('encrypted data changed upstream');", + ); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + ); + }); + + it("fails when the dispatcher block drifts even if encryptedStream is patchable", async () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-hotfix-dispatcher-drifted-"); + const targetPath = path.join( + repoRoot, + "node_modules", + "@whiskeysockets", + "baileys", + "lib", + "Utils", + "messages-media.js", + ); + writeRepoFile( + repoRoot, + "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", + createBaileysMessagesMediaSource({ dispatcherHeaderDrifted: true }), + ); + + const originalText = fs.readFileSync(targetPath, "utf8"); + const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); + const result = applyBaileysEncryptedStreamFinishHotfix({ packageRoot: repoRoot }); + + expect(result).toEqual({ + applied: false, + reason: "unexpected_content", + targetPath, + }); + expect(fs.readFileSync(targetPath, "utf8")).toBe(originalText); + }); + + it("patches the Baileys dispatcher guard when sequential awaits include comments", async () => { + const repoRoot = makeRepoRoot( + "openclaw-stage-bundled-runtime-hotfix-dispatcher-sequential-comments-", + ); + const targetPath = path.join( + repoRoot, + "node_modules", + "@whiskeysockets", + "baileys", + "lib", + "Utils", + "messages-media.js", + ); + writeRepoFile( + repoRoot, + "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", + createBaileysMessagesMediaSource({ encryptedStreamPatchedSequentiallyWithComments: true }), + ); + + const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); + const result = applyBaileysEncryptedStreamFinishHotfix({ packageRoot: repoRoot }); + + expect(result).toEqual({ + applied: true, + reason: "patched", + targetPath, + }); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + ); + expect(fs.readFileSync(targetPath, "utf8")).toContain("await encFinishPromise;"); + expect(fs.readFileSync(targetPath, "utf8")).toContain("await originalFinishPromise;"); + }); + + it("patches the Baileys dispatcher guard when the flush hotfix uses sequential awaits", async () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-hotfix-dispatcher-sequential-"); + const targetPath = path.join( + repoRoot, + "node_modules", + "@whiskeysockets", + "baileys", + "lib", + "Utils", + "messages-media.js", + ); + writeRepoFile( + repoRoot, + "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", + createBaileysMessagesMediaSource({ encryptedStreamPatchedSequentially: true }), + ); + + const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); + const result = applyBaileysEncryptedStreamFinishHotfix({ packageRoot: repoRoot }); + + expect(result).toEqual({ + applied: true, + reason: "patched", + targetPath, + }); + expect(fs.readFileSync(targetPath, "utf8")).toContain("await encFinishPromise;"); + expect(fs.readFileSync(targetPath, "utf8")).toContain("await originalFinishPromise;"); + expect(fs.readFileSync(targetPath, "utf8")).toContain( + "...(typeof fetchAgent?.dispatch === 'function' ? { dispatcher: fetchAgent } : {}),", + ); }); it("preserves the original module read mode when replacing Baileys", async () => { @@ -250,16 +517,7 @@ describe("stageBundledPluginRuntimeDeps", () => { writeRepoFile( repoRoot, "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", - [ - "import { once } from 'events';", - "const encryptedStream = async () => {", - " encFileWriteStream.write(mac);", - " encFileWriteStream.end();", - " originalFileStream?.end?.();", - " stream.destroy();", - " logger?.debug('encrypted data successfully');", - "};", - ].join("\n"), + createBaileysMessagesMediaSource(), ); fs.chmodSync(targetPath, 0o644); @@ -315,16 +573,7 @@ describe("stageBundledPluginRuntimeDeps", () => { writeRepoFile( repoRoot, "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", - [ - "import { once } from 'events';", - "const encryptedStream = async () => {", - " encFileWriteStream.write(mac);", - " encFileWriteStream.end();", - " originalFileStream?.end?.();", - " stream.destroy();", - " logger?.debug('encrypted data successfully');", - "};", - ].join("\n"), + createBaileysMessagesMediaSource(), ); const { applyBaileysEncryptedStreamFinishHotfix } = await loadPostinstallBundledPluginsModule(); @@ -341,7 +590,7 @@ describe("stageBundledPluginRuntimeDeps", () => { targetPath, error: "read-only filesystem", }); - expect(fs.readFileSync(targetPath, "utf8")).toContain("encFileWriteStream.end();"); + expect(fs.readFileSync(targetPath, "utf8")).toContain("dispatcher: fetchAgent,"); }); it("refuses pre-created symlink temp paths instead of following them", async () => { @@ -363,16 +612,7 @@ describe("stageBundledPluginRuntimeDeps", () => { writeRepoFile( repoRoot, "node_modules/@whiskeysockets/baileys/lib/Utils/messages-media.js", - [ - "import { once } from 'events';", - "const encryptedStream = async () => {", - " encFileWriteStream.write(mac);", - " encFileWriteStream.end();", - " originalFileStream?.end?.();", - " stream.destroy();", - " logger?.debug('encrypted data successfully');", - "};", - ].join("\n"), + createBaileysMessagesMediaSource(), ); writeRepoFile(repoRoot, "redirected-temp-target.js", "const untouched = true;\n"); fs.symlinkSync(redirectedTarget, attackerTempPath); @@ -389,6 +629,6 @@ describe("stageBundledPluginRuntimeDeps", () => { expect(result.reason).toBe("error"); expect(result.error).toContain("EEXIST"); expect(fs.readFileSync(redirectedTarget, "utf8")).toBe("const untouched = true;\n"); - expect(fs.readFileSync(targetPath, "utf8")).toContain("encFileWriteStream.end();"); + expect(fs.readFileSync(targetPath, "utf8")).toContain("dispatcher: fetchAgent,"); }); }); diff --git a/test/scripts/bundled-plugin-staged-runtime-deps.test.ts b/test/scripts/bundled-plugin-staged-runtime-deps.test.ts index e6d72636d8a..501de4cc4a7 100644 --- a/test/scripts/bundled-plugin-staged-runtime-deps.test.ts +++ b/test/scripts/bundled-plugin-staged-runtime-deps.test.ts @@ -62,4 +62,20 @@ describe("collectBuiltBundledPluginStagedRuntimeDependencyErrors", () => { }), ).toEqual([]); }); + + it("keeps the WhatsApp bundled plugin opted into staged runtime dependencies", () => { + const packageJson = JSON.parse( + fs.readFileSync(path.join(process.cwd(), "extensions/whatsapp/package.json"), "utf8"), + ) as { + dependencies?: Record; + openclaw?: { + bundle?: { + stageRuntimeDependencies?: boolean; + }; + }; + }; + + expect(packageJson.dependencies?.["@whiskeysockets/baileys"]).toBe("7.0.0-rc.9"); + expect(packageJson.openclaw?.bundle?.stageRuntimeDependencies).toBe(true); + }); }); diff --git a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts index 1363a19cf3b..4a365b31e0c 100644 --- a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts +++ b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts @@ -123,6 +123,77 @@ describe("stageBundledPluginRuntimeDeps", () => { expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe("2\n"); }); + it("restages when the root pnpm lockfile changes", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { "left-pad": "1.3.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + fs.writeFileSync(path.join(repoRoot, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n", "utf8"); + + let installCount = 0; + const stageOnce = () => + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint }: { fingerprint: string }) => { + installCount += 1; + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync(path.join(nodeModulesDir, "marker.txt"), `${installCount}\n`, "utf8"); + fs.writeFileSync( + path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"), + `${JSON.stringify({ fingerprint }, null, 2)}\n`, + "utf8", + ); + }, + }); + + stageOnce(); + fs.writeFileSync( + path.join(repoRoot, "pnpm-lock.yaml"), + "lockfileVersion: '9.0'\npatchedDependencies:\n left-pad@1.3.0: patches/left-pad.patch\n", + "utf8", + ); + stageOnce(); + + expect(installCount).toBe(2); + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe("2\n"); + }); + + it("restages when installed root runtime dependency contents change", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + fs.mkdirSync(directDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'first';\n", "utf8"); + + stageBundledPluginRuntimeDeps({ cwd: repoRoot }); + expect( + fs.readFileSync(path.join(pluginDir, "node_modules", "direct", "index.js"), "utf8"), + ).toBe("module.exports = 'first';\n"); + + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'second';\n", "utf8"); + stageBundledPluginRuntimeDeps({ cwd: repoRoot }); + + expect( + fs.readFileSync(path.join(pluginDir, "node_modules", "direct", "index.js"), "utf8"), + ).toBe("module.exports = 'second';\n"); + }); + it("stages runtime deps from the root node_modules when already installed", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: { @@ -189,6 +260,269 @@ describe("stageBundledPluginRuntimeDeps", () => { ).toBe("module.exports = 'transitive';\n"); }); + it("stages nested dependency trees from installed direct package roots", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + const nestedDir = path.join(directDir, "node_modules", "nested"); + fs.mkdirSync(nestedDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0", "dependencies": { "nested": "^1.0.0" } }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + fs.writeFileSync( + path.join(nestedDir, "package.json"), + '{ "name": "nested", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(nestedDir, "index.js"), "module.exports = 'nested';\n", "utf8"); + + stageBundledPluginRuntimeDeps({ cwd: repoRoot }); + + expect( + fs.readFileSync(path.join(pluginDir, "node_modules", "direct", "index.js"), "utf8"), + ).toBe("module.exports = 'direct';\n"); + expect( + fs.readFileSync( + path.join(pluginDir, "node_modules", "direct", "node_modules", "nested", "index.js"), + "utf8", + ), + ).toBe("module.exports = 'nested';\n"); + }); + + it("falls back to install when a dependency tree contains an unowned symlinked directory", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + const linkedTargetDir = path.join(repoRoot, "linked-target"); + const linkedPath = path.join(directDir, "node_modules", "linked"); + fs.mkdirSync(path.join(directDir, "node_modules"), { recursive: true }); + fs.mkdirSync(linkedTargetDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + fs.writeFileSync(path.join(linkedTargetDir, "marker.txt"), "first\n", "utf8"); + fs.symlinkSync(linkedTargetDir, linkedPath); + + let installCount = 0; + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint }: { fingerprint: string }) => { + installCount += 1; + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync(path.join(nodeModulesDir, "marker.txt"), "installed\n", "utf8"); + fs.writeFileSync( + path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"), + `${JSON.stringify({ fingerprint }, null, 2)}\n`, + "utf8", + ); + }, + }); + + expect(installCount).toBe(1); + expect( + fs.existsSync(path.join(pluginDir, "node_modules", "direct", "node_modules", "linked")), + ).toBe(false); + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe( + "installed\n", + ); + }); + + it("dedupes cyclic dependency aliases by canonical root", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { a: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const rootNodeModulesDir = path.join(repoRoot, "node_modules"); + const storeDir = path.join(repoRoot, ".store"); + const aStoreDir = path.join(storeDir, "a"); + const bStoreDir = path.join(storeDir, "b"); + fs.mkdirSync(path.join(aStoreDir, "node_modules"), { recursive: true }); + fs.mkdirSync(path.join(bStoreDir, "node_modules"), { recursive: true }); + fs.writeFileSync( + path.join(aStoreDir, "package.json"), + '{ "name": "a", "version": "1.0.0", "dependencies": { "b": "1.0.0" } }\n', + "utf8", + ); + fs.writeFileSync(path.join(aStoreDir, "index.js"), "module.exports = 'a';\n", "utf8"); + fs.writeFileSync( + path.join(bStoreDir, "package.json"), + '{ "name": "b", "version": "1.0.0", "dependencies": { "a": "1.0.0" } }\n', + "utf8", + ); + fs.writeFileSync(path.join(bStoreDir, "index.js"), "module.exports = 'b';\n", "utf8"); + fs.mkdirSync(rootNodeModulesDir, { recursive: true }); + fs.symlinkSync(aStoreDir, path.join(rootNodeModulesDir, "a")); + fs.symlinkSync(bStoreDir, path.join(rootNodeModulesDir, "b")); + fs.symlinkSync(bStoreDir, path.join(aStoreDir, "node_modules", "b")); + fs.symlinkSync(aStoreDir, path.join(bStoreDir, "node_modules", "a")); + + stageBundledPluginRuntimeDeps({ cwd: repoRoot }); + + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "a", "index.js"), "utf8")).toBe( + "module.exports = 'a';\n", + ); + expect( + fs.readFileSync( + path.join(pluginDir, "node_modules", "a", "node_modules", "b", "index.js"), + "utf8", + ), + ).toBe("module.exports = 'b';\n"); + }); + + it("falls back to install when a dependency name escapes node_modules", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { "../escape": "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + + let installCount = 0; + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint }: { fingerprint: string }) => { + installCount += 1; + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync(path.join(nodeModulesDir, "marker.txt"), "installed\n", "utf8"); + fs.writeFileSync( + path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"), + `${JSON.stringify({ fingerprint }, null, 2)}\n`, + "utf8", + ); + }, + }); + + expect(installCount).toBe(1); + expect(fs.existsSync(path.join(pluginDir, "escape"))).toBe(false); + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe( + "installed\n", + ); + }); + + it("falls back to install when a staged dependency tree contains a symlink outside copied roots", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + const escapedDir = path.join(repoRoot, "outside-root"); + fs.mkdirSync(path.join(directDir, "node_modules"), { recursive: true }); + fs.mkdirSync(escapedDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + fs.writeFileSync(path.join(escapedDir, "secret.txt"), "host secret\n", "utf8"); + fs.symlinkSync(escapedDir, path.join(directDir, "node_modules", "escaped")); + + let installCount = 0; + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint }: { fingerprint: string }) => { + installCount += 1; + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync(path.join(nodeModulesDir, "marker.txt"), "installed\n", "utf8"); + fs.writeFileSync( + path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"), + `${JSON.stringify({ fingerprint }, null, 2)}\n`, + "utf8", + ); + }, + }); + + expect(installCount).toBe(1); + expect( + fs.existsSync( + path.join(pluginDir, "node_modules", "direct", "node_modules", "escaped", "secret.txt"), + ), + ).toBe(false); + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe( + "installed\n", + ); + }); + + it("falls back to install when the root transitive closure is incomplete", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { direct: "1.0.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const directDir = path.join(repoRoot, "node_modules", "direct"); + fs.mkdirSync(directDir, { recursive: true }); + fs.writeFileSync( + path.join(directDir, "package.json"), + '{ "name": "direct", "version": "1.0.0", "dependencies": { "missing-transitive": "^1.0.0" } }\n', + "utf8", + ); + fs.writeFileSync(path.join(directDir, "index.js"), "module.exports = 'direct';\n", "utf8"); + + let installCount = 0; + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint }: { fingerprint: string }) => { + installCount += 1; + const nodeModulesDir = path.join(pluginDir, "node_modules", "direct"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync( + path.join(nodeModulesDir, "package.json"), + '{ "name": "direct", "version": "1.0.0" }\n', + "utf8", + ); + fs.writeFileSync( + path.join(nodeModulesDir, "index.js"), + "module.exports = 'installed';\n", + "utf8", + ); + fs.writeFileSync( + path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"), + `${JSON.stringify({ fingerprint }, null, 2)}\n`, + "utf8", + ); + }, + }); + + expect(installCount).toBe(1); + expect( + fs.readFileSync(path.join(pluginDir, "node_modules", "direct", "index.js"), "utf8"), + ).toBe("module.exports = 'installed';\n"); + }); + it("removes global non-runtime suffixes from staged runtime dependencies", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: {