fix(plugins): reuse unchanged runtime mirrors

This commit is contained in:
Peter Steinberger
2026-04-27 23:44:57 +01:00
parent 323030594e
commit 6f09039b0c
6 changed files with 407 additions and 94 deletions

View File

@@ -0,0 +1,219 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
const BUNDLED_RUNTIME_MIRROR_METADATA_FILE = ".openclaw-runtime-mirror.json";
const BUNDLED_RUNTIME_MIRROR_METADATA_VERSION = 1;
type BundledRuntimeMirrorMetadata = {
version: number;
pluginId: string;
sourceRoot: string;
sourceFingerprint: string;
};
export function refreshBundledPluginRuntimeMirrorRoot(params: {
pluginId: string;
sourceRoot: string;
targetRoot: string;
tempDirParent?: string;
}): boolean {
if (path.resolve(params.sourceRoot) === path.resolve(params.targetRoot)) {
return false;
}
const metadata = createBundledRuntimeMirrorMetadata(params);
if (isBundledRuntimeMirrorRootFresh(params.targetRoot, metadata)) {
return false;
}
const tempDir = fs.mkdtempSync(
path.join(
params.tempDirParent ?? path.dirname(params.targetRoot),
`.plugin-${sanitizeBundledRuntimeMirrorTempId(params.pluginId)}-`,
),
);
const stagedRoot = path.join(tempDir, "plugin");
try {
copyBundledPluginRuntimeRoot(params.sourceRoot, stagedRoot);
writeBundledRuntimeMirrorMetadata(stagedRoot, metadata);
fs.rmSync(params.targetRoot, { recursive: true, force: true });
fs.renameSync(stagedRoot, params.targetRoot);
return true;
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
export function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void {
if (path.resolve(sourceRoot) === path.resolve(targetRoot)) {
return;
}
fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 });
for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) {
if (shouldIgnoreBundledRuntimeMirrorEntry(entry.name)) {
continue;
}
const sourcePath = path.join(sourceRoot, entry.name);
const targetPath = path.join(targetRoot, entry.name);
if (entry.isDirectory()) {
copyBundledPluginRuntimeRoot(sourcePath, targetPath);
continue;
}
if (entry.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath);
continue;
}
if (!entry.isFile()) {
continue;
}
fs.copyFileSync(sourcePath, targetPath);
try {
const sourceMode = fs.statSync(sourcePath).mode;
fs.chmodSync(targetPath, sourceMode | 0o600);
} catch {
// Readable copied files are enough for plugin loading.
}
}
}
function createBundledRuntimeMirrorMetadata(params: {
pluginId: string;
sourceRoot: string;
}): BundledRuntimeMirrorMetadata {
return {
version: BUNDLED_RUNTIME_MIRROR_METADATA_VERSION,
pluginId: params.pluginId,
sourceRoot: resolveBundledRuntimeMirrorSourceRootId(params.sourceRoot),
sourceFingerprint: fingerprintBundledRuntimeMirrorSourceRoot(params.sourceRoot),
};
}
function isBundledRuntimeMirrorRootFresh(
targetRoot: string,
expected: BundledRuntimeMirrorMetadata,
): boolean {
try {
if (!fs.lstatSync(targetRoot).isDirectory()) {
return false;
}
} catch {
return false;
}
const actual = readBundledRuntimeMirrorMetadata(targetRoot);
return (
actual?.version === expected.version &&
actual.pluginId === expected.pluginId &&
actual.sourceRoot === expected.sourceRoot &&
actual.sourceFingerprint === expected.sourceFingerprint
);
}
function readBundledRuntimeMirrorMetadata(targetRoot: string): BundledRuntimeMirrorMetadata | null {
try {
const parsed = JSON.parse(
fs.readFileSync(path.join(targetRoot, BUNDLED_RUNTIME_MIRROR_METADATA_FILE), "utf8"),
) as Partial<BundledRuntimeMirrorMetadata>;
if (
parsed.version !== BUNDLED_RUNTIME_MIRROR_METADATA_VERSION ||
typeof parsed.pluginId !== "string" ||
typeof parsed.sourceRoot !== "string" ||
typeof parsed.sourceFingerprint !== "string"
) {
return null;
}
return parsed as BundledRuntimeMirrorMetadata;
} catch {
return null;
}
}
function writeBundledRuntimeMirrorMetadata(
targetRoot: string,
metadata: BundledRuntimeMirrorMetadata,
): void {
fs.writeFileSync(
path.join(targetRoot, BUNDLED_RUNTIME_MIRROR_METADATA_FILE),
`${JSON.stringify(metadata, null, 2)}\n`,
"utf8",
);
}
function fingerprintBundledRuntimeMirrorSourceRoot(sourceRoot: string): string {
const hash = createHash("sha256");
hashBundledRuntimeMirrorDirectory(hash, sourceRoot, sourceRoot);
return hash.digest("hex");
}
function hashBundledRuntimeMirrorDirectory(
hash: ReturnType<typeof createHash>,
sourceRoot: string,
directory: string,
): void {
const entries = fs
.readdirSync(directory, { withFileTypes: true })
.filter((entry) => !shouldIgnoreBundledRuntimeMirrorEntry(entry.name))
.toSorted((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const sourcePath = path.join(directory, entry.name);
const relativePath = path.relative(sourceRoot, sourcePath).replaceAll(path.sep, "/");
const stat = fs.lstatSync(sourcePath, { bigint: true });
if (entry.isDirectory()) {
updateBundledRuntimeMirrorHash(hash, [
"dir",
relativePath,
formatBundledRuntimeMirrorMode(stat.mode),
]);
hashBundledRuntimeMirrorDirectory(hash, sourceRoot, sourcePath);
continue;
}
if (entry.isSymbolicLink()) {
updateBundledRuntimeMirrorHash(hash, [
"symlink",
relativePath,
formatBundledRuntimeMirrorMode(stat.mode),
stat.ctimeNs.toString(),
fs.readlinkSync(sourcePath),
]);
continue;
}
if (!entry.isFile()) {
continue;
}
updateBundledRuntimeMirrorHash(hash, [
"file",
relativePath,
formatBundledRuntimeMirrorMode(stat.mode),
stat.size.toString(),
stat.mtimeNs.toString(),
stat.ctimeNs.toString(),
]);
}
}
function updateBundledRuntimeMirrorHash(
hash: ReturnType<typeof createHash>,
fields: readonly string[],
): void {
hash.update(JSON.stringify(fields));
hash.update("\n");
}
function formatBundledRuntimeMirrorMode(mode: bigint): string {
return (mode & 0o7777n).toString(8);
}
function resolveBundledRuntimeMirrorSourceRootId(sourceRoot: string): string {
try {
return fs.realpathSync.native(sourceRoot);
} catch {
return path.resolve(sourceRoot);
}
}
function shouldIgnoreBundledRuntimeMirrorEntry(name: string): boolean {
return name === "node_modules" || name === BUNDLED_RUNTIME_MIRROR_METADATA_FILE;
}
function sanitizeBundledRuntimeMirrorTempId(pluginId: string): string {
return pluginId.replaceAll(/[^a-zA-Z0-9._-]/g, "_");
}

View File

@@ -19,6 +19,10 @@ afterEach(() => {
}
});
async function waitForFilesystemTimestampTick(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 50));
}
describe("prepareBundledPluginRuntimeRoot", () => {
it("materializes root JavaScript chunks in external mirrors", () => {
const packageRoot = makeTempRoot();
@@ -167,4 +171,122 @@ describe("prepareBundledPluginRuntimeRoot", () => {
expect(prepared.modulePath).toBe(path.join(pluginRoot, "index.js"));
expect(fs.readFileSync(distChunk, "utf8")).toContain("same-root");
});
it("reuses unchanged external runtime mirrors from the original plugin root", async () => {
const packageRoot = makeTempRoot();
const stageDir = makeTempRoot();
const pluginRoot = path.join(packageRoot, "dist", "extensions", "whatsapp");
const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir };
fs.mkdirSync(pluginRoot, { recursive: true });
fs.writeFileSync(
path.join(packageRoot, "package.json"),
JSON.stringify({ name: "openclaw", version: "2026.4.27", type: "module" }),
"utf8",
);
fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v1';\n", "utf8");
fs.writeFileSync(
path.join(pluginRoot, "package.json"),
JSON.stringify(
{
name: "@openclaw/whatsapp",
version: "1.0.0",
type: "module",
dependencies: { "whatsapp-runtime": "1.0.0" },
openclaw: { extensions: ["./index.js"] },
},
null,
2,
),
"utf8",
);
const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env });
fs.mkdirSync(path.join(installRoot, "node_modules", "whatsapp-runtime"), { recursive: true });
fs.writeFileSync(
path.join(installRoot, "node_modules", "whatsapp-runtime", "package.json"),
JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }),
"utf8",
);
const prepared = prepareBundledPluginRuntimeRoot({
pluginId: "whatsapp",
pluginRoot,
modulePath: path.join(pluginRoot, "index.js"),
env,
});
const mirrorEntry = path.join(prepared.pluginRoot, "index.js");
const initialStat = fs.statSync(mirrorEntry);
await waitForFilesystemTimestampTick();
const preparedAgain = prepareBundledPluginRuntimeRoot({
pluginId: "whatsapp",
pluginRoot,
modulePath: path.join(pluginRoot, "index.js"),
env,
});
const reusedStat = fs.statSync(mirrorEntry);
expect(preparedAgain).toEqual(prepared);
expect(reusedStat.mtimeMs).toBe(initialStat.mtimeMs);
expect(fs.readFileSync(mirrorEntry, "utf8")).toContain("v1");
});
it("refreshes external runtime mirrors when source files change", async () => {
const packageRoot = makeTempRoot();
const stageDir = makeTempRoot();
const pluginRoot = path.join(packageRoot, "dist", "extensions", "whatsapp");
const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir };
fs.mkdirSync(pluginRoot, { recursive: true });
fs.writeFileSync(
path.join(packageRoot, "package.json"),
JSON.stringify({ name: "openclaw", version: "2026.4.27", type: "module" }),
"utf8",
);
fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v1';\n", "utf8");
fs.writeFileSync(
path.join(pluginRoot, "package.json"),
JSON.stringify(
{
name: "@openclaw/whatsapp",
version: "1.0.0",
type: "module",
dependencies: { "whatsapp-runtime": "1.0.0" },
openclaw: { extensions: ["./index.js"] },
},
null,
2,
),
"utf8",
);
const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env });
fs.mkdirSync(path.join(installRoot, "node_modules", "whatsapp-runtime"), { recursive: true });
fs.writeFileSync(
path.join(installRoot, "node_modules", "whatsapp-runtime", "package.json"),
JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }),
"utf8",
);
const prepared = prepareBundledPluginRuntimeRoot({
pluginId: "whatsapp",
pluginRoot,
modulePath: path.join(pluginRoot, "index.js"),
env,
});
const mirrorEntry = path.join(prepared.pluginRoot, "index.js");
const initialStat = fs.statSync(mirrorEntry);
await waitForFilesystemTimestampTick();
fs.writeFileSync(path.join(pluginRoot, "index.js"), "export const marker = 'v2';\n", "utf8");
prepareBundledPluginRuntimeRoot({
pluginId: "whatsapp",
pluginRoot,
modulePath: path.join(pluginRoot, "index.js"),
env,
});
const refreshedStat = fs.statSync(mirrorEntry);
expect(refreshedStat.mtimeMs).toBeGreaterThan(initialStat.mtimeMs);
expect(fs.readFileSync(mirrorEntry, "utf8")).toContain("v2");
});
});

View File

@@ -9,6 +9,10 @@ import {
shouldMaterializeBundledRuntimeMirrorDistFile,
withBundledRuntimeDepsFilesystemLock,
} from "./bundled-runtime-deps.js";
import {
copyBundledPluginRuntimeRoot,
refreshBundledPluginRuntimeMirrorRoot,
} from "./bundled-runtime-mirror.js";
const bundledRuntimeDepsRetainSpecsByInstallRoot = new Map<string, readonly string[]>();
const BUNDLED_RUNTIME_MIRROR_LOCK_DIR = ".openclaw-runtime-mirror.lock";
@@ -117,15 +121,12 @@ function mirrorBundledPluginRuntimeRoot(params: {
if (path.resolve(mirrorRoot) === path.resolve(params.pluginRoot)) {
return mirrorRoot;
}
const tempDir = fs.mkdtempSync(path.join(mirrorParent, `.plugin-${params.pluginId}-`));
const stagedRoot = path.join(tempDir, "plugin");
try {
copyBundledPluginRuntimeRoot(params.pluginRoot, stagedRoot);
fs.rmSync(mirrorRoot, { recursive: true, force: true });
fs.renameSync(stagedRoot, mirrorRoot);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
refreshBundledPluginRuntimeMirrorRoot({
pluginId: params.pluginId,
sourceRoot: params.pluginRoot,
targetRoot: mirrorRoot,
tempDirParent: mirrorParent,
});
return mirrorRoot;
},
);
@@ -182,38 +183,6 @@ function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void {
writeRuntimeJsonFile(packageJsonPath, { type: "module" });
}
function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void {
if (path.resolve(sourceRoot) === path.resolve(targetRoot)) {
return;
}
fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 });
for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) {
if (entry.name === "node_modules") {
continue;
}
const sourcePath = path.join(sourceRoot, entry.name);
const targetPath = path.join(targetRoot, entry.name);
if (entry.isDirectory()) {
copyBundledPluginRuntimeRoot(sourcePath, targetPath);
continue;
}
if (entry.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath);
continue;
}
if (!entry.isFile()) {
continue;
}
fs.copyFileSync(sourcePath, targetPath);
try {
const sourceMode = fs.statSync(sourcePath).mode;
fs.chmodSync(targetPath, sourceMode | 0o600);
} catch {
// Readable copied files are enough for plugin loading.
}
}
}
function writeRuntimeJsonFile(targetPath: string, value: unknown): void {
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
fs.writeFileSync(targetPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");

View File

@@ -149,6 +149,10 @@ const RESERVED_ADMIN_PLUGIN_METHOD = "config.plugin.inspect";
const RESERVED_ADMIN_SCOPE_WARNING =
"gateway method scope coerced to operator.admin for reserved core namespace";
async function waitForFilesystemTimestampTick(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 50));
}
function writeBundledPlugin(params: {
id: string;
body?: string;
@@ -2021,7 +2025,7 @@ module.exports = {
expect(registry.plugins.find((entry) => entry.id === "discord")?.status).toBe("loaded");
});
it("loads dist-runtime wrappers from an external stage dir", () => {
it("loads dist-runtime wrappers from an external stage dir", async () => {
const packageRoot = makeTempDir();
const stageDir = makeTempDir();
const bundledDir = path.join(packageRoot, "dist-runtime", "extensions");
@@ -2143,6 +2147,40 @@ module.exports = {
expect(fs.lstatSync(path.join(actualInstallRoot, "dist", "pw-ai.js")).isSymbolicLink()).toBe(
false,
);
const runtimeMirrorEntry = path.join(
actualInstallRoot,
"dist-runtime",
"extensions",
"acpx",
"index.js",
);
const canonicalMirrorEntry = path.join(
actualInstallRoot,
"dist",
"extensions",
"acpx",
"index.js",
);
const runtimeMirrorStat = fs.statSync(runtimeMirrorEntry);
const canonicalMirrorStat = fs.statSync(canonicalMirrorEntry);
await waitForFilesystemTimestampTick();
const reloadedRegistry = loadOpenClawPlugins({
cache: false,
config: {
plugins: {
enabled: true,
},
},
});
const reloadedRecord = reloadedRegistry.plugins.find((entry) => entry.id === "acpx");
expect(reloadedRecord?.error).toBeUndefined();
expect(reloadedRecord?.status).toBe("loaded");
expect(fs.statSync(runtimeMirrorEntry).mtimeMs).toBe(runtimeMirrorStat.mtimeMs);
expect(fs.statSync(canonicalMirrorEntry).mtimeMs).toBe(canonicalMirrorStat.mtimeMs);
});
it("loads native ESM deps from a layered baseline stage dir", () => {

View File

@@ -43,6 +43,10 @@ import {
withBundledRuntimeDepsFilesystemLock,
type BundledRuntimeDepsInstallParams,
} from "./bundled-runtime-deps.js";
import {
copyBundledPluginRuntimeRoot,
refreshBundledPluginRuntimeMirrorRoot,
} from "./bundled-runtime-mirror.js";
import {
clearPluginCommands,
listRegisteredPluginCommands,
@@ -703,15 +707,12 @@ function mirrorBundledPluginRuntimeRoot(params: {
if (path.resolve(mirrorRoot) === path.resolve(params.pluginRoot)) {
return mirrorRoot;
}
const tempDir = fs.mkdtempSync(path.join(mirrorParent, `.plugin-${params.pluginId}-`));
const stagedRoot = path.join(tempDir, "plugin");
try {
copyBundledPluginRuntimeRoot(params.pluginRoot, stagedRoot);
fs.rmSync(mirrorRoot, { recursive: true, force: true });
fs.renameSync(stagedRoot, mirrorRoot);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
refreshBundledPluginRuntimeMirrorRoot({
pluginId: params.pluginId,
sourceRoot: params.pluginRoot,
targetRoot: mirrorRoot,
tempDirParent: mirrorParent,
});
return mirrorRoot;
},
);
@@ -819,17 +820,12 @@ function mirrorCanonicalBundledRuntimeDistRoot(params: {
return;
}
const targetCanonicalPluginRoot = path.join(targetCanonicalDistRoot, "extensions", pluginId);
const tempDir = fs.mkdtempSync(
path.join(path.dirname(targetCanonicalPluginRoot), `.plugin-${pluginId}-`),
);
const stagedRoot = path.join(tempDir, "plugin");
try {
copyBundledPluginRuntimeRoot(sourceCanonicalPluginRoot, stagedRoot);
fs.rmSync(targetCanonicalPluginRoot, { recursive: true, force: true });
fs.renameSync(stagedRoot, targetCanonicalPluginRoot);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
refreshBundledPluginRuntimeMirrorRoot({
pluginId,
sourceRoot: sourceCanonicalPluginRoot,
targetRoot: targetCanonicalPluginRoot,
tempDirParent: path.dirname(targetCanonicalPluginRoot),
});
}
function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void {
@@ -840,38 +836,6 @@ function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void {
writeRuntimeJsonFile(packageJsonPath, { type: "module" });
}
function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void {
if (path.resolve(sourceRoot) === path.resolve(targetRoot)) {
return;
}
fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 });
for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) {
if (entry.name === "node_modules") {
continue;
}
const sourcePath = path.join(sourceRoot, entry.name);
const targetPath = path.join(targetRoot, entry.name);
if (entry.isDirectory()) {
copyBundledPluginRuntimeRoot(sourcePath, targetPath);
continue;
}
if (entry.isSymbolicLink()) {
fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath);
continue;
}
if (!entry.isFile()) {
continue;
}
fs.copyFileSync(sourcePath, targetPath);
try {
const sourceMode = fs.statSync(sourcePath).mode;
fs.chmodSync(targetPath, sourceMode | 0o600);
} catch {
// Readable copied files are enough for plugin loading.
}
}
}
function writeRuntimeJsonFile(targetPath: string, value: unknown): void {
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
fs.writeFileSync(targetPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");