mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 09:30:43 +00:00
perf(plugins): cache runtime mirror file decisions
This commit is contained in:
@@ -21,6 +21,7 @@ Docs: https://docs.openclaw.ai
|
|||||||
- Tasks: keep terminal mirrored TaskFlow timestamps pinned to task completion time and let maintenance repair stale mirrors, so ACP terminal delivery updates no longer leave inconsistent flow audits. Refs #73609. Thanks @joerod26.
|
- Tasks: keep terminal mirrored TaskFlow timestamps pinned to task completion time and let maintenance repair stale mirrors, so ACP terminal delivery updates no longer leave inconsistent flow audits. Refs #73609. Thanks @joerod26.
|
||||||
- Gateway/sessions: add conservative stuck-session recovery that releases only stale session lanes while active embedded runs, reply operations, and lane tasks remain serialized, so queued follow-ups can drain without aborting legitimate long-running turns. Refs #73581, #73655, #73652, #73705, #73647, #73602, #73592, and #73601. Thanks @WS-Q0758, @bryangauvin, @spenceryang1996-dot, @bmilne1981, @mattmcintyre, @Vksh07, and @Spolen23.
|
- Gateway/sessions: add conservative stuck-session recovery that releases only stale session lanes while active embedded runs, reply operations, and lane tasks remain serialized, so queued follow-ups can drain without aborting legitimate long-running turns. Refs #73581, #73655, #73652, #73705, #73647, #73602, #73592, and #73601. Thanks @WS-Q0758, @bryangauvin, @spenceryang1996-dot, @bmilne1981, @mattmcintyre, @Vksh07, and @Spolen23.
|
||||||
- Plugins: cache unchanged plugin manifest loads by file signature, reducing repeated JSON/JSON5 parsing and manifest normalization in bursty startup and runtime registry paths. Refs #73532 and #73647; carries forward #73678. Thanks @TheDutchRuler.
|
- Plugins: cache unchanged plugin manifest loads by file signature, reducing repeated JSON/JSON5 parsing and manifest normalization in bursty startup and runtime registry paths. Refs #73532 and #73647; carries forward #73678. Thanks @TheDutchRuler.
|
||||||
|
- Plugins/runtime-deps: cache unchanged bundled runtime mirror dist-file materialization decisions and close file-lock handles on owner-write failures, reducing repeated startup chunk scans and avoiding FileHandle-GC recovery stalls. Refs #73532. Thanks @oadiazp and @bstanbury.
|
||||||
- CLI/TUI: keep `chat.history` off model-catalog discovery so initial Gateway-backed TUI history loads cannot block behind slow provider/plugin model scans on low-core hosts. Refs #73524. Thanks @harshcatsystems-collab.
|
- CLI/TUI: keep `chat.history` off model-catalog discovery so initial Gateway-backed TUI history loads cannot block behind slow provider/plugin model scans on low-core hosts. Refs #73524. Thanks @harshcatsystems-collab.
|
||||||
- Channels/WhatsApp: flag recently reconnected linked accounts in channel status even when the socket is currently healthy, so flapping WhatsApp Web sessions no longer look clean after a brief reconnect. Refs #73602. Thanks @Vksh07.
|
- Channels/WhatsApp: flag recently reconnected linked accounts in channel status even when the socket is currently healthy, so flapping WhatsApp Web sessions no longer look clean after a brief reconnect. Refs #73602. Thanks @Vksh07.
|
||||||
- Agents/model selection: resolve slash-form aliases before provider/model parsing and keep alias-resolved primary models subject to transient provider cooldowns, so cron and persisted sessions do not retry cooled-down raw aliases. Fixes #73573 and #73657. Thanks @akai-shuuichi and @hashslingers.
|
- Agents/model selection: resolve slash-form aliases before provider/model parsing and keep alias-resolved primary models subject to transient provider cooldowns, so cron and persisted sessions do not retry cooled-down raw aliases. Fixes #73573 and #73657. Thanks @akai-shuuichi and @hashslingers.
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import os from "node:os";
|
import os from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import {
|
import {
|
||||||
acquireFileLock,
|
acquireFileLock,
|
||||||
drainFileLockStateForTest,
|
drainFileLockStateForTest,
|
||||||
@@ -55,4 +55,28 @@ describe("acquireFileLock", () => {
|
|||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
}, 5_000);
|
}, 5_000);
|
||||||
|
|
||||||
|
it("closes an opened lock handle when writing the owner payload fails", async () => {
|
||||||
|
const filePath = path.join(tempDir, "write-fails");
|
||||||
|
const writeError = new Error("owner write failed");
|
||||||
|
const close = vi.fn().mockResolvedValue(undefined);
|
||||||
|
vi.spyOn(fs, "open").mockResolvedValue({
|
||||||
|
close,
|
||||||
|
writeFile: vi.fn().mockRejectedValue(writeError),
|
||||||
|
} as unknown as Awaited<ReturnType<typeof fs.open>>);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
acquireFileLock(filePath, {
|
||||||
|
retries: {
|
||||||
|
retries: 0,
|
||||||
|
factor: 1,
|
||||||
|
minTimeout: 1,
|
||||||
|
maxTimeout: 1,
|
||||||
|
},
|
||||||
|
stale: 100,
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(writeError);
|
||||||
|
|
||||||
|
expect(close).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -183,10 +183,16 @@ export async function acquireFileLock(
|
|||||||
for (let attempt = 0; attempt <= options.retries.retries; attempt += 1) {
|
for (let attempt = 0; attempt <= options.retries.retries; attempt += 1) {
|
||||||
try {
|
try {
|
||||||
const handle = await fs.open(lockPath, "wx");
|
const handle = await fs.open(lockPath, "wx");
|
||||||
await handle.writeFile(
|
try {
|
||||||
JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() }, null, 2),
|
await handle.writeFile(
|
||||||
"utf8",
|
JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() }, null, 2),
|
||||||
);
|
"utf8",
|
||||||
|
);
|
||||||
|
} catch (writeError) {
|
||||||
|
await handle.close().catch(() => undefined);
|
||||||
|
await fs.rm(lockPath, { force: true }).catch(() => undefined);
|
||||||
|
throw writeError;
|
||||||
|
}
|
||||||
HELD_LOCKS.set(normalizedFile, { count: 1, handle, lockPath });
|
HELD_LOCKS.set(normalizedFile, { count: 1, handle, lockPath });
|
||||||
return {
|
return {
|
||||||
lockPath,
|
lockPath,
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import {
|
|||||||
resolveBundledRuntimeDependencyInstallRootPlan,
|
resolveBundledRuntimeDependencyInstallRootPlan,
|
||||||
resolveBundledRuntimeDepsNpmRunner,
|
resolveBundledRuntimeDepsNpmRunner,
|
||||||
scanBundledPluginRuntimeDeps,
|
scanBundledPluginRuntimeDeps,
|
||||||
|
shouldMaterializeBundledRuntimeMirrorDistFile,
|
||||||
type BundledRuntimeDepsInstallParams,
|
type BundledRuntimeDepsInstallParams,
|
||||||
} from "./bundled-runtime-deps.js";
|
} from "./bundled-runtime-deps.js";
|
||||||
|
|
||||||
@@ -99,11 +100,42 @@ afterEach(() => {
|
|||||||
spawnMock.mockReset();
|
spawnMock.mockReset();
|
||||||
spawnSyncMock.mockReset();
|
spawnSyncMock.mockReset();
|
||||||
bundledRuntimeDepsActivityTesting.resetBundledRuntimeDepsInstallActivity();
|
bundledRuntimeDepsActivityTesting.resetBundledRuntimeDepsInstallActivity();
|
||||||
|
bundledRuntimeDepsTesting.clearBundledRuntimeMirrorMaterializeCache();
|
||||||
for (const dir of tempDirs.splice(0)) {
|
for (const dir of tempDirs.splice(0)) {
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("shouldMaterializeBundledRuntimeMirrorDistFile", () => {
|
||||||
|
it("reuses unchanged root dist file decisions without rereading source", () => {
|
||||||
|
const root = makeTempDir();
|
||||||
|
const sourcePath = path.join(root, "shared-runtime.js");
|
||||||
|
fs.writeFileSync(
|
||||||
|
sourcePath,
|
||||||
|
[
|
||||||
|
`//#region extensions/browser/src/runtime.ts`,
|
||||||
|
`export const marker = "shared-runtime";`,
|
||||||
|
`//#endregion`,
|
||||||
|
"",
|
||||||
|
].join("\n"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
const realReadFileSync = fs.readFileSync.bind(fs);
|
||||||
|
let sourceReads = 0;
|
||||||
|
vi.spyOn(fs, "readFileSync").mockImplementation(((target, options) => {
|
||||||
|
if (path.resolve(target.toString()) === path.resolve(sourcePath)) {
|
||||||
|
sourceReads += 1;
|
||||||
|
}
|
||||||
|
return realReadFileSync(target, options as never);
|
||||||
|
}) as typeof fs.readFileSync);
|
||||||
|
|
||||||
|
expect(shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)).toBe(true);
|
||||||
|
expect(shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)).toBe(true);
|
||||||
|
|
||||||
|
expect(sourceReads).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("resolveBundledRuntimeDepsNpmRunner", () => {
|
describe("resolveBundledRuntimeDepsNpmRunner", () => {
|
||||||
it("ignores npm_execpath and uses the Node-adjacent npm CLI on Windows", () => {
|
it("ignores npm_execpath and uses the Node-adjacent npm CLI on Windows", () => {
|
||||||
const execPath = "C:\\Program Files\\nodejs\\node.exe";
|
const execPath = "C:\\Program Files\\nodejs\\node.exe";
|
||||||
|
|||||||
@@ -78,6 +78,10 @@ const BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE =
|
|||||||
const NPM_EXECPATH_ENV_KEY = "npm_execpath";
|
const NPM_EXECPATH_ENV_KEY = "npm_execpath";
|
||||||
|
|
||||||
const registeredBundledRuntimeDepNodePaths = new Set<string>();
|
const registeredBundledRuntimeDepNodePaths = new Set<string>();
|
||||||
|
const bundledRuntimeMirrorMaterializeCache = new Map<
|
||||||
|
string,
|
||||||
|
{ signature: string; materialize: boolean }
|
||||||
|
>();
|
||||||
|
|
||||||
export type BundledRuntimeDepsNpmRunner = {
|
export type BundledRuntimeDepsNpmRunner = {
|
||||||
command: string;
|
command: string;
|
||||||
@@ -85,10 +89,15 @@ export type BundledRuntimeDepsNpmRunner = {
|
|||||||
env?: NodeJS.ProcessEnv;
|
env?: NodeJS.ProcessEnv;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string): boolean {
|
function clearBundledRuntimeMirrorMaterializeCache(): void {
|
||||||
if (!BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath))) {
|
bundledRuntimeMirrorMaterializeCache.clear();
|
||||||
return false;
|
}
|
||||||
}
|
|
||||||
|
function statSignature(stat: Pick<fs.Stats, "dev" | "ino" | "size" | "mtimeMs">): string {
|
||||||
|
return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function computeBundledRuntimeMirrorDistFileMaterialization(sourcePath: string): boolean {
|
||||||
let source: string;
|
let source: string;
|
||||||
try {
|
try {
|
||||||
source = fs.readFileSync(sourcePath, "utf8");
|
source = fs.readFileSync(sourcePath, "utf8");
|
||||||
@@ -113,6 +122,27 @@ export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string): boolean {
|
||||||
|
if (!BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const cacheKey = path.resolve(sourcePath);
|
||||||
|
let signature: string;
|
||||||
|
try {
|
||||||
|
signature = statSignature(fs.statSync(sourcePath));
|
||||||
|
} catch {
|
||||||
|
bundledRuntimeMirrorMaterializeCache.delete(cacheKey);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const cached = bundledRuntimeMirrorMaterializeCache.get(cacheKey);
|
||||||
|
if (cached?.signature === signature) {
|
||||||
|
return cached.materialize;
|
||||||
|
}
|
||||||
|
const materialize = computeBundledRuntimeMirrorDistFileMaterialization(sourcePath);
|
||||||
|
bundledRuntimeMirrorMaterializeCache.set(cacheKey, { signature, materialize });
|
||||||
|
return materialize;
|
||||||
|
}
|
||||||
|
|
||||||
export function materializeBundledRuntimeMirrorDistFile(
|
export function materializeBundledRuntimeMirrorDistFile(
|
||||||
sourcePath: string,
|
sourcePath: string,
|
||||||
targetPath: string,
|
targetPath: string,
|
||||||
@@ -404,6 +434,7 @@ function formatRuntimeDepsLockTimeoutMessage(params: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const __testing = {
|
export const __testing = {
|
||||||
|
clearBundledRuntimeMirrorMaterializeCache,
|
||||||
formatRuntimeDepsLockTimeoutMessage,
|
formatRuntimeDepsLockTimeoutMessage,
|
||||||
shouldRemoveRuntimeDepsLock,
|
shouldRemoveRuntimeDepsLock,
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user