mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-05 14:00:26 +00:00
perf(core): speed up routing, pairing, slack, and security scans
This commit is contained in:
@@ -1,13 +1,15 @@
|
||||
import crypto from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveOAuthDir } from "../config/paths.js";
|
||||
import { DEFAULT_ACCOUNT_ID } from "../routing/session-key.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import {
|
||||
addChannelAllowFromStoreEntry,
|
||||
clearPairingAllowFromReadCacheForTest,
|
||||
approveChannelPairingCode,
|
||||
listChannelPairingRequests,
|
||||
readChannelAllowFromStore,
|
||||
@@ -31,6 +33,10 @@ afterAll(async () => {
|
||||
}
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
clearPairingAllowFromReadCacheForTest();
|
||||
});
|
||||
|
||||
async function withTempStateDir<T>(fn: (stateDir: string) => Promise<T>) {
|
||||
const dir = path.join(fixtureRoot, `case-${caseId++}`);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
@@ -412,4 +418,62 @@ describe("pairing store", () => {
|
||||
expect(syncScoped).toEqual(["1002", "1001"]);
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses cached async allowFrom reads and invalidates on file updates", async () => {
|
||||
await withTempStateDir(async (stateDir) => {
|
||||
await writeAllowFromFixture({
|
||||
stateDir,
|
||||
channel: "telegram",
|
||||
accountId: "yy",
|
||||
allowFrom: ["1001"],
|
||||
});
|
||||
const readSpy = vi.spyOn(fs, "readFile");
|
||||
|
||||
const first = await readChannelAllowFromStore("telegram", process.env, "yy");
|
||||
const second = await readChannelAllowFromStore("telegram", process.env, "yy");
|
||||
expect(first).toEqual(["1001"]);
|
||||
expect(second).toEqual(["1001"]);
|
||||
expect(readSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
await writeAllowFromFixture({
|
||||
stateDir,
|
||||
channel: "telegram",
|
||||
accountId: "yy",
|
||||
allowFrom: ["1002"],
|
||||
});
|
||||
const third = await readChannelAllowFromStore("telegram", process.env, "yy");
|
||||
expect(third).toEqual(["1002"]);
|
||||
expect(readSpy).toHaveBeenCalledTimes(2);
|
||||
readSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses cached sync allowFrom reads and invalidates on file updates", async () => {
|
||||
await withTempStateDir(async (stateDir) => {
|
||||
await writeAllowFromFixture({
|
||||
stateDir,
|
||||
channel: "telegram",
|
||||
accountId: "yy",
|
||||
allowFrom: ["1001"],
|
||||
});
|
||||
const readSpy = vi.spyOn(fsSync, "readFileSync");
|
||||
|
||||
const first = readChannelAllowFromStoreSync("telegram", process.env, "yy");
|
||||
const second = readChannelAllowFromStoreSync("telegram", process.env, "yy");
|
||||
expect(first).toEqual(["1001"]);
|
||||
expect(second).toEqual(["1001"]);
|
||||
expect(readSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
await writeAllowFromFixture({
|
||||
stateDir,
|
||||
channel: "telegram",
|
||||
accountId: "yy",
|
||||
allowFrom: ["1002"],
|
||||
});
|
||||
const third = readChannelAllowFromStoreSync("telegram", process.env, "yy");
|
||||
expect(third).toEqual(["1002"]);
|
||||
expect(readSpy).toHaveBeenCalledTimes(2);
|
||||
readSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,14 @@ const PAIRING_STORE_LOCK_OPTIONS = {
|
||||
},
|
||||
stale: 30_000,
|
||||
} as const;
|
||||
type AllowFromReadCacheEntry = {
|
||||
exists: boolean;
|
||||
mtimeMs: number | null;
|
||||
size: number | null;
|
||||
entries: string[];
|
||||
};
|
||||
|
||||
const allowFromReadCache = new Map<string, AllowFromReadCacheEntry>();
|
||||
|
||||
export type PairingChannel = ChannelId;
|
||||
|
||||
@@ -278,15 +286,86 @@ async function readAllowFromStateForPath(
|
||||
return (await readAllowFromStateForPathWithExists(channel, filePath)).entries;
|
||||
}
|
||||
|
||||
function cloneAllowFromCacheEntry(entry: AllowFromReadCacheEntry): AllowFromReadCacheEntry {
|
||||
return {
|
||||
exists: entry.exists,
|
||||
mtimeMs: entry.mtimeMs,
|
||||
size: entry.size,
|
||||
entries: entry.entries.slice(),
|
||||
};
|
||||
}
|
||||
|
||||
function setAllowFromReadCache(filePath: string, entry: AllowFromReadCacheEntry): void {
|
||||
allowFromReadCache.set(filePath, cloneAllowFromCacheEntry(entry));
|
||||
}
|
||||
|
||||
function resolveAllowFromReadCacheHit(params: {
|
||||
filePath: string;
|
||||
exists: boolean;
|
||||
mtimeMs: number | null;
|
||||
size: number | null;
|
||||
}): AllowFromReadCacheEntry | null {
|
||||
const cached = allowFromReadCache.get(params.filePath);
|
||||
if (!cached) {
|
||||
return null;
|
||||
}
|
||||
if (cached.exists !== params.exists) {
|
||||
return null;
|
||||
}
|
||||
if (!params.exists) {
|
||||
return cloneAllowFromCacheEntry(cached);
|
||||
}
|
||||
if (cached.mtimeMs !== params.mtimeMs || cached.size !== params.size) {
|
||||
return null;
|
||||
}
|
||||
return cloneAllowFromCacheEntry(cached);
|
||||
}
|
||||
|
||||
async function readAllowFromStateForPathWithExists(
|
||||
channel: PairingChannel,
|
||||
filePath: string,
|
||||
): Promise<{ entries: string[]; exists: boolean }> {
|
||||
let stat: Awaited<ReturnType<typeof fs.promises.stat>> | null = null;
|
||||
try {
|
||||
stat = await fs.promises.stat(filePath);
|
||||
} catch (err) {
|
||||
const code = (err as { code?: string }).code;
|
||||
if (code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const cached = resolveAllowFromReadCacheHit({
|
||||
filePath,
|
||||
exists: Boolean(stat),
|
||||
mtimeMs: stat?.mtimeMs ?? null,
|
||||
size: stat?.size ?? null,
|
||||
});
|
||||
if (cached) {
|
||||
return { entries: cached.entries, exists: cached.exists };
|
||||
}
|
||||
|
||||
if (!stat) {
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists: false,
|
||||
mtimeMs: null,
|
||||
size: null,
|
||||
entries: [],
|
||||
});
|
||||
return { entries: [], exists: false };
|
||||
}
|
||||
|
||||
const { value, exists } = await readJsonFile<AllowFromStore>(filePath, {
|
||||
version: 1,
|
||||
allowFrom: [],
|
||||
});
|
||||
const entries = normalizeAllowFromList(channel, value);
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
entries,
|
||||
});
|
||||
return { entries, exists };
|
||||
}
|
||||
|
||||
@@ -298,6 +377,36 @@ function readAllowFromStateForPathSyncWithExists(
|
||||
channel: PairingChannel,
|
||||
filePath: string,
|
||||
): { entries: string[]; exists: boolean } {
|
||||
let stat: fs.Stats | null = null;
|
||||
try {
|
||||
stat = fs.statSync(filePath);
|
||||
} catch (err) {
|
||||
const code = (err as { code?: string }).code;
|
||||
if (code !== "ENOENT") {
|
||||
return { entries: [], exists: false };
|
||||
}
|
||||
}
|
||||
|
||||
const cached = resolveAllowFromReadCacheHit({
|
||||
filePath,
|
||||
exists: Boolean(stat),
|
||||
mtimeMs: stat?.mtimeMs ?? null,
|
||||
size: stat?.size ?? null,
|
||||
});
|
||||
if (cached) {
|
||||
return { entries: cached.entries, exists: cached.exists };
|
||||
}
|
||||
|
||||
if (!stat) {
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists: false,
|
||||
mtimeMs: null,
|
||||
size: null,
|
||||
entries: [],
|
||||
});
|
||||
return { entries: [], exists: false };
|
||||
}
|
||||
|
||||
let raw = "";
|
||||
try {
|
||||
raw = fs.readFileSync(filePath, "utf8");
|
||||
@@ -311,9 +420,21 @@ function readAllowFromStateForPathSyncWithExists(
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as AllowFromStore;
|
||||
const entries = normalizeAllowFromList(channel, parsed);
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists: true,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
entries,
|
||||
});
|
||||
return { entries, exists: true };
|
||||
} catch {
|
||||
// Keep parity with async reads: malformed JSON still means the file exists.
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists: true,
|
||||
mtimeMs: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
entries: [],
|
||||
});
|
||||
return { entries: [], exists: true };
|
||||
}
|
||||
}
|
||||
@@ -337,6 +458,16 @@ async function writeAllowFromState(filePath: string, allowFrom: string[]): Promi
|
||||
version: 1,
|
||||
allowFrom,
|
||||
} satisfies AllowFromStore);
|
||||
let stat: Awaited<ReturnType<typeof fs.promises.stat>> | null = null;
|
||||
try {
|
||||
stat = await fs.promises.stat(filePath);
|
||||
} catch {}
|
||||
setAllowFromReadCache(filePath, {
|
||||
exists: true,
|
||||
mtimeMs: stat?.mtimeMs ?? null,
|
||||
size: stat?.size ?? null,
|
||||
entries: allowFrom.slice(),
|
||||
});
|
||||
}
|
||||
|
||||
async function readNonDefaultAccountAllowFrom(params: {
|
||||
@@ -448,6 +579,10 @@ export function readChannelAllowFromStoreSync(
|
||||
return dedupePreserveOrder([...scopedEntries, ...legacyEntries]);
|
||||
}
|
||||
|
||||
export function clearPairingAllowFromReadCacheForTest(): void {
|
||||
allowFromReadCache.clear();
|
||||
}
|
||||
|
||||
type AllowFromStoreEntryUpdateParams = {
|
||||
channel: PairingChannel;
|
||||
entry: string | number;
|
||||
|
||||
Reference in New Issue
Block a user