mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 06:50:43 +00:00
refactor: streamline plugin cache helpers
This commit is contained in:
@@ -6,10 +6,7 @@ import { normalizeChannelId as normalizeBundledChannelId } from "../channels/reg
|
||||
import { isRouteBinding, listRouteBindings } from "../config/bindings.js";
|
||||
import type { AgentRouteBinding } from "../config/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
listPluginContributionIds,
|
||||
loadPluginRegistrySnapshot,
|
||||
} from "../plugins/plugin-registry.js";
|
||||
import { listManifestChannelContributionIds } from "../plugins/manifest-channel-contributions.js";
|
||||
import { DEFAULT_ACCOUNT_ID, normalizeAgentId } from "../routing/session-key.js";
|
||||
import { normalizeOptionalString } from "../shared/string-coerce.js";
|
||||
import { normalizeStringEntries } from "../shared/string-normalization.js";
|
||||
@@ -220,16 +217,11 @@ function resolveDefaultAccountId(cfg: OpenClawConfig, provider: ChannelId): stri
|
||||
}
|
||||
|
||||
function listManifestChannelIds(config: OpenClawConfig): Set<string> {
|
||||
const index = loadPluginRegistrySnapshot({
|
||||
config,
|
||||
env: process.env,
|
||||
});
|
||||
return new Set(
|
||||
listPluginContributionIds({
|
||||
index,
|
||||
contribution: "channels",
|
||||
listManifestChannelContributionIds({
|
||||
includeDisabled: true,
|
||||
config,
|
||||
env: process.env,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,10 +8,7 @@ import type { ChannelMeta } from "../../channels/plugins/types.public.js";
|
||||
import { isStaticallyChannelConfigured } from "../../config/channel-configured-shared.js";
|
||||
import { applyPluginAutoEnable } from "../../config/plugin-auto-enable.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
listPluginContributionIds,
|
||||
loadPluginRegistrySnapshot,
|
||||
} from "../../plugins/plugin-registry.js";
|
||||
import { listManifestChannelContributionIds } from "../../plugins/manifest-channel-contributions.js";
|
||||
import type { ChannelChoice } from "../onboard-types.js";
|
||||
import {
|
||||
listSetupDiscoveryChannelPluginCatalogEntries,
|
||||
@@ -51,15 +48,8 @@ export function listManifestInstalledChannelIds(params: {
|
||||
env: params.env ?? process.env,
|
||||
}).config;
|
||||
const workspaceDir = resolveWorkspaceDir(resolvedConfig, params.workspaceDir);
|
||||
const index = loadPluginRegistrySnapshot({
|
||||
config: resolvedConfig,
|
||||
workspaceDir,
|
||||
env: params.env ?? process.env,
|
||||
});
|
||||
return new Set(
|
||||
listPluginContributionIds({
|
||||
index,
|
||||
contribution: "channels",
|
||||
listManifestChannelContributionIds({
|
||||
config: resolvedConfig,
|
||||
workspaceDir,
|
||||
env: params.env ?? process.env,
|
||||
|
||||
@@ -3,10 +3,7 @@ import { normalizeChannelId as normalizeBundledChannelId } from "../../channels/
|
||||
import { getResolvedLoggerSettings } from "../../logging.js";
|
||||
import { resolveLogFile } from "../../logging/log-tail.js";
|
||||
import { parseLogLine } from "../../logging/parse-log-line.js";
|
||||
import {
|
||||
listPluginContributionIds,
|
||||
loadPluginRegistrySnapshot,
|
||||
} from "../../plugins/plugin-registry.js";
|
||||
import { listManifestChannelContributionIds } from "../../plugins/manifest-channel-contributions.js";
|
||||
import { defaultRuntime, type RuntimeEnv, writeRuntimeJson } from "../../runtime.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js";
|
||||
import { theme } from "../../terminal/theme.js";
|
||||
@@ -23,14 +20,10 @@ const DEFAULT_LIMIT = 200;
|
||||
const MAX_BYTES = 1_000_000;
|
||||
|
||||
function listManifestChannelIds(): Set<string> {
|
||||
const index = loadPluginRegistrySnapshot({
|
||||
env: process.env,
|
||||
});
|
||||
return new Set(
|
||||
listPluginContributionIds({
|
||||
index,
|
||||
contribution: "channels",
|
||||
listManifestChannelContributionIds({
|
||||
includeDisabled: true,
|
||||
env: process.env,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,30 +4,12 @@ import type {
|
||||
DocumentExtractionResult,
|
||||
} from "../plugins/document-extractor-types.js";
|
||||
import { resolvePluginDocumentExtractors } from "../plugins/document-extractors.runtime.js";
|
||||
import { createConfigScopedPromiseLoader } from "../plugins/plugin-cache-primitives.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js";
|
||||
|
||||
let extractorPromise: Promise<ReturnType<typeof resolvePluginDocumentExtractors>> | undefined;
|
||||
const extractorPromisesByConfig = new WeakMap<
|
||||
OpenClawConfig,
|
||||
Promise<ReturnType<typeof resolvePluginDocumentExtractors>>
|
||||
>();
|
||||
|
||||
async function loadDocumentExtractors(config?: OpenClawConfig) {
|
||||
if (config) {
|
||||
const cached = extractorPromisesByConfig.get(config);
|
||||
if (cached) {
|
||||
return await cached;
|
||||
}
|
||||
const promise = Promise.resolve().then(() => resolvePluginDocumentExtractors({ config }));
|
||||
extractorPromisesByConfig.set(config, promise);
|
||||
void promise.catch(() => {
|
||||
extractorPromisesByConfig.delete(config);
|
||||
});
|
||||
return await promise;
|
||||
}
|
||||
extractorPromise ??= Promise.resolve(resolvePluginDocumentExtractors());
|
||||
return await extractorPromise;
|
||||
}
|
||||
const documentExtractorLoader = createConfigScopedPromiseLoader((config?: OpenClawConfig) =>
|
||||
resolvePluginDocumentExtractors(config ? { config } : undefined),
|
||||
);
|
||||
|
||||
export async function extractDocumentContent(
|
||||
params: DocumentExtractionRequest & {
|
||||
@@ -35,7 +17,7 @@ export async function extractDocumentContent(
|
||||
},
|
||||
): Promise<(DocumentExtractionResult & { extractor: string }) | null> {
|
||||
const mimeType = normalizeLowercaseStringOrEmpty(params.mimeType);
|
||||
const extractors = await loadDocumentExtractors(params.config);
|
||||
const extractors = await documentExtractorLoader.load(params.config);
|
||||
const request: DocumentExtractionRequest = {
|
||||
buffer: params.buffer,
|
||||
mimeType: params.mimeType,
|
||||
|
||||
26
src/plugins/manifest-channel-contributions.ts
Normal file
26
src/plugins/manifest-channel-contributions.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { listPluginContributionIds, loadPluginRegistrySnapshot } from "./plugin-registry.js";
|
||||
|
||||
export function listManifestChannelContributionIds(
|
||||
params: {
|
||||
config?: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
includeDisabled?: boolean;
|
||||
} = {},
|
||||
): readonly string[] {
|
||||
const env = params.env ?? process.env;
|
||||
const index = loadPluginRegistrySnapshot({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env,
|
||||
});
|
||||
return listPluginContributionIds({
|
||||
index,
|
||||
contribution: "channels",
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env,
|
||||
includeDisabled: params.includeDisabled,
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
PluginLruCache,
|
||||
createConfigScopedPromiseLoader,
|
||||
resolveConfigScopedRuntimeCacheValue,
|
||||
type ConfigScopedRuntimeCache,
|
||||
} from "./plugin-cache-primitives.js";
|
||||
@@ -84,3 +85,64 @@ describe("resolveConfigScopedRuntimeCacheValue", () => {
|
||||
expect(load).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
describe("createConfigScopedPromiseLoader", () => {
|
||||
it("dedupes concurrent default loads", async () => {
|
||||
let calls = 0;
|
||||
const loader = createConfigScopedPromiseLoader(async () => `loaded-${++calls}`);
|
||||
|
||||
await expect(Promise.all([loader.load(), loader.load()])).resolves.toEqual([
|
||||
"loaded-1",
|
||||
"loaded-1",
|
||||
]);
|
||||
await expect(loader.load()).resolves.toBe("loaded-1");
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
|
||||
it("caches loads by config object", async () => {
|
||||
const firstConfig = { plugins: { load: { disabled: true } } } as OpenClawConfig;
|
||||
const secondConfig = { plugins: { load: { disabled: false } } } as OpenClawConfig;
|
||||
const load = vi.fn(async (config?: OpenClawConfig) =>
|
||||
config === firstConfig ? "first" : "second",
|
||||
);
|
||||
const loader = createConfigScopedPromiseLoader(load);
|
||||
|
||||
await expect(loader.load(firstConfig)).resolves.toBe("first");
|
||||
await expect(loader.load(firstConfig)).resolves.toBe("first");
|
||||
await expect(loader.load(secondConfig)).resolves.toBe("second");
|
||||
|
||||
expect(load).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("evicts rejected loads so retries can recover", async () => {
|
||||
const config = {} as OpenClawConfig;
|
||||
let calls = 0;
|
||||
const loader = createConfigScopedPromiseLoader(async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
throw new Error("transient");
|
||||
}
|
||||
return "recovered";
|
||||
});
|
||||
|
||||
await expect(loader.load(config)).rejects.toThrow("transient");
|
||||
await expect(loader.load(config)).resolves.toBe("recovered");
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
it("clears default and config-scoped entries", async () => {
|
||||
const config = {} as OpenClawConfig;
|
||||
let calls = 0;
|
||||
const loader = createConfigScopedPromiseLoader(
|
||||
async (owner?: OpenClawConfig) => `${owner ? "config" : "default"}-${++calls}`,
|
||||
);
|
||||
|
||||
await expect(loader.load()).resolves.toBe("default-1");
|
||||
await expect(loader.load(config)).resolves.toBe("config-2");
|
||||
|
||||
loader.clear();
|
||||
|
||||
await expect(loader.load()).resolves.toBe("default-3");
|
||||
await expect(loader.load(config)).resolves.toBe("config-4");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -68,6 +68,11 @@ export class PluginLruCache<T> {
|
||||
|
||||
export type ConfigScopedRuntimeCache<T> = WeakMap<OpenClawConfig, Map<string, T>>;
|
||||
|
||||
export type ConfigScopedPromiseLoader<T> = {
|
||||
load(config?: OpenClawConfig): Promise<T>;
|
||||
clear(): void;
|
||||
};
|
||||
|
||||
export function resolveConfigScopedRuntimeCacheValue<T>(params: {
|
||||
cache: ConfigScopedRuntimeCache<T>;
|
||||
config?: OpenClawConfig;
|
||||
@@ -94,6 +99,45 @@ export function createPluginCacheKey(parts: readonly unknown[]): string {
|
||||
return JSON.stringify(parts);
|
||||
}
|
||||
|
||||
export function createConfigScopedPromiseLoader<T>(
|
||||
load: (config?: OpenClawConfig) => T | Promise<T>,
|
||||
): ConfigScopedPromiseLoader<T> {
|
||||
let defaultPromise: Promise<T> | undefined;
|
||||
let promisesByConfig = new WeakMap<OpenClawConfig, Promise<T>>();
|
||||
|
||||
const createPromise = (config?: OpenClawConfig): Promise<T> => {
|
||||
const promise = Promise.resolve().then(() => load(config));
|
||||
void promise.catch(() => {
|
||||
if (config) {
|
||||
promisesByConfig.delete(config);
|
||||
} else if (defaultPromise === promise) {
|
||||
defaultPromise = undefined;
|
||||
}
|
||||
});
|
||||
return promise;
|
||||
};
|
||||
|
||||
return {
|
||||
async load(config?: OpenClawConfig): Promise<T> {
|
||||
if (!config) {
|
||||
defaultPromise ??= createPromise();
|
||||
return await defaultPromise;
|
||||
}
|
||||
const cached = promisesByConfig.get(config);
|
||||
if (cached) {
|
||||
return await cached;
|
||||
}
|
||||
const promise = createPromise(config);
|
||||
promisesByConfig.set(config, promise);
|
||||
return await promise;
|
||||
},
|
||||
clear(): void {
|
||||
defaultPromise = undefined;
|
||||
promisesByConfig = new WeakMap<OpenClawConfig, Promise<T>>();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMaxEntries(value: number, fallback: number): number {
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
return fallback;
|
||||
|
||||
@@ -2,8 +2,8 @@ import { resolveProviderIdForAuth } from "../agents/provider-auth-aliases.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { sanitizeForLog } from "../terminal/ansi.js";
|
||||
import { normalizePluginsConfig, resolveEffectiveEnableState } from "./config-state.js";
|
||||
import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js";
|
||||
import type { PluginManifestRecord } from "./manifest-registry.js";
|
||||
import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js";
|
||||
import type { PluginOrigin } from "./plugin-origin.types.js";
|
||||
|
||||
export type ProviderAuthChoiceMetadata = {
|
||||
@@ -180,7 +180,7 @@ function resolveManifestProviderAuthChoiceCandidates(params?: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
includeUntrustedWorkspacePlugins?: boolean;
|
||||
}): ProviderAuthChoiceCandidate[] {
|
||||
const metadataSnapshot = loadPluginMetadataSnapshot({
|
||||
const metadataSnapshot = loadManifestMetadataSnapshot({
|
||||
config: params?.config ?? {},
|
||||
workspaceDir: params?.workspaceDir,
|
||||
env: params?.env ?? process.env,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js";
|
||||
import type { PluginManifestRecord } from "./manifest-registry.js";
|
||||
import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js";
|
||||
import type { PluginMetadataRegistryView } from "./plugin-metadata-snapshot.types.js";
|
||||
import { resolveDiscoveredProviderPluginIds } from "./providers.js";
|
||||
import { resolvePluginProviders } from "./providers.runtime.js";
|
||||
@@ -80,7 +80,7 @@ function resolveProviderDiscoveryEntryPlugins(params: {
|
||||
}): ProviderDiscoveryEntryResult {
|
||||
const metadataSnapshot =
|
||||
params.pluginMetadataSnapshot ??
|
||||
loadPluginMetadataSnapshot({
|
||||
loadManifestMetadataSnapshot({
|
||||
config: params.config ?? {},
|
||||
env: params.env ?? process.env,
|
||||
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { normalizeProviderId } from "../agents/provider-id.js";
|
||||
import { isInstalledPluginEnabled } from "./installed-plugin-index.js";
|
||||
import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js";
|
||||
import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js";
|
||||
|
||||
type SetupRegistryRuntimeModule = Pick<
|
||||
typeof import("./setup-registry.js"),
|
||||
@@ -30,7 +30,7 @@ export const __testing = {
|
||||
};
|
||||
|
||||
function resolveBundledSetupCliBackends(): SetupCliBackendRuntimeEntry[] {
|
||||
const snapshot = loadPluginMetadataSnapshot({ config: {}, env: process.env });
|
||||
const snapshot = loadManifestMetadataSnapshot({ config: {}, env: process.env });
|
||||
return snapshot.plugins.flatMap((plugin) => {
|
||||
if (plugin.origin !== "bundled" || !isInstalledPluginEnabled(snapshot.index, plugin.id)) {
|
||||
return [];
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveBundledPluginCompatibleLoadValues } from "./activation-context.js";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js";
|
||||
import type { PluginManifestRecord } from "./manifest-registry.js";
|
||||
import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js";
|
||||
import { createPluginIdScopeSet, normalizePluginIdScope } from "./plugin-scope.js";
|
||||
|
||||
export type WebProviderContract = "webSearchProviders" | "webFetchProviders";
|
||||
@@ -66,7 +66,7 @@ function loadInstalledWebProviderManifestRecords(params: {
|
||||
env?: PluginLoadOptions["env"];
|
||||
pluginIds?: readonly string[];
|
||||
}): readonly PluginManifestRecord[] {
|
||||
const records = loadPluginMetadataSnapshot({
|
||||
const records = loadManifestMetadataSnapshot({
|
||||
config: params.config ?? {},
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env ?? process.env,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js";
|
||||
import type { PluginManifestRecord } from "./manifest-registry.js";
|
||||
import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js";
|
||||
|
||||
function hasConfiguredCredentialValue(value: unknown): boolean {
|
||||
if (typeof value === "string") {
|
||||
@@ -42,7 +42,7 @@ function hasManifestWebSearchEnvCredentialCandidate(params: {
|
||||
if (!env) {
|
||||
return false;
|
||||
}
|
||||
return loadPluginMetadataSnapshot({
|
||||
return loadManifestMetadataSnapshot({
|
||||
config: params.config,
|
||||
env,
|
||||
}).plugins.some((plugin) => {
|
||||
|
||||
@@ -1,32 +1,14 @@
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { createConfigScopedPromiseLoader } from "../plugins/plugin-cache-primitives.js";
|
||||
import type {
|
||||
WebContentExtractionResult,
|
||||
WebContentExtractMode,
|
||||
} from "../plugins/web-content-extractor-types.js";
|
||||
import { resolvePluginWebContentExtractors } from "../plugins/web-content-extractors.runtime.js";
|
||||
|
||||
let extractorPromise: Promise<ReturnType<typeof resolvePluginWebContentExtractors>> | undefined;
|
||||
const extractorPromisesByConfig = new WeakMap<
|
||||
OpenClawConfig,
|
||||
Promise<ReturnType<typeof resolvePluginWebContentExtractors>>
|
||||
>();
|
||||
|
||||
async function loadWebContentExtractors(config?: OpenClawConfig) {
|
||||
if (config) {
|
||||
const cached = extractorPromisesByConfig.get(config);
|
||||
if (cached) {
|
||||
return await cached;
|
||||
}
|
||||
const promise = Promise.resolve().then(() => resolvePluginWebContentExtractors({ config }));
|
||||
extractorPromisesByConfig.set(config, promise);
|
||||
void promise.catch(() => {
|
||||
extractorPromisesByConfig.delete(config);
|
||||
});
|
||||
return await promise;
|
||||
}
|
||||
extractorPromise ??= Promise.resolve(resolvePluginWebContentExtractors());
|
||||
return await extractorPromise;
|
||||
}
|
||||
const webContentExtractorLoader = createConfigScopedPromiseLoader((config?: OpenClawConfig) =>
|
||||
resolvePluginWebContentExtractors(config ? { config } : undefined),
|
||||
);
|
||||
|
||||
export async function extractReadableContent(params: {
|
||||
html: string;
|
||||
@@ -34,9 +16,9 @@ export async function extractReadableContent(params: {
|
||||
extractMode: WebContentExtractMode;
|
||||
config?: OpenClawConfig;
|
||||
}): Promise<(WebContentExtractionResult & { extractor: string }) | null> {
|
||||
let extractors: Awaited<ReturnType<typeof loadWebContentExtractors>>;
|
||||
let extractors: Awaited<ReturnType<typeof webContentExtractorLoader.load>>;
|
||||
try {
|
||||
extractors = await loadWebContentExtractors(params.config);
|
||||
extractors = await webContentExtractorLoader.load(params.config);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user