mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 06:30:42 +00:00
perf: speed up boundary and provider tests
This commit is contained in:
@@ -13,10 +13,8 @@ import {
|
||||
diffInventoryEntries,
|
||||
normalizeRepoPath,
|
||||
resolveRepoSpecifier,
|
||||
visitModuleSpecifiers,
|
||||
writeLine,
|
||||
} from "./lib/guard-inventory-utils.mjs";
|
||||
import { toLine } from "./lib/ts-guard-utils.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const extensionsRoot = path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR);
|
||||
@@ -49,7 +47,7 @@ const baselinePathByMode = {
|
||||
};
|
||||
|
||||
let allInventoryByModePromise;
|
||||
let parsedExtensionSourceFilesPromise;
|
||||
let extensionModuleReferencesPromise;
|
||||
|
||||
const ruleTextByMode = {
|
||||
"src-outside-plugin-sdk":
|
||||
@@ -97,36 +95,26 @@ async function collectExtensionSourceFiles(rootDir) {
|
||||
);
|
||||
}
|
||||
|
||||
async function collectParsedExtensionSourceFiles() {
|
||||
if (!parsedExtensionSourceFilesPromise) {
|
||||
parsedExtensionSourceFilesPromise = (async () => {
|
||||
async function collectExtensionModuleReferences() {
|
||||
if (!extensionModuleReferencesPromise) {
|
||||
extensionModuleReferencesPromise = (async () => {
|
||||
const files = await collectExtensionSourceFiles(extensionsRoot);
|
||||
const parsed = await Promise.all(
|
||||
const referenced = await Promise.all(
|
||||
files.map(async (filePath) => {
|
||||
const source = await fs.readFile(filePath, "utf8");
|
||||
if (!mayContainModuleSpecifier(source)) {
|
||||
return null;
|
||||
}
|
||||
const scriptKind =
|
||||
filePath.endsWith(".tsx") || filePath.endsWith(".jsx")
|
||||
? ts.ScriptKind.TSX
|
||||
: ts.ScriptKind.TS;
|
||||
return {
|
||||
filePath,
|
||||
sourceFile: ts.createSourceFile(
|
||||
filePath,
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
scriptKind,
|
||||
),
|
||||
references: collectModuleReferences(source),
|
||||
};
|
||||
}),
|
||||
);
|
||||
return parsed.filter(Boolean);
|
||||
return referenced.filter((entry) => entry && entry.references.length > 0);
|
||||
})();
|
||||
}
|
||||
return await parsedExtensionSourceFilesPromise;
|
||||
return await extensionModuleReferencesPromise;
|
||||
}
|
||||
|
||||
function mayContainModuleSpecifier(source) {
|
||||
@@ -137,6 +125,49 @@ function mayContainModuleSpecifier(source) {
|
||||
);
|
||||
}
|
||||
|
||||
function collectModuleReferences(source) {
|
||||
const lineStarts = computeLineStarts(source);
|
||||
return ts.preProcessFile(source, true, true).importedFiles.map((reference) => ({
|
||||
kind: inferModuleReferenceKind(source, reference.pos),
|
||||
line: lineFromPosition(lineStarts, reference.pos),
|
||||
specifier: reference.fileName,
|
||||
}));
|
||||
}
|
||||
|
||||
function computeLineStarts(source) {
|
||||
const lineStarts = [0];
|
||||
for (let index = 0; index < source.length; index += 1) {
|
||||
if (source.charCodeAt(index) === 10) {
|
||||
lineStarts.push(index + 1);
|
||||
}
|
||||
}
|
||||
return lineStarts;
|
||||
}
|
||||
|
||||
function lineFromPosition(lineStarts, position) {
|
||||
let low = 0;
|
||||
let high = lineStarts.length - 1;
|
||||
while (low <= high) {
|
||||
const middle = Math.floor((low + high) / 2);
|
||||
if (lineStarts[middle] <= position) {
|
||||
low = middle + 1;
|
||||
} else {
|
||||
high = middle - 1;
|
||||
}
|
||||
}
|
||||
return high + 1;
|
||||
}
|
||||
|
||||
function inferModuleReferenceKind(source, specifierStart) {
|
||||
const importIndex = source.lastIndexOf("import", specifierStart);
|
||||
const exportIndex = source.lastIndexOf("export", specifierStart);
|
||||
if (exportIndex > importIndex) {
|
||||
return "export";
|
||||
}
|
||||
const importPrefix = source.slice(importIndex, specifierStart);
|
||||
return /\bimport\s*\(\s*["']?$/.test(importPrefix) ? "dynamic-import" : "import";
|
||||
}
|
||||
|
||||
function resolveExtensionRoot(filePath) {
|
||||
const relativePath = normalizeRepoPath(repoRoot, filePath);
|
||||
const segments = relativePath.split("/");
|
||||
@@ -198,7 +229,7 @@ function shouldReport(mode, resolvedPath) {
|
||||
return !resolvedPath.startsWith("src/plugin-sdk/");
|
||||
}
|
||||
|
||||
function collectEntriesByModeFromSourceFile(sourceFile, filePath) {
|
||||
function collectEntriesByModeFromModuleReferences(filePath, references) {
|
||||
const entriesByMode = {
|
||||
"src-outside-plugin-sdk": [],
|
||||
"plugin-sdk-internal": [],
|
||||
@@ -207,11 +238,11 @@ function collectEntriesByModeFromSourceFile(sourceFile, filePath) {
|
||||
const extensionRoot = resolveExtensionRoot(filePath);
|
||||
const relativeFile = normalizeRepoPath(repoRoot, filePath);
|
||||
|
||||
function push(kind, specifierNode, specifier) {
|
||||
function push(kind, line, specifier) {
|
||||
const resolvedPath = resolveRepoSpecifier(repoRoot, specifier, filePath);
|
||||
const baseEntry = {
|
||||
file: relativeFile,
|
||||
line: toLine(sourceFile, specifierNode),
|
||||
line,
|
||||
kind,
|
||||
specifier,
|
||||
resolvedPath,
|
||||
@@ -237,9 +268,9 @@ function collectEntriesByModeFromSourceFile(sourceFile, filePath) {
|
||||
}
|
||||
}
|
||||
|
||||
visitModuleSpecifiers(ts, sourceFile, ({ kind, specifier, specifierNode }) => {
|
||||
push(kind, specifierNode, specifier);
|
||||
});
|
||||
for (const { kind, line, specifier } of references) {
|
||||
push(kind, line, specifier);
|
||||
}
|
||||
return entriesByMode;
|
||||
}
|
||||
|
||||
@@ -249,14 +280,14 @@ export async function collectExtensionPluginSdkBoundaryInventory(mode) {
|
||||
}
|
||||
if (!allInventoryByModePromise) {
|
||||
allInventoryByModePromise = (async () => {
|
||||
const files = await collectParsedExtensionSourceFiles();
|
||||
const files = await collectExtensionModuleReferences();
|
||||
const inventoryByMode = {
|
||||
"src-outside-plugin-sdk": [],
|
||||
"plugin-sdk-internal": [],
|
||||
"relative-outside-package": [],
|
||||
};
|
||||
for (const { filePath, sourceFile } of files) {
|
||||
const entriesByMode = collectEntriesByModeFromSourceFile(sourceFile, filePath);
|
||||
for (const { filePath, references } of files) {
|
||||
const entriesByMode = collectEntriesByModeFromModuleReferences(filePath, references);
|
||||
for (const inventoryMode of MODES) {
|
||||
inventoryByMode[inventoryMode].push(...entriesByMode[inventoryMode]);
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMissingProviderApiKey } from "./models-config.providers.secret-helpers.js";
|
||||
|
||||
describe("models-config", () => {
|
||||
it("fills missing provider.apiKey from env var name when models exist", () => {
|
||||
const provider = resolveMissingProviderApiKey({
|
||||
providerKey: "minimax",
|
||||
provider: {
|
||||
baseUrl: "https://api.minimax.io/anthropic",
|
||||
api: "anthropic-messages",
|
||||
models: [
|
||||
{
|
||||
id: "MiniMax-M2.7",
|
||||
name: "MiniMax M2.7",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
],
|
||||
},
|
||||
env: { MINIMAX_API_KEY: "sk-minimax-test" } as NodeJS.ProcessEnv,
|
||||
profileApiKey: undefined,
|
||||
});
|
||||
|
||||
expect(provider.apiKey).toBe("MINIMAX_API_KEY"); // pragma: allowlist secret
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,39 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ModelProviderConfig } from "../config/types.models.js";
|
||||
import { applyProviderNativeStreamingUsageCompat } from "../plugin-sdk/provider-catalog-shared.js";
|
||||
import { resolveMissingProviderApiKey } from "./models-config.providers.secret-helpers.js";
|
||||
|
||||
vi.mock("../plugins/setup-registry.js", () => ({
|
||||
resolvePluginSetupProvider: () => undefined,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/shell-env.js", () => ({
|
||||
getShellEnvAppliedKeys: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("./provider-auth-aliases.js", () => ({
|
||||
resolveProviderAuthAliasMap: () => ({}),
|
||||
resolveProviderIdForAuth: (provider: string) => provider.trim().toLowerCase(),
|
||||
}));
|
||||
|
||||
vi.mock("./model-auth-env-vars.js", () => {
|
||||
const candidates = {
|
||||
moonshot: ["MOONSHOT_API_KEY"],
|
||||
} as const;
|
||||
return {
|
||||
PROVIDER_ENV_API_KEY_CANDIDATES: candidates,
|
||||
listKnownProviderEnvApiKeyNames: () => [...new Set(Object.values(candidates).flat())],
|
||||
resolveProviderEnvApiKeyCandidates: () => candidates,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../plugin-sdk/provider-http.js", () => ({
|
||||
resolveProviderRequestCapabilities: (params: { provider: string; baseUrl?: string }) => ({
|
||||
supportsNativeStreamingUsageCompat:
|
||||
params.provider === "moonshot" && params.baseUrl === "https://api.moonshot.cn/v1",
|
||||
}),
|
||||
}));
|
||||
|
||||
const MOONSHOT_BASE_URL = "https://api.moonshot.ai/v1";
|
||||
const MOONSHOT_CN_BASE_URL = "https://api.moonshot.cn/v1";
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ModelDefinitionConfig, ModelProviderConfig } from "../config/types.models.js";
|
||||
import { resolveEnvApiKey } from "./model-auth-env.js";
|
||||
import {
|
||||
@@ -6,6 +6,33 @@ import {
|
||||
resolveMissingProviderApiKey,
|
||||
} from "./models-config.providers.secret-helpers.js";
|
||||
|
||||
vi.mock("../plugins/setup-registry.js", () => ({
|
||||
resolvePluginSetupProvider: () => undefined,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/shell-env.js", () => ({
|
||||
getShellEnvAppliedKeys: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("./provider-auth-aliases.js", () => ({
|
||||
resolveProviderAuthAliasMap: () => ({}),
|
||||
resolveProviderIdForAuth: (provider: string) => provider.trim().toLowerCase(),
|
||||
}));
|
||||
|
||||
vi.mock("./model-auth-env-vars.js", () => {
|
||||
const candidates = {
|
||||
minimax: ["MINIMAX_API_KEY"],
|
||||
"minimax-portal": ["MINIMAX_OAUTH_TOKEN"],
|
||||
nvidia: ["NVIDIA_API_KEY"],
|
||||
vllm: ["VLLM_API_KEY"],
|
||||
} as const;
|
||||
return {
|
||||
PROVIDER_ENV_API_KEY_CANDIDATES: candidates,
|
||||
listKnownProviderEnvApiKeyNames: () => [...new Set(Object.values(candidates).flat())],
|
||||
resolveProviderEnvApiKeyCandidates: () => candidates,
|
||||
};
|
||||
});
|
||||
|
||||
const NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1";
|
||||
const MINIMAX_BASE_URL = "https://api.minimax.io/anthropic";
|
||||
const VLLM_DEFAULT_BASE_URL = "http://127.0.0.1:8000/v1";
|
||||
|
||||
Reference in New Issue
Block a user