perf: trim plugin and skills test overhead

This commit is contained in:
Peter Steinberger
2026-04-18 16:22:40 +01:00
parent aa73df571d
commit 464cbbc9f9
10 changed files with 165 additions and 171 deletions

View File

@@ -16,6 +16,10 @@ const PLUGIN_SDK_ROOT = ["openclaw", "plugin-sdk"].join("/");
const SCOPED_PLUGIN_SDK_ROOT = ["@openclaw", "plugin-sdk"].join("/");
const GROUP_ACCESS_SUBPATH = `${PLUGIN_SDK_ROOT}/group-access`;
const SCOPED_GROUP_ACCESS_SUBPATH = `${SCOPED_PLUGIN_SDK_ROOT}/group-access`;
const MATRIX_RUNTIME_WRAPPER_SOURCE = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
const PACKAGED_RUNTIME_STUB = [
"export async function ensureMatrixCryptoRuntime() {}",
"export async function handleVerifyRecoveryKey() {}",
@@ -147,14 +151,14 @@ afterEach(() => {
it("loads the source-checkout runtime wrapper through native ESM import", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-source-runtime-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
writeOpenClawPackageFixture(fixtureRoot);
writeJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",
@@ -176,14 +180,14 @@ it("loads the source-checkout runtime wrapper through native ESM import", async
it("loads the packaged runtime wrapper without recursing through the stable root alias", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
writeOpenClawPackageFixture(fixtureRoot);
writeJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "dist/plugin-entry.runtime-C88YIa_v.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"dist/plugin-entry.runtime-C88YIa_v.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"dist/plugin-entry.runtime.js",
@@ -210,15 +214,15 @@ it("loads the packaged runtime wrapper without recursing through the stable root
it("builds scoped and unscoped plugin-sdk aliases for the wrapper jiti loader", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-aliases-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
delete matrixWrapperGlobal.__openclawMatrixWrapperJitiOptions;
writeOpenClawAliasFixture(fixtureRoot);
writeCapturingJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",
@@ -242,16 +246,16 @@ it("builds scoped and unscoped plugin-sdk aliases for the wrapper jiti loader",
it("resolves extension-api aliases through the same source extension family", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-extension-api-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
delete matrixWrapperGlobal.__openclawMatrixWrapperJitiOptions;
writeOpenClawAliasFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "src/extensionAPI.mts", "export {};\n");
writeCapturingJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",
@@ -272,10 +276,6 @@ it("resolves extension-api aliases through the same source extension family", as
it("keeps wrapper plugin-sdk aliases deterministic and ignores unsafe subpaths", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-alias-order-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
delete matrixWrapperGlobal.__openclawMatrixWrapperJitiOptions;
writeOpenClawAliasFixture(fixtureRoot, {
@@ -286,7 +286,11 @@ it("keeps wrapper plugin-sdk aliases deterministic and ignores unsafe subpaths",
writeFixtureFile(fixtureRoot, "src/plugin-sdk/alpha.ts", "export {};\n");
writeFixtureFile(fixtureRoot, "src/plugin-sdk/zeta.ts", "export {};\n");
writeCapturingJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",
@@ -319,10 +323,6 @@ it("keeps wrapper plugin-sdk aliases deterministic and ignores unsafe subpaths",
it("ignores nearby untrusted openclaw package stubs when resolving the wrapper root", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-trusted-root-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
delete matrixWrapperGlobal.__openclawMatrixWrapperJitiOptions;
writeOpenClawAliasFixture(fixtureRoot);
@@ -349,7 +349,11 @@ it("ignores nearby untrusted openclaw package stubs when resolving the wrapper r
);
writeFixtureFile(fixtureRoot, "extensions/src/plugin-sdk/group-access.ts", "export {};\n");
writeCapturingJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",
@@ -373,15 +377,15 @@ it("ignores nearby untrusted openclaw package stubs when resolving the wrapper r
it("treats string bin hints case-insensitively when trusting wrapper package roots", async () => {
const fixtureRoot = makeFixtureRoot(".tmp-matrix-runtime-bin-root-");
const wrapperSource = fs.readFileSync(
path.join(REPO_ROOT, "extensions", "matrix", "src", "plugin-entry.runtime.js"),
"utf8",
);
delete matrixWrapperGlobal.__openclawMatrixWrapperJitiOptions;
writeTrustedOpenClawBinFixture(fixtureRoot, "OpenClaw.MJS");
writeCapturingJitiFixture(fixtureRoot);
writeFixtureFile(fixtureRoot, "extensions/matrix/src/plugin-entry.runtime.js", wrapperSource);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/src/plugin-entry.runtime.js",
MATRIX_RUNTIME_WRAPPER_SOURCE,
);
writeFixtureFile(
fixtureRoot,
"extensions/matrix/plugin-entry.handlers.runtime.js",

View File

@@ -109,37 +109,32 @@ describe("session MCP runtime", () => {
]);
});
it("reuses the same session runtime across repeated materialization", async () => {
it("reuses repeated materialization and recreates after explicit disposal", async () => {
const workspaceDir = await makeTempDir("openclaw-bundle-mcp-tools-");
const startupCounterPath = path.join(workspaceDir, "bundle-starts.txt");
const pluginRoot = path.join(workspaceDir, ".openclaw", "extensions", "bundle-probe");
const serverScriptPath = path.join(pluginRoot, "servers", "bundle-probe.mjs");
await writeBundleProbeMcpServer(serverScriptPath, { startupCounterPath });
await writeClaudeBundle({ pluginRoot, serverScriptPath });
const cfg = {
plugins: {
entries: {
"bundle-probe": { enabled: true },
},
},
};
const runtimeA = await getOrCreateSessionMcpRuntime({
sessionId: "session-a",
sessionKey: "agent:test:session-a",
workspaceDir,
cfg: {
plugins: {
entries: {
"bundle-probe": { enabled: true },
},
},
},
cfg,
});
const runtimeB = await getOrCreateSessionMcpRuntime({
sessionId: "session-a",
sessionKey: "agent:test:session-a",
workspaceDir,
cfg: {
plugins: {
entries: {
"bundle-probe": { enabled: true },
},
},
},
cfg,
});
const materializedA = await materializeBundleMcpToolsForRun({ runtime: runtimeA });
@@ -153,42 +148,18 @@ describe("session MCP runtime", () => {
expect(materializedB.tools.map((tool) => tool.name)).toEqual(["bundleProbe__bundle_probe"]);
expect(await fs.readFile(startupCounterPath, "utf8")).toBe("1");
expect(__testing.getCachedSessionIds()).toEqual(["session-a"]);
});
it("recreates the session runtime after explicit disposal", async () => {
const workspaceDir = await makeTempDir("openclaw-bundle-mcp-tools-");
const startupCounterPath = path.join(workspaceDir, "bundle-starts.txt");
const pluginRoot = path.join(workspaceDir, ".openclaw", "extensions", "bundle-probe");
const serverScriptPath = path.join(pluginRoot, "servers", "bundle-probe.mjs");
await writeBundleProbeMcpServer(serverScriptPath, { startupCounterPath });
await writeClaudeBundle({ pluginRoot, serverScriptPath });
await disposeSessionMcpRuntime("session-a");
const cfg = {
plugins: {
entries: {
"bundle-probe": { enabled: true },
},
},
};
const runtimeA = await getOrCreateSessionMcpRuntime({
sessionId: "session-b",
sessionKey: "agent:test:session-b",
const runtimeC = await getOrCreateSessionMcpRuntime({
sessionId: "session-a",
sessionKey: "agent:test:session-a",
workspaceDir,
cfg,
});
await materializeBundleMcpToolsForRun({ runtime: runtimeA });
await disposeSessionMcpRuntime("session-b");
await materializeBundleMcpToolsForRun({ runtime: runtimeC });
const runtimeB = await getOrCreateSessionMcpRuntime({
sessionId: "session-b",
sessionKey: "agent:test:session-b",
workspaceDir,
cfg,
});
await materializeBundleMcpToolsForRun({ runtime: runtimeB });
expect(runtimeA).not.toBe(runtimeB);
expect(runtimeC).not.toBe(runtimeA);
expect(await fs.readFile(startupCounterPath, "utf8")).toBe("2");
});

View File

@@ -9,7 +9,11 @@ import {
writeClaudeBundle,
writeExecutable,
} from "./pi-bundle-mcp-test-harness.js";
import { createBundleMcpToolRuntime } from "./pi-bundle-mcp-tools.js";
import {
createBundleMcpToolRuntime,
materializeBundleMcpToolsForRun,
} from "./pi-bundle-mcp-tools.js";
import type { SessionMcpRuntime } from "./pi-bundle-mcp-types.js";
const require = createRequire(import.meta.url);
const SDK_SERVER_MCP_PATH = require.resolve("@modelcontextprotocol/sdk/server/mcp.js");
@@ -39,6 +43,43 @@ async function createBundleProbeRuntime(params?: { reservedToolNames?: string[]
});
}
function makeSingleToolRuntime(): SessionMcpRuntime {
return {
sessionId: "session-collision",
workspaceDir: "/tmp",
configFingerprint: "fingerprint",
createdAt: 0,
lastUsedAt: 0,
markUsed: () => {},
getCatalog: async () => ({
version: 1,
generatedAt: 0,
servers: {
bundleProbe: {
serverName: "bundleProbe",
launchSummary: "bundleProbe",
toolCount: 1,
},
},
tools: [
{
serverName: "bundleProbe",
safeServerName: "bundleProbe",
toolName: "bundle_probe",
description: "Bundle probe",
inputSchema: { type: "object", properties: {} },
fallbackDescription: "Bundle probe",
},
],
}),
callTool: async () => ({
content: [{ type: "text", text: "FROM-BUNDLE" }],
isError: false,
}),
dispose: async () => {},
};
}
describe("createBundleMcpToolRuntime", () => {
it("loads bundle MCP tools and executes them", async () => {
const runtime = await createBundleProbeRuntime();
@@ -60,15 +101,12 @@ describe("createBundleMcpToolRuntime", () => {
});
it("disambiguates bundle MCP tools that collide with existing tool names", async () => {
const runtime = await createBundleProbeRuntime({
const runtime = await materializeBundleMcpToolsForRun({
runtime: makeSingleToolRuntime(),
reservedToolNames: ["bundleProbe__bundle_probe"],
});
try {
expect(runtime.tools.map((tool) => tool.name)).toEqual(["bundleProbe__bundle_probe-2"]);
} finally {
await runtime.dispose();
}
expect(runtime.tools.map((tool) => tool.name)).toEqual(["bundleProbe__bundle_probe-2"]);
});
it("loads configured stdio MCP tools without a bundle", async () => {

View File

@@ -12,43 +12,43 @@ type SpawnCall = {
const spawnCalls: SpawnCall[] = [];
vi.mock("node:child_process", async () => {
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
return {
...actual,
spawn: (command: string, args: string[]) => {
spawnCalls.push({ command, args });
const child = new EventEmitter() as {
stdout?: Readable;
stderr?: Readable;
on: (event: string, cb: (...args: unknown[]) => void) => void;
emit: (event: string, ...args: unknown[]) => boolean;
};
child.stdout = new Readable({ read() {} });
child.stderr = new Readable({ read() {} });
vi.mock("node:child_process", () => ({
execFile: (...args: unknown[]) => {
const callback = args.findLast(
(arg): arg is (error: null, stdout: string, stderr: string) => void =>
typeof arg === "function",
);
queueMicrotask(() => callback?.(null, "", ""));
return new EventEmitter();
},
spawn: (command: string, args: string[]) => {
spawnCalls.push({ command, args });
const child = new EventEmitter() as {
stdout?: Readable;
stderr?: Readable;
on: (event: string, cb: (...args: unknown[]) => void) => void;
emit: (event: string, ...args: unknown[]) => boolean;
};
child.stdout = new Readable({ read() {} });
child.stderr = new Readable({ read() {} });
const dockerArgs = command === "docker" ? args : [];
const shouldFailContainerInspect =
dockerArgs[0] === "inspect" &&
dockerArgs[1] === "-f" &&
dockerArgs[2] === "{{.State.Running}}";
const shouldSucceedImageInspect = dockerArgs[0] === "image" && dockerArgs[1] === "inspect";
const dockerArgs = command === "docker" ? args : [];
const shouldFailContainerInspect =
dockerArgs[0] === "inspect" &&
dockerArgs[1] === "-f" &&
dockerArgs[2] === "{{.State.Running}}";
const shouldSucceedImageInspect = dockerArgs[0] === "image" && dockerArgs[1] === "inspect";
queueMicrotask(() =>
child.emit("close", shouldFailContainerInspect && !shouldSucceedImageInspect ? 1 : 0),
);
return child;
},
};
});
queueMicrotask(() =>
child.emit("close", shouldFailContainerInspect && !shouldSucceedImageInspect ? 1 : 0),
);
return child;
},
}));
vi.mock("./skills.js", async () => {
const actual = await vi.importActual<typeof import("./skills.js")>("./skills.js");
return {
...actual,
syncSkillsToWorkspace: vi.fn(async () => undefined),
};
});
vi.mock("./skills.js", () => ({
syncSkillsToWorkspace: vi.fn(async () => undefined),
}));
let resolveSandboxContext: typeof import("./sandbox/context.js").resolveSandboxContext;
let resolveSandboxConfigForAgent: typeof import("./sandbox/config.js").resolveSandboxConfigForAgent;

View File

@@ -1,9 +1,9 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { captureFullEnv } from "../test-utils/env.js";
import { captureEnv } from "../test-utils/env.js";
import { resolveSandboxContext } from "./sandbox/context.js";
import { writeSkill } from "./skills.e2e-test-helpers.js";
@@ -20,23 +20,34 @@ vi.mock("./sandbox/prune.js", () => ({
}));
describe("sandbox skill mirroring", () => {
let envSnapshot: ReturnType<typeof captureFullEnv>;
let envSnapshot: ReturnType<typeof captureEnv>;
let tempRoot = "";
beforeAll(async () => {
tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-sandbox-skills-"));
});
beforeEach(() => {
envSnapshot = captureFullEnv();
envSnapshot = captureEnv(["OPENCLAW_BUNDLED_SKILLS_DIR"]);
});
afterEach(() => {
envSnapshot.restore();
});
afterAll(async () => {
if (tempRoot) {
await fs.rm(tempRoot, { recursive: true, force: true });
}
});
const runContext = async (workspaceAccess: "none" | "ro") => {
const bundledDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-bundled-skills-"));
const bundledDir = await fs.mkdtemp(path.join(tempRoot, "bundled-"));
await fs.mkdir(bundledDir, { recursive: true });
process.env.OPENCLAW_BUNDLED_SKILLS_DIR = bundledDir;
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
const workspaceDir = await fs.mkdtemp(path.join(tempRoot, "workspace-"));
await writeSkill({
dir: path.join(workspaceDir, "skills", "demo-skill"),
name: "demo-skill",

View File

@@ -17,10 +17,7 @@ vi.mock("../infra/net/fetch-guard.js", () => ({
fetchWithSsrFGuard: vi.fn(),
}));
vi.mock("../security/skill-scanner.js", async () => ({
...(await vi.importActual<typeof import("../security/skill-scanner.js")>(
"../security/skill-scanner.js",
)),
vi.mock("../security/skill-scanner.js", () => ({
scanDirectoryWithSummary: (...args: unknown[]) => scanDirectoryWithSummaryMock(...args),
}));

View File

@@ -21,8 +21,7 @@ vi.mock("../infra/net/fetch-guard.js", () => ({
fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args),
}));
vi.mock("./skills.js", async () => ({
...(await vi.importActual<typeof import("./skills.js")>("./skills.js")),
vi.mock("./skills.js", () => ({
hasBinary: (bin: string) => hasBinaryMock(bin),
}));

View File

@@ -4,29 +4,3 @@ export const runCommandWithTimeoutMock: Mock<(...args: unknown[]) => unknown> =
export const scanDirectoryWithSummaryMock: Mock<(...args: unknown[]) => unknown> = vi.fn();
export const fetchWithSsrFGuardMock: Mock<(...args: unknown[]) => unknown> = vi.fn();
export const hasBinaryMock: Mock<(bin: string) => boolean> = vi.fn();
export function runCommandWithTimeoutFromMock(...args: unknown[]) {
return runCommandWithTimeoutMock(...args);
}
export function fetchWithSsrFGuardFromMock(...args: unknown[]) {
return fetchWithSsrFGuardMock(...args);
}
export function hasBinaryFromMock(bin: string) {
return hasBinaryMock(bin);
}
export function scanDirectoryWithSummaryFromMock(...args: unknown[]) {
return scanDirectoryWithSummaryMock(...args);
}
export async function mockSkillScannerModule(
loadActual: () => Promise<typeof import("../security/skill-scanner.js")>,
) {
const actual = await loadActual();
return {
...actual,
scanDirectoryWithSummary: scanDirectoryWithSummaryFromMock,
};
}

View File

@@ -19,10 +19,7 @@ vi.mock("../process/exec.js", () => ({
runCommandWithTimeout: (...args: unknown[]) => runCommandWithTimeoutMock(...args),
}));
vi.mock("../security/skill-scanner.js", async () => ({
...(await vi.importActual<typeof import("../security/skill-scanner.js")>(
"../security/skill-scanner.js",
)),
vi.mock("../security/skill-scanner.js", () => ({
scanDirectoryWithSummary: (...args: unknown[]) => scanDirectoryWithSummaryMock(...args),
}));

View File

@@ -1,7 +1,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { resetLogger, setLoggerOverride } from "../logging/logger.js";
import { loggingState } from "../logging/state.js";
import { withPathResolutionEnv } from "../test-utils/env.js";
@@ -29,9 +29,8 @@ function withWorkspaceHome<T>(workspaceDir: string, cb: () => T): T {
return withPathResolutionEnv(workspaceDir, { PATH: "" }, () => cb());
}
beforeEach(async () => {
beforeAll(async () => {
fakeHome = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-home-"));
tempDirs.push(fakeHome);
envSnapshot = setMockSkillsHomeEnv(fakeHome);
});
@@ -39,12 +38,16 @@ afterEach(async () => {
setLoggerOverride(null);
loggingState.rawConsole = null;
resetLogger();
await Promise.all(
tempDirs.splice(0, tempDirs.length).map((dir) => fs.rm(dir, { recursive: true, force: true })),
);
});
afterAll(async () => {
await restoreMockSkillsHomeEnv(envSnapshot, async () => {
await Promise.all(
tempDirs
.splice(0, tempDirs.length)
.map((dir) => fs.rm(dir, { recursive: true, force: true })),
);
if (fakeHome) {
await fs.rm(fakeHome, { recursive: true, force: true });
}
});
});