mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 18:50:42 +00:00
perf(sessions): stop persisting skillsSnapshot.resolvedSkills
Each SessionEntry carried the fully parsed Skill[] (including each SKILL.md body) inside skillsSnapshot.resolvedSkills, multiplied across every active session. Strip the field at the persistence chokepoint — normalizeSessionStore in store-load.ts — so every load and every save naturally drops it. The runtime already falls back to a workspace skill scan when resolvedSkills is absent (see src/agents/pi-embedded-runner/skills-runtime.ts:14), so prompts and session resume behavior are unchanged. Legacy sessions.json files self-heal on first load: normalize strips the in-memory store, the next write rewrites the file in stripped form. Test fixture (100 sessions × 50 skills × ~3KB body) goes from ~32MB to under 2MB on disk. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
committed by
Peter Steinberger
parent
03df3539e9
commit
a3cacf3b67
@@ -26,6 +26,11 @@ import { normalizeOptionalString } from "../../shared/string-coerce.js";
|
||||
import { buildSessionEndHookPayload, buildSessionStartHookPayload } from "./session-hooks.js";
|
||||
export { drainFormattedSystemEvents } from "./session-system-events.js";
|
||||
|
||||
// nextEntry.skillsSnapshot may carry resolvedSkills (full Skill[] with
|
||||
// SKILL.md bodies) for in-turn use. The persistence layer in
|
||||
// src/config/sessions/store-load.ts strips resolvedSkills before serializing,
|
||||
// so the on-disk sessions.json stays small. The in-memory params.sessionStore
|
||||
// reference still carries the runtime cache for the rest of this turn.
|
||||
async function persistSessionEntryUpdate(params: {
|
||||
sessionStore?: Record<string, SessionEntry>;
|
||||
sessionKey?: string;
|
||||
|
||||
@@ -65,13 +65,30 @@ function normalizeSessionEntryDelivery(entry: SessionEntry): SessionEntry {
|
||||
};
|
||||
}
|
||||
|
||||
// resolvedSkills carries the full parsed Skill[] (including each SKILL.md body)
|
||||
// and is only used as an in-turn cache by the runtime — see
|
||||
// src/agents/pi-embedded-runner/skills-runtime.ts. Persisting it bloats
|
||||
// sessions.json by orders of magnitude when many sessions are active. Strip
|
||||
// it from every entry that flows through normalize, so neither the in-memory
|
||||
// store reloaded from disk nor the JSON serialized back to disk carries it.
|
||||
function stripPersistedSkillsCache(entry: SessionEntry): SessionEntry {
|
||||
const snapshot = entry.skillsSnapshot;
|
||||
if (!snapshot || snapshot.resolvedSkills === undefined) {
|
||||
return entry;
|
||||
}
|
||||
const { resolvedSkills: _drop, ...rest } = snapshot;
|
||||
return { ...entry, skillsSnapshot: rest };
|
||||
}
|
||||
|
||||
export function normalizeSessionStore(store: Record<string, SessionEntry>): boolean {
|
||||
let changed = false;
|
||||
for (const [key, entry] of Object.entries(store)) {
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeSessionEntryDelivery(normalizeSessionRuntimeModelFields(entry));
|
||||
const normalized = stripPersistedSkillsCache(
|
||||
normalizeSessionEntryDelivery(normalizeSessionRuntimeModelFields(entry)),
|
||||
);
|
||||
if (normalized !== entry) {
|
||||
store[key] = normalized;
|
||||
changed = true;
|
||||
|
||||
205
src/config/sessions/store.skills-stripping.test.ts
Normal file
205
src/config/sessions/store.skills-stripping.test.ts
Normal file
@@ -0,0 +1,205 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveEmbeddedRunSkillEntries } from "../../agents/pi-embedded-runner/skills-runtime.js";
|
||||
import { createCanonicalFixtureSkill } from "../../agents/skills.test-helpers.js";
|
||||
import type { Skill } from "../../agents/skills/skill-contract.js";
|
||||
import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js";
|
||||
import type { SessionEntry, SessionSkillSnapshot } from "./types.js";
|
||||
|
||||
vi.mock("../config.js", async () => ({
|
||||
...(await vi.importActual<typeof import("../config.js")>("../config.js")),
|
||||
getRuntimeConfig: vi.fn().mockReturnValue({}),
|
||||
}));
|
||||
|
||||
import {
|
||||
clearSessionStoreCacheForTest,
|
||||
loadSessionStore,
|
||||
saveSessionStore,
|
||||
updateSessionStore,
|
||||
} from "./store.js";
|
||||
|
||||
const suiteRootTracker = createSuiteTempRootTracker({ prefix: "openclaw-skills-strip-" });
|
||||
|
||||
function makeFixtureSkill(name: string, bodySize = 3000): Skill {
|
||||
// 3KB body simulates a realistic SKILL.md.
|
||||
const source = `# ${name}\n\n${"x".repeat(bodySize)}`;
|
||||
return createCanonicalFixtureSkill({
|
||||
name,
|
||||
description: `${name} skill description`,
|
||||
filePath: `/skills/${name}/SKILL.md`,
|
||||
baseDir: `/skills/${name}`,
|
||||
source,
|
||||
});
|
||||
}
|
||||
|
||||
function makeSnapshot(skillCount: number): SessionSkillSnapshot {
|
||||
const resolved = Array.from({ length: skillCount }, (_, i) => makeFixtureSkill(`skill-${i}`));
|
||||
return {
|
||||
prompt: "<available_skills>...</available_skills>",
|
||||
skills: resolved.map((s) => ({ name: s.name })),
|
||||
skillFilter: undefined,
|
||||
resolvedSkills: resolved,
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function makeEntry(sessionId: string, snapshot?: SessionSkillSnapshot): SessionEntry {
|
||||
return {
|
||||
sessionId,
|
||||
updatedAt: Date.now(),
|
||||
skillsSnapshot: snapshot,
|
||||
};
|
||||
}
|
||||
|
||||
describe("session store strips resolvedSkills from persistence", () => {
|
||||
let testDir: string;
|
||||
let storePath: string;
|
||||
let savedCacheTtl: string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
await suiteRootTracker.setup();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await suiteRootTracker.cleanup();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
testDir = await suiteRootTracker.make("case");
|
||||
storePath = path.join(testDir, "sessions.json");
|
||||
savedCacheTtl = process.env.OPENCLAW_SESSION_CACHE_TTL_MS;
|
||||
process.env.OPENCLAW_SESSION_CACHE_TTL_MS = "0";
|
||||
clearSessionStoreCacheForTest();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearSessionStoreCacheForTest();
|
||||
if (savedCacheTtl === undefined) {
|
||||
delete process.env.OPENCLAW_SESSION_CACHE_TTL_MS;
|
||||
} else {
|
||||
process.env.OPENCLAW_SESSION_CACHE_TTL_MS = savedCacheTtl;
|
||||
}
|
||||
});
|
||||
|
||||
it("does not write resolvedSkills to disk", async () => {
|
||||
const store = {
|
||||
"agent:main:test:1": makeEntry("session-1", makeSnapshot(5)),
|
||||
};
|
||||
|
||||
await saveSessionStore(storePath, store, { skipMaintenance: true });
|
||||
|
||||
const raw = await fs.readFile(storePath, "utf-8");
|
||||
expect(raw).not.toContain("resolvedSkills");
|
||||
expect(raw).not.toContain("xxxxx"); // none of the skill source bodies leaked
|
||||
const parsed = JSON.parse(raw) as Record<string, SessionEntry>;
|
||||
expect(parsed["agent:main:test:1"]?.skillsSnapshot?.resolvedSkills).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves prompt, skills, skillFilter, and version on roundtrip", async () => {
|
||||
const snapshot = makeSnapshot(3);
|
||||
snapshot.skillFilter = ["skill-0"];
|
||||
const store = {
|
||||
"agent:main:test:1": makeEntry("session-1", snapshot),
|
||||
};
|
||||
|
||||
await saveSessionStore(storePath, store, { skipMaintenance: true });
|
||||
const loaded = loadSessionStore(storePath, { skipCache: true });
|
||||
|
||||
const persistedSnapshot = loaded["agent:main:test:1"]?.skillsSnapshot;
|
||||
expect(persistedSnapshot).toBeDefined();
|
||||
expect(persistedSnapshot?.prompt).toBe(snapshot.prompt);
|
||||
expect(persistedSnapshot?.skills).toEqual(snapshot.skills);
|
||||
expect(persistedSnapshot?.skillFilter).toEqual(["skill-0"]);
|
||||
expect(persistedSnapshot?.version).toBe(1);
|
||||
expect(persistedSnapshot?.resolvedSkills).toBeUndefined();
|
||||
});
|
||||
|
||||
it("strips resolvedSkills from a legacy sessions.json on load", async () => {
|
||||
// Hand-craft a pre-fix file with embedded resolvedSkills.
|
||||
const legacy = {
|
||||
"agent:main:test:1": makeEntry("session-1", makeSnapshot(4)),
|
||||
};
|
||||
await fs.mkdir(path.dirname(storePath), { recursive: true });
|
||||
const rawLegacy = JSON.stringify(legacy, null, 2);
|
||||
expect(rawLegacy).toContain("resolvedSkills");
|
||||
await fs.writeFile(storePath, rawLegacy, "utf-8");
|
||||
|
||||
const loaded = loadSessionStore(storePath, { skipCache: true });
|
||||
expect(loaded["agent:main:test:1"]?.skillsSnapshot?.resolvedSkills).toBeUndefined();
|
||||
expect(loaded["agent:main:test:1"]?.skillsSnapshot?.prompt).toBe(
|
||||
legacy["agent:main:test:1"].skillsSnapshot?.prompt,
|
||||
);
|
||||
|
||||
// Saving the loaded record should rewrite the file in stripped form.
|
||||
await saveSessionStore(storePath, loaded, { skipMaintenance: true });
|
||||
const rawAfter = await fs.readFile(storePath, "utf-8");
|
||||
expect(rawAfter).not.toContain("resolvedSkills");
|
||||
});
|
||||
|
||||
it("strips resolvedSkills written via updateSessionStore mutator", async () => {
|
||||
// Simulate the production hot path where ensureSkillSnapshot puts a
|
||||
// freshly-built snapshot (with resolvedSkills) into the store via mutator.
|
||||
await updateSessionStore(
|
||||
storePath,
|
||||
(store) => {
|
||||
store["agent:main:test:1"] = makeEntry("session-1", makeSnapshot(6));
|
||||
},
|
||||
{ skipMaintenance: true },
|
||||
);
|
||||
|
||||
const raw = await fs.readFile(storePath, "utf-8");
|
||||
expect(raw).not.toContain("resolvedSkills");
|
||||
const reloaded = loadSessionStore(storePath, { skipCache: true });
|
||||
expect(reloaded["agent:main:test:1"]?.skillsSnapshot?.resolvedSkills).toBeUndefined();
|
||||
expect(reloaded["agent:main:test:1"]?.skillsSnapshot?.skills).toHaveLength(6);
|
||||
});
|
||||
|
||||
it("keeps the on-disk file small with many sessions and skills", async () => {
|
||||
const SESSION_COUNT = 100;
|
||||
const SKILLS_PER_SESSION = 50;
|
||||
const store: Record<string, SessionEntry> = {};
|
||||
for (let i = 0; i < SESSION_COUNT; i += 1) {
|
||||
store[`agent:main:scale:${i}`] = makeEntry(`session-${i}`, makeSnapshot(SKILLS_PER_SESSION));
|
||||
}
|
||||
|
||||
await saveSessionStore(storePath, store, { skipMaintenance: true });
|
||||
|
||||
const stat = await fs.stat(storePath);
|
||||
// Pre-fix: ~SESSION_COUNT * SKILLS_PER_SESSION * ~3KB ≈ 15MB.
|
||||
// Post-fix: only the lightweight `skills` array + prompt per entry.
|
||||
// Conservative budget that comfortably covers metadata growth.
|
||||
expect(stat.size).toBeLessThan(2 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe("embedded runner falls back to disk when resolvedSkills is absent", () => {
|
||||
it("signals shouldLoadSkillEntries when the persisted snapshot has no resolvedSkills", () => {
|
||||
const result = resolveEmbeddedRunSkillEntries({
|
||||
workspaceDir: "/nonexistent-workspace-for-test",
|
||||
skillsSnapshot: {
|
||||
prompt: "",
|
||||
skills: [{ name: "x" }],
|
||||
version: 1,
|
||||
// resolvedSkills intentionally omitted — this is the post-fix shape.
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.shouldLoadSkillEntries).toBe(true);
|
||||
});
|
||||
|
||||
it("skips loading when resolvedSkills is present (in-turn cache hot path)", () => {
|
||||
const result = resolveEmbeddedRunSkillEntries({
|
||||
workspaceDir: "/nonexistent-workspace-for-test",
|
||||
skillsSnapshot: {
|
||||
prompt: "",
|
||||
skills: [{ name: "x" }],
|
||||
resolvedSkills: [makeFixtureSkill("x", 100)],
|
||||
version: 1,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.shouldLoadSkillEntries).toBe(false);
|
||||
expect(result.skillEntries).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -530,6 +530,14 @@ export type SessionSkillSnapshot = {
|
||||
skills: Array<{ name: string; primaryEnv?: string; requiredEnv?: string[] }>;
|
||||
/** Normalized agent-level filter used to build this snapshot; undefined means unrestricted. */
|
||||
skillFilter?: string[];
|
||||
/**
|
||||
* Runtime-only, never persisted. Carries the full parsed Skill[] (including
|
||||
* each SKILL.md body) so the embedded runner can skip a workspace skill
|
||||
* scan within a turn. Stripped from sessions.json on every read and write
|
||||
* via normalizeSessionStore — see store-load.ts. On a cold session resume
|
||||
* this is undefined and src/agents/pi-embedded-runner/skills-runtime.ts
|
||||
* rebuilds it by reloading skill entries from disk.
|
||||
*/
|
||||
resolvedSkills?: Skill[];
|
||||
version?: number;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user