mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-20 01:51:37 +00:00
* refactor(sessions): migrate runtime storage to sqlite * test(sessions): fix sqlite CI regressions * test(sessions): align remaining sqlite fixtures * fix(codex): require sqlite trajectory recorder * test(sessions): align orphan recovery sqlite fixture * test(sessions): align sqlite rebase fixtures * fix(sessions): finish current-main integration of the sqlite flip Resolve the whole-store SDK removal across its owner boundary: drop the loadSessionStore re-export and the registry whole-store wrappers, wire hasTrackedActiveSessionRun into gateway chat, complete the preserveLockedHarnessIds cleanup contract, flip the codex thread-history import to storePath targets, and port remaining main-side tests from file-store helpers to session accessor reads. * chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain bump gets its own review, and regenerate docs_map, the plugin SDK API baseline, and the export-surface ratchet for the merged tree. * feat(sessions): keep archived transcripts by default with zstd cold storage Codex-style retention: deleting or resetting a session archives its transcript as a zstd-compressed JSONL artifact (plain when the runtime lacks node:zlib zstd) and keeps it until the disk budget evicts oldest first. resetArchiveRetention now governs both deleted and reset archives and defaults to keep; maxDiskBytes defaults to 2gb so retention stays bounded, with archives evicted before live sessions. The cron reaper follows the same knob instead of deleting archives on its own timer. * fix(state): converge agent DB migration lineages and bound database growth Merge coherence: run both structure-gated legacy memory-schema repairs (flip-lineage drop, main-lineage identity rebuild) before the flip migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all converge, and hoist foreign_keys=OFF outside the schema transaction where the pragma was silently ignored and the v1 sessions rebuild cascade-deleted session_entries. Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL maintenance releases freed pages in bounded passes (never a blocking full VACUUM), and doctor reports state/agent DB bloat from freelist stats. * fix(codex): resolve the store path for thread-history import via the SDK The supervision catalog passed the legacy sessionFile locator to the storePath-targeted transcript mirror; resolve the agent store path with the session-store SDK helper instead of a runtime-object seam so test fakes and headless callers need no extra surface. Drop the obsolete missing-session-id preprocessing case: sessions rows are NOT NULL on session_id and upsert repairs id-less patches at write time. * fix(sessions): fail safe on malformed disk-budget config and doctor stat errors A malformed explicit maxDiskBytes disables the budget instead of falling back to the destructive 2gb default the user never chose, and the doctor bloat check skips databases whose paths stat-fail instead of aborting doctor. * fix(sessions): complete sqlite conflict translations * test(sqlite): align hardening checks with maintenance * test(sessions): inspect compressed transcript archives * fix(tests): await session seeds and drop unused helpers flagged by CI lint The five unawaited writeSessionStoreSeed calls raced their SQLite seeds against the assertions, failing compact shards; the bloat probe drops a useless initializer and the merged tests drop now-unused helpers. * test(sessions): type legacy proof events directly * test(sessions): align hardening contracts * perf(sessions): read usage transcript sizes from SQL aggregates Usage/cost scans walked every session and materialized every transcript event just to re-stringify it for a byte estimate — the #86718 stall class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes in SQLite without loading a single row. * fix(sessions): re-root foreign-root transcript paths onto the current sessions dir Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry absolute sessionFile paths from the old root; the containment fallback kept those foreign paths, so migration read (and would archive) files in the original root and reported local copies missing. Re-root the canonical agents/<id>/sessions suffix onto the current dir when the file exists there; genuine cross-root layouts still fall through unchanged. * test(agents): seed harness admission through sqlite * fix(sqlite): close agent db on pragma setup failure * fix(doctor): compact and retrofit incremental auto-vacuum after session import The migration is the sanctioned offline window: post-import compact reclaims import churn and applies auto_vacuum=INCREMENTAL to databases created before the fresh-DB pragma existed, so runtime maintenance can release pages in bounded passes on every install. --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
223 lines
8.3 KiB
TypeScript
223 lines
8.3 KiB
TypeScript
// Tests session update lifecycle ordering and active-session state transitions.
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import type { OpenClawConfig } from "../../config/config.js";
|
|
import type { SessionEntry } from "../../config/sessions.js";
|
|
import {
|
|
applySessionEntryLifecycleMutation,
|
|
loadSessionEntry,
|
|
replaceSessionEntry,
|
|
} from "../../config/sessions/session-accessor.js";
|
|
import type { HookRunner } from "../../plugins/hooks.js";
|
|
import {
|
|
getActiveGatewayRootWorkCount,
|
|
markGatewayRestartDraining,
|
|
resetGatewayWorkAdmission,
|
|
tryBeginGatewayRootWorkAdmission,
|
|
} from "../../process/gateway-work-admission.js";
|
|
|
|
const hookRunnerMocks = vi.hoisted(() => ({
|
|
hasHooks: vi.fn<HookRunner["hasHooks"]>(),
|
|
runSessionEnd: vi.fn<HookRunner["runSessionEnd"]>(),
|
|
runSessionStart: vi.fn<HookRunner["runSessionStart"]>(),
|
|
}));
|
|
|
|
let incrementCompactionCount: typeof import("./session-updates.js").incrementCompactionCount;
|
|
const tempDirs: string[] = [];
|
|
|
|
async function createFixture() {
|
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-updates-"));
|
|
tempDirs.push(root);
|
|
const storePath = path.join(root, "sessions.json");
|
|
const sessionKey = "agent:main:forum:direct:compaction";
|
|
const transcriptPath = path.join(root, "s1.jsonl");
|
|
await fs.writeFile(transcriptPath, '{"type":"message"}\n', "utf-8");
|
|
const entry = {
|
|
sessionId: "s1",
|
|
sessionFile: transcriptPath,
|
|
updatedAt: Date.now(),
|
|
compactionCount: 0,
|
|
} as SessionEntry;
|
|
const sessionStore: Record<string, SessionEntry> = {
|
|
[sessionKey]: entry,
|
|
};
|
|
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
|
return { storePath, sessionKey, sessionStore, entry, transcriptPath };
|
|
}
|
|
|
|
function firstSessionEndCall() {
|
|
return hookRunnerMocks.runSessionEnd.mock.calls[0] ?? [];
|
|
}
|
|
|
|
function firstSessionStartCall() {
|
|
return hookRunnerMocks.runSessionStart.mock.calls[0] ?? [];
|
|
}
|
|
|
|
describe("session-updates lifecycle hooks", () => {
|
|
beforeEach(async () => {
|
|
resetGatewayWorkAdmission();
|
|
vi.resetModules();
|
|
vi.doMock("../../plugins/hook-runner-global.js", () => ({
|
|
getGlobalHookRunner: () =>
|
|
({
|
|
hasHooks: hookRunnerMocks.hasHooks,
|
|
runSessionEnd: hookRunnerMocks.runSessionEnd,
|
|
runSessionStart: hookRunnerMocks.runSessionStart,
|
|
}) as unknown as HookRunner,
|
|
}));
|
|
hookRunnerMocks.hasHooks.mockReset();
|
|
hookRunnerMocks.runSessionEnd.mockReset();
|
|
hookRunnerMocks.runSessionStart.mockReset();
|
|
hookRunnerMocks.hasHooks.mockImplementation(
|
|
(hookName) => hookName === "session_end" || hookName === "session_start",
|
|
);
|
|
hookRunnerMocks.runSessionEnd.mockResolvedValue(undefined);
|
|
hookRunnerMocks.runSessionStart.mockResolvedValue(undefined);
|
|
({ incrementCompactionCount } = await import("./session-updates.js"));
|
|
});
|
|
|
|
afterEach(async () => {
|
|
resetGatewayWorkAdmission();
|
|
vi.restoreAllMocks();
|
|
await Promise.all(
|
|
tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })),
|
|
);
|
|
});
|
|
|
|
it("emits compaction lifecycle hooks when newSessionId replaces the session", async () => {
|
|
const { storePath, sessionKey, sessionStore, entry, transcriptPath } = await createFixture();
|
|
const cfg = { session: { store: storePath } } as OpenClawConfig;
|
|
|
|
await incrementCompactionCount({
|
|
cfg,
|
|
sessionEntry: entry,
|
|
sessionStore,
|
|
sessionKey,
|
|
storePath,
|
|
newSessionId: "s2",
|
|
});
|
|
|
|
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
|
|
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
|
|
|
|
const [endEvent, endContext] = firstSessionEndCall();
|
|
const [startEvent, startContext] = firstSessionStartCall();
|
|
|
|
expect(endEvent?.sessionId).toBe("s1");
|
|
expect(endEvent?.sessionKey).toBe(sessionKey);
|
|
expect(endEvent?.reason).toBe("compaction");
|
|
expect(endEvent?.transcriptArchived).toBe(false);
|
|
expect(endEvent?.sessionFile).toBe(await fs.realpath(transcriptPath));
|
|
expect(endContext?.sessionId).toBe("s1");
|
|
expect(endContext?.sessionKey).toBe(sessionKey);
|
|
expect(endContext?.agentId).toBe("main");
|
|
expect(endEvent?.nextSessionId).toBe(startEvent?.sessionId);
|
|
expect(startEvent?.sessionId).toBe("s2");
|
|
expect(startEvent?.sessionKey).toBe(sessionKey);
|
|
expect(startEvent?.resumedFrom).toBe("s1");
|
|
expect(startContext?.sessionId).toBe("s2");
|
|
expect(startContext?.sessionKey).toBe(sessionKey);
|
|
expect(startContext?.agentId).toBe("main");
|
|
});
|
|
|
|
it("keeps compaction lifecycle hooks root-admitted until both settle", async () => {
|
|
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
|
const releases: Array<() => void> = [];
|
|
const heldHook = () =>
|
|
new Promise<void>((resolve) => {
|
|
releases.push(resolve);
|
|
});
|
|
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
|
|
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
|
|
|
|
await incrementCompactionCount({
|
|
cfg: { session: { store: storePath } } as OpenClawConfig,
|
|
sessionEntry: entry,
|
|
sessionStore,
|
|
sessionKey,
|
|
storePath,
|
|
newSessionId: "s2",
|
|
});
|
|
|
|
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(2));
|
|
await vi.waitFor(() => expect(releases).toHaveLength(2));
|
|
for (const release of releases) {
|
|
release();
|
|
}
|
|
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
|
});
|
|
|
|
it("hands compaction lifecycle hooks off after restart drain closes admission", async () => {
|
|
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
|
const releases: Array<() => void> = [];
|
|
const heldHook = () =>
|
|
new Promise<void>((resolve) => {
|
|
releases.push(resolve);
|
|
});
|
|
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
|
|
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
|
|
const admission = tryBeginGatewayRootWorkAdmission();
|
|
expect(admission).not.toBeNull();
|
|
|
|
await admission?.run(async () => {
|
|
markGatewayRestartDraining();
|
|
await incrementCompactionCount({
|
|
cfg: { session: { store: storePath } } as OpenClawConfig,
|
|
sessionEntry: entry,
|
|
sessionStore,
|
|
sessionKey,
|
|
storePath,
|
|
newSessionId: "s2",
|
|
});
|
|
await vi.waitFor(() => expect(releases).toHaveLength(2));
|
|
expect(getActiveGatewayRootWorkCount()).toBe(3);
|
|
});
|
|
|
|
admission?.release();
|
|
expect(getActiveGatewayRootWorkCount()).toBe(2);
|
|
for (const release of releases) {
|
|
release();
|
|
}
|
|
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
|
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
|
|
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("recreates a complete persisted row when compaction updates a missing store row", async () => {
|
|
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
|
await applySessionEntryLifecycleMutation({
|
|
storePath,
|
|
removals: [{ sessionKey }],
|
|
skipMaintenance: true,
|
|
});
|
|
|
|
await incrementCompactionCount({
|
|
sessionEntry: entry,
|
|
sessionStore,
|
|
sessionKey,
|
|
storePath,
|
|
newSessionId: "s2",
|
|
tokensAfter: 123,
|
|
now: 456,
|
|
});
|
|
|
|
const persisted = loadSessionEntry({ storePath, sessionKey });
|
|
expect(sessionStore[sessionKey]?.sessionId).toBe("s2");
|
|
expect(sessionStore[sessionKey]?.sessionFile).toContain("s2.jsonl");
|
|
expect(sessionStore[sessionKey]?.usageFamilyKey).toBe(sessionKey);
|
|
expect(sessionStore[sessionKey]?.usageFamilySessionIds).toEqual(["s1", "s2"]);
|
|
expect(sessionStore[sessionKey]?.compactionCount).toBe(1);
|
|
expect(sessionStore[sessionKey]?.totalTokens).toBe(123);
|
|
expect(sessionStore[sessionKey]?.updatedAt).toBeGreaterThanOrEqual(entry.updatedAt);
|
|
expect(persisted?.sessionId).toBe("s2");
|
|
expect(persisted?.sessionFile).toContain("s2.jsonl");
|
|
expect(persisted?.usageFamilyKey).toBe(sessionKey);
|
|
expect(persisted?.usageFamilySessionIds).toEqual(["s1", "s2"]);
|
|
expect(persisted?.compactionCount).toBe(1);
|
|
expect(persisted?.totalTokens).toBe(123);
|
|
expect(persisted?.updatedAt).toBeGreaterThanOrEqual(entry.updatedAt);
|
|
});
|
|
});
|