mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-14 15:26:18 +00:00
* refactor(sessions): migrate runtime storage to sqlite * test(sessions): fix sqlite CI regressions * test(sessions): align remaining sqlite fixtures * fix(codex): require sqlite trajectory recorder * test(sessions): align orphan recovery sqlite fixture * test(sessions): align sqlite rebase fixtures * fix(sessions): finish current-main integration of the sqlite flip Resolve the whole-store SDK removal across its owner boundary: drop the loadSessionStore re-export and the registry whole-store wrappers, wire hasTrackedActiveSessionRun into gateway chat, complete the preserveLockedHarnessIds cleanup contract, flip the codex thread-history import to storePath targets, and port remaining main-side tests from file-store helpers to session accessor reads. * chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain bump gets its own review, and regenerate docs_map, the plugin SDK API baseline, and the export-surface ratchet for the merged tree. * feat(sessions): keep archived transcripts by default with zstd cold storage Codex-style retention: deleting or resetting a session archives its transcript as a zstd-compressed JSONL artifact (plain when the runtime lacks node:zlib zstd) and keeps it until the disk budget evicts oldest first. resetArchiveRetention now governs both deleted and reset archives and defaults to keep; maxDiskBytes defaults to 2gb so retention stays bounded, with archives evicted before live sessions. The cron reaper follows the same knob instead of deleting archives on its own timer. * fix(state): converge agent DB migration lineages and bound database growth Merge coherence: run both structure-gated legacy memory-schema repairs (flip-lineage drop, main-lineage identity rebuild) before the flip migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all converge, and hoist foreign_keys=OFF outside the schema transaction where the pragma was silently ignored and the v1 sessions rebuild cascade-deleted session_entries. Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL maintenance releases freed pages in bounded passes (never a blocking full VACUUM), and doctor reports state/agent DB bloat from freelist stats. * fix(codex): resolve the store path for thread-history import via the SDK The supervision catalog passed the legacy sessionFile locator to the storePath-targeted transcript mirror; resolve the agent store path with the session-store SDK helper instead of a runtime-object seam so test fakes and headless callers need no extra surface. Drop the obsolete missing-session-id preprocessing case: sessions rows are NOT NULL on session_id and upsert repairs id-less patches at write time. * fix(sessions): fail safe on malformed disk-budget config and doctor stat errors A malformed explicit maxDiskBytes disables the budget instead of falling back to the destructive 2gb default the user never chose, and the doctor bloat check skips databases whose paths stat-fail instead of aborting doctor. * fix(sessions): complete sqlite conflict translations * test(sqlite): align hardening checks with maintenance * test(sessions): inspect compressed transcript archives * fix(tests): await session seeds and drop unused helpers flagged by CI lint The five unawaited writeSessionStoreSeed calls raced their SQLite seeds against the assertions, failing compact shards; the bloat probe drops a useless initializer and the merged tests drop now-unused helpers. * test(sessions): type legacy proof events directly * test(sessions): align hardening contracts * perf(sessions): read usage transcript sizes from SQL aggregates Usage/cost scans walked every session and materialized every transcript event just to re-stringify it for a byte estimate — the #86718 stall class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes in SQLite without loading a single row. * fix(sessions): re-root foreign-root transcript paths onto the current sessions dir Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry absolute sessionFile paths from the old root; the containment fallback kept those foreign paths, so migration read (and would archive) files in the original root and reported local copies missing. Re-root the canonical agents/<id>/sessions suffix onto the current dir when the file exists there; genuine cross-root layouts still fall through unchanged. * test(agents): seed harness admission through sqlite * fix(sqlite): close agent db on pragma setup failure * fix(doctor): compact and retrofit incremental auto-vacuum after session import The migration is the sanctioned offline window: post-import compact reclaims import churn and applies auto_vacuum=INCREMENTAL to databases created before the fresh-DB pragma existed, so runtime maintenance can release pages in bounded passes on every install. --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
319 lines
10 KiB
TypeScript
319 lines
10 KiB
TypeScript
// Session transcript hit tests cover transcript match formatting and path resolution.
|
|
import fs from "node:fs";
|
|
import { describe, expect, it } from "vitest";
|
|
import type { SessionEntry } from "../config/sessions/types.js";
|
|
import {
|
|
extractTranscriptIdentityFromSessionsMemoryHit,
|
|
extractTranscriptStemFromSessionsMemoryHit,
|
|
formatSessionTranscriptMemoryHitKey,
|
|
parseSessionTranscriptMemoryHitKey,
|
|
resolveSessionTranscriptMemoryHitKeyToSessionKeys,
|
|
resolveTranscriptStemToSessionKeys,
|
|
} from "./session-transcript-hit.js";
|
|
|
|
describe("extractTranscriptStemFromSessionsMemoryHit", () => {
|
|
it("strips sessions/ and .jsonl for builtin paths", () => {
|
|
expect(extractTranscriptStemFromSessionsMemoryHit("sessions/abc-uuid.jsonl")).toBe("abc-uuid");
|
|
});
|
|
|
|
it("handles plain basename jsonl", () => {
|
|
expect(extractTranscriptStemFromSessionsMemoryHit("def-topic-thread.jsonl")).toBe(
|
|
"def-topic-thread",
|
|
);
|
|
});
|
|
|
|
it("uses .md basename for QMD exports", () => {
|
|
expect(extractTranscriptStemFromSessionsMemoryHit("qmd/sessions/x/y/z.md")).toBe("z");
|
|
});
|
|
|
|
it("strips .jsonl.reset.<iso> archive suffix so rotated transcripts resolve to the live stem", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"sessions/abc-uuid.jsonl.reset.2026-02-16T22-26-33.000Z",
|
|
),
|
|
).toBe("abc-uuid");
|
|
});
|
|
|
|
it("strips .jsonl.deleted.<iso> archive suffix the same way", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"sessions/def-uuid.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
|
),
|
|
).toBe("def-uuid");
|
|
});
|
|
|
|
it("handles archive suffix on bare basenames without the sessions/ prefix", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit("ghi-thread.jsonl.reset.2026-02-16T22-28-33.000Z"),
|
|
).toBe("ghi-thread");
|
|
});
|
|
|
|
it("recognizes QMD-normalized archived reset transcript .md stems", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc-uuid-jsonl-reset-2026-02-16T22-26-33.000Z.md",
|
|
),
|
|
).toBe("abc-uuid");
|
|
});
|
|
|
|
it("recognizes QMD-normalized archived deleted transcript .md stems", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"qmd/sessions-main/def-uuid-jsonl-deleted-2026-02-16T22-27-33.000Z.md",
|
|
),
|
|
).toBe("def-uuid");
|
|
});
|
|
|
|
it("recognizes real QMD-slugified archived reset transcript .md stems", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc-uuid-jsonl-reset-2026-02-16t22-26-33-000z.md",
|
|
),
|
|
).toBe("abc-uuid");
|
|
});
|
|
|
|
it("returns non-archived identity for QMD .md stems that are not archive patterns", () => {
|
|
const identity = extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"qmd/sessions-main/normal-session.md",
|
|
);
|
|
expect(identity).toEqual({ stem: "normal-session", archived: false });
|
|
});
|
|
|
|
it("returns archived identity for QMD-normalized reset .md stems", () => {
|
|
const identity = extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc-uuid-jsonl-reset-2026-02-16T22-26-33.000Z.md",
|
|
);
|
|
expect(identity).toEqual({
|
|
stem: "abc-uuid",
|
|
liveStem: "abc-uuid-jsonl-reset-2026-02-16T22-26-33.000Z",
|
|
archived: true,
|
|
});
|
|
});
|
|
|
|
it("recognizes QMD-exported dot-form archived reset .md paths", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc-uuid.jsonl.reset.2026-02-16T22-26-33.000Z.md",
|
|
),
|
|
).toBe("abc-uuid");
|
|
});
|
|
|
|
it("recognizes QMD-exported dot-form archived deleted .md paths", () => {
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit(
|
|
"qmd/sessions-main/def-uuid.jsonl.deleted.2026-02-16T22-27-33.000Z.md",
|
|
),
|
|
).toBe("def-uuid");
|
|
});
|
|
|
|
it("returns archived identity for QMD-exported dot-form reset .md paths", () => {
|
|
const identity = extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc-uuid.jsonl.reset.2026-02-16T22-26-33.000Z.md",
|
|
);
|
|
expect(identity).toEqual({
|
|
stem: "abc-uuid",
|
|
liveStem: "abc-uuid.jsonl.reset.2026-02-16T22-26-33.000Z",
|
|
archived: true,
|
|
});
|
|
});
|
|
|
|
it("does not treat QMD .md names with invalid archive timestamps as archives", () => {
|
|
const identity = extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"qmd/sessions-main/abc.jsonl.reset.not-a-timestamp.md",
|
|
);
|
|
expect(identity).toEqual({
|
|
stem: "abc.jsonl.reset.not-a-timestamp",
|
|
archived: false,
|
|
});
|
|
});
|
|
|
|
it("does not treat non-QMD .md names with archive-looking timestamps as archives", () => {
|
|
const identity = extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"abc-uuid-jsonl-reset-2026-02-16t22-26-33-000z.md",
|
|
);
|
|
expect(identity).toEqual({
|
|
stem: "abc-uuid-jsonl-reset-2026-02-16t22-26-33-000z",
|
|
archived: false,
|
|
});
|
|
});
|
|
|
|
it("does not mistake arbitrary suffixes containing .jsonl. for archives", () => {
|
|
// Not a real archive pattern: suffix after .jsonl. must be `reset` or `deleted`.
|
|
expect(
|
|
extractTranscriptStemFromSessionsMemoryHit("sessions/weird.jsonl.backup.2026-01-01.zst"),
|
|
).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("extractTranscriptIdentityFromSessionsMemoryHit", () => {
|
|
it("extracts owner metadata from agent-scoped session archive paths", () => {
|
|
expect(
|
|
extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"sessions/main/deleted-uuid.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
|
),
|
|
).toEqual({
|
|
stem: "deleted-uuid",
|
|
ownerAgentId: "main",
|
|
archived: true,
|
|
});
|
|
});
|
|
|
|
it("does not derive owner metadata from lossy QMD session collection names", () => {
|
|
expect(
|
|
extractTranscriptIdentityFromSessionsMemoryHit(
|
|
"qmd/sessions-main/deleted-uuid-jsonl-deleted-2026-02-16t22-27-33-000z.md",
|
|
),
|
|
).toEqual({
|
|
stem: "deleted-uuid",
|
|
liveStem: "deleted-uuid-jsonl-deleted-2026-02-16t22-27-33-000z",
|
|
archived: true,
|
|
});
|
|
});
|
|
|
|
it("does not invent owner metadata for legacy basename-only paths", () => {
|
|
expect(extractTranscriptIdentityFromSessionsMemoryHit("sessions/abc-uuid.jsonl")).toEqual({
|
|
stem: "abc-uuid",
|
|
archived: false,
|
|
});
|
|
});
|
|
|
|
it("extracts owner metadata from agent-scoped live session paths", () => {
|
|
expect(extractTranscriptIdentityFromSessionsMemoryHit("sessions/main/abc-uuid.jsonl")).toEqual({
|
|
stem: "abc-uuid",
|
|
ownerAgentId: "main",
|
|
archived: false,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("resolveTranscriptStemToSessionKeys", () => {
|
|
const baseEntry = (overrides: Partial<SessionEntry> = {}): SessionEntry => ({
|
|
sessionId: "stem-a",
|
|
updatedAt: 1,
|
|
...overrides,
|
|
});
|
|
|
|
it("returns keys for every agent whose store entry matches the stem", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:s1": baseEntry({
|
|
sessionFile: "/data/sessions/stem-a.jsonl",
|
|
}),
|
|
"agent:peer:s2": baseEntry({
|
|
sessionFile: "/other/volume/stem-a.jsonl",
|
|
}),
|
|
};
|
|
const keys = resolveTranscriptStemToSessionKeys({ store, stem: "stem-a" }).toSorted();
|
|
expect(keys).toEqual(["agent:main:s1", "agent:peer:s2"]);
|
|
});
|
|
|
|
it("falls back to archived owner metadata when deleted archives are gone from the live store", () => {
|
|
const keys = resolveTranscriptStemToSessionKeys({
|
|
store: {},
|
|
stem: "deleted-stem",
|
|
archivedOwnerAgentId: "main",
|
|
});
|
|
|
|
expect(keys).toEqual(["agent:main:deleted-stem"]);
|
|
});
|
|
|
|
it("matches QMD-slugified stems to unique session ids with safe punctuation", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:s1": baseEntry({ sessionId: "foo_bar.v1" }),
|
|
};
|
|
|
|
expect(
|
|
resolveTranscriptStemToSessionKeys({
|
|
store,
|
|
stem: "foo-bar-v1",
|
|
allowQmdSlugFallback: true,
|
|
}),
|
|
).toEqual(["agent:main:s1"]);
|
|
});
|
|
|
|
it("ignores store entries without session ids during QMD-slugified fallback", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:non-session": {
|
|
updatedAt: 1,
|
|
} as SessionEntry,
|
|
"agent:main:s1": baseEntry({ sessionId: "foo_bar.v1" }),
|
|
};
|
|
|
|
expect(
|
|
resolveTranscriptStemToSessionKeys({
|
|
store,
|
|
stem: "foo-bar-v1",
|
|
allowQmdSlugFallback: true,
|
|
}),
|
|
).toEqual(["agent:main:s1"]);
|
|
});
|
|
|
|
it("does not use QMD-slugified fallback unless requested", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:s1": baseEntry({ sessionId: "foo_bar.v1" }),
|
|
};
|
|
|
|
expect(resolveTranscriptStemToSessionKeys({ store, stem: "foo-bar-v1" })).toEqual([]);
|
|
});
|
|
|
|
it("prefers exact stem matches before QMD-slugified fallback matches", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:exact": baseEntry({ sessionId: "foo-bar" }),
|
|
"agent:main:slug": baseEntry({ sessionId: "foo_bar" }),
|
|
};
|
|
|
|
expect(
|
|
resolveTranscriptStemToSessionKeys({
|
|
store,
|
|
stem: "foo-bar",
|
|
allowQmdSlugFallback: true,
|
|
}),
|
|
).toEqual(["agent:main:exact"]);
|
|
});
|
|
|
|
it("does not guess when QMD-slugified fallback matches multiple sessions", () => {
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:dot": baseEntry({ sessionId: "foo.bar" }),
|
|
"agent:main:underscore": baseEntry({ sessionId: "foo_bar" }),
|
|
};
|
|
|
|
expect(
|
|
resolveTranscriptStemToSessionKeys({
|
|
store,
|
|
stem: "foo-bar",
|
|
allowQmdSlugFallback: true,
|
|
}),
|
|
).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("session transcript memory hit key compatibility exports", () => {
|
|
it("keeps hit-subpath memory helpers off the runtime writer import path", () => {
|
|
const source = fs.readFileSync(new URL("./session-transcript-hit.ts", import.meta.url), "utf8");
|
|
|
|
expect(source).not.toContain("session-transcript-runtime.js");
|
|
});
|
|
|
|
it("exports storage-neutral memory hit key helpers from the legacy hit subpath", () => {
|
|
const key = formatSessionTranscriptMemoryHitKey({
|
|
agentId: "main",
|
|
sessionId: "session:legacy",
|
|
});
|
|
const store: Record<string, SessionEntry> = {
|
|
"agent:main:discord:direct:42": {
|
|
sessionFile: "/tmp/not-the-identity.jsonl",
|
|
sessionId: "session:legacy",
|
|
updatedAt: 10,
|
|
},
|
|
};
|
|
|
|
expect(key).toBe("transcript:main:session%3Alegacy");
|
|
expect(parseSessionTranscriptMemoryHitKey(key)).toMatchObject({
|
|
agentId: "main",
|
|
sessionId: "session:legacy",
|
|
});
|
|
expect(resolveSessionTranscriptMemoryHitKeyToSessionKeys({ key, store })).toEqual([
|
|
"agent:main:discord:direct:42",
|
|
]);
|
|
});
|
|
});
|