mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-25 18:21:12 +00:00
* refactor(sessions): migrate runtime storage to sqlite * test(sessions): fix sqlite CI regressions * test(sessions): align remaining sqlite fixtures * fix(codex): require sqlite trajectory recorder * test(sessions): align orphan recovery sqlite fixture * test(sessions): align sqlite rebase fixtures * fix(sessions): finish current-main integration of the sqlite flip Resolve the whole-store SDK removal across its owner boundary: drop the loadSessionStore re-export and the registry whole-store wrappers, wire hasTrackedActiveSessionRun into gateway chat, complete the preserveLockedHarnessIds cleanup contract, flip the codex thread-history import to storePath targets, and port remaining main-side tests from file-store helpers to session accessor reads. * chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain bump gets its own review, and regenerate docs_map, the plugin SDK API baseline, and the export-surface ratchet for the merged tree. * feat(sessions): keep archived transcripts by default with zstd cold storage Codex-style retention: deleting or resetting a session archives its transcript as a zstd-compressed JSONL artifact (plain when the runtime lacks node:zlib zstd) and keeps it until the disk budget evicts oldest first. resetArchiveRetention now governs both deleted and reset archives and defaults to keep; maxDiskBytes defaults to 2gb so retention stays bounded, with archives evicted before live sessions. The cron reaper follows the same knob instead of deleting archives on its own timer. * fix(state): converge agent DB migration lineages and bound database growth Merge coherence: run both structure-gated legacy memory-schema repairs (flip-lineage drop, main-lineage identity rebuild) before the flip migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all converge, and hoist foreign_keys=OFF outside the schema transaction where the pragma was silently ignored and the v1 sessions rebuild cascade-deleted session_entries. Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL maintenance releases freed pages in bounded passes (never a blocking full VACUUM), and doctor reports state/agent DB bloat from freelist stats. * fix(codex): resolve the store path for thread-history import via the SDK The supervision catalog passed the legacy sessionFile locator to the storePath-targeted transcript mirror; resolve the agent store path with the session-store SDK helper instead of a runtime-object seam so test fakes and headless callers need no extra surface. Drop the obsolete missing-session-id preprocessing case: sessions rows are NOT NULL on session_id and upsert repairs id-less patches at write time. * fix(sessions): fail safe on malformed disk-budget config and doctor stat errors A malformed explicit maxDiskBytes disables the budget instead of falling back to the destructive 2gb default the user never chose, and the doctor bloat check skips databases whose paths stat-fail instead of aborting doctor. * fix(sessions): complete sqlite conflict translations * test(sqlite): align hardening checks with maintenance * test(sessions): inspect compressed transcript archives * fix(tests): await session seeds and drop unused helpers flagged by CI lint The five unawaited writeSessionStoreSeed calls raced their SQLite seeds against the assertions, failing compact shards; the bloat probe drops a useless initializer and the merged tests drop now-unused helpers. * test(sessions): type legacy proof events directly * test(sessions): align hardening contracts * perf(sessions): read usage transcript sizes from SQL aggregates Usage/cost scans walked every session and materialized every transcript event just to re-stringify it for a byte estimate — the #86718 stall class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes in SQLite without loading a single row. * fix(sessions): re-root foreign-root transcript paths onto the current sessions dir Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry absolute sessionFile paths from the old root; the containment fallback kept those foreign paths, so migration read (and would archive) files in the original root and reported local copies missing. Re-root the canonical agents/<id>/sessions suffix onto the current dir when the file exists there; genuine cross-root layouts still fall through unchanged. * test(agents): seed harness admission through sqlite * fix(sqlite): close agent db on pragma setup failure * fix(doctor): compact and retrofit incremental auto-vacuum after session import The migration is the sanctioned offline window: post-import compact reclaims import churn and applies auto_vacuum=INCREMENTAL to databases created before the fresh-DB pragma existed, so runtime maintenance can release pages in bounded passes on every install. --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
129 lines
4.5 KiB
TypeScript
129 lines
4.5 KiB
TypeScript
// SQLite trajectory runtime tests cover session-scoped event row storage.
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import { replaceSessionEntry } from "../config/sessions/session-accessor.js";
|
|
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
|
|
import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
|
|
import {
|
|
closeOpenClawAgentDatabasesForTest,
|
|
openOpenClawAgentDatabase,
|
|
} from "../state/openclaw-agent-db.js";
|
|
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
|
import {
|
|
appendSqliteTrajectoryRuntimeEvents,
|
|
loadSqliteTrajectoryRuntimeEvents,
|
|
readSqliteTrajectoryRuntimeStatsSync,
|
|
} from "./runtime-store.sqlite.js";
|
|
import type { TrajectoryEvent } from "./types.js";
|
|
|
|
type TrajectoryRuntimeTestDatabase = Pick<OpenClawAgentKyselyDatabase, "trajectory_runtime_events">;
|
|
|
|
describe("SQLite trajectory runtime store", () => {
|
|
let tempDir: string;
|
|
let storePath: string;
|
|
|
|
beforeEach(async () => {
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-trajectory-sqlite-"));
|
|
storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json");
|
|
await replaceSessionEntry(
|
|
{ sessionKey: "agent:main:main", storePath },
|
|
{ sessionId: "session-1", updatedAt: 10 },
|
|
);
|
|
});
|
|
|
|
afterEach(() => {
|
|
closeOpenClawAgentDatabasesForTest();
|
|
closeOpenClawStateDatabaseForTest();
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("appends events in database order without trusting recorder-local seq", async () => {
|
|
appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }, [
|
|
createTrajectoryEvent({ seq: 1, type: "model.started" }),
|
|
createTrajectoryEvent({ seq: 1, type: "model.completed" }),
|
|
]);
|
|
|
|
await expect(
|
|
loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }),
|
|
).resolves.toEqual([
|
|
expect.objectContaining({ seq: 1, type: "model.started" }),
|
|
expect.objectContaining({ seq: 1, type: "model.completed" }),
|
|
]);
|
|
|
|
const database = openOpenClawAgentDatabase({ agentId: "main", path: sqlitePath() });
|
|
const db = getNodeSqliteKysely<TrajectoryRuntimeTestDatabase>(database.db);
|
|
const rows = executeSqliteQuerySync(
|
|
database.db,
|
|
db
|
|
.selectFrom("trajectory_runtime_events")
|
|
.select(["seq", "run_id"])
|
|
.where("session_id", "=", "session-1")
|
|
.orderBy("seq", "asc"),
|
|
).rows;
|
|
expect(rows).toEqual([
|
|
{ run_id: "run-1", seq: 0 },
|
|
{ run_id: "run-1", seq: 1 },
|
|
]);
|
|
});
|
|
|
|
it("trims oldest rows beyond the configured byte window", async () => {
|
|
appendSqliteTrajectoryRuntimeEvents(
|
|
{ maxRuntimeBytes: 900, sessionId: "session-1", storePath },
|
|
[
|
|
createTrajectoryEvent({ type: "event-1" }),
|
|
createTrajectoryEvent({ type: "event-2" }),
|
|
createTrajectoryEvent({ type: "event-3" }),
|
|
createTrajectoryEvent({ type: "event-4" }),
|
|
],
|
|
);
|
|
|
|
const events = await loadSqliteTrajectoryRuntimeEvents({
|
|
sessionId: "session-1",
|
|
storePath,
|
|
});
|
|
|
|
expect(events.map((event) => event.type)).toEqual(["event-3", "event-4"]);
|
|
expect(readSqliteTrajectoryRuntimeStatsSync({ sessionId: "session-1", storePath })).toEqual({
|
|
eventCount: 2,
|
|
maxSeq: 3,
|
|
sizeBytes: expect.any(Number),
|
|
});
|
|
});
|
|
|
|
it("cascades trajectory rows when the session row is deleted", async () => {
|
|
appendSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }, [
|
|
createTrajectoryEvent({ type: "model.started" }),
|
|
]);
|
|
|
|
const database = openOpenClawAgentDatabase({ agentId: "main", path: sqlitePath() });
|
|
database.db.prepare("DELETE FROM sessions WHERE session_id = ?").run("session-1");
|
|
|
|
await expect(
|
|
loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }),
|
|
).resolves.toEqual([]);
|
|
});
|
|
|
|
function sqlitePath(): string {
|
|
return path.join(tempDir, "agents", "main", "agent", "openclaw-agent.sqlite");
|
|
}
|
|
});
|
|
|
|
function createTrajectoryEvent(options: { seq?: number; type: string }): TrajectoryEvent {
|
|
return {
|
|
traceSchema: "openclaw-trajectory",
|
|
schemaVersion: 1,
|
|
traceId: "session-1",
|
|
source: "runtime",
|
|
type: options.type,
|
|
ts: "2026-07-03T00:00:00.000Z",
|
|
seq: options.seq ?? 1,
|
|
sourceSeq: options.seq ?? 1,
|
|
sessionId: "session-1",
|
|
sessionKey: "agent:main:main",
|
|
runId: "run-1",
|
|
data: { payload: "x".repeat(120) },
|
|
};
|
|
}
|