Files
openclaw/scripts/check-sqlite-transaction-boundary.mjs
Josh Lehman 0a8e3604ba refactor: flip sessions and transcripts to sqlite storage (#98236)
* refactor(sessions): migrate runtime storage to sqlite

* test(sessions): fix sqlite CI regressions

* test(sessions): align remaining sqlite fixtures

* fix(codex): require sqlite trajectory recorder

* test(sessions): align orphan recovery sqlite fixture

* test(sessions): align sqlite rebase fixtures

* fix(sessions): finish current-main integration of the sqlite flip

Resolve the whole-store SDK removal across its owner boundary: drop the
loadSessionStore re-export and the registry whole-store wrappers, wire
hasTrackedActiveSessionRun into gateway chat, complete the
preserveLockedHarnessIds cleanup contract, flip the codex thread-history
import to storePath targets, and port remaining main-side tests from
file-store helpers to session accessor reads.

* chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs

Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore
the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain
bump gets its own review, and regenerate docs_map, the plugin SDK API baseline,
and the export-surface ratchet for the merged tree.

* feat(sessions): keep archived transcripts by default with zstd cold storage

Codex-style retention: deleting or resetting a session archives its
transcript as a zstd-compressed JSONL artifact (plain when the runtime
lacks node:zlib zstd) and keeps it until the disk budget evicts oldest
first. resetArchiveRetention now governs both deleted and reset archives
and defaults to keep; maxDiskBytes defaults to 2gb so retention stays
bounded, with archives evicted before live sessions. The cron reaper
follows the same knob instead of deleting archives on its own timer.

* fix(state): converge agent DB migration lineages and bound database growth

Merge coherence: run both structure-gated legacy memory-schema repairs
(flip-lineage drop, main-lineage identity rebuild) before the flip
migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all
converge, and hoist foreign_keys=OFF outside the schema transaction
where the pragma was silently ignored and the v1 sessions rebuild
cascade-deleted session_entries.

Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL
maintenance releases freed pages in bounded passes (never a blocking
full VACUUM), and doctor reports state/agent DB bloat from freelist
stats.

* fix(codex): resolve the store path for thread-history import via the SDK

The supervision catalog passed the legacy sessionFile locator to the
storePath-targeted transcript mirror; resolve the agent store path with
the session-store SDK helper instead of a runtime-object seam so test
fakes and headless callers need no extra surface. Drop the obsolete
missing-session-id preprocessing case: sessions rows are NOT NULL on
session_id and upsert repairs id-less patches at write time.

* fix(sessions): fail safe on malformed disk-budget config and doctor stat errors

A malformed explicit maxDiskBytes disables the budget instead of
falling back to the destructive 2gb default the user never chose, and
the doctor bloat check skips databases whose paths stat-fail instead of
aborting doctor.

* fix(sessions): complete sqlite conflict translations

* test(sqlite): align hardening checks with maintenance

* test(sessions): inspect compressed transcript archives

* fix(tests): await session seeds and drop unused helpers flagged by CI lint

The five unawaited writeSessionStoreSeed calls raced their SQLite seeds
against the assertions, failing compact shards; the bloat probe drops a
useless initializer and the merged tests drop now-unused helpers.

* test(sessions): type legacy proof events directly

* test(sessions): align hardening contracts

* perf(sessions): read usage transcript sizes from SQL aggregates

Usage/cost scans walked every session and materialized every transcript
event just to re-stringify it for a byte estimate — the #86718 stall
class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes
in SQLite without loading a single row.

* fix(sessions): re-root foreign-root transcript paths onto the current sessions dir

Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry
absolute sessionFile paths from the old root; the containment fallback
kept those foreign paths, so migration read (and would archive) files in
the original root and reported local copies missing. Re-root the
canonical agents/<id>/sessions suffix onto the current dir when the file
exists there; genuine cross-root layouts still fall through unchanged.

* test(agents): seed harness admission through sqlite

* fix(sqlite): close agent db on pragma setup failure

* fix(doctor): compact and retrofit incremental auto-vacuum after session import

The migration is the sanctioned offline window: post-import compact
reclaims import churn and applies auto_vacuum=INCREMENTAL to databases
created before the fresh-DB pragma existed, so runtime maintenance can
release pages in bounded passes on every install.

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
2026-07-11 14:50:37 -07:00

174 lines
5.8 KiB
JavaScript

#!/usr/bin/env node
import ts from "typescript";
import {
collectFileViolations,
resolveRepoRoot,
resolveSourceRoots,
runAsScript,
toLine,
unwrapExpression,
} from "./lib/ts-guard-utils.mjs";
const removedAsyncTransactionNames = new Set([
"runOpenClawAgentWriteTransactionAsync",
"runSqliteImmediateTransactionAsync",
]);
const synchronousTransactionCallbackIndexes = new Map([
["runOpenClawAgentWriteTransaction", 0],
["runSqliteImmediateTransactionSync", 1],
]);
function expressionName(expression) {
const unwrapped = unwrapExpression(expression);
if (ts.isIdentifier(unwrapped)) {
return unwrapped.text;
}
if (ts.isPropertyAccessExpression(unwrapped)) {
return unwrapped.name.text;
}
if (ts.isElementAccessExpression(unwrapped) && ts.isStringLiteral(unwrapped.argumentExpression)) {
return unwrapped.argumentExpression.text;
}
return undefined;
}
function hasAsyncModifier(node) {
return node.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword) === true;
}
function collectLocalAsyncFunctionNames(sourceFile) {
const names = new Set();
const visit = (node) => {
if (ts.isFunctionDeclaration(node) && node.name && hasAsyncModifier(node)) {
names.add(node.name.text);
}
if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) {
const initializer = unwrapExpression(node.initializer);
if (
(ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer)) &&
hasAsyncModifier(initializer)
) {
names.add(node.name.text);
}
}
ts.forEachChild(node, visit);
};
visit(sourceFile);
return names;
}
function collectSynchronousTransactionAliases(sourceFile) {
const aliases = new Map();
for (const statement of sourceFile.statements) {
if (!ts.isImportDeclaration(statement)) {
continue;
}
const bindings = statement.importClause?.namedBindings;
if (!bindings || !ts.isNamedImports(bindings)) {
continue;
}
for (const specifier of bindings.elements) {
const importedName = specifier.propertyName?.text ?? specifier.name.text;
if (synchronousTransactionCallbackIndexes.has(importedName)) {
aliases.set(specifier.name.text, importedName);
}
}
}
return aliases;
}
function isAsyncCallback(expression, localAsyncFunctionNames) {
const unwrapped = unwrapExpression(expression);
if (ts.isArrowFunction(unwrapped) || ts.isFunctionExpression(unwrapped)) {
return hasAsyncModifier(unwrapped);
}
return ts.isIdentifier(unwrapped) && localAsyncFunctionNames.has(unwrapped.text);
}
export function findSqliteTransactionBoundaryViolations(content, fileName = "source.ts") {
const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true);
const localAsyncFunctionNames = collectLocalAsyncFunctionNames(sourceFile);
const transactionAliases = collectSynchronousTransactionAliases(sourceFile);
const violations = [];
const visit = (node) => {
if (ts.isImportDeclaration(node)) {
const bindings = node.importClause?.namedBindings;
if (bindings && ts.isNamedImports(bindings)) {
for (const specifier of bindings.elements) {
const importedName = specifier.propertyName?.text ?? specifier.name.text;
if (removedAsyncTransactionNames.has(importedName)) {
violations.push({
line: toLine(sourceFile, specifier),
reason: `imports removed async SQLite transaction primitive "${importedName}"`,
});
}
}
}
}
if (
((ts.isFunctionDeclaration(node) && node.name) ||
(ts.isVariableDeclaration(node) && ts.isIdentifier(node.name))) &&
removedAsyncTransactionNames.has(node.name.text)
) {
violations.push({
line: toLine(sourceFile, node.name),
reason: `declares removed async SQLite transaction primitive "${node.name.text}"`,
});
}
if (ts.isCallExpression(node)) {
const calledName = expressionName(node.expression);
if (calledName && removedAsyncTransactionNames.has(calledName)) {
violations.push({
line: toLine(sourceFile, node.expression),
reason: `calls removed async SQLite transaction primitive "${calledName}"`,
});
}
const canonicalName = calledName
? (transactionAliases.get(calledName) ?? calledName)
: undefined;
const callbackIndex = canonicalName
? synchronousTransactionCallbackIndexes.get(canonicalName)
: undefined;
const callback = callbackIndex === undefined ? undefined : node.arguments[callbackIndex];
if (canonicalName && callback && isAsyncCallback(callback, localAsyncFunctionNames)) {
violations.push({
line: toLine(sourceFile, callback),
reason: `passes an async callback to synchronous SQLite transaction helper "${canonicalName}"`,
});
}
}
ts.forEachChild(node, visit);
};
visit(sourceFile);
return violations;
}
async function main() {
const repoRoot = resolveRepoRoot(import.meta.url);
const violations = await collectFileViolations({
repoRoot,
sourceRoots: resolveSourceRoots(repoRoot, ["src", "extensions", "packages"]),
skipFile: (filePath) => /(^|[\\/])dist([\\/]|$)/.test(filePath),
findViolations: findSqliteTransactionBoundaryViolations,
});
if (violations.length === 0) {
console.log("SQLite transaction boundary guard passed.");
return;
}
console.error("Found asynchronous SQLite transaction work:");
for (const violation of violations) {
console.error(`- ${violation.path}:${violation.line}: ${violation.reason}`);
}
console.error(
"Complete asynchronous preparation before the transaction, then validate and apply inside a synchronous transaction callback.",
);
process.exit(1);
}
runAsScript(import.meta.url, main);