diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 25fd7c96ee4..f84b984f35e 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -1,8 +1,11 @@ // Assertions for upgrade-survivor E2E scenarios. import fs from "node:fs"; +import { createRequire } from "node:module"; import path from "node:path"; import { readPluginInstallIndex } from "../plugin-index-sqlite.mjs"; +const require = createRequire(import.meta.url); + const command = process.argv[2]; const SCENARIOS = new Set([ "base", @@ -23,6 +26,10 @@ const PERSONA_FILES = new Map([ ["MEMORY.md", "# Existing Memory\n\nUpgrade reports came from real users.\n"], ]); +const LEGACY_SESSION_MAIN_ID = "upgrade-main-session"; +const LEGACY_SESSION_DIRECT_ID = "upgrade-direct-session"; +const LEGACY_SESSION_GROUP_ID = "upgrade-group-session"; + function requireEnv(name) { const value = process.env[name]; if (!value) { @@ -83,6 +90,71 @@ function assert(condition, message) { } } +function readSessionRowsFromAgentSqlite(stateDir, agentId = "main") { + const dbPath = path.join(stateDir, "agents", agentId, "agent", "openclaw-agent.sqlite"); + assert(fs.existsSync(dbPath), `agent SQLite session database missing: ${dbPath}`); + const { DatabaseSync } = require("node:sqlite"); + const db = new DatabaseSync(dbPath, { readOnly: true }); + try { + const rows = db + .prepare("SELECT key, value_json FROM cache_entries WHERE scope = ? ORDER BY key ASC") + .all("session_entries"); + return Object.fromEntries( + rows.map((row) => { + assert(typeof row.key === "string", "session SQLite row key was not a string"); + assert( + typeof row.value_json === "string", + `session SQLite row ${String(row.key)} had no JSON payload`, + ); + return [row.key, JSON.parse(row.value_json)]; + }), + ); + } finally { + db.close(); + } +} + +function seedLegacySessionMetadata(stateDir) { + const legacySessionsDir = path.join(stateDir, "sessions"); + writeJson(path.join(legacySessionsDir, "sessions.json"), { + main: { + sessionId: LEGACY_SESSION_MAIN_ID, + sessionFile: path.join(legacySessionsDir, `${LEGACY_SESSION_MAIN_ID}.jsonl`), + provider: "openai", + model: "gpt-5.5", + updatedAt: 1710000000000, + skillsSnapshot: { + prompt: "legacy prompt survives as metadata", + resolvedSkills: [ + { + name: "legacy-heavy-skill-cache", + filePath: "/tmp/openclaw-old-package/skills/legacy-heavy-skill-cache/SKILL.md", + }, + ], + }, + }, + "+15551234567": { + sessionId: LEGACY_SESSION_DIRECT_ID, + sessionFile: path.join(legacySessionsDir, `${LEGACY_SESSION_DIRECT_ID}.jsonl`), + provider: "openai", + model: "gpt-5.5", + updatedAt: 1710000000100, + }, + "slack:channel:CUPGRADE": { + sessionId: LEGACY_SESSION_GROUP_ID, + sessionFile: path.join(legacySessionsDir, `${LEGACY_SESSION_GROUP_ID}.jsonl`), + provider: "openai", + model: "gpt-5.5", + updatedAt: 1710000000200, + lastChannel: "slack", + lastTo: "CUPGRADE", + }, + }); + write(path.join(legacySessionsDir, `${LEGACY_SESSION_MAIN_ID}.jsonl`), '{"type":"main"}\n'); + write(path.join(legacySessionsDir, `${LEGACY_SESSION_DIRECT_ID}.jsonl`), '{"type":"direct"}\n'); + write(path.join(legacySessionsDir, `${LEGACY_SESSION_GROUP_ID}.jsonl`), '{"type":"group"}\n'); +} + function getScenario() { const scenario = process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO || "base"; assert(SCENARIOS.has(scenario), `unknown upgrade survivor scenario: ${scenario}`); @@ -139,6 +211,7 @@ function seedState() { agentId: "main", title: "Existing user session", }); + seedLegacySessionMetadata(stateDir); const runtimeRoot = path.join(stateDir, "plugin-runtime-deps"); for (const plugin of ["discord", "telegram", "whatsapp"]) { @@ -357,12 +430,15 @@ function assertStateSurvived() { const stateDir = requireEnv("OPENCLAW_STATE_DIR"); const workspace = requireEnv("OPENCLAW_TEST_WORKSPACE_DIR"); const scenario = getScenario(); + const stage = process.env.OPENCLAW_UPGRADE_SURVIVOR_ASSERT_STAGE || "survival"; assert(fs.existsSync(path.join(workspace, "IDENTITY.md")), "workspace identity file missing"); assert( fs.existsSync(path.join(stateDir, "agents", "main", "sessions", "legacy-session.json")), "legacy session file missing", ); - const stage = process.env.OPENCLAW_UPGRADE_SURVIVOR_ASSERT_STAGE || "survival"; + if (stage !== "baseline") { + assertSessionMetadataMigratedToSqlite(stateDir); + } const legacyRuntimeRoot = path.join(stateDir, "plugin-runtime-deps"); if (stage === "baseline") { if (fs.existsSync(legacyRuntimeRoot)) { @@ -406,6 +482,59 @@ function assertStateSurvived() { } } +function assertSessionMetadataMigratedToSqlite(stateDir) { + const legacyStorePath = path.join(stateDir, "sessions", "sessions.json"); + const agentSessionsDir = path.join(stateDir, "agents", "main", "sessions"); + assert( + !fs.existsSync(legacyStorePath), + `legacy sessions.json survived migration: ${legacyStorePath}`, + ); + for (const sessionId of [ + LEGACY_SESSION_MAIN_ID, + LEGACY_SESSION_DIRECT_ID, + LEGACY_SESSION_GROUP_ID, + ]) { + assert( + fs.existsSync(path.join(agentSessionsDir, `${sessionId}.jsonl`)), + `legacy session transcript was not moved for ${sessionId}`, + ); + } + + const store = readSessionRowsFromAgentSqlite(stateDir); + const main = store["agent:main:main"]; + const direct = store["agent:main:+15551234567"]; + const group = store["agent:main:slack:channel:cupgrade"]; + assert(main?.sessionId === LEGACY_SESSION_MAIN_ID, "main legacy session row missing from SQLite"); + assert( + direct?.sessionId === LEGACY_SESSION_DIRECT_ID, + "direct legacy session row missing from SQLite", + ); + assert( + group?.sessionId === LEGACY_SESSION_GROUP_ID, + "channel legacy session row missing from SQLite", + ); + assert( + main?.sessionFile === path.join(agentSessionsDir, `${LEGACY_SESSION_MAIN_ID}.jsonl`), + "main legacy session row still points at the old sessions directory", + ); + assert( + direct?.sessionFile === path.join(agentSessionsDir, `${LEGACY_SESSION_DIRECT_ID}.jsonl`), + "direct legacy session row still points at the old sessions directory", + ); + assert( + group?.sessionFile === path.join(agentSessionsDir, `${LEGACY_SESSION_GROUP_ID}.jsonl`), + "channel legacy session row still points at the old sessions directory", + ); + assert( + main.skillsSnapshot?.prompt === "legacy prompt survives as metadata", + "legacy session metadata prompt was not preserved", + ); + assert( + main.skillsSnapshot?.resolvedSkills === undefined, + "heavy resolvedSkills cache was persisted into SQLite session metadata", + ); +} + function readInstalledPluginIndex() { const stateDir = requireEnv("OPENCLAW_STATE_DIR"); const index = readPluginInstallIndex({ stateDir }); diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index f4052bc506d..e56ddf92d53 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { resolveSqliteSessionStoreDatabasePath } from "../config/sessions/store-sqlite.js"; import { loadSessionStore } from "../config/sessions/store.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { @@ -18,6 +19,7 @@ import { writePersistedInstalledPluginIndex, } from "../plugins/installed-plugin-index-store.js"; import type { InstalledPluginInstallRecordInfo } from "../plugins/installed-plugin-index.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { loadTaskFlowRegistryStateFromSqlite } from "../tasks/task-flow-registry.store.sqlite.js"; import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; @@ -245,6 +247,7 @@ async function runTelegramAllowFromMigration(params: { root: string; cfg: OpenCl afterEach(async () => { resetAutoMigrateLegacyStateForTest(); resetAutoMigrateLegacyStateDirForTest(); + closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); setMaxPluginStateEntriesPerPluginForTests(); resetPluginStateStoreForTests(); @@ -534,7 +537,7 @@ async function withStateDir(root: string, run: () => Promise): Promise function readSessionsStore(targetDir: string) { return loadSessionStore(path.join(targetDir, "sessions.json"), { skipCache: true }) as Record< string, - { sessionId: string } + { sessionId: string; sessionFile?: string } >; } @@ -629,7 +632,7 @@ describe("doctor legacy state migrations", () => { result: Awaited>; targetDir: string; legacySessionsDir: string; - store: Record; + store: Record; }; beforeAll(async () => { @@ -638,8 +641,16 @@ describe("doctor legacy state migrations", () => { const legacySessionsDir = writeLegacySessionsFixture({ root, sessions: { - "+1555": { sessionId: "a", updatedAt: 10 }, - "+1666": { sessionId: "b", updatedAt: 20 }, + "+1555": { + sessionId: "a", + sessionFile: path.join(root, "sessions", "a.jsonl"), + updatedAt: 10, + }, + "+1666": { + sessionId: "b", + sessionFile: path.join(root, "sessions", "b.jsonl"), + updatedAt: 20, + }, "slack:channel:C123": { sessionId: "c", updatedAt: 30 }, "group:abc": { sessionId: "d", updatedAt: 40 }, "subagent:xyz": { sessionId: "e", updatedAt: 50 }, @@ -672,8 +683,11 @@ describe("doctor legacy state migrations", () => { expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false); expect(store["agent:main:main"]?.sessionId).toBe("b"); + expect(store["agent:main:main"]?.sessionFile).toBe(path.join(targetDir, "b.jsonl")); expect(store["agent:main:+1555"]?.sessionId).toBe("a"); + expect(store["agent:main:+1555"]?.sessionFile).toBe(path.join(targetDir, "a.jsonl")); expect(store["agent:main:+1666"]?.sessionId).toBe("b"); + expect(store["agent:main:+1666"]?.sessionFile).toBe(path.join(targetDir, "b.jsonl")); expect(store["+1555"]).toBeUndefined(); expect(store["+1666"]).toBeUndefined(); expect(store["agent:main:slack:channel:c123"]?.sessionId).toBe("c"); @@ -681,6 +695,360 @@ describe("doctor legacy state migrations", () => { expect(store["agent:main:subagent:xyz"]?.sessionId).toBe("e"); }); + it("keeps migrated sessionFile metadata aligned with conflicted transcript moves", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + updatedAt: 10, + }, + "+1666": { + sessionId: "b", + sessionFile: "b.jsonl", + updatedAt: 20, + }, + "+1777": { + sessionId: "legacy-collision", + sessionFile: "a.legacy-123.jsonl", + updatedAt: 15, + }, + }, + transcripts: { + "a.jsonl": "legacy transcript", + "b.jsonl": "legacy relative transcript", + "a.legacy-123.jsonl": "legacy default-destination transcript", + }, + }); + const legacyMtime = new Date("2024-01-02T03:04:05.000Z"); + const legacyRelativeMtime = new Date("2024-01-03T03:04:05.000Z"); + fs.utimesSync(path.join(root, "sessions", "a.jsonl"), legacyMtime, legacyMtime); + fs.utimesSync(path.join(root, "sessions", "b.jsonl"), legacyRelativeMtime, legacyRelativeMtime); + const targetDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync(path.join(targetDir, "a.jsonl"), "existing transcript", "utf-8"); + fs.writeFileSync(path.join(targetDir, "b.jsonl"), "existing relative transcript", "utf-8"); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 123, + }); + + const movedPath = path.join(targetDir, "a.legacy-123-1.jsonl"); + const movedCollisionPath = path.join(targetDir, "a.legacy-123.jsonl"); + const movedRelativePath = path.join(targetDir, "b.legacy-123.jsonl"); + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(path.join(targetDir, "a.jsonl"), "utf-8")).toBe("existing transcript"); + expect(fs.readFileSync(path.join(targetDir, "b.jsonl"), "utf-8")).toBe( + "existing relative transcript", + ); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("legacy transcript"); + expect(fs.readFileSync(movedCollisionPath, "utf-8")).toBe( + "legacy default-destination transcript", + ); + expect(fs.readFileSync(movedRelativePath, "utf-8")).toBe("legacy relative transcript"); + expect(Math.abs(fs.statSync(movedPath).mtimeMs - legacyMtime.getTime())).toBeLessThan(5); + expect( + Math.abs(fs.statSync(movedRelativePath).mtimeMs - legacyRelativeMtime.getTime()), + ).toBeLessThan(5); + expect(store["agent:main:+1555"]?.sessionFile).toBe(movedPath); + expect(store["agent:main:+1666"]?.sessionFile).toBe(movedRelativePath); + expect(store["agent:main:+1777"]?.sessionFile).toBe(movedCollisionPath); + expect(store["agent:main:main"]?.sessionFile).toBe(movedRelativePath); + }); + + it("keeps case-only target transcript names from sharing a moved legacy path", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1888": { + sessionId: "Case", + sessionFile: "Case.jsonl", + updatedAt: 10, + }, + }, + transcripts: { + "Case.jsonl": "legacy transcript", + }, + }); + const targetDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync(path.join(targetDir, "case.jsonl"), "existing target transcript", "utf-8"); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 123, + }); + + const movedPath = path.join(targetDir, "Case.legacy-123.jsonl"); + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(path.join(targetDir, "case.jsonl"), "utf-8")).toBe( + "existing target transcript", + ); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("legacy transcript"); + expect(store["agent:main:+1888"]?.sessionFile).toBe(movedPath); + expect(store["agent:main:main"]?.sessionFile).toBe(movedPath); + }); + + it("rewrites case-mismatched legacy sessionFile metadata when the source match is unique", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1888": { + sessionId: "case-mismatch", + sessionFile: "casemismatch.jsonl", + updatedAt: 10, + }, + }, + transcripts: { + "CaseMismatch.jsonl": "case-mismatched transcript", + }, + }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 123, + }); + + const targetDir = path.join(root, "agents", "main", "sessions"); + const movedPath = path.join(targetDir, "CaseMismatch.jsonl"); + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("case-mismatched transcript"); + expect(store["agent:main:+1888"]?.sessionFile).toBe(movedPath); + expect(store["agent:main:main"]?.sessionFile).toBe(movedPath); + }); + + it("does not rewrite newer target session rows to older copied legacy transcripts", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + updatedAt: 10, + }, + }, + transcripts: { + "a.jsonl": "older legacy transcript", + }, + }); + const targetDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync(path.join(targetDir, "a.jsonl"), "newer target transcript", "utf-8"); + writeJson5(path.join(targetDir, "sessions.json"), { + "+1555": { sessionId: "a", updatedAt: 20 }, + }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 123, + }); + + const movedPath = path.join(targetDir, "a.legacy-123.jsonl"); + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(path.join(targetDir, "a.jsonl"), "utf-8")).toBe( + "newer target transcript", + ); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("older legacy transcript"); + expect(store["agent:main:+1555"]?.sessionId).toBe("a"); + expect(store["agent:main:+1555"]?.sessionFile).toBeUndefined(); + expect(store["agent:main:main"]?.sessionFile).toBe(movedPath); + }); + + it("keeps legacy transcripts retryable when SQLite session import fails", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + sessionFile: path.join(root, "sessions", "a.jsonl"), + updatedAt: 10, + }, + }, + transcripts: { + "a.jsonl": "legacy transcript", + }, + }); + const dbPath = path.join(root, "agents", "main", "agent", "openclaw-agent.sqlite"); + fs.mkdirSync(dbPath, { recursive: true }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + + await expect(runLegacyStateMigrations({ detected, now: () => 123 })).rejects.toThrow(); + expect(fs.readFileSync(path.join(root, "sessions", "a.jsonl"), "utf-8")).toBe( + "legacy transcript", + ); + expect(fs.existsSync(path.join(root, "agents", "main", "sessions", "a.jsonl"))).toBe(false); + expect(fs.existsSync(path.join(root, "sessions", "sessions.json"))).toBe(true); + }); + + it("keeps legacy transcripts retryable after a partial SQLite session import artifact", async () => { + const root = await makeTempRoot(); + writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + sessionFile: path.join(root, "sessions", "a.jsonl"), + updatedAt: 10, + }, + }, + transcripts: { + "a.jsonl": "legacy transcript", + }, + }); + const targetStorePath = path.join(root, "agents", "main", "sessions", "sessions.json"); + const databasePath = resolveSqliteSessionStoreDatabasePath(targetStorePath); + fs.mkdirSync(path.dirname(databasePath), { recursive: true }); + fs.writeFileSync(databasePath, "partial sqlite artifact from failed import", "utf-8"); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + + await expect(runLegacyStateMigrations({ detected, now: () => 123 })).rejects.toThrow(); + expect(fs.readFileSync(databasePath, "utf-8")).toBe( + "partial sqlite artifact from failed import", + ); + expect(fs.readFileSync(path.join(root, "sessions", "a.jsonl"), "utf-8")).toBe( + "legacy transcript", + ); + expect(fs.existsSync(path.join(root, "agents", "main", "sessions", "a.jsonl"))).toBe(false); + expect(fs.existsSync(path.join(root, "sessions", "sessions.json"))).toBe(true); + expect( + fs.existsSync(path.join(root, "sessions", ".openclaw-session-migration-plan.json")), + ).toBe(true); + }); + + it("retries session metadata import from a persisted transcript move plan", async () => { + const root = await makeTempRoot(); + const legacyDir = writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + sessionFile: path.join(root, "sessions", "a.jsonl"), + updatedAt: 10, + }, + }, + transcripts: { + "a.jsonl": "legacy transcript", + }, + }); + const targetDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync(path.join(targetDir, "a.jsonl"), "existing target transcript", "utf-8"); + const movedPath = path.join(targetDir, "a.legacy-123.jsonl"); + fs.renameSync(path.join(legacyDir, "a.jsonl"), movedPath); + writeJson5(path.join(legacyDir, ".openclaw-session-migration-plan.json"), { + version: 1, + moves: [ + { + from: path.join(legacyDir, "a.jsonl"), + to: movedPath, + name: "a.jsonl", + }, + ], + }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 456, + }); + + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(path.join(targetDir, "a.jsonl"), "utf-8")).toBe( + "existing target transcript", + ); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("legacy transcript"); + expect(store["agent:main:+1555"]?.sessionFile).toBe(movedPath); + expect(store["agent:main:main"]?.sessionFile).toBe(movedPath); + expect(fs.existsSync(path.join(legacyDir, ".openclaw-session-migration-plan.json"))).toBe( + false, + ); + }); + + it("does not overwrite newer target transcripts when replaying a persisted move plan", async () => { + const root = await makeTempRoot(); + const legacyDir = writeLegacySessionsFixture({ + root, + sessions: { + "+1555": { + sessionId: "a", + sessionFile: path.join(root, "sessions", "a.jsonl"), + updatedAt: 10, + }, + }, + transcripts: { + "a.jsonl": "legacy transcript", + }, + }); + const targetDir = path.join(root, "agents", "main", "sessions"); + fs.mkdirSync(targetDir, { recursive: true }); + const stalePlannedPath = path.join(targetDir, "a.legacy-123.jsonl"); + fs.writeFileSync(stalePlannedPath, "newer target transcript", "utf-8"); + writeJson5(path.join(legacyDir, ".openclaw-session-migration-plan.json"), { + version: 1, + moves: [ + { + from: path.join(legacyDir, "a.jsonl"), + to: stalePlannedPath, + name: "a.jsonl", + }, + ], + }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ + detected, + now: () => 456, + }); + + const movedPath = path.join(targetDir, "a.legacy-456.jsonl"); + const store = readSessionsStore(targetDir); + expect(result.warnings).toStrictEqual([]); + expect(fs.readFileSync(stalePlannedPath, "utf-8")).toBe("newer target transcript"); + expect(fs.readFileSync(movedPath, "utf-8")).toBe("legacy transcript"); + expect(store["agent:main:+1555"]?.sessionFile).toBe(movedPath); + expect(store["agent:main:main"]?.sessionFile).toBe(movedPath); + }); + it("imports detected non-default configured session stores into SQLite", async () => { const root = await makeTempRoot(); const storeTemplate = path.join(root, "stores", "sessions-{agentId}.json"); diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index b11a92cad01..2905a3cee0b 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -180,6 +180,7 @@ type DetectedPluginDoctorStateMigrationPlan = { const PLUGIN_STATE_SQLITE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; const TASK_STATE_SQLITE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; +const LEGACY_SESSION_FILE_MOVE_PLAN_NAME = ".openclaw-session-migration-plan.json"; const LEGACY_DELIVERY_QUEUE_DIRS = [ { label: "outbound delivery queue", queueName: "outbound", dirName: "delivery-queue" }, { label: "session delivery queue", queueName: "session", dirName: "session-delivery-queue" }, @@ -1931,6 +1932,342 @@ function mergeSessionEntry(params: { return params.preferIncomingOnTie ? params.incoming : params.existing; } +function rewriteLegacySessionFilePaths(params: { + store: Record; + legacyDir: string; + movedFiles: MovedSessionFiles; +}): Record { + const rewritten: Record = {}; + const legacyDir = path.resolve(params.legacyDir); + for (const [key, entry] of Object.entries(params.store)) { + const rawSessionFile = (entry as { sessionFile?: unknown }).sessionFile; + const movedSessionFile = + typeof rawSessionFile === "string" + ? lookupMovedSessionFile( + params.movedFiles, + path.isAbsolute(rawSessionFile) + ? path.resolve(rawSessionFile) + : path.resolve(legacyDir, rawSessionFile), + ) + : resolveMovedSessionFileFromSessionId({ + entry, + legacyDir, + movedFiles: params.movedFiles, + }); + if (!movedSessionFile) { + rewritten[key] = entry; + continue; + } + rewritten[key] = { + ...entry, + sessionFile: movedSessionFile, + }; + } + return rewritten; +} + +function resolveMovedSessionFileFromSessionId(params: { + entry: SessionEntryLike; + legacyDir: string; + movedFiles: MovedSessionFiles; +}): string | undefined { + const rawSessionId = (params.entry as { sessionId?: unknown }).sessionId; + if (typeof rawSessionId !== "string") { + return undefined; + } + try { + const sessionId = validateSessionId(rawSessionId); + return lookupMovedSessionFile( + params.movedFiles, + path.join(params.legacyDir, `${sessionId}.jsonl`), + ); + } catch { + return undefined; + } +} + +type LegacySessionFileMove = { + from: string; + to: string; + name: string; +}; + +type MovedSessionFiles = { + exact: Map; + folded: Map; + ambiguousFolded: Set; +}; + +// Case-insensitive filesystems can report a source file with casing that differs +// from stored sessionFile metadata. Folded aliases are safe only when exactly +// one moved source owns that spelling; otherwise keep lookup exact. +function buildMovedSessionFiles(moves: LegacySessionFileMove[]): MovedSessionFiles { + const foldedCounts = new Map(); + for (const move of moves) { + const folded = sessionMovePathKey(move.from); + foldedCounts.set(folded, (foldedCounts.get(folded) ?? 0) + 1); + } + + const movedFiles: MovedSessionFiles = { + exact: new Map(), + folded: new Map(), + ambiguousFolded: new Set(), + }; + for (const [folded, count] of foldedCounts) { + if (count > 1) { + movedFiles.ambiguousFolded.add(folded); + } + } + return movedFiles; +} + +function recordMovedSessionFile(params: { + movedFiles: MovedSessionFiles; + move: LegacySessionFileMove; +}): void { + const exact = path.resolve(params.move.from); + const folded = sessionMovePathKey(params.move.from); + params.movedFiles.exact.set(exact, params.move.to); + if (!params.movedFiles.ambiguousFolded.has(folded)) { + params.movedFiles.folded.set(folded, params.move.to); + } +} + +function lookupMovedSessionFile( + movedFiles: MovedSessionFiles, + filePath: string, +): string | undefined { + const exact = movedFiles.exact.get(path.resolve(filePath)); + if (exact) { + return exact; + } + const folded = sessionMovePathKey(filePath); + if (movedFiles.ambiguousFolded.has(folded)) { + return undefined; + } + return movedFiles.folded.get(folded); +} + +function resolveLegacySessionFileMovePlanPath(legacyDir: string): string { + return path.join(legacyDir, LEGACY_SESSION_FILE_MOVE_PLAN_NAME); +} + +function isLegacySessionFileMovePlanName(name: string): boolean { + return ( + name === LEGACY_SESSION_FILE_MOVE_PLAN_NAME || + name === `${LEGACY_SESSION_FILE_MOVE_PLAN_NAME}.tmp` + ); +} + +// Transcript moves happen before SQLite import so metadata can point at final +// paths. Persist the plan first so a crash in that window can retry without +// guessing conflict-renamed transcript names. +function parseLegacySessionFileMovePlan(raw: string): LegacySessionFileMove[] | null { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (!parsed || typeof parsed !== "object") { + return null; + } + const moves = (parsed as { moves?: unknown }).moves; + if (!Array.isArray(moves)) { + return null; + } + const plan: LegacySessionFileMove[] = []; + for (const move of moves) { + if (!move || typeof move !== "object") { + return null; + } + const rec = move as { from?: unknown; to?: unknown; name?: unknown }; + if ( + typeof rec.from !== "string" || + typeof rec.to !== "string" || + typeof rec.name !== "string" + ) { + return null; + } + plan.push({ + from: rec.from, + to: rec.to, + name: rec.name, + }); + } + return plan; +} + +function readLegacySessionFileMovePlan(params: { + legacyDir: string; + targetDir: string; +}): LegacySessionFileMove[] | null { + const legacyDir = path.resolve(params.legacyDir); + const targetDir = path.resolve(params.targetDir); + const planPath = resolveLegacySessionFileMovePlanPath(legacyDir); + if (!fileExists(planPath)) { + return null; + } + try { + const moves = parseLegacySessionFileMovePlan(fs.readFileSync(planPath, "utf-8")); + if (!moves) { + return null; + } + for (const move of moves) { + if ( + !isWithinDir(legacyDir, move.from) || + !isWithinDir(targetDir, move.to) || + path.basename(move.from) !== move.name + ) { + return null; + } + } + return moves; + } catch { + return null; + } +} + +function writeLegacySessionFileMovePlan(params: { + legacyDir: string; + moves: LegacySessionFileMove[]; +}): void { + if (params.moves.length === 0) { + return; + } + const planPath = resolveLegacySessionFileMovePlanPath(params.legacyDir); + const tempPath = `${planPath}.tmp`; + fs.writeFileSync( + tempPath, + JSON.stringify( + { + version: 1, + moves: params.moves, + }, + null, + 2, + ), + "utf-8", + ); + fs.renameSync(tempPath, planPath); +} + +function revalidateLegacySessionFileMovePlan(params: { + moves: LegacySessionFileMove[]; + targetDir: string; + now: () => number; +}): { moves: LegacySessionFileMove[]; changed: boolean } { + let changed = false; + const reservedPaths = new Set( + safeReadDir(params.targetDir) + .filter((entry) => entry.isFile()) + .map((entry) => sessionMovePathKey(path.join(params.targetDir, entry.name))), + ); + const moves: LegacySessionFileMove[] = []; + for (const move of params.moves) { + let to = move.to; + const sourceExists = fileExists(move.from); + const targetKey = sessionMovePathKey(to); + if (sourceExists && reservedPaths.has(targetKey)) { + to = nextLegacySessionConflictPath({ + targetDir: params.targetDir, + name: move.name, + now: params.now, + reservedPaths, + }); + changed = true; + } + reservedPaths.add(sessionMovePathKey(to)); + moves.push(to === move.to ? move : { ...move, to }); + } + return { moves, changed }; +} + +function nextLegacySessionConflictPath(params: { + targetDir: string; + name: string; + now: () => number; + reservedPaths: Set; +}): string { + const parsed = path.parse(params.name); + const baseName = parsed.name || "session"; + const ext = parsed.ext || ".jsonl"; + const suffix = `.legacy-${params.now()}`; + let index = 0; + while (true) { + const numbered = index === 0 ? "" : `-${index}`; + const candidate = path.join(params.targetDir, `${baseName}${suffix}${numbered}${ext}`); + if (!fileExists(candidate) && !params.reservedPaths.has(sessionMovePathKey(candidate))) { + return candidate; + } + index++; + } +} + +function sessionMovePathKey(filePath: string): string { + return normalizeLowercaseStringOrEmpty(path.resolve(filePath)); +} + +function buildLegacySessionFileMovePlan(params: { + legacyDir: string; + targetDir: string; + now: () => number; +}): LegacySessionFileMove[] { + const moves: LegacySessionFileMove[] = []; + const entries = safeReadDir(params.legacyDir) + .filter( + (entry) => + entry.isFile() && + entry.name !== "sessions.json" && + !isLegacySessionFileMovePlanName(entry.name), + ) + .toSorted((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0)); + const existingTargetPaths = new Set( + safeReadDir(params.targetDir) + .filter((entry) => entry.isFile()) + .map((entry) => sessionMovePathKey(path.join(params.targetDir, entry.name))), + ); + const defaultTargetPaths = new Set( + entries.map((entry) => sessionMovePathKey(path.join(params.targetDir, entry.name))), + ); + const plannedTargetPaths = new Set(); + for (const entry of entries) { + if (!entry.isFile() || entry.name === "sessions.json") { + continue; + } + const from = path.join(params.legacyDir, entry.name); + const defaultTo = path.join(params.targetDir, entry.name); + const resolvedDefaultTo = sessionMovePathKey(defaultTo); + const mustUseConflictName = + fileExists(defaultTo) || + existingTargetPaths.has(resolvedDefaultTo) || + plannedTargetPaths.has(resolvedDefaultTo); + const reservedPaths = new Set([ + ...existingTargetPaths, + ...defaultTargetPaths, + ...plannedTargetPaths, + ]); + if (!mustUseConflictName) { + reservedPaths.delete(resolvedDefaultTo); + } + const to = mustUseConflictName + ? nextLegacySessionConflictPath({ + targetDir: params.targetDir, + name: entry.name, + now: params.now, + reservedPaths, + }) + : defaultTo; + plannedTargetPaths.add(sessionMovePathKey(to)); + moves.push({ + from, + to, + name: entry.name, + }); + } + return moves; +} + function canonicalizeSessionStore(params: { store: Record; agentId: string; @@ -2687,8 +3024,10 @@ export async function detectLegacyStateMigrations(params: { const sessionsTargetDir = path.join(stateDir, "agents", targetAgentId, "sessions"); const sessionsTargetStorePath = path.join(sessionsTargetDir, "sessions.json"); const legacySessionEntries = safeReadDir(sessionsLegacyDir); + const legacySessionMovePlanPath = resolveLegacySessionFileMovePlanPath(sessionsLegacyDir); const hasLegacySessions = fileExists(sessionsLegacyStorePath) || + fileExists(legacySessionMovePlanPath) || legacySessionEntries.some((e) => e.isFile() && e.name.endsWith(".jsonl")); const targetSessionParsed = fileExists(sessionsTargetStorePath) @@ -2909,28 +3248,6 @@ async function migrateLegacySessions( scope: detected.targetScope, }); - const merged: Record = { ...canonicalizedTarget.store }; - for (const [key, entry] of Object.entries(canonicalizedLegacy.store)) { - merged[key] = mergeSessionEntry({ - existing: merged[key], - incoming: entry, - preferIncomingOnTie: false, - }); - } - - const mainKey = buildAgentMainSessionKey({ - agentId: detected.targetAgentId, - mainKey: detected.targetMainKey, - }); - let migratedDirectChatKey: string | undefined; - if (!merged[mainKey]) { - const latest = pickLatestLegacyDirectEntry(legacyStore); - if (latest?.sessionId) { - merged[mainKey] = latest; - migratedDirectChatKey = mainKey; - } - } - if (!legacyParsed.ok) { warnings.push( `Legacy sessions store unreadable; left in place at ${detected.sessions.legacyStorePath}`, @@ -2958,20 +3275,121 @@ async function migrateLegacySessions( } } + if (!targetReadable) { + return { changes, warnings }; + } + + const persistedSessionFileMovePlans = readLegacySessionFileMovePlan({ + legacyDir: detected.sessions.legacyDir, + targetDir: detected.sessions.targetDir, + }); + const revalidatedSessionFileMovePlans = persistedSessionFileMovePlans + ? revalidateLegacySessionFileMovePlan({ + moves: persistedSessionFileMovePlans, + targetDir: detected.sessions.targetDir, + now, + }) + : null; + const movedSessionFilePlans = + revalidatedSessionFileMovePlans?.moves ?? + buildLegacySessionFileMovePlan({ + legacyDir: detected.sessions.legacyDir, + targetDir: detected.sessions.targetDir, + now, + }); + if (!persistedSessionFileMovePlans || revalidatedSessionFileMovePlans?.changed) { + writeLegacySessionFileMovePlan({ + legacyDir: detected.sessions.legacyDir, + moves: movedSessionFilePlans, + }); + } + const movedSessionFiles = buildMovedSessionFiles(movedSessionFilePlans); + const completedMovedSessionFilePlans: LegacySessionFileMove[] = []; + for (const move of movedSessionFilePlans) { + try { + if (fileExists(move.from)) { + fs.renameSync(move.from, move.to); + } else if (!fileExists(move.to)) { + warnings.push(`Skipped missing legacy transcript ${move.from}`); + continue; + } + recordMovedSessionFile({ + movedFiles: movedSessionFiles, + move, + }); + completedMovedSessionFilePlans.push(move); + } catch (err) { + warnings.push(`Failed moving ${move.from}: ${String(err)}`); + } + } + + const rewrittenLegacyStore = rewriteLegacySessionFilePaths({ + store: canonicalizedLegacy.store, + legacyDir: detected.sessions.legacyDir, + movedFiles: movedSessionFiles, + }); + const merged: Record = { ...canonicalizedTarget.store }; + for (const [key, entry] of Object.entries(rewrittenLegacyStore)) { + merged[key] = mergeSessionEntry({ + existing: merged[key], + incoming: entry, + preferIncomingOnTie: false, + }); + } + + const mainKey = buildAgentMainSessionKey({ + agentId: detected.targetAgentId, + mainKey: detected.targetMainKey, + }); + let migratedDirectChatKey: string | undefined; + if (!merged[mainKey]) { + const latest = pickLatestLegacyDirectEntry(legacyStore); + if (latest?.sessionId) { + const latestStore = rewriteLegacySessionFilePaths({ + store: { latest }, + legacyDir: detected.sessions.legacyDir, + movedFiles: movedSessionFiles, + }); + merged[mainKey] = latestStore.latest ?? latest; + migratedDirectChatKey = mainKey; + } + } + if ( - targetReadable && (legacyParsed.ok || targetParsed.ok) && (targetExists || fileExists(detected.sessions.legacyStorePath) || Object.keys(legacyStore).length > 0 || Object.keys(targetStore).length > 0) ) { - const { imported, acpMigrated } = importNormalizedSessionsIntoSqlite({ - storePath: detected.sessions.targetStorePath, - store: merged, - stateDir: detected.stateDir, - now, - }); + let imported: number; + let acpMigrated: number; + try { + const result = importNormalizedSessionsIntoSqlite({ + storePath: detected.sessions.targetStorePath, + store: merged, + stateDir: detected.stateDir, + now, + }); + imported = result.imported; + acpMigrated = result.acpMigrated; + } catch (err) { + const rollbackFailures: string[] = []; + for (const move of completedMovedSessionFilePlans.toReversed()) { + try { + fs.renameSync(move.to, move.from); + } catch (rollbackErr) { + rollbackFailures.push(`${move.to}: ${String(rollbackErr)}`); + } + } + if (rollbackFailures.length > 0) { + throw new Error( + `Failed importing session metadata: ${String(err)}; additionally failed rolling back moved transcript(s): ${rollbackFailures.join("; ")}`, + { cause: err }, + ); + } + throw err; + } if (migratedDirectChatKey) { changes.push(`Migrated latest direct-chat session → ${migratedDirectChatKey}`); } @@ -2995,29 +3413,13 @@ async function migrateLegacySessions( } } - if (!targetReadable) { - return { changes, warnings }; - } - - const entries = safeReadDir(detected.sessions.legacyDir); - for (const entry of entries) { - if (!entry.isFile()) { - continue; - } - if (entry.name === "sessions.json") { - continue; - } - const from = path.join(detected.sessions.legacyDir, entry.name); - const to = path.join(detected.sessions.targetDir, entry.name); - if (fileExists(to)) { - continue; - } - try { - fs.renameSync(from, to); - changes.push(`Moved ${entry.name} → agents/${detected.targetAgentId}/sessions`); - } catch (err) { - warnings.push(`Failed moving ${from}: ${String(err)}`); - } + for (const move of completedMovedSessionFilePlans) { + const movedName = path.basename(move.to); + changes.push( + movedName === move.name + ? `Moved ${move.name} → agents/${detected.targetAgentId}/sessions` + : `Moved ${move.name} → agents/${detected.targetAgentId}/sessions/${movedName}`, + ); } if (legacyParsed.ok && targetReadable) { @@ -3030,6 +3432,17 @@ async function migrateLegacySessions( } } + try { + const movePlanPath = resolveLegacySessionFileMovePlanPath(detected.sessions.legacyDir); + if (fileExists(movePlanPath)) { + fs.rmSync(movePlanPath, { force: true }); + } + } catch (err) { + warnings.push( + `Migrated legacy sessions, but failed removing ${LEGACY_SESSION_FILE_MOVE_PLAN_NAME}: ${String(err)}`, + ); + } + removeDirIfEmpty(detected.sessions.legacyDir); const legacyLeft = safeReadDir(detected.sessions.legacyDir).filter((e) => e.isFile()); if (legacyLeft.length > 0) {