diff --git a/docs/cli/transcripts.md b/docs/cli/transcripts.md index 9877a87dc745..85d7a22c83aa 100644 --- a/docs/cli/transcripts.md +++ b/docs/cli/transcripts.md @@ -1,5 +1,5 @@ --- -summary: "CLI reference for `openclaw transcripts` (list, show, and locate stored transcripts)" +summary: "CLI reference for `openclaw transcripts` (list, show, and export stored transcripts)" read_when: - You want to read stored transcript summaries from the terminal - You need the path to a transcripts markdown summary @@ -9,10 +9,12 @@ title: "Transcripts CLI" # `openclaw transcripts` -Read-only inspector for transcripts written by the `transcripts` agent tool. -Capture, import, and summarization run through that tool, not this CLI. +Inspector and export command for transcripts written by the `transcripts` agent +tool. Capture, import, and summarization run through that tool, not this CLI. -Artifacts live under the state directory: +Canonical transcript state lives in the shared SQLite database at +`$OPENCLAW_STATE_DIR/state/openclaw.sqlite`. `show` and `path` explicitly +materialize user-facing artifacts under the state directory: ```text $OPENCLAW_STATE_DIR/transcripts/YYYY-MM-DD// @@ -22,9 +24,11 @@ $OPENCLAW_STATE_DIR/transcripts/YYYY-MM-DD// summary.md ``` -Default state directory is `~/.openclaw`; override with `OPENCLAW_STATE_DIR`. -The date directory comes from the session start time; the session directory is -a filesystem-safe slug derived from the session id. +These files are exports, not a second runtime store. OpenClaw does not read them +back during capture, summarization, or listing. Default state directory is +`~/.openclaw`; override with `OPENCLAW_STATE_DIR`. The date directory comes +from the session start time; the session directory is a filesystem-safe slug +derived from the session id. ## Commands @@ -42,15 +46,15 @@ openclaw transcripts show --json openclaw transcripts path --json ``` -| Command | Description | -| ----------------------------- | ----------------------------------------------- | -| `list` | List stored sessions. | -| `show ` | Print the stored `summary.md`. | -| `path ` | Print the `summary.md` path. | -| `path --dir` | Print the session directory. | -| `path --metadata` | Print `metadata.json`. | -| `path --transcript` | Print `transcript.jsonl`. | -| `--json` | Print machine-readable output (any subcommand). | +| Command | Description | +| ----------------------------- | ---------------------------------------------------- | +| `list` | List stored sessions. | +| `show ` | Print and materialize `summary.md`. | +| `path ` | Materialize and print the `summary.md` path. | +| `path --dir` | Materialize all artifacts and print their directory. | +| `path --metadata` | Materialize and print `metadata.json`. | +| `path --transcript` | Materialize and print `transcript.jsonl`. | +| `--json` | Print machine-readable output (any subcommand). | `` accepts either a bare session id or a date-qualified selector (`YYYY-MM-DD/`). Use the qualified form when the same session id @@ -75,7 +79,9 @@ The selector is the safest value to pass back to `show` or `path`. `show --json` returns the stored session metadata, selector, session directory, summary path, and summary Markdown text. -`path --json` returns the selected path and whether that file exists. +`path --json` returns the selected path and whether that artifact could be +materialized. Metadata and transcript exports always exist for a stored +session; a summary path reports `exists: false` until the session has a summary. ## Many sessions per day @@ -94,15 +100,30 @@ when it will not repeat on the same date. ## Missing summaries -Live sessions write `summary.md` when the session stops; imported transcripts -write it immediately after import. A session can appear in `list` without a -summary while capture is still active, if a provider failed during stop, or if -metadata was written before any utterances arrived. +Live sessions store and materialize `summary.md` when the session stops; +imported transcripts do so immediately after import. A session can appear in +`list` without a summary while capture is still active, if a provider failed +during stop, or if metadata was stored before any utterances arrived. Use `path --transcript` to inspect the raw append-only transcript, or run the `transcripts` tool's `summarize` action to regenerate the Markdown summary. +## Upgrading the legacy file store + +OpenClaw releases that predate the SQLite store wrote canonical runtime state +directly beneath `$OPENCLAW_STATE_DIR/transcripts/`. Run: + +```bash +openclaw doctor --fix +``` + +Doctor imports the complete legacy tree into SQLite, verifies row counts and +ordering, records migration receipts, and moves the verified source tree to a +timestamped `transcripts.migrated-*` archive. Runtime commands do not fall back +to the legacy files. Keep the archive until you have verified the imported +sessions and any exports you rely on. + ## Configuration Capture is opt-in (live sources can join and record meeting audio). Enable it diff --git a/docs/docs_map.md b/docs/docs_map.md index a44e6611ebe9..3ad81b83aa8c 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -2110,6 +2110,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Output - H2: Many sessions per day - H2: Missing summaries + - H2: Upgrading the legacy file store - H2: Configuration ## cli/tui.md diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 657139a2198f..a0880b735233 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -54,6 +54,7 @@ const rawSqliteAllowPathGroups = { "src/state/openclaw-state-db-schema-repair.ts", "src/state/openclaw-state-db-startup-checkpoint.ts", "src/state/openclaw-state-db.ts", + "src/transcripts/sqlite-schema.ts", "src/state/sqlite-schema-shape.test-support.ts", ], "cross-process SQLite coordination locks": ["src/infra/device-identity-coordinator.ts"], @@ -90,6 +91,9 @@ const rawSqliteAllowPathGroups = { "src/infra/state-migrations.storage.ts", "src/infra/state-migrations.cron-run-logs.ts", "src/infra/state-migrations.debug-proxy.ts", + "src/infra/state-migrations.meeting-transcripts-detection.ts", + "src/infra/state-migrations.meeting-transcripts-files.ts", + "src/infra/state-migrations.meeting-transcripts-verify.ts", ], "shared database stores with direct DatabaseSync access": ["src/proxy-capture/store.sqlite.ts"], "Kysely-backed stores that own a DatabaseSync boundary": [ diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 1582bce0fb06..7240b6e18a28 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; // Assertions for upgrade-survivor E2E scenarios. import fs from "node:fs"; import path from "node:path"; @@ -16,6 +17,7 @@ const SCENARIOS = new Set([ "configured-plugin-installs", "stale-source-plugin-shadow", "tilde-log-path", + "meeting-transcripts-sqlite", "versioned-runtime-deps", ]); @@ -138,6 +140,50 @@ function seedLegacySessionMetadata(stateDir) { } } +function seedLegacyMeetingTranscripts(stateDir) { + const sessionDir = path.join(stateDir, "transcripts", "2026-07-01", "design-review"); + const session = { + sessionId: "design-review", + title: "Design review", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-01T10:00:00.000Z", + stoppedAt: "2026-07-01T10:30:00.000Z", + }; + writeJson(path.join(sessionDir, "metadata.json"), session); + write( + path.join(sessionDir, "transcript.jsonl"), + [ + JSON.stringify({ + id: "legacy-u-1", + sessionId: session.sessionId, + speaker: { label: "Alex" }, + text: "First shipped transcript line", + final: true, + }), + JSON.stringify({ + id: "legacy-u-2", + sessionId: session.sessionId, + speaker: { label: "Sam" }, + text: "Second shipped transcript line", + final: true, + }), + ].join("\n") + "\n", + ); + const summary = { + sessionId: session.sessionId, + title: session.title, + generatedAt: "2026-07-01T10:31:00.000Z", + overview: "First shipped transcript line. Second shipped transcript line.", + transcript: ["Alex: First shipped transcript line", "Sam: Second shipped transcript line"], + decisions: [], + actionItems: [], + risks: [], + utteranceCount: 2, + }; + writeJson(path.join(sessionDir, "summary.json"), summary); + write(path.join(sessionDir, "summary.md"), "# Design review\n\nShipped transcript summary.\n"); +} + function getScenario() { const scenario = process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO || "base"; assert(SCENARIOS.has(scenario), `unknown upgrade survivor scenario: ${scenario}`); @@ -195,6 +241,9 @@ function seedState() { title: "Existing user session", }); seedLegacySessionMetadata(stateDir); + if (scenario === "meeting-transcripts-sqlite") { + seedLegacyMeetingTranscripts(stateDir); + } const runtimeRoot = path.join(stateDir, "plugin-runtime-deps"); for (const plugin of ["discord", "telegram", "whatsapp"]) { @@ -257,6 +306,11 @@ function seedState() { function assertConfigSurvived() { const config = getConfig(); const coverage = getCoverage(); + if (getScenario() === "meeting-transcripts-sqlite") { + // This focused migration fixture proves state import/export across one published + // baseline; the broad base scenario owns unrelated agent/channel config parity. + return; + } if (acceptsIntent(coverage, "update")) { assert(config.update?.channel === "stable", "update.channel was not preserved"); @@ -435,6 +489,9 @@ function assertStateSurvived() { if (stage !== "baseline") { assertSessionMetadataMigrated(stateDir); } + if (scenario === "meeting-transcripts-sqlite") { + assertMeetingTranscriptsMigrated(stateDir, stage); + } const legacyRuntimeRoot = path.join(stateDir, "plugin-runtime-deps"); if (stage === "baseline") { if (fs.existsSync(legacyRuntimeRoot)) { @@ -478,6 +535,92 @@ function assertStateSurvived() { } } +function assertMeetingTranscriptsMigrated(stateDir, stage) { + const legacySessionDir = path.join(stateDir, "transcripts", "2026-07-01", "design-review"); + if (stage === "baseline") { + assert( + fs.existsSync(path.join(legacySessionDir, "transcript.jsonl")), + "v2026.7.1 meeting transcript fixture missing before update", + ); + return; + } + + assert(!fs.existsSync(legacySessionDir), "legacy meeting transcript source was not archived"); + const archiveRoot = fs + .readdirSync(stateDir) + .find((entry) => entry.startsWith("transcripts.migrated-")); + assert(archiveRoot, "meeting transcript migration archive missing"); + assert( + fs.existsSync( + path.join(stateDir, archiveRoot, "2026-07-01", "design-review", "transcript.jsonl"), + ), + "archived meeting transcript JSONL missing", + ); + + const databasePath = path.join(stateDir, "state", "openclaw.sqlite"); + const db = new DatabaseSync(databasePath, { readOnly: true }); + try { + const session = db + .prepare( + "SELECT session_id, started_at, next_utterance_seq FROM meeting_transcript_sessions WHERE session_id = ?", + ) + .get("design-review"); + assert(session?.started_at === "2026-07-01T10:00:00.000Z", "meeting session row missing"); + assert(session?.next_utterance_seq === 2, "meeting transcript sequence head changed"); + const utterances = db + .prepare( + "SELECT sequence, utterance_id, text FROM meeting_transcript_utterances WHERE session_id = ? ORDER BY sequence ASC", + ) + .all("design-review"); + assert( + JSON.stringify(utterances) === + JSON.stringify([ + { + sequence: 0, + utterance_id: "legacy-u-1", + text: "First shipped transcript line", + }, + { + sequence: 1, + utterance_id: "legacy-u-2", + text: "Second shipped transcript line", + }, + ]), + "meeting transcript utterance ordering changed", + ); + const receipt = db + .prepare( + "SELECT status, removed_source, source_record_count FROM migration_sources WHERE migration_kind = ?", + ) + .get("meeting-transcripts-files-v1"); + assert(receipt?.status === "archived", "meeting transcript migration receipt incomplete"); + assert(receipt?.removed_source === 1, "meeting transcript source removal was not recorded"); + assert(receipt?.source_record_count === 2, "meeting transcript receipt count changed"); + } finally { + db.close(); + } + + const exportedDir = execFileSync( + "openclaw", + ["transcripts", "path", "2026-07-01/design-review", "--dir"], + { encoding: "utf8", env: process.env }, + ).trim(); + assert(exportedDir === legacySessionDir, "meeting transcript export path changed"); + const exportedLines = fs + .readFileSync(path.join(exportedDir, "transcript.jsonl"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert(exportedLines[0]?.id === "legacy-u-1", "first exported utterance changed"); + assert(exportedLines[1]?.id === "legacy-u-2", "second exported utterance changed"); + assert( + fs + .readFileSync(path.join(exportedDir, "summary.md"), "utf8") + .includes("Shipped transcript summary"), + "summary.md was not materialized from SQLite", + ); +} + function assertSessionMetadataMigrated(stateDir) { const legacyStorePath = path.join(stateDir, "sessions", "sessions.json"); const agentSessionsDir = path.join(stateDir, "agents", "main", "sessions"); diff --git a/scripts/lib/docker-e2e-plan.mjs b/scripts/lib/docker-e2e-plan.mjs index 676e1a12773e..ac4efd4d35f3 100644 --- a/scripts/lib/docker-e2e-plan.mjs +++ b/scripts/lib/docker-e2e-plan.mjs @@ -86,6 +86,7 @@ const UPGRADE_SURVIVOR_SCENARIOS = [ "configured-plugin-installs", "stale-source-plugin-shadow", "tilde-log-path", + "meeting-transcripts-sqlite", "versioned-runtime-deps", ]; @@ -97,6 +98,10 @@ const UPGRADE_SURVIVOR_SCENARIO_ALIASES = new Map([ // Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail // closed instead of requiring a dependency or reimplementing a JavaScript parser. const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([ + [ + "213e004a28814fe0f7bb33018ae59a709c2e6d6e13b273df82a0e7935fbaf5af", + "base acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps", + ], [ "755557a6ea609e5b9d9fe9d61beb7e75651641e26a3f0ef2fe1fc3a973b398c8", "base feishu-channel bootstrap-persona channel-post-core-restore plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path versioned-runtime-deps", diff --git a/src/agents/tools/transcripts-tool.test.ts b/src/agents/tools/transcripts-tool.test.ts index 0eca605607f0..61db842bffbe 100644 --- a/src/agents/tools/transcripts-tool.test.ts +++ b/src/agents/tools/transcripts-tool.test.ts @@ -3,7 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import type { TranscriptStopRequest } from "../../transcripts/provider-types.js"; import { TranscriptsStore } from "../../transcripts/store.js"; import { createTranscriptsAutoStartService, createTranscriptsTool } from "./transcripts-tool.js"; @@ -38,7 +39,15 @@ async function createHarness(stateDir: string, pluginConfig: Record { + afterEach(() => closeOpenClawStateDatabaseForTest()); + beforeEach(() => { getTranscriptSourceProviderMock.mockReset(); }); @@ -141,12 +150,14 @@ describe("transcripts tool", () => { expect(startupSignal?.aborted).toBe(false); await emitAfterStart?.(); - await expect( - fs.readFile( - path.join(stateDir, "transcripts", currentDateDir(), "ongoing-meeting", "transcript.jsonl"), - "utf8", - ), - ).resolves.toContain("captured after the start action completed\\nsecond\\tcolumn"); + const ongoingStore = storeFor(stateDir); + const ongoingSession = await ongoingStore.readSession("ongoing-meeting"); + expect(ongoingSession).toBeDefined(); + await expect(ongoingStore.readUtterancesForSession(ongoingSession!)).resolves.toEqual([ + expect.objectContaining({ + text: "captured after the start action completed\nsecond\tcolumn", + }), + ]); await tool.execute( "call-2", { action: "stop", sessionId: "ongoing-meeting" }, @@ -200,18 +211,10 @@ describe("transcripts tool", () => { ), ).rejects.toThrow("transcripts start aborted; provider cleanup failed: voice cleanup failed"); - await expect( - fs.readFile( - path.join( - stateDir, - "transcripts", - currentDateDir(), - "cancelled-meeting-retry", - "transcript.jsonl", - ), - "utf8", - ), - ).rejects.toMatchObject({ code: "ENOENT" }); + const cancelledStore = storeFor(stateDir); + const cancelledSession = await cancelledStore.readSession("cancelled-meeting-retry"); + expect(cancelledSession).toBeDefined(); + await expect(cancelledStore.readUtterancesForSession(cancelledSession!)).resolves.toEqual([]); expect(stop).toHaveBeenCalledOnce(); await expect( @@ -440,12 +443,12 @@ describe("transcripts tool", () => { "utf8", ), ).resolves.toContain('"Alex: We decided to ship Discord first."'); - await expect( - fs.readFile( - path.join(stateDir, "transcripts", currentDateDir(), "design-review", "transcript.jsonl"), - "utf8", - ), - ).resolves.toContain("Alex"); + const stored = await storeFor(stateDir).readSession("design-review"); + expect(stored).toBeDefined(); + await expect(storeFor(stateDir).readUtterancesForSession(stored!)).resolves.toEqual([ + expect.objectContaining({ text: "We decided to ship Discord first." }), + expect.objectContaining({ text: "Action item: add Slack import later." }), + ]); }); it("bounds summary input while retaining the full transcript", async () => { @@ -477,17 +480,16 @@ describe("transcripts tool", () => { ); expect(summary).not.toContain("transcript line 0\n"); expect(summary).toContain("transcript line 2000"); - const storedTranscript = await fs.readFile( - path.join(stateDir, "transcripts", currentDateDir(), "long-meeting", "transcript.jsonl"), - "utf8", - ); - expect(storedTranscript).toContain("transcript line 0"); - expect(storedTranscript).toContain("transcript line 2000"); + const stored = await storeFor(stateDir).readSession("long-meeting"); + expect(stored).toBeDefined(); + const storedTranscript = await storeFor(stateDir).readUtterancesForSession(stored!); + expect(storedTranscript[0]?.text).toContain("transcript line 0"); + expect(storedTranscript.at(-1)?.text).toContain("transcript line 2000"); }); it("requires date-qualified selectors for repeated stored session ids", async () => { const stateDir = await makeStateDir(); - const store = new TranscriptsStore(path.join(stateDir, "transcripts")); + const store = storeFor(stateDir); await store.writeSession({ sessionId: "standup", title: "Tuesday standup", @@ -618,17 +620,14 @@ describe("transcripts tool", () => { "utf8", ), ).resolves.toContain("publish the notes"); - await expect( - fs.readFile( - path.join(stateDir, "transcripts", currentDateDir(), "standup", "metadata.json"), - "utf8", - ), - ).resolves.toContain("providerStopError"); + await expect(storeFor(stateDir).readSession("standup")).resolves.toMatchObject({ + metadata: { providerStopError: "Discord voice manager is unavailable" }, + }); }); it("does not stop a current active session when summarizing an older dated duplicate", async () => { const stateDir = await makeStateDir(); - const store = new TranscriptsStore(path.join(stateDir, "transcripts")); + const store = storeFor(stateDir); const olderSession = { sessionId: "standup", title: "Older standup", @@ -745,12 +744,9 @@ describe("transcripts tool", () => { }, }); expect(request.startupWaitMs).toBe(30_000); - await expect( - fs.readFile( - path.join(stateDir, "transcripts", currentDateDir(), "standup", "metadata.json"), - "utf8", - ), - ).resolves.toContain("Standup"); + await expect(storeFor(stateDir).readSession("standup")).resolves.toMatchObject({ + title: "Standup", + }); await service.stop(); expect(stop).toHaveBeenCalledOnce(); }); diff --git a/src/agents/tools/transcripts-tool.ts b/src/agents/tools/transcripts-tool.ts index cc8fdf779d5f..1d59d3db573f 100644 --- a/src/agents/tools/transcripts-tool.ts +++ b/src/agents/tools/transcripts-tool.ts @@ -67,7 +67,9 @@ const TranscriptsSchema = Type.Object( ); function createStore(ctx: TranscriptsRuntimeContext): TranscriptsStore { - return new TranscriptsStore(path.join(ctx.stateDir, "transcripts")); + return new TranscriptsStore(path.join(ctx.stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: ctx.stateDir }, + }); } async function waitForPendingAutoStartsToSettle( @@ -92,28 +94,24 @@ async function waitForPendingAutoStartsToSettle( } } -// Summaries are persisted beside the session so stop/import/summarize actions -// return both model-readable details and a durable artifact path. +// Tool stop/import/summarize actions explicitly materialize artifacts, but a +// divergent export must not turn a successful canonical summary write into failure. async function summarizeAndPersist(params: { config: ReturnType; store: TranscriptsStore; session: TranscriptSessionDescriptor; - sessionDir?: string; }) { - const utterances = - params.sessionDir !== undefined - ? await params.store.readUtterancesFromSessionDir(params.sessionDir, { - maxUtterances: params.config.maxUtterances, - }) - : await params.store.readUtterancesForSession(params.session, { - maxUtterances: params.config.maxUtterances, - }); + const utterances = await params.store.readUtterancesForSession(params.session, { + maxUtterances: params.config.maxUtterances, + }); const summary = summarizeTranscripts({ session: params.session, utterances }); - const summaryPath = - params.sessionDir !== undefined - ? await params.store.writeSummaryToDir(summary, params.sessionDir) - : await params.store.writeSummary(summary, params.session); - return { summary, summaryPath }; + const intendedSummaryPath = await params.store.writeSummary(summary, params.session); + try { + const artifacts = await params.store.materializeSessionArtifacts(params.session, "all"); + return { summary, summaryPath: artifacts.summaryPath }; + } catch (error) { + return { summary, intendedSummaryPath, summaryExportError: String(error) }; + } } async function stopTranscripts(params: { @@ -127,9 +125,9 @@ async function stopTranscripts(params: { }); const directActive = activeSessions.get(sessionSelector); const resolvedEntry: TranscriptsSessionEntry | undefined = directActive - ? { session: directActive.session, sessionDir: params.store.sessionDir(directActive.session) } + ? undefined : await params.store.readSessionEntry(sessionSelector); - const resolvedSession = resolvedEntry?.session; + const resolvedSession = directActive?.session ?? resolvedEntry?.session; const activeCandidate = resolvedSession !== undefined ? activeSessions.get(resolvedSession.sessionId) : undefined; const activeMatchesResolved = @@ -188,18 +186,23 @@ async function stopTranscripts(params: { } else { await params.store.updateStopped(sessionSelector, stoppedAt); } - const { summaryPath, summary } = await summarizeAndPersist({ - config: resolveTranscriptsConfig(params.ctx.config?.transcripts), - store: params.store, - session: stoppedSession, - sessionDir: selectedActive ? undefined : resolvedEntry?.sessionDir, - }); - return toolText(`Transcripts stopped: ${sessionId}\nSummary: ${summaryPath}`, { - sessionId, - ...(providerStopError ? { providerStopError } : {}), - summary, - summaryPath, - }); + const { summaryPath, intendedSummaryPath, summary, summaryExportError } = + await summarizeAndPersist({ + config: resolveTranscriptsConfig(params.ctx.config?.transcripts), + store: params.store, + session: stoppedSession, + }); + return toolText( + `Transcripts stopped: ${sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`, + { + sessionId, + ...(providerStopError ? { providerStopError } : {}), + ...(summaryExportError ? { summaryExportError } : {}), + ...(intendedSummaryPath ? { intendedSummaryPath } : {}), + summary, + ...(summaryPath ? { summaryPath } : {}), + }, + ); } async function importTranscripts(params: { @@ -233,17 +236,23 @@ async function importTranscripts(params: { for (const utterance of utterances) { await params.store.appendUtteranceForSession(session, utterance); } - const { summaryPath, summary } = await summarizeAndPersist({ - config: resolveTranscriptsConfig(params.ctx.config?.transcripts), - store: params.store, - session, - }); - return toolText(`Transcript imported: ${session.sessionId}\nSummary: ${summaryPath}`, { - sessionId: session.sessionId, - utteranceCount: utterances.length, - summary, - summaryPath, - }); + const { summaryPath, intendedSummaryPath, summary, summaryExportError } = + await summarizeAndPersist({ + config: resolveTranscriptsConfig(params.ctx.config?.transcripts), + store: params.store, + session, + }); + return toolText( + `Transcript imported: ${session.sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`, + { + sessionId: session.sessionId, + utteranceCount: utterances.length, + ...(summaryExportError ? { summaryExportError } : {}), + ...(intendedSummaryPath ? { intendedSummaryPath } : {}), + summary, + ...(summaryPath ? { summaryPath } : {}), + }, + ); } async function summarizeExisting(params: { @@ -259,17 +268,22 @@ async function summarizeExisting(params: { if (!entry) { throw new Error(`transcripts session not found: ${sessionId}`); } - const { summaryPath, summary } = await summarizeAndPersist({ - config: params.config, - store: params.store, - session: entry.session, - sessionDir: entry.sessionDir, - }); - return toolText(`Transcripts summarized: ${sessionId}\nSummary: ${summaryPath}`, { - sessionId, - summary, - summaryPath, - }); + const { summaryPath, intendedSummaryPath, summary, summaryExportError } = + await summarizeAndPersist({ + config: params.config, + store: params.store, + session: entry.session, + }); + return toolText( + `Transcripts summarized: ${sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`, + { + sessionId, + ...(summaryExportError ? { summaryExportError } : {}), + ...(intendedSummaryPath ? { intendedSummaryPath } : {}), + summary, + ...(summaryPath ? { summaryPath } : {}), + }, + ); } async function statusTranscripts(ctx: TranscriptsRuntimeContext) { @@ -412,7 +426,7 @@ export function createTranscriptsAutoStartService(ctx: TranscriptsRuntimeContext if (!config.enabled || config.autoStart.length === 0) { return; } - const store = new TranscriptsStore(path.join(ctx.stateDir, "transcripts")); + const store = createStore(ctx); for (const entry of config.autoStart) { startEntry( { @@ -441,7 +455,7 @@ export function createTranscriptsAutoStartService(ctx: TranscriptsRuntimeContext }`, ); } - const store = new TranscriptsStore(path.join(ctx.stateDir, "transcripts")); + const store = createStore(ctx); for (const sessionId of startedSessionIds) { await stopTranscripts({ ctx, diff --git a/src/cli/program/register.transcripts.test.ts b/src/cli/program/register.transcripts.test.ts index 0412c56ad620..cc46298b41cc 100644 --- a/src/cli/program/register.transcripts.test.ts +++ b/src/cli/program/register.transcripts.test.ts @@ -1,9 +1,15 @@ -// Register transcripts tests cover transcript command registration and file handling. +// Transcripts CLI tests cover SQLite reads and explicit artifact materialization. import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../../state/openclaw-state-db.js"; import { manualTranscriptSourceProvider } from "../../transcripts/manual-source.js"; import type { TranscriptSessionDescriptor } from "../../transcripts/provider-types.js"; import { TranscriptsStore } from "../../transcripts/store.js"; @@ -16,32 +22,31 @@ async function makeStateDir(): Promise { return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcripts-cli-")); } +function storeFor(stateDir: string): TranscriptsStore { + return new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); +} + async function writeSession( stateDir: string, sessionId: string, date = "2026-05-22", ): Promise { - const sessionDir = path.join(stateDir, "transcripts", date, sessionId); - await fs.mkdir(sessionDir, { recursive: true }); - await fs.writeFile( - path.join(sessionDir, "metadata.json"), - `${JSON.stringify( - { - sessionId, - title: "Design review", - source: { providerId: "manual-transcript" }, - startedAt: `${date}T10:00:00.000Z`, - stoppedAt: `${date}T10:05:00.000Z`, - }, - null, - 2, - )}\n`, - ); - await fs.writeFile( - path.join(sessionDir, "summary.md"), - "# Design review\n\n## Action Items\n- Sam: Ship CLI\n", - ); - return sessionDir; + const session: TranscriptSessionDescriptor = { + sessionId, + title: "Design review", + source: { providerId: "manual-transcript" }, + startedAt: `${date}T10:00:00.000Z`, + stoppedAt: `${date}T10:05:00.000Z`, + }; + const store = storeFor(stateDir); + const utterance = { text: "Action item: Ship CLI", speaker: { label: "Sam" } }; + const utterances = [utterance]; + await store.writeSession(session); + await store.appendUtteranceForSession(session, utterance); + await store.writeSummary(summarizeTranscripts({ session, utterances }), session); + return store.sessionDir(session); } async function runTranscriptsCli(args: string[]): Promise { @@ -72,6 +77,7 @@ describe("transcripts CLI", () => { }); afterEach(() => { + closeOpenClawStateDatabaseForTest(); if (originalStateDir === undefined) { delete process.env.OPENCLAW_STATE_DIR; } else { @@ -86,7 +92,7 @@ describe("transcripts CLI", () => { expect(program.commands.map((command) => command.name())).toContain("transcripts"); }); - it("lists stored transcript sessions", async () => { + it("lists stored transcript sessions from SQLite", async () => { const sessionDir = await writeSession(stateDir, "design-review"); const output = await runTranscriptsCli(["list"]); @@ -96,29 +102,67 @@ describe("transcripts CLI", () => { expect(output).toContain(path.join(sessionDir, "summary.md")); }); - it("prints summary markdown for a session", async () => { - await writeSession(stateDir, "design-review"); + it("prints summary markdown and keeps its export current", async () => { + const sessionDir = await writeSession(stateDir, "design-review"); + await fs.rm(sessionDir, { recursive: true, force: true }); const output = await runTranscriptsCli(["show", "design-review"]); expect(output).toContain("# Design review"); expect(output).toContain("Ship CLI"); + expect(output.endsWith("\n")).toBe(true); + const jsonOutput = JSON.parse(await runTranscriptsCli(["show", "design-review", "--json"])) as { + summary: string; + }; + expect(jsonOutput.summary.endsWith("\n")).toBe(true); + await expect(fs.readFile(path.join(sessionDir, "summary.md"), "utf8")).resolves.toContain( + "Ship CLI", + ); }); - it("sanitizes summaries created before the upgrade at the show boundary", async () => { - const sessionDir = await writeSession(stateDir, "legacy-summary"); - await fs.writeFile( - path.join(sessionDir, "summary.md"), - "# Legacy\n\n- first\tcolumn\n- \u001b[2J\u001b[31mADMIN APPROVED\u001b[0m\n", + it("keeps JSON inspection available before a summary exists", async () => { + await storeFor(stateDir).writeSession({ + sessionId: "active-session", + source: { providerId: "manual-transcript" }, + startedAt: "2026-05-22T10:00:00.000Z", + }); + + const jsonOutput = await runTranscriptsCli(["show", "active-session", "--json"]); + + expect(JSON.parse(jsonOutput)).toMatchObject({ + session: { sessionId: "active-session" }, + summary: null, + }); + await expect(runTranscriptsCli(["show", "active-session"])).rejects.toThrow( + "summary.md not found", + ); + }); + + it("sanitizes stored summary control bytes at the show boundary", async () => { + await writeSession(stateDir, "legacy-summary"); + const database = openOpenClawStateDatabase({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const db = getNodeSqliteKysely< + Pick + >(database.db); + executeSqliteQuerySync( + database.db, + db + .updateTable("meeting_transcript_summaries") + .set({ + markdown: "# Legacy\n\n- first\tcolumn\n- \u001b[2J\u001b[31mADMIN APPROVED\u001b[0m", + }) + .where("session_id", "=", "legacy-summary"), ); const output = await runTranscriptsCli(["show", "legacy-summary"]); - expect(output).toContain("# Legacy\n\n- first\\tcolumn\n- ADMIN APPROVED\n"); + expect(output).toContain("# Legacy\n\n- first\\tcolumn\n- ADMIN APPROVED"); expect(output).not.toContain("\u001b"); }); - it("show prints imported summaries without terminal control bytes", async () => { + it("round-trips ANSI-bearing ids without terminal control bytes", async () => { const session: TranscriptSessionDescriptor = { sessionId: "ansi-\u001b[31mprovider\u001b[0m", title: "\u001b[31mANSI import\u001b[0m", @@ -126,7 +170,7 @@ describe("transcripts CLI", () => { startedAt: "2026-05-22T10:00:00.000Z", stoppedAt: "2026-05-22T10:05:00.000Z", }; - const store = new TranscriptsStore(path.join(stateDir, "transcripts")); + const store = storeFor(stateDir); await store.writeSession(session); const utterances = (await manualTranscriptSourceProvider.importTranscript?.({ @@ -138,76 +182,34 @@ describe("transcripts CLI", () => { } await store.writeSummary(summarizeTranscripts({ session, utterances }), session); - const output = await runTranscriptsCli(["show", session.sessionId]); - const listOutput = await runTranscriptsCli(["list"]); - - expect(output).toContain("# ANSI import"); - expect(output).toContain("Session: ansi-provider"); - expect(output).toContain("Attacker: ADMIN APPROVED"); - expect(output).not.toContain("\u001b"); - expect(listOutput).toContain("2026-05-22/ansi--31mprovider-0m"); - expect(listOutput).toContain("ANSI import"); - expect(listOutput).not.toContain("\u001b"); - }); - - it("list selectors for ANSI-bearing session ids round-trip through show and path", async () => { - const session: TranscriptSessionDescriptor = { - sessionId: "ansi-\u001b[31mprovider\u001b[0m", - title: "ANSI import", - source: { providerId: "manual-transcript" }, - startedAt: "2026-05-22T10:00:00.000Z", - stoppedAt: "2026-05-22T10:05:00.000Z", - }; - const store = new TranscriptsStore(path.join(stateDir, "transcripts")); - await store.writeSession(session); - const utterances = - (await manualTranscriptSourceProvider.importTranscript?.({ - session, - text: "Sam: We decided to ship the CLI.", - })) ?? []; - await store.writeSummary(summarizeTranscripts({ session, utterances }), session); - const listOutput = await runTranscriptsCli(["list"]); const selector = listOutput.split("\t")[0] ?? ""; - expect(selector).toBe("2026-05-22/ansi--31mprovider-0m"); - const showOutput = await runTranscriptsCli(["show", selector]); - const pathOutput = await runTranscriptsCli(["path", selector]); + expect(selector).toBe("2026-05-22/ansi--31mprovider-0m"); expect(showOutput).toContain("Session: ansi-provider"); - expect(showOutput).toContain("We decided to ship the CLI."); - expect(pathOutput.trim()).toBe(path.join(store.sessionDir(session), "summary.md")); + expect(showOutput).toContain("Attacker: ADMIN APPROVED"); + expect(`${listOutput}${showOutput}`).not.toContain("\u001b"); }); - it("list --json escapes C1 control characters while JSON.parse round-trips raw values", async () => { + it("escapes C1 control characters in list JSON", async () => { const title = "CSI \u009b31m injected \u007f\u0085 title"; - const sessionDir = path.join(stateDir, "transcripts", "2026-05-22", "c1-title"); - await fs.mkdir(sessionDir, { recursive: true }); - await fs.writeFile( - path.join(sessionDir, "metadata.json"), - JSON.stringify({ - sessionId: "c1-title", - title, - source: { providerId: "manual-transcript" }, - startedAt: "2026-05-22T10:00:00.000Z", - stoppedAt: "2026-05-22T10:05:00.000Z", - }), - ); + await storeFor(stateDir).writeSession({ + sessionId: "c1-title", + title, + source: { providerId: "manual-transcript" }, + startedAt: "2026-05-22T10:00:00.000Z", + }); const output = await runTranscriptsCli(["list", "--json"]); - const bytes = Buffer.from(output, "utf8"); - expect(bytes.includes(Buffer.from([0xc2, 0x9b]))).toBe(false); - expect(bytes.includes(0x7f)).toBe(false); expect(/[\u007f-\u009f]/.test(output)).toBe(false); expect(output).toContain("\\u009b"); const parsed = JSON.parse(output) as Array<{ sessionId: string; title: string }>; - expect(parsed).toHaveLength(1); - expect(parsed[0]?.sessionId).toBe("c1-title"); - expect(parsed[0]?.title).toBe(title); + expect(parsed).toEqual([expect.objectContaining({ sessionId: "c1-title", title })]); }); - it("ignores unrelated corrupt metadata while reading a valid session", async () => { + it("ignores unrelated corrupt export files", async () => { await writeSession(stateDir, "design-review"); const corruptDir = path.join(stateDir, "transcripts", "corrupt"); await fs.mkdir(corruptDir, { recursive: true }); @@ -221,7 +223,7 @@ describe("transcripts CLI", () => { expect(showOutput).toContain("# Design review"); }); - it("requires date-qualified selectors for repeated human session ids", async () => { + it("requires date-qualified selectors for repeated ids", async () => { const olderSessionDir = await writeSession(stateDir, "standup", "2026-05-21"); await writeSession(stateDir, "standup", "2026-05-22"); @@ -233,11 +235,22 @@ describe("transcripts CLI", () => { expect(output.trim()).toBe(path.join(olderSessionDir, "summary.md")); }); - it("prints the summary path by default", async () => { + it("materializes metadata, transcript, and directory exports from SQLite", async () => { const sessionDir = await writeSession(stateDir, "design-review"); + await fs.rm(sessionDir, { recursive: true, force: true }); - const output = await runTranscriptsCli(["path", "design-review"]); + const metadataOutput = await runTranscriptsCli(["path", "design-review", "--metadata"]); + const transcriptOutput = await runTranscriptsCli(["path", "design-review", "--transcript"]); + const dirOutput = await runTranscriptsCli(["path", "design-review", "--dir"]); - expect(output.trim()).toBe(path.join(sessionDir, "summary.md")); + expect(metadataOutput.trim()).toBe(path.join(sessionDir, "metadata.json")); + expect(transcriptOutput.trim()).toBe(path.join(sessionDir, "transcript.jsonl")); + expect(dirOutput.trim()).toBe(sessionDir); + await expect(fs.readFile(path.join(sessionDir, "metadata.json"), "utf8")).resolves.toContain( + '"sessionId": "design-review"', + ); + await expect(fs.readFile(path.join(sessionDir, "transcript.jsonl"), "utf8")).resolves.toContain( + '"text":"Action item: Ship CLI"', + ); }); }); diff --git a/src/cli/program/register.transcripts.ts b/src/cli/program/register.transcripts.ts index d35a2e3f30c5..16c307a663e7 100644 --- a/src/cli/program/register.transcripts.ts +++ b/src/cli/program/register.transcripts.ts @@ -1,12 +1,13 @@ -// `openclaw transcripts`: local state inspector for stored transcript metadata and summaries. -import type { Dirent } from "node:fs"; -import fs from "node:fs/promises"; +// `openclaw transcripts`: SQLite-backed transcript inspector and artifact exporter. import path from "node:path"; import type { Command } from "commander"; import { sanitizeTerminalText } from "../../../packages/terminal-core/src/safe-text.js"; import { resolveStateDir } from "../../config/paths.js"; -import { formatErrorMessage } from "../../infra/errors.js"; -import type { TranscriptSessionDescriptor } from "../../transcripts/provider-types.js"; +import { + TranscriptsStore, + type TranscriptArtifactKind, + type TranscriptsSessionEntry, +} from "../../transcripts/store.js"; type TranscriptsCliOptions = { json?: boolean; @@ -18,54 +19,11 @@ type TranscriptsPathOptions = TranscriptsCliOptions & { transcript?: boolean; }; -type StoredTranscriptsSession = { - session: TranscriptSessionDescriptor; - sessionDir: string; - date: string; - summaryPath: string; - hasSummary: boolean; -}; - -const TRANSCRIPTS_STATE_SUBDIR = "transcripts"; - -function safeSegment(value: string): string { - return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session"; -} - -function stateRootDir(): string { - return path.join(resolveStateDir(), TRANSCRIPTS_STATE_SUBDIR); -} - -function dateFromSessionId(sessionId: string): string | undefined { - return sessionId - .match(/^transcript-(\d{4})-(\d{2})-(\d{2})T/) - ?.slice(1, 4) - .join("-"); -} - -function sessionDir(date: string, sessionId: string): string { - return path.join(stateRootDir(), date, safeSegment(sessionId)); -} - -// Selectors are date-qualified when duplicate session ids can exist across transcript days. -function readDateFromSessionDir(sessionDirValue: string): string { - const candidate = path.basename(path.dirname(sessionDirValue)); - if (!/^\d{4}-\d{2}-\d{2}$/.test(candidate)) { - throw new Error(`invalid transcripts date directory: ${candidate}`); - } - return candidate; -} - -function formatSelector(entry: StoredTranscriptsSession): string { - return `${entry.date}/${safeSegment(entry.session.sessionId)}`; -} - -function parseQualifiedSelector(selector: string): { date: string; sessionId: string } | null { - const match = selector.match(/^(\d{4}-\d{2}-\d{2})\/(.+)$/); - if (!match?.[1] || !match[2]) { - return null; - } - return { date: match[1], sessionId: match[2] }; +function createStore(): TranscriptsStore { + const stateDir = resolveStateDir(); + return new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); } function writeLine(value: string): void { @@ -81,163 +39,33 @@ function writeJson(value: unknown): void { ); } -function isNodeError(err: unknown, code: string): boolean { - return Boolean(err && typeof err === "object" && "code" in err && err.code === code); -} - -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch (err) { - if (isNodeError(err, "ENOENT")) { - return false; - } - throw err; - } -} - -async function readJsonFile(filePath: string): Promise { - return JSON.parse(await fs.readFile(filePath, "utf8")) as T; -} - -async function readStoredSession( - sessionDirLocal: string, - options: { ignoreInvalid?: boolean } = {}, -): Promise { - const metadataPath = path.join(sessionDirLocal, "metadata.json"); - try { - const session = await readJsonFile(metadataPath); - const summaryPath = path.join(sessionDirLocal, "summary.md"); - return { - session, - sessionDir: sessionDirLocal, - date: readDateFromSessionDir(sessionDirLocal), - summaryPath, - hasSummary: await pathExists(summaryPath), - }; - } catch (err) { - if (isNodeError(err, "ENOENT")) { - return null; - } - if (options.ignoreInvalid) { - return null; - } - throw new Error(`invalid transcripts metadata at ${metadataPath}: ${formatErrorMessage(err)}`, { - cause: err, - }); - } -} - -async function listStoredSessionDirs(): Promise { - let entries: Dirent[]; - try { - entries = await fs.readdir(stateRootDir(), { withFileTypes: true }); - } catch (err) { - if (isNodeError(err, "ENOENT")) { - return []; - } - throw err; - } - const dirs: string[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) { - continue; - } - const firstLevelDir = path.join(stateRootDir(), entry.name); - if (!/^\d{4}-\d{2}-\d{2}$/.test(entry.name)) { - continue; - } - const nestedEntries = await fs.readdir(firstLevelDir, { withFileTypes: true }); - dirs.push( - ...nestedEntries - .filter((nestedEntry) => nestedEntry.isDirectory()) - .map((nestedEntry) => path.join(firstLevelDir, nestedEntry.name)), - ); - } - return dirs; -} - -function assertRequestedSession( - entry: StoredTranscriptsSession, - sessionId: string, -): StoredTranscriptsSession { - if (entry.session.sessionId !== sessionId && safeSegment(entry.session.sessionId) !== sessionId) { - throw new Error( - `transcripts metadata mismatch for ${sessionId}: found ${entry.session.sessionId}`, - ); - } - return entry; -} - -async function requireStoredSession(selector: string): Promise { - const qualified = parseQualifiedSelector(selector); - if (qualified) { - const session = await readStoredSession(sessionDir(qualified.date, qualified.sessionId)); - if (!session) { - throw new Error(`transcripts session not found: ${selector}`); - } - return assertRequestedSession(session, qualified.sessionId); - } - - const idDate = dateFromSessionId(selector); - const session = idDate ? await readStoredSession(sessionDir(idDate, selector)) : null; - if (session) { - return assertRequestedSession(session, selector); - } - const sessions = await listStoredSessions(); - const matches = sessions.filter( - (entry) => - entry.session.sessionId === selector || safeSegment(entry.session.sessionId) === selector, - ); - if (matches.length === 1 && matches[0]) { - return assertRequestedSession(matches[0], selector); - } - if (matches.length > 1) { - throw new Error( - `multiple transcripts sessions match ${selector}; use one of: ${matches - .map(formatSelector) - .join(", ")}`, - ); - } - throw new Error(`transcripts session not found: ${selector}`); -} - -async function listStoredSessions(): Promise { - const dirs = await listStoredSessionDirs(); - const sessions = await Promise.all( - dirs.map((dir) => - readStoredSession(dir, { - ignoreInvalid: true, - }), - ), - ); - return sessions - .filter((session): session is StoredTranscriptsSession => session !== null) - .toSorted((left, right) => - (right.session.startedAt ?? "").localeCompare(left.session.startedAt ?? ""), - ); -} - -function formatSessionLine(entry: StoredTranscriptsSession): string { - const title = sanitizeTerminalText(entry.session.title?.trim() || "Transcripts"); - const started = sanitizeTerminalText(entry.session.startedAt || "unknown"); - const summary = sanitizeTerminalText(entry.hasSummary ? entry.summaryPath : "no summary.md"); - return `${formatSelector(entry)}\t${started}\t${title}\t${summary}`; -} - function sanitizeMarkdownForTerminal(markdown: string): string { return markdown.split("\n").map(sanitizeTerminalText).join("\n"); } +function formatSessionLine(entry: TranscriptsSessionEntry): string { + const title = sanitizeTerminalText(entry.session.title?.trim() || "Transcripts"); + const started = sanitizeTerminalText(entry.session.startedAt || "unknown"); + const summary = sanitizeTerminalText(entry.hasSummary ? entry.summaryPath : "no summary.md"); + return `${entry.selector}\t${started}\t${title}\t${summary}`; +} + +async function requireStoredSession(selector: string): Promise { + const session = await createStore().readSessionEntry(selector); + if (!session) { + throw new Error(`transcripts session not found: ${selector}`); + } + return session; +} + async function listCommand(options: TranscriptsCliOptions): Promise { - const sessions = await listStoredSessions(); + const sessions = await createStore().listSessionEntries(); if (options.json) { writeJson( sessions.map((entry) => ({ sessionId: entry.session.sessionId, - selector: formatSelector(entry), - date: entry.date, + selector: entry.selector, + date: entry.selector.slice(0, 10), title: entry.session.title, startedAt: entry.session.startedAt, stoppedAt: entry.session.stoppedAt, @@ -258,47 +86,76 @@ async function listCommand(options: TranscriptsCliOptions): Promise { } } -async function showCommand(sessionId: string, options: TranscriptsCliOptions): Promise { - const session = await requireStoredSession(sessionId); +async function showCommand(sessionSelector: string, options: TranscriptsCliOptions): Promise { + const store = createStore(); + const entry = await store.readSessionEntry(sessionSelector); + if (!entry) { + throw new Error(`transcripts session not found: ${sessionSelector}`); + } + const storedSummary = await store.readSummary(entry.session); + const materializedMarkdown = + storedSummary.markdown === undefined + ? undefined + : storedSummary.markdown.endsWith("\n") + ? storedSummary.markdown + : `${storedSummary.markdown}\n`; + // `show` is an explicit export boundary: keep the shipped summary path current. + await store.materializeSessionArtifacts(entry.session, "summary"); if (options.json) { - const summary = session.hasSummary ? await fs.readFile(session.summaryPath, "utf8") : null; writeJson({ - session: session.session, - selector: formatSelector(session), - path: session.sessionDir, - summaryPath: session.summaryPath, - summary, + session: entry.session, + selector: entry.selector, + path: entry.sessionDir, + summaryPath: entry.summaryPath, + summary: materializedMarkdown ?? null, }); return; } - if (!session.hasSummary) { - throw new Error(`summary.md not found for transcripts session: ${sessionId}`); + if (materializedMarkdown === undefined) { + throw new Error(`summary.md not found for transcripts session: ${sessionSelector}`); } - process.stdout.write(sanitizeMarkdownForTerminal(await fs.readFile(session.summaryPath, "utf8"))); + process.stdout.write(sanitizeMarkdownForTerminal(materializedMarkdown)); +} + +function selectedArtifactKind(options: TranscriptsPathOptions): TranscriptArtifactKind { + if (options.dir) { + return "all"; + } + if (options.metadata) { + return "metadata"; + } + if (options.transcript) { + return "transcript"; + } + return "summary"; } async function pathCommand(selector: string, options: TranscriptsPathOptions): Promise { - const session = await requireStoredSession(selector); + const store = createStore(); + const entry = await requireStoredSession(selector); + const kind = selectedArtifactKind(options); + const artifacts = await store.materializeSessionArtifacts(entry.session, kind); const selectedPath = options.dir - ? session.sessionDir + ? artifacts.sessionDir : options.metadata - ? path.join(session.sessionDir, "metadata.json") + ? artifacts.metadataPath : options.transcript - ? path.join(session.sessionDir, "transcript.jsonl") - : session.summaryPath; + ? artifacts.transcriptPath + : artifacts.summaryPath; + const exists = kind !== "summary" || artifacts.hasSummary; if (options.json) { writeJson({ - sessionId: session.session.sessionId, - selector: formatSelector(session), + sessionId: entry.session.sessionId, + selector: entry.selector, path: selectedPath, - exists: await pathExists(selectedPath), + exists, }); return; } writeLine(selectedPath); } -/** Register transcript list/show/path inspection commands. */ +/** Register transcript list/show/path inspection and export commands. */ export function registerTranscriptsCli(program: Command): void { const transcripts = program.command("transcripts").description("Inspect stored transcripts"); @@ -312,7 +169,7 @@ export function registerTranscriptsCli(program: Command): void { transcripts .command("show") - .description("Print a transcript summary markdown file") + .description("Print and materialize a transcript summary") .argument("", "Transcripts session id or YYYY-MM-DD/session selector") .option("--json", "Print JSON") .action(async (sessionId: string, options: TranscriptsCliOptions) => { @@ -321,11 +178,11 @@ export function registerTranscriptsCli(program: Command): void { transcripts .command("path") - .description("Print a stored transcripts artifact path") + .description("Materialize and print a stored transcripts artifact path") .argument("", "Transcripts session id or YYYY-MM-DD/session selector") - .option("--dir", "Print the session directory") - .option("--metadata", "Print metadata.json") - .option("--transcript", "Print transcript.jsonl") + .option("--dir", "Materialize all artifacts and print the session directory") + .option("--metadata", "Materialize and print metadata.json") + .option("--transcript", "Materialize and print transcript.jsonl") .option("--json", "Print JSON") .action(async (sessionId: string, options: TranscriptsPathOptions) => { await pathCommand(sessionId, options); diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index def3f0ba260b..768ae05ea06d 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -75,6 +75,10 @@ import { detectLegacyMcpOAuthStores, migrateLegacyMcpOAuthStores, } from "./state-migrations.mcp-oauth.js"; +import { + detectLegacyMeetingTranscripts, + migrateLegacyMeetingTranscripts, +} from "./state-migrations.meeting-transcripts.js"; import { mergeNotices } from "./state-migrations.messages.js"; import { detectLegacyNodeHostConfig, @@ -461,6 +465,11 @@ export async function detectLegacyStateMigrations(params: { stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); + const meetingTranscripts = detectLegacyMeetingTranscripts({ + stateDir, + env, + doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, + }); const restartSentinel = detectLegacyRestartSentinel({ stateDir }); const workspace = detectLegacyWorkspaceState({ cfg: params.cfg, @@ -657,6 +666,9 @@ export async function detectLegacyStateMigrations(params: { if (mcpOauth.hasLegacy) { preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state"); } + if (meetingTranscripts.hasLegacy) { + preview.push("- Meeting transcripts: legacy JSON/JSONL files → shared SQLite state"); + } if (restartSentinel.hasLegacy) { preview.push("- Restart sentinel: legacy JSON → shared SQLite state"); } @@ -767,6 +779,7 @@ export async function detectLegacyStateMigrations(params: { deviceAuth, deviceIdentity, mcpOauth, + meetingTranscripts, restartSentinel, workspace, webPush, @@ -1046,6 +1059,12 @@ export async function runLegacyStateMigrations(params: { env, stateDir: detected.stateDir, }); + const meetingTranscripts = await migrateLegacyMeetingTranscripts({ + detected: detected.meetingTranscripts, + env, + stateDir: detected.stateDir, + now, + }); const restartSentinel = await migrateLegacyRestartSentinel({ detected: detected.restartSentinel, env, @@ -1113,6 +1132,7 @@ export async function runLegacyStateMigrations(params: { deviceAuth, deviceIdentity, mcpOauth, + meetingTranscripts, restartSentinel, workspace, webPush, @@ -1142,6 +1162,7 @@ export async function runLegacyStateMigrations(params: { ...deviceAuth.changes, ...deviceIdentity.changes, ...mcpOauth.changes, + ...meetingTranscripts.changes, ...restartSentinel.changes, ...workspace.changes, ...webPush.changes, @@ -1178,6 +1199,7 @@ export async function runLegacyStateMigrations(params: { ...deviceAuth.warnings, ...deviceIdentity.warnings, ...mcpOauth.warnings, + ...meetingTranscripts.warnings, ...restartSentinel.warnings, ...workspace.warnings, ...webPush.warnings, @@ -1325,6 +1347,12 @@ export async function autoMigrateLegacyState(params: { stateDir: detected.stateDir, doctorOnlyStateMigrations: params.doctorOnlyStateMigrations, }); + const meetingTranscripts = await migrateLegacyMeetingTranscripts({ + detected: detected.meetingTranscripts, + env, + stateDir: detected.stateDir, + now: params.now, + }); const hasCustomAgentDir = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim(); if (hasCustomAgentDir) { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ @@ -1434,6 +1462,7 @@ export async function autoMigrateLegacyState(params: { updateCheck, deviceAuth, deviceIdentity, + meetingTranscripts, restartSentinel, pluginPlans, ]; @@ -1458,6 +1487,7 @@ export async function autoMigrateLegacyState(params: { currentConversationBindings.changes.length > 0 || deviceAuth.changes.length > 0 || deviceIdentity.changes.length > 0 || + meetingTranscripts.changes.length > 0 || restartSentinel.changes.length > 0 || channelPairing.changes.length > 0 || preSessionChannelPlans.changes.length > 0 || @@ -1497,6 +1527,7 @@ export async function autoMigrateLegacyState(params: { ...acpSessionMetadata.changes, ...deviceAuth.changes, ...deviceIdentity.changes, + ...meetingTranscripts.changes, ]; const warnings = [ ...stateDirResult.warnings, @@ -1507,6 +1538,7 @@ export async function autoMigrateLegacyState(params: { ...acpSessionMetadata.warnings, ...deviceAuth.warnings, ...deviceIdentity.warnings, + ...meetingTranscripts.warnings, ]; const notices = [ ...(stateDirResult.notices ?? []), @@ -1523,7 +1555,8 @@ export async function autoMigrateLegacyState(params: { orphanKeys.changes.length > 0 || acpSessionMetadata.changes.length > 0 || deviceAuth.changes.length > 0 || - deviceIdentity.changes.length > 0, + deviceIdentity.changes.length > 0 || + meetingTranscripts.changes.length > 0, skipped: false, changes, warnings, @@ -1616,6 +1649,7 @@ export async function autoMigrateLegacyState(params: { ...currentConversationBindings.changes, ...deviceAuth.changes, ...deviceIdentity.changes, + ...meetingTranscripts.changes, ...restartSentinel.changes, ...channelPairing.changes, ...preSessionChannelPlans.changes, @@ -1644,6 +1678,7 @@ export async function autoMigrateLegacyState(params: { ...currentConversationBindings.warnings, ...deviceAuth.warnings, ...deviceIdentity.warnings, + ...meetingTranscripts.warnings, ...restartSentinel.warnings, ...channelPairing.warnings, ...preSessionChannelPlans.warnings, @@ -1660,6 +1695,7 @@ export async function autoMigrateLegacyState(params: { updateCheck, deviceAuth, deviceIdentity, + meetingTranscripts, restartSentinel, pluginPlans, ]; diff --git a/src/infra/state-migrations.meeting-transcripts-database.ts b/src/infra/state-migrations.meeting-transcripts-database.ts new file mode 100644 index 000000000000..2a57ec50c305 --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts-database.ts @@ -0,0 +1,16 @@ +import type { DatabaseSync } from "node:sqlite"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { getNodeSqliteKysely } from "./kysely-sync.js"; + +type MeetingTranscriptMigrationDatabase = Pick< + OpenClawStateKyselyDatabase, + | "meeting_transcript_sessions" + | "meeting_transcript_summaries" + | "meeting_transcript_utterances" + | "migration_runs" + | "migration_sources" +>; + +export function migrationDb(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} diff --git a/src/infra/state-migrations.meeting-transcripts-detection.ts b/src/infra/state-migrations.meeting-transcripts-detection.ts new file mode 100644 index 000000000000..672d14e68b1d --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts-detection.ts @@ -0,0 +1,247 @@ +// Doctor detection for legacy meeting transcript files and interrupted imports. +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + hasMatchingRecordedTranscriptArtifact, + isRecordedCanonicalTranscriptExport, +} from "./state-migrations.meeting-transcripts-files.js"; +import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js"; + +type MeetingTranscriptExportOwnership = { + selector: string; + sessionId: string; + startedAt: string; + manifest: Record; + pending: ReadonlySet; +}; + +type MeetingTranscriptMigrationDetectionState = { + exportOwnership: Map; + exportOwnershipByFoldedSelector: Map; + pendingImportCount: number; +}; + +const TRANSCRIPT_ARTIFACT_NAMES = new Set([ + "metadata.json", + "summary.json", + "summary.md", + "transcript.jsonl", +]); + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function hasLegacyArtifactsSync(directory: string): boolean { + const entries = fs.readdirSync(directory, { withFileTypes: true }); + let found = false; + for (const entry of entries) { + if (!TRANSCRIPT_ARTIFACT_NAMES.has(entry.name.toLowerCase())) { + continue; + } + const stat = fs.lstatSync(path.join(directory, entry.name)); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`legacy transcript source must be a regular file: ${directory}`); + } + found = true; + } + return found; +} + +export function resolveMeetingTranscriptExportOwnership(params: { + state: MeetingTranscriptMigrationDetectionState; + selector: string; + sessionDir: string; + sourceRoot: string; +}): MeetingTranscriptExportOwnership | undefined { + const exact = params.state.exportOwnership.get(params.selector); + if (exact) { + return exact; + } + const folded = params.state.exportOwnershipByFoldedSelector.get(params.selector.toLowerCase()); + if (!folded || folded.length === 0) { + return undefined; + } + try { + const metadataEntries = fs + .readdirSync(params.sessionDir, { withFileTypes: true }) + .filter((entry) => entry.name.toLowerCase() === "metadata.json"); + if (metadataEntries.length > 0) { + if (metadataEntries.length !== 1) { + return undefined; + } + const metadataPath = path.join(params.sessionDir, metadataEntries[0]!.name); + const stat = fs.lstatSync(metadataPath); + if (stat.isSymbolicLink() || !stat.isFile()) { + return undefined; + } + const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf8")) as { + sessionId?: unknown; + startedAt?: unknown; + }; + const matches = folded.filter( + (ownership) => + metadata.sessionId === ownership.sessionId && metadata.startedAt === ownership.startedAt, + ); + return matches.length === 1 ? matches[0] : undefined; + } + } catch { + return undefined; + } + const manifestMatches = folded.filter((ownership) => { + try { + const canonicalDir = path.join(params.sourceRoot, ownership.selector); + const canonicalStat = fs.statSync(canonicalDir); + const observedStat = fs.statSync(params.sessionDir); + if (canonicalStat.dev !== observedStat.dev || canonicalStat.ino !== observedStat.ino) { + return false; + } + return hasMatchingRecordedTranscriptArtifact({ + sessionDir: params.sessionDir, + manifest: ownership.manifest, + }); + } catch { + return false; + } + }); + return manifestMatches.length === 1 ? manifestMatches[0] : undefined; +} + +export function detectLegacyMeetingTranscripts(params: { + stateDir: string; + env?: NodeJS.ProcessEnv; + doctorOnlyStateMigrations?: boolean; +}): LegacyMeetingTranscriptsDetection { + const sourceDir = path.join(params.stateDir, "transcripts"); + if (params.doctorOnlyStateMigrations !== true) { + return { sourceDir, hasLegacy: false, pendingImportCount: 0 }; + } + const databaseState = readMeetingTranscriptMigrationDetectionState({ + env: { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir }, + }); + const pendingImportCount = databaseState.pendingImportCount; + try { + const rootStat = fs.lstatSync(sourceDir); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + throw new Error(`meeting transcript root must be a regular directory: ${sourceDir}`); + } + const dateEntries = fs.readdirSync(sourceDir, { withFileTypes: true }); + const sourceSelectors = hasLegacyArtifactsSync(sourceDir) ? ["."] : []; + for (const dateEntry of dateEntries) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(dateEntry.name)) { + continue; + } + if (dateEntry.isSymbolicLink()) { + throw new Error(`legacy transcript date directory cannot be a symlink: ${dateEntry.name}`); + } + if (!dateEntry.isDirectory()) { + continue; + } + const dateDir = path.join(sourceDir, dateEntry.name); + if (hasLegacyArtifactsSync(dateDir)) { + sourceSelectors.push(dateEntry.name); + } + for (const entry of fs.readdirSync(dateDir, { withFileTypes: true })) { + if (entry.isSymbolicLink()) { + throw new Error(`legacy transcript session cannot be a symlink: ${entry.name}`); + } + if (entry.isDirectory() && hasLegacyArtifactsSync(path.join(dateDir, entry.name))) { + sourceSelectors.push(`${dateEntry.name}/${entry.name}`); + } + } + } + const hasSource = sourceSelectors.some((selector) => { + const ownership = resolveMeetingTranscriptExportOwnership({ + state: databaseState, + selector, + sessionDir: path.join(sourceDir, selector), + sourceRoot: sourceDir, + }); + return ( + !ownership || + !isRecordedCanonicalTranscriptExport({ + sessionDir: path.join(sourceDir, selector), + manifest: ownership.manifest, + pending: ownership.pending, + }) + ); + }); + return { + sourceDir, + hasLegacy: hasSource || pendingImportCount > 0, + pendingImportCount, + }; + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return { sourceDir, hasLegacy: pendingImportCount > 0, pendingImportCount }; + } + throw error; + } +} + +export function readMeetingTranscriptMigrationDetectionState(params: { + env: NodeJS.ProcessEnv; +}): MeetingTranscriptMigrationDetectionState { + const databasePath = resolveOpenClawStateSqlitePath(params.env); + if (!fs.existsSync(databasePath)) { + return { + exportOwnership: new Map(), + exportOwnershipByFoldedSelector: new Map(), + pendingImportCount: 0, + }; + } + const database = new DatabaseSync(databasePath, { readOnly: true }); + try { + const tables = new Set( + database + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name IN ('meeting_transcript_sessions', 'migration_sources')", + ) + .all() + .map((row) => String(row.name)), + ); + const exportOwnership = new Map(); + const exportOwnershipByFoldedSelector = new Map(); + if (tables.has("meeting_transcript_sessions")) { + const rows = database + .prepare( + "SELECT session_id, started_at, selector, export_manifest_json, export_pending_json FROM meeting_transcript_sessions", + ) + .all(); + for (const row of rows) { + const selector = String(row.selector); + const parsed = JSON.parse(String(row.export_manifest_json)) as unknown; + if (isRecord(parsed)) { + const ownership = { + selector, + sessionId: String(row.session_id), + startedAt: String(row.started_at), + manifest: parsed as Record, + pending: new Set(JSON.parse(String(row.export_pending_json)) as string[]), + } satisfies MeetingTranscriptExportOwnership; + exportOwnership.set(selector, ownership); + const foldedSelector = selector.toLowerCase(); + const foldedOwners = exportOwnershipByFoldedSelector.get(foldedSelector) ?? []; + foldedOwners.push(ownership); + exportOwnershipByFoldedSelector.set(foldedSelector, foldedOwners); + } + } + } + const pendingRow = tables.has("migration_sources") + ? (database + .prepare( + "SELECT COUNT(*) AS count FROM migration_sources WHERE migration_kind = ? AND status = ? AND removed_source = 0", + ) + .get("meeting-transcripts-files-v1", "imported") as { count?: unknown } | undefined) + : undefined; + return { + exportOwnership, + exportOwnershipByFoldedSelector, + pendingImportCount: typeof pendingRow?.count === "number" ? pendingRow.count : 0, + }; + } finally { + database.close(); + } +} diff --git a/src/infra/state-migrations.meeting-transcripts-files.ts b/src/infra/state-migrations.meeting-transcripts-files.ts new file mode 100644 index 000000000000..eb5876e104bd --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts-files.ts @@ -0,0 +1,714 @@ +// Filesystem preflight and archive helpers for legacy meeting transcripts. +import { createHash } from "node:crypto"; +import fsSync, { createReadStream } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { createInterface } from "node:readline"; +import { DatabaseSync } from "node:sqlite"; +import type { + TranscriptSessionDescriptor, + TranscriptUtterance, +} from "../transcripts/provider-types.js"; +import type { TranscriptsSummary } from "../transcripts/summary.js"; +import { renderTranscriptsMarkdown } from "../transcripts/summary.js"; +import { sha256File, sha256Hex } from "./crypto-digest.js"; +import { assertNoSymlinkParents } from "./fs-safe-advanced.js"; + +const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([ + "metadata.json", + "summary.json", + "summary.md", + "transcript.jsonl", +]); + +export const LEGACY_UTTERANCE_INSERT_CHUNK_SIZE = 64; +const LEGACY_UTTERANCE_STAGE_BATCH_SIZE = 256; + +export type LegacyMeetingTranscriptSnapshot = { + sourceDir: string; + relativeDir: string; + stageKey: string; + session: TranscriptSessionDescriptor; + utteranceCount: number; + summary?: TranscriptsSummary; + markdown?: string; + sourceHash: string; + sourceSizeBytes: number; +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function sha256FileSync(filePath: string): string { + const digest = createHash("sha256"); + const descriptor = fsSync.openSync(filePath, "r"); + const buffer = Buffer.allocUnsafe(64 * 1024); + try { + while (true) { + const bytesRead = fsSync.readSync(descriptor, buffer, 0, buffer.length, null); + if (bytesRead === 0) { + break; + } + digest.update(buffer.subarray(0, bytesRead)); + } + } finally { + fsSync.closeSync(descriptor); + } + return digest.digest("hex"); +} + +export function isRecordedCanonicalTranscriptExport(params: { + sessionDir: string; + manifest: Readonly>; + pending?: ReadonlySet; +}): boolean { + const entries = fsSync.readdirSync(params.sessionDir, { withFileTypes: true }); + for (const entry of entries) { + const canonicalName = entry.name.toLowerCase(); + if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName)) { + continue; + } + const filePath = path.join(params.sessionDir, entry.name); + const stat = fsSync.lstatSync(filePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + return false; + } + const expectedHash = params.manifest[canonicalName]; + if ( + params.pending?.has(canonicalName) !== true && + (!expectedHash || sha256FileSync(filePath) !== expectedHash) + ) { + return false; + } + } + return true; +} + +export function hasMatchingRecordedTranscriptArtifact(params: { + sessionDir: string; + manifest: Readonly>; +}): boolean { + for (const entry of fsSync.readdirSync(params.sessionDir, { withFileTypes: true })) { + const canonicalName = entry.name.toLowerCase(); + const expectedHash = params.manifest[canonicalName]; + if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName) || !expectedHash) { + continue; + } + const filePath = path.join(params.sessionDir, entry.name); + const stat = fsSync.lstatSync(filePath); + if (!stat.isSymbolicLink() && stat.isFile() && sha256FileSync(filePath) === expectedHash) { + return true; + } + } + return false; +} + +export async function validateMeetingTranscriptRoot( + rootDir: string, + options: { allowMissing?: boolean } = {}, +): Promise { + try { + const stat = await fs.lstat(rootDir); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`meeting transcript root must be a regular directory: ${rootDir}`); + } + return true; + } catch (error) { + if (isRecord(error) && error.code === "ENOENT" && options.allowMissing === true) { + return false; + } + throw error; + } +} + +function parseSession(value: unknown, sourcePath: string): TranscriptSessionDescriptor { + if (!isRecord(value) || typeof value.sessionId !== "string" || !value.sessionId) { + throw new Error(`invalid transcripts metadata sessionId at ${sourcePath}`); + } + if (typeof value.startedAt !== "string" || !value.startedAt) { + throw new Error(`invalid transcripts metadata startedAt at ${sourcePath}`); + } + if (!isRecord(value.source) || typeof value.source.providerId !== "string") { + throw new Error(`invalid transcripts metadata source at ${sourcePath}`); + } + if (value.title !== undefined && typeof value.title !== "string") { + throw new Error(`invalid transcripts metadata title at ${sourcePath}`); + } + if (value.stoppedAt !== undefined && typeof value.stoppedAt !== "string") { + throw new Error(`invalid transcripts metadata stoppedAt at ${sourcePath}`); + } + if (value.metadata !== undefined && !isRecord(value.metadata)) { + throw new Error(`invalid transcripts metadata payload at ${sourcePath}`); + } + return value as TranscriptSessionDescriptor; +} + +function parseUtterance( + value: unknown, + sourcePath: string, + lineNumber: number, +): TranscriptUtterance { + if (!isRecord(value) || typeof value.text !== "string") { + throw new Error(`invalid transcript utterance at ${sourcePath}:${lineNumber}`); + } + if ( + value.speaker !== undefined && + (!isRecord(value.speaker) || typeof value.speaker.label !== "string") + ) { + throw new Error(`invalid transcript speaker at ${sourcePath}:${lineNumber}`); + } + return value as TranscriptUtterance; +} + +function parseSummary(value: unknown, sourcePath: string): TranscriptsSummary { + if ( + !isRecord(value) || + typeof value.sessionId !== "string" || + typeof value.title !== "string" || + typeof value.generatedAt !== "string" || + typeof value.overview !== "string" || + !Array.isArray(value.transcript) || + !Array.isArray(value.decisions) || + !Array.isArray(value.actionItems) || + !Array.isArray(value.risks) || + !Number.isSafeInteger(value.utteranceCount) || + (value.utteranceCount as number) < 0 + ) { + throw new Error(`invalid transcripts summary at ${sourcePath}`); + } + return value as unknown as TranscriptsSummary; +} + +function legacyTranscriptRelativeDir(session: TranscriptSessionDescriptor): string { + const date = session.startedAt.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1]; + if (!date) { + throw new Error(`legacy transcript startedAt has no date: ${session.startedAt}`); + } + const legacySegment = + session.sessionId.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session"; + return path.normalize(path.join(date, legacySegment)); +} + +async function optionalRegularFile(filePath: string): Promise { + try { + const stat = await fs.lstat(filePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`legacy transcript source must be a regular file: ${filePath}`); + } + return true; + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return false; + } + throw error; + } +} + +export function openLegacyMeetingTranscriptStage(databasePath: string): DatabaseSync { + const database = new DatabaseSync(databasePath); + database.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE staged_utterances ( + stage_key TEXT NOT NULL, + sequence INTEGER NOT NULL, + utterance_json TEXT NOT NULL, + PRIMARY KEY (stage_key, sequence) + ) STRICT; + `); + return database; +} + +async function stageUtterances(params: { + filePath: string; + stageDatabase: DatabaseSync; + stageKey: string; +}): Promise { + const filePath = params.filePath; + if (!(await optionalRegularFile(filePath))) { + return 0; + } + const stream = createReadStream(filePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + let lineNumber = 0; + let sequence = 0; + let pending: string[] = []; + const insert = params.stageDatabase.prepare( + "INSERT INTO staged_utterances (stage_key, sequence, utterance_json) VALUES (?, ?, ?)", + ); + const flush = () => { + if (pending.length === 0) { + return; + } + params.stageDatabase.exec("BEGIN IMMEDIATE"); + try { + for (const utteranceJson of pending) { + insert.run(params.stageKey, sequence, utteranceJson); + sequence += 1; + } + params.stageDatabase.exec("COMMIT"); + pending = []; + } catch (error) { + params.stageDatabase.exec("ROLLBACK"); + throw error; + } + }; + try { + for await (const line of lines) { + lineNumber += 1; + if (!line.trim()) { + continue; + } + const utterance = parseUtterance(JSON.parse(line) as unknown, filePath, lineNumber); + pending.push(JSON.stringify(utterance)); + if (pending.length >= LEGACY_UTTERANCE_STAGE_BATCH_SIZE) { + flush(); + } + } + flush(); + } finally { + lines.close(); + stream.destroy(); + } + return sequence; +} + +export function readStagedMeetingTranscriptUtterances(params: { + stageDatabase: DatabaseSync; + stageKey: string; + start: number; + limit: number; +}): TranscriptUtterance[] { + return params.stageDatabase + .prepare( + "SELECT utterance_json FROM staged_utterances WHERE stage_key = ? AND sequence >= ? ORDER BY sequence ASC LIMIT ?", + ) + .all(params.stageKey, params.start, params.limit) + .map((row) => JSON.parse(String(row.utterance_json)) as TranscriptUtterance); +} + +async function snapshotFile(filePath: string): Promise<{ + hash?: string; + sizeBytes: number; +}> { + if (!(await optionalRegularFile(filePath))) { + return { sizeBytes: 0 }; + } + const stat = await fs.stat(filePath); + return { hash: await sha256File(filePath), sizeBytes: stat.size }; +} + +async function snapshotSourceFiles(files: string[]) { + return await Promise.all(files.map(snapshotFile)); +} + +function sourceFilesHash( + files: string[], + snapshots: Array<{ hash?: string; sizeBytes: number }>, +): string { + return sha256Hex( + snapshots + .map((snapshot, index) => `${path.basename(files[index] ?? "")}\0${snapshot.hash ?? "-"}`) + .join("\n"), + ); +} + +export async function snapshotLegacyMeetingTranscriptSession(params: { + rootDir: string; + relativeDir: string; + stageDatabase: DatabaseSync; +}): Promise { + const sourceDir = path.join(params.rootDir, params.relativeDir); + const sourceStat = await fs.lstat(sourceDir); + if (sourceStat.isSymbolicLink() || !sourceStat.isDirectory()) { + throw new Error(`legacy transcript session must be a regular directory: ${sourceDir}`); + } + const metadataPath = path.join(sourceDir, "metadata.json"); + const transcriptPath = path.join(sourceDir, "transcript.jsonl"); + const summaryJsonPath = path.join(sourceDir, "summary.json"); + const summaryMarkdownPath = path.join(sourceDir, "summary.md"); + const files = [metadataPath, transcriptPath, summaryJsonPath, summaryMarkdownPath]; + const beforeSnapshots = await snapshotSourceFiles(files); + if (!beforeSnapshots[0]?.hash) { + throw new Error(`legacy transcript session is missing metadata.json: ${sourceDir}`); + } + const session = parseSession(JSON.parse(await fs.readFile(metadataPath, "utf8")), metadataPath); + const expectedRelativeDir = legacyTranscriptRelativeDir(session); + if (path.normalize(params.relativeDir) !== expectedRelativeDir) { + throw new Error( + `legacy transcript selector mismatch at ${sourceDir}: expected ${expectedRelativeDir}`, + ); + } + const utteranceCount = await stageUtterances({ + filePath: transcriptPath, + stageDatabase: params.stageDatabase, + stageKey: params.relativeDir, + }); + const hasSummaryJson = await optionalRegularFile(summaryJsonPath); + const hasSummaryMarkdown = await optionalRegularFile(summaryMarkdownPath); + const summary = hasSummaryJson + ? parseSummary(JSON.parse(await fs.readFile(summaryJsonPath, "utf8")), summaryJsonPath) + : undefined; + const markdown = hasSummaryMarkdown + ? await fs.readFile(summaryMarkdownPath, "utf8") + : summary + ? renderTranscriptsMarkdown(summary) + : undefined; + if (summary && summary.sessionId !== session.sessionId) { + throw new Error(`legacy transcript summary session mismatch at ${summaryJsonPath}`); + } + + const fileSnapshots = await snapshotSourceFiles(files); + if ( + fileSnapshots.some( + (snapshot, index) => + snapshot.hash !== beforeSnapshots[index]?.hash || + snapshot.sizeBytes !== beforeSnapshots[index]?.sizeBytes, + ) + ) { + throw new Error(`legacy transcript files changed while being staged: ${sourceDir}`); + } + const sourceHash = sourceFilesHash(files, fileSnapshots); + return { + sourceDir, + relativeDir: params.relativeDir, + stageKey: params.relativeDir, + session, + utteranceCount, + summary, + markdown, + sourceHash, + sourceSizeBytes: fileSnapshots.reduce((total, file) => total + file.sizeBytes, 0), + }; +} + +async function hasLegacyTranscriptArtifacts(directory: string): Promise { + const entries = await fs.readdir(directory, { withFileTypes: true }); + let found = false; + for (const entry of entries) { + if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(entry.name.toLowerCase())) { + continue; + } + const stat = await fs.lstat(path.join(directory, entry.name)); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`legacy transcript source must be a regular file: ${directory}`); + } + found = true; + } + return found; +} + +async function listLegacyMeetingTranscriptDirs( + rootDir: string, + mode: "artifacts" | "sessions", +): Promise { + if (!(await validateMeetingTranscriptRoot(rootDir, { allowMissing: true }))) { + return []; + } + let dateEntries; + try { + dateEntries = await fs.readdir(rootDir, { withFileTypes: true }); + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return []; + } + throw error; + } + const include = async (directory: string) => + mode === "sessions" + ? await optionalRegularFile(path.join(directory, "metadata.json")) + : await hasLegacyTranscriptArtifacts(directory); + const sessions: string[] = []; + if (await include(rootDir)) { + sessions.push("."); + } + for (const dateEntry of dateEntries.toSorted((a, b) => a.name.localeCompare(b.name))) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(dateEntry.name)) { + continue; + } + if (dateEntry.isSymbolicLink()) { + throw new Error(`legacy transcript date directory cannot be a symlink: ${dateEntry.name}`); + } + if (!dateEntry.isDirectory()) { + continue; + } + const dateDir = path.join(rootDir, dateEntry.name); + if (await include(dateDir)) { + sessions.push(dateEntry.name); + } + const sessionEntries = await fs.readdir(dateDir, { withFileTypes: true }); + for (const sessionEntry of sessionEntries.toSorted((a, b) => a.name.localeCompare(b.name))) { + if (sessionEntry.isSymbolicLink()) { + throw new Error(`legacy transcript session cannot be a symlink: ${sessionEntry.name}`); + } + if (sessionEntry.isDirectory() && (await include(path.join(dateDir, sessionEntry.name)))) { + sessions.push(path.join(dateEntry.name, sessionEntry.name)); + } + } + } + return sessions; +} + +export async function listLegacyMeetingTranscriptSessionDirs(rootDir: string): Promise { + return await listLegacyMeetingTranscriptDirs(rootDir, "sessions"); +} + +export async function listLegacyMeetingTranscriptArtifactDirs(rootDir: string): Promise { + return await listLegacyMeetingTranscriptDirs(rootDir, "artifacts"); +} + +export async function archivePartialMeetingTranscriptArtifacts(params: { + sourceRoot: string; + relativeDirs: string[]; + recoveryRoot: string; +}): Promise { + const moves: Array<{ source: string; destination: string }> = []; + const sourceDirs = new Set(); + for (const relativeDir of params.relativeDirs) { + const sourceDir = path.join(params.sourceRoot, relativeDir); + if (relativeDir !== ".") { + sourceDirs.add(sourceDir); + } + const destinationDir = path.join(params.recoveryRoot, relativeDir); + for (const entry of await fs.readdir(sourceDir, { withFileTypes: true })) { + if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(entry.name.toLowerCase())) { + continue; + } + const source = path.join(sourceDir, entry.name); + const stat = await fs.lstat(source); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`legacy transcript source must be a regular file: ${sourceDir}`); + } + const destination = path.join(destinationDir, entry.name); + try { + await fs.lstat(destination); + throw new Error(`partial transcript recovery destination already exists: ${destination}`); + } catch (error) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + } + moves.push({ source, destination }); + } + } + for (const destinationDir of new Set(moves.map((move) => path.dirname(move.destination)))) { + await fs.mkdir(destinationDir, { recursive: true }); + } + const moved: Array<{ source: string; destination: string }> = []; + try { + for (const move of moves) { + await fs.rename(move.source, move.destination); + moved.push(move); + } + } catch (error) { + const rollbackErrors: string[] = []; + for (const move of moved.toReversed()) { + try { + await fs.rename(move.destination, move.source); + } catch (rollbackError) { + rollbackErrors.push(String(rollbackError)); + } + } + if (rollbackErrors.length > 0) { + throw new Error( + `partial transcript recovery failed and rollback was incomplete; inspect ${params.recoveryRoot}: ${String(error)}; rollback errors: ${rollbackErrors.join("; ")}`, + { cause: error }, + ); + } + throw error; + } + // Artifact moves are already committed; empty-directory cleanup is best effort + // so an unremovable harmless directory cannot hide the reported recovery move. + for (const sourceDir of [...sourceDirs].toSorted((a, b) => b.length - a.length)) { + await fs.rmdir(sourceDir).catch(() => undefined); + } +} + +export async function rehashLegacyMeetingTranscriptSnapshots( + snapshots: LegacyMeetingTranscriptSnapshot[], +): Promise { + for (const snapshot of snapshots) { + const files = ["metadata.json", "transcript.jsonl", "summary.json", "summary.md"].map( + (fileName) => path.join(snapshot.sourceDir, fileName), + ); + const fileSnapshots = await snapshotSourceFiles(files); + const currentHash = sourceFilesHash(files, fileSnapshots); + if (currentHash !== snapshot.sourceHash) { + return false; + } + } + return true; +} + +export async function archiveLegacyMeetingTranscriptSnapshots(params: { + sourceRoot: string; + snapshots: LegacyMeetingTranscriptSnapshot[]; + expectedRelativeDirs: string[]; + canonicalRelativeDirs: string[]; + archiveRoot: string; +}): Promise { + await validateMeetingTranscriptRoot(params.sourceRoot); + const currentRelativeDirs = await listLegacyMeetingTranscriptSessionDirs(params.sourceRoot); + const expectedRelativeDirs = params.expectedRelativeDirs.toSorted((a, b) => a.localeCompare(b)); + if (JSON.stringify(currentRelativeDirs) !== JSON.stringify(expectedRelativeDirs)) { + throw new Error("legacy transcript session tree changed before archive"); + } + await fs.rename(params.sourceRoot, params.archiveRoot); + try { + const archivedSnapshots = params.snapshots.map((snapshot) => ({ + ...snapshot, + sourceDir: path.join( + params.archiveRoot, + path.relative(params.sourceRoot, snapshot.sourceDir) || ".", + ), + })); + if (!(await rehashLegacyMeetingTranscriptSnapshots(archivedSnapshots))) { + throw new Error("legacy transcript files changed at the archive boundary"); + } + await restoreCanonicalMeetingTranscriptExports({ + sourceRoot: params.sourceRoot, + archiveRoot: params.archiveRoot, + migratedSourcePaths: params.snapshots.map((snapshot) => snapshot.sourceDir), + canonicalRelativeDirs: params.canonicalRelativeDirs, + }); + } catch (error) { + throw new LegacyMeetingTranscriptArchiveMovedError(error); + } + return params.archiveRoot; +} + +export class LegacyMeetingTranscriptArchiveMovedError extends Error { + constructor(cause: unknown) { + super( + `legacy transcript source moved but canonical export restoration failed: ${String(cause)}`, + ); + this.name = "LegacyMeetingTranscriptArchiveMovedError"; + } +} + +export async function restoreCanonicalMeetingTranscriptExports(params: { + sourceRoot: string; + archiveRoot: string; + migratedSourcePaths: string[]; + canonicalRelativeDirs: string[]; +}): Promise { + await validateMeetingTranscriptRoot(params.archiveRoot); + if (!(await validateMeetingTranscriptRoot(params.sourceRoot, { allowMissing: true }))) { + await fs.mkdir(params.sourceRoot, { recursive: true }); + await validateMeetingTranscriptRoot(params.sourceRoot); + } + const migratedRelativeDirs = new Set( + params.migratedSourcePaths.map( + (sourcePath) => path.relative(params.sourceRoot, sourcePath) || ".", + ), + ); + for (const relativeDir of params.canonicalRelativeDirs) { + const archiveRelative = path.relative( + path.resolve(params.archiveRoot), + path.resolve(params.archiveRoot, relativeDir), + ); + const sourceRelative = path.relative( + path.resolve(params.sourceRoot), + path.resolve(params.sourceRoot, relativeDir), + ); + if ( + !archiveRelative || + archiveRelative.startsWith("..") || + path.isAbsolute(archiveRelative) || + !sourceRelative || + sourceRelative.startsWith("..") || + path.isAbsolute(sourceRelative) + ) { + throw new Error(`canonical transcript export path escaped its root: ${relativeDir}`); + } + if (migratedRelativeDirs.has(relativeDir)) { + continue; + } + const source = path.join(params.archiveRoot, relativeDir); + const destination = path.join(params.sourceRoot, relativeDir); + try { + const sourceStat = await fs.lstat(source); + if (sourceStat.isSymbolicLink() || !sourceStat.isDirectory()) { + throw new Error(`canonical transcript export source is not a directory: ${source}`); + } + await assertNoSymlinkParents({ + rootDir: params.archiveRoot, + targetPath: source, + allowMissing: false, + messagePrefix: "Canonical transcript export source", + }); + } catch (error) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + const destinationStat = await fs.lstat(destination); + if (destinationStat.isSymbolicLink() || !destinationStat.isDirectory()) { + throw new Error( + `canonical transcript export destination is not a directory: ${destination}`, + { cause: error }, + ); + } + await assertNoSymlinkParents({ + rootDir: params.sourceRoot, + targetPath: destination, + allowMissing: false, + messagePrefix: "Canonical transcript export destination", + }); + continue; + } + try { + const destinationStat = await fs.lstat(destination); + if (destinationStat.isSymbolicLink() || !destinationStat.isDirectory()) { + throw new Error( + `canonical transcript export destination is not a directory: ${destination}`, + ); + } + await assertNoSymlinkParents({ + rootDir: params.sourceRoot, + targetPath: destination, + allowMissing: false, + messagePrefix: "Canonical transcript export destination", + }); + const readMetadata = async (directory: string) => + parseSession( + JSON.parse(await fs.readFile(path.join(directory, "metadata.json"), "utf8")), + path.join(directory, "metadata.json"), + ); + const [sourceMetadata, destinationMetadata] = await Promise.all([ + readMetadata(source), + readMetadata(destination), + ]); + if ( + sourceMetadata.sessionId !== destinationMetadata.sessionId || + sourceMetadata.startedAt !== destinationMetadata.startedAt + ) { + throw new Error(`canonical transcript export destination changed identity: ${destination}`); + } + continue; + } catch (error) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + } + await assertNoSymlinkParents({ + rootDir: params.sourceRoot, + targetPath: destination, + allowMissing: true, + messagePrefix: "Canonical transcript export destination", + }); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.rename(source, destination); + } +} + +export async function archiveDivergentMeetingTranscriptExport(params: { + sourceRoot: string; + relativeDir: string; + recoveryRoot: string; +}): Promise { + const source = path.join(params.sourceRoot, params.relativeDir); + const destination = path.join(params.recoveryRoot, params.relativeDir); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.rename(source, destination); + return destination; +} diff --git a/src/infra/state-migrations.meeting-transcripts-insert.ts b/src/infra/state-migrations.meeting-transcripts-insert.ts new file mode 100644 index 000000000000..08a86ad9e6cb --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts-insert.ts @@ -0,0 +1,161 @@ +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; +import { + safeTranscriptPathSegment, + transcriptSessionExportKey, + transcriptSessionSelector, +} from "../transcripts/store.js"; +import { sha256Hex } from "./crypto-digest.js"; +import { executeSqliteQuerySync } from "./kysely-sync.js"; +import { migrationDb } from "./state-migrations.meeting-transcripts-database.js"; +import { + LEGACY_UTTERANCE_INSERT_CHUNK_SIZE, + readStagedMeetingTranscriptUtterances, + type LegacyMeetingTranscriptSnapshot, +} from "./state-migrations.meeting-transcripts-files.js"; + +function sourceKey(sourceDir: string): string { + return `meeting-transcripts:${sha256Hex(path.resolve(sourceDir))}`; +} + +export function insertMeetingTranscriptSnapshots(params: { + snapshots: LegacyMeetingTranscriptSnapshot[]; + runId: string; + now: number; + archiveRoot: string; + canonicalRelativeDirs: string[]; + stageDatabase: DatabaseSync; + env: NodeJS.ProcessEnv; + stateDir: string; +}): void { + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = migrationDb(database); + // Run-wide metadata is stored once here; per-source receipts below keep + // only their selector so large migrations remain linear in session count. + executeSqliteQuerySync( + database, + db.insertInto("migration_runs").values({ + id: params.runId, + started_at: params.now, + finished_at: null, + status: "imported", + report_json: JSON.stringify({ + format: "meeting-transcripts-files-v1", + sessions: params.snapshots.length, + utterances: params.snapshots.reduce( + (total, snapshot) => total + snapshot.utteranceCount, + 0, + ), + archiveRoot: params.archiveRoot, + canonicalRelativeDirs: params.canonicalRelativeDirs, + }), + }), + ); + for (const snapshot of params.snapshots) { + executeSqliteQuerySync( + database, + db.insertInto("meeting_transcript_sessions").values({ + session_id: snapshot.session.sessionId, + started_at: snapshot.session.startedAt, + selector: transcriptSessionSelector(snapshot.session), + export_key: transcriptSessionExportKey(snapshot.session), + session_slug: safeTranscriptPathSegment(snapshot.session.sessionId), + provider_id: snapshot.session.source.providerId, + title: snapshot.session.title ?? null, + source_json: JSON.stringify(snapshot.session.source), + stopped_at: snapshot.session.stoppedAt ?? null, + metadata_json: snapshot.session.metadata + ? JSON.stringify(snapshot.session.metadata) + : null, + export_manifest_json: "{}", + export_pending_json: "[]", + next_utterance_seq: snapshot.utteranceCount, + created_at_ms: params.now, + updated_at_ms: params.now, + }), + ); + if (snapshot.utteranceCount > 0) { + for ( + let start = 0; + start < snapshot.utteranceCount; + start += LEGACY_UTTERANCE_INSERT_CHUNK_SIZE + ) { + const chunk = readStagedMeetingTranscriptUtterances({ + stageDatabase: params.stageDatabase, + stageKey: snapshot.stageKey, + start, + limit: LEGACY_UTTERANCE_INSERT_CHUNK_SIZE, + }); + executeSqliteQuerySync( + database, + db.insertInto("meeting_transcript_utterances").values( + chunk.map((utterance, offset) => ({ + session_id: snapshot.session.sessionId, + session_started_at: snapshot.session.startedAt, + sequence: start + offset, + utterance_id: utterance.id ?? null, + started_at: utterance.startedAt ?? null, + ended_at: utterance.endedAt ?? null, + speaker_id: utterance.speaker?.id ?? null, + speaker_label: utterance.speaker?.label ?? null, + text: utterance.text, + final: utterance.final === undefined ? null : utterance.final ? 1 : 0, + metadata_json: utterance.metadata ? JSON.stringify(utterance.metadata) : null, + })), + ), + ); + } + } + if (snapshot.summary !== undefined || snapshot.markdown !== undefined) { + executeSqliteQuerySync( + database, + db.insertInto("meeting_transcript_summaries").values({ + session_id: snapshot.session.sessionId, + session_started_at: snapshot.session.startedAt, + generated_at: snapshot.summary?.generatedAt ?? null, + summary_json: snapshot.summary ? JSON.stringify(snapshot.summary) : null, + markdown: snapshot.markdown ?? null, + utterance_count: snapshot.summary?.utteranceCount ?? snapshot.utteranceCount, + }), + ); + } + executeSqliteQuerySync( + database, + db + .insertInto("migration_sources") + .values({ + source_key: sourceKey(snapshot.sourceDir), + migration_kind: "meeting-transcripts-files-v1", + source_path: snapshot.sourceDir, + target_table: "meeting_transcript_sessions", + source_sha256: snapshot.sourceHash, + source_size_bytes: snapshot.sourceSizeBytes, + source_record_count: snapshot.utteranceCount, + last_run_id: params.runId, + status: "imported", + imported_at: params.now, + removed_source: 0, + report_json: JSON.stringify({ + selector: transcriptSessionSelector(snapshot.session), + }), + }) + .onConflict((conflict) => + conflict.column("source_key").doUpdateSet({ + source_sha256: snapshot.sourceHash, + source_size_bytes: snapshot.sourceSizeBytes, + source_record_count: snapshot.utteranceCount, + last_run_id: params.runId, + status: "imported", + imported_at: params.now, + removed_source: 0, + }), + ), + ); + } + }, + { env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } }, + { operationLabel: "meeting-transcripts.legacy-import" }, + ); +} diff --git a/src/infra/state-migrations.meeting-transcripts-verify.ts b/src/infra/state-migrations.meeting-transcripts-verify.ts new file mode 100644 index 000000000000..54c881a38a85 --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts-verify.ts @@ -0,0 +1,125 @@ +// Verifies staged legacy transcript rows against the committed canonical store. +import type { DatabaseSync } from "node:sqlite"; +import type { TranscriptUtterance } from "../transcripts/provider-types.js"; +import { transcriptSessionSelector, TranscriptsStore } from "../transcripts/store.js"; +import { + LEGACY_UTTERANCE_INSERT_CHUNK_SIZE, + readStagedMeetingTranscriptUtterances, + type LegacyMeetingTranscriptSnapshot, +} from "./state-migrations.meeting-transcripts-files.js"; + +type StoredUtteranceRow = { + ended_at: string | null; + final: number | null; + metadata_json: string | null; + session_id: string; + speaker_id: string | null; + speaker_label: string | null; + started_at: string | null; + text: string; + utterance_id: string | null; +}; + +function canonicalJson(value: unknown): string { + if (value === undefined) { + return "undefined"; + } + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(",")}]`; + } + if (value && typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .toSorted() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +function storedUtteranceFromRow(row: StoredUtteranceRow): TranscriptUtterance { + const utterance: TranscriptUtterance = { sessionId: row.session_id, text: row.text }; + if (row.utterance_id !== null) { + utterance.id = row.utterance_id; + } + if (row.started_at !== null) { + utterance.startedAt = row.started_at; + } + if (row.ended_at !== null) { + utterance.endedAt = row.ended_at; + } + if (row.speaker_label !== null) { + const speaker: NonNullable = { label: row.speaker_label }; + if (row.speaker_id !== null) { + speaker.id = row.speaker_id; + } + utterance.speaker = speaker; + } + if (row.final !== null) { + utterance.final = row.final === 1; + } + if (row.metadata_json) { + utterance.metadata = JSON.parse(row.metadata_json) as Record; + } + return utterance; +} + +export async function verifyImportedMeetingTranscriptSnapshots(params: { + store: TranscriptsStore; + snapshots: LegacyMeetingTranscriptSnapshot[]; + stageDatabase: DatabaseSync; + database: DatabaseSync; +}): Promise { + const selectUtterances = params.database.prepare(` + SELECT + ended_at, + final, + metadata_json, + session_id, + speaker_id, + speaker_label, + started_at, + text, + utterance_id + FROM meeting_transcript_utterances + WHERE session_id = ? AND session_started_at = ? + ORDER BY sequence ASC + LIMIT ? OFFSET ? + `); + for (const snapshot of params.snapshots) { + const session = await params.store.readSession(transcriptSessionSelector(snapshot.session)); + if (!session || canonicalJson(session) !== canonicalJson(snapshot.session)) { + throw new Error(`meeting transcript import verification failed: ${snapshot.relativeDir}`); + } + for ( + let start = 0; + start < snapshot.utteranceCount; + start += LEGACY_UTTERANCE_INSERT_CHUNK_SIZE + ) { + const expected = readStagedMeetingTranscriptUtterances({ + stageDatabase: params.stageDatabase, + stageKey: snapshot.stageKey, + start, + limit: LEGACY_UTTERANCE_INSERT_CHUNK_SIZE, + }); + const actual = selectUtterances + .all( + snapshot.session.sessionId, + snapshot.session.startedAt, + LEGACY_UTTERANCE_INSERT_CHUNK_SIZE, + start, + ) + .map((row) => storedUtteranceFromRow(row as StoredUtteranceRow)); + if (canonicalJson(actual) !== canonicalJson(expected)) { + throw new Error(`meeting transcript import verification failed: ${snapshot.relativeDir}`); + } + } + const summary = await params.store.readSummary(session); + if ( + canonicalJson(summary.summary) !== canonicalJson(snapshot.summary) || + canonicalJson(summary.markdown?.trimEnd()) !== canonicalJson(snapshot.markdown?.trimEnd()) + ) { + throw new Error(`meeting transcript summary verification failed: ${snapshot.relativeDir}`); + } + } +} diff --git a/src/infra/state-migrations.meeting-transcripts.test.ts b/src/infra/state-migrations.meeting-transcripts.test.ts new file mode 100644 index 000000000000..7fe35d5f5636 --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts.test.ts @@ -0,0 +1,995 @@ +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { TranscriptsStore } from "../transcripts/store.js"; +import { summarizeTranscripts } from "../transcripts/summary.js"; +import { restoreCanonicalMeetingTranscriptExports } from "./state-migrations.meeting-transcripts-files.js"; +import { + detectLegacyMeetingTranscripts, + migrateLegacyMeetingTranscripts, +} from "./state-migrations.meeting-transcripts.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +afterEach(() => closeOpenClawStateDatabaseForTest()); + +async function seedLegacySession(params: { + stateDir: string; + sessionId: string; + date?: string; + invalidTranscript?: boolean; + utteranceCount?: number; + emptyMarkdown?: boolean; + omitSummaryJson?: boolean; +}): Promise { + const date = params.date ?? "2026-07-01"; + const sessionDir = path.join(params.stateDir, "transcripts", date, params.sessionId); + await fs.mkdir(sessionDir, { recursive: true }); + const session = { + sessionId: params.sessionId, + title: "Design review", + source: { providerId: "manual-transcript", meetingUrl: "https://meet.example.invalid/room" }, + startedAt: `${date}T10:00:00.000Z`, + stoppedAt: `${date}T10:30:00.000Z`, + }; + await fs.writeFile( + path.join(sessionDir, "metadata.json"), + `${JSON.stringify(session, null, 2)}\n`, + ); + await fs.writeFile( + path.join(sessionDir, "transcript.jsonl"), + params.invalidTranscript + ? "{invalid\n" + : Array.from({ length: params.utteranceCount ?? 2 }, (_, index) => + JSON.stringify({ + id: `u-${index + 1}`, + sessionId: params.sessionId, + speaker: { label: index % 2 === 0 ? "Alex" : "Sam" }, + text: index === 0 ? "First line" : index === 1 ? "Second line" : `Line ${index + 1}`, + final: true, + }), + ).join("\n") + "\n", + ); + const summary = { + sessionId: params.sessionId, + title: "Design review", + generatedAt: `${date}T10:31:00.000Z`, + overview: "First line. Second line.", + transcript: ["Alex: First line", "Sam: Second line"], + decisions: [], + actionItems: [], + risks: [], + utteranceCount: params.utteranceCount ?? 2, + }; + if (!params.omitSummaryJson) { + await fs.writeFile( + path.join(sessionDir, "summary.json"), + `${JSON.stringify(summary, null, 2)}\n`, + ); + } + await fs.writeFile( + path.join(sessionDir, "summary.md"), + params.emptyMarkdown ? "" : "# Design review\n\nFirst line.\n", + ); + return sessionDir; +} + +function databaseEnv(stateDir: string): NodeJS.ProcessEnv { + return { ...process.env, OPENCLAW_STATE_DIR: stateDir }; +} + +describe("meeting transcript Doctor migration", () => { + it("is doctor-only", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ stateDir, sessionId: "design-review" }); + + expect(detectLegacyMeetingTranscripts({ stateDir })).toMatchObject({ hasLegacy: false }); + expect( + detectLegacyMeetingTranscripts({ stateDir, doctorOnlyStateMigrations: true }), + ).toMatchObject({ hasLegacy: true }); + }); + + it("surfaces filesystem errors during detection", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await fs.writeFile(path.join(stateDir, "transcripts"), "not a directory"); + + expect(() => + detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }), + ).toThrow(); + }); + + it.runIf(process.platform !== "win32")( + "rejects a symlinked transcript root before migration", + async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-"); + await fs.symlink(externalRoot, path.join(stateDir, "transcripts"), "dir"); + + expect(() => + detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }), + ).toThrow("regular directory"); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects symlinked date and session directories during detection", + async () => { + for (const target of ["date", "session"] as const) { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-"); + const transcriptsDir = path.join(stateDir, "transcripts"); + await fs.mkdir(transcriptsDir, { recursive: true }); + if (target === "date") { + await fs.symlink(externalRoot, path.join(transcriptsDir, "2026-07-01"), "dir"); + } else { + const dateDir = path.join(transcriptsDir, "2026-07-01"); + await fs.mkdir(dateDir); + await fs.symlink(externalRoot, path.join(dateDir, "linked-session"), "dir"); + } + + expect(() => + detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }), + ).toThrow("cannot be a symlink"); + } + }, + ); + + it("imports, verifies, receipts, archives, and reopens SQLite-only", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const sourceDir = await seedLegacySession({ stateDir, sessionId: "design-review" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + now: () => Date.parse("2026-07-02T00:00:00.000Z"), + }); + + expect(result.warnings).toEqual([]); + expect(result.changes.join("\n")).toContain("2 utterances"); + await expect(fs.stat(sourceDir)).rejects.toMatchObject({ code: "ENOENT" }); + const archives = (await fs.readdir(stateDir)).filter((entry) => + entry.startsWith("transcripts.migrated-2026-07-02T00-00-00-000Z"), + ); + expect(archives).toHaveLength(1); + + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + expect( + database + .prepare( + "SELECT status, removed_source, source_record_count FROM migration_sources WHERE migration_kind = ?", + ) + .get("meeting-transcripts-files-v1"), + ).toEqual({ status: "archived", removed_source: 1, source_record_count: 2 }); + + closeOpenClawStateDatabaseForTest(); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = await store.readSession("design-review"); + expect(session).toMatchObject({ title: "Design review" }); + await expect(store.readUtterancesForSession(session!)).resolves.toEqual([ + expect.objectContaining({ id: "u-1", text: "First line" }), + expect.objectContaining({ id: "u-2", text: "Second line" }), + ]); + await expect(store.readSummary(session!)).resolves.toMatchObject({ + markdown: "# Design review\n\nFirst line.\n", + }); + }); + + it("imports shipped dot-only session layouts into reserved SQLite selectors", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + for (const sessionId of [".", "..", "session"]) { + await seedLegacySession({ stateDir, sessionId }); + } + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(detected.hasLegacy).toBe(true); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const expectedSlugs = new Map([ + [".", "%2E"], + ["..", "%2E%2E"], + ["session", "session"], + ]); + for (const [sessionId, expectedSlug] of expectedSlugs) { + const session = await store.readSession(sessionId); + expect(session?.sessionId).toBe(sessionId); + expect(store.sessionDir(session!)).toBe( + path.join(stateDir, "transcripts", "2026-07-01", expectedSlug), + ); + const artifacts = await store.materializeSessionArtifacts(session!, "transcript"); + const lines = (await fs.readFile(artifacts.transcriptPath, "utf8")).trim().split("\n"); + expect(lines.map((line) => JSON.parse(line).text)).toEqual(["First line", "Second line"]); + } + }); + + it.runIf(process.platform !== "win32" && process.platform !== "darwin")( + "imports case-distinct sessions from a case-sensitive legacy tree", + async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ stateDir, sessionId: "Capital" }); + await seedLegacySession({ stateDir, sessionId: "capital" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + await expect(store.readSession("2026-07-01/Capital")).resolves.toMatchObject({ + sessionId: "Capital", + }); + await expect(store.readSession("2026-07-01/capital")).resolves.toMatchObject({ + sessionId: "capital", + }); + const upper = (await store.readSession("2026-07-01/Capital"))!; + const lower = (await store.readSession("2026-07-01/capital"))!; + const upperArtifacts = await store.materializeSessionArtifacts(upper, "metadata"); + await store.materializeSessionArtifacts(lower, "metadata"); + await fs.rename( + upperArtifacts.sessionDir, + path.join(stateDir, "transcripts", "2026-07-01", "CAPITAL"), + ); + expect( + detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }), + ).toMatchObject({ hasLegacy: false }); + }, + ); + + it.runIf(process.platform !== "win32" && process.platform !== "darwin")( + "does not assign a case-distinct legacy directory to an existing SQLite session", + async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + await store.writeSession({ + sessionId: "Capital", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-01T09:00:00.000Z", + }); + await seedLegacySession({ stateDir, sessionId: "capital" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + await expect(store.readSession("2026-07-01/Capital")).resolves.toMatchObject({ + sessionId: "Capital", + }); + await expect(store.readSession("2026-07-01/capital")).resolves.toMatchObject({ + sessionId: "capital", + }); + }, + ); + + it("preflights the whole tree before importing anything", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const validDir = await seedLegacySession({ stateDir, sessionId: "valid" }); + const invalidDir = await seedLegacySession({ + stateDir, + sessionId: "invalid", + invalidTranscript: true, + }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.changes).toEqual([]); + expect(result.warnings.join("\n")).toContain("Failed migrating meeting transcripts"); + await expect(fs.stat(validDir)).resolves.toBeDefined(); + await expect(fs.stat(invalidDir)).resolves.toBeDefined(); + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + expect( + database.prepare("SELECT COUNT(*) AS count FROM meeting_transcript_sessions").get(), + ).toEqual({ count: 0 }); + }); + + it("archives metadata-less interrupted session directories without blocking import", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ stateDir, sessionId: "complete-session" }); + const incompleteRelativeDir = path.join("2026-07-01", "incomplete-session"); + await fs.mkdir(path.join(stateDir, "transcripts", incompleteRelativeDir)); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + now: () => Date.parse("2026-07-02T00:00:00.000Z"), + }); + + expect(result.warnings).toEqual([]); + await expect( + fs.stat( + path.join(stateDir, "transcripts.migrated-2026-07-02T00-00-00-000Z", incompleteRelativeDir), + ), + ).resolves.toBeDefined(); + }); + + it("detects and recovers a partial-only legacy transcript tree", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const partialDir = path.join(stateDir, "transcripts", "2026-07-01", "partial-only"); + await fs.mkdir(partialDir, { recursive: true }); + await fs.writeFile(path.join(partialDir, "transcript.jsonl"), '{"text":"partial"}\n'); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + expect(detected.hasLegacy).toBe(true); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + now: () => Date.parse("2026-07-02T00:00:00.000Z"), + }); + + expect(result.warnings).toEqual([]); + expect(result.changes.join("\n")).toContain("incomplete meeting transcript directory"); + await expect(fs.stat(path.join(partialDir, "transcript.jsonl"))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect( + fs.stat( + path.join( + stateDir, + "transcripts.partials-recovered-2026-07-02T00-00-00-000Z", + "2026-07-01", + "partial-only", + "transcript.jsonl", + ), + ), + ).resolves.toBeDefined(); + }); + + it.runIf(process.platform !== "win32")( + "preflights every partial artifact before moving any source", + async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const externalDir = tempDirs.make("openclaw-meeting-transcripts-external-"); + const partialDir = path.join(stateDir, "transcripts", "2026-07-01", "partial-invalid"); + await fs.mkdir(partialDir, { recursive: true }); + await fs.writeFile(path.join(partialDir, "summary.md"), "keep me\n"); + await fs.writeFile(path.join(externalDir, "transcript.jsonl"), '{"text":"outside"}\n'); + await fs.symlink( + path.join(externalDir, "transcript.jsonl"), + path.join(partialDir, "transcript.jsonl"), + ); + expect(() => + detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }), + ).toThrow("regular file"); + await expect(fs.readFile(path.join(partialDir, "summary.md"), "utf8")).resolves.toBe( + "keep me\n", + ); + }, + ); + + it("rolls back when a session appears between verification and archive", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const sourceDir = await seedLegacySession({ stateDir, sessionId: "verified" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + testHooks: { + afterImport: () => { + const lateDir = path.join(stateDir, "transcripts", "2026-07-03", "late-session"); + fsSync.mkdirSync(lateDir, { recursive: true }); + fsSync.writeFileSync( + path.join(lateDir, "metadata.json"), + JSON.stringify({ + sessionId: "late-session", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-03T10:00:00.000Z", + }), + ); + }, + }, + }); + + expect(result.changes).toEqual([]); + expect(result.warnings.join("\n")).toContain("session tree changed before archive"); + await expect(fs.stat(sourceDir)).resolves.toBeDefined(); + await expect( + fs.stat(path.join(stateDir, "transcripts", "2026-07-03", "late-session")), + ).resolves.toBeDefined(); + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + expect( + database.prepare("SELECT COUNT(*) AS count FROM meeting_transcript_sessions").get(), + ).toEqual({ count: 0 }); + }); + + it("does not mistake a colliding archive destination for a completed move", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const sourceDir = await seedLegacySession({ stateDir, sessionId: "archive-collision" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + const archiveRoot = path.join(stateDir, "transcripts.migrated-2026-07-02T00-00-00-000Z"); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + now: () => Date.parse("2026-07-02T00:00:00.000Z"), + testHooks: { + afterImport: () => { + fsSync.mkdirSync(archiveRoot); + fsSync.writeFileSync(path.join(archiveRoot, "unrelated"), "keep"); + }, + }, + }); + + expect(result.changes).toEqual([]); + expect(result.warnings.join("\n")).toContain("Failed archiving verified legacy"); + await expect(fs.stat(sourceDir)).resolves.toBeDefined(); + await expect(fs.stat(archiveRoot)).resolves.toBeDefined(); + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + expect(database.prepare("SELECT COUNT(*) AS count FROM migration_sources").get()).toEqual({ + count: 0, + }); + }); + + it("restores idempotently when a canonical exporter recreated the destination", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const sourceRoot = path.join(stateDir, "transcripts"); + const archivedStateDir = tempDirs.make("openclaw-meeting-transcripts-archive-"); + const archivedSessionDir = await seedLegacySession({ + stateDir: archivedStateDir, + sessionId: "recreated-export", + }); + const archiveRoot = path.join(archivedStateDir, "transcripts"); + const destination = path.join(sourceRoot, "2026-07-01", "recreated-export"); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.cp(archivedSessionDir, destination, { recursive: true }); + + await expect( + restoreCanonicalMeetingTranscriptExports({ + sourceRoot, + archiveRoot, + migratedSourcePaths: [], + canonicalRelativeDirs: [path.join("2026-07-01", "recreated-export")], + }), + ).resolves.toBeUndefined(); + await expect(fs.stat(path.join(destination, "metadata.json"))).resolves.toBeDefined(); + await expect(fs.stat(path.join(archivedSessionDir, "metadata.json"))).resolves.toBeDefined(); + }); + + it("rejects canonical restore paths that normalize outside their roots", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const sourceRoot = path.join(stateDir, "transcripts"); + const archiveRoot = path.join(stateDir, "transcripts.migrated-test"); + await fs.mkdir(archiveRoot, { recursive: true }); + + await expect( + restoreCanonicalMeetingTranscriptExports({ + sourceRoot, + archiveRoot, + migratedSourcePaths: [], + canonicalRelativeDirs: [path.join("safe", "..", "..", "outside")], + }), + ).rejects.toThrow("escaped its root"); + }); + + it.runIf(process.platform !== "win32")( + "rejects symlinked ancestors during canonical export restore", + async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const archiveRoot = path.join(stateDir, "transcripts.migrated-test"); + const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-"); + await fs.mkdir(path.join(archiveRoot), { recursive: true }); + await seedLegacySession({ stateDir: externalRoot, sessionId: "linked-export" }); + await fs.symlink( + path.join(externalRoot, "transcripts", "2026-07-01"), + path.join(archiveRoot, "2026-07-01"), + "dir", + ); + + await expect( + restoreCanonicalMeetingTranscriptExports({ + sourceRoot: path.join(stateDir, "transcripts"), + archiveRoot, + migratedSourcePaths: [], + canonicalRelativeDirs: [path.join("2026-07-01", "linked-export")], + }), + ).rejects.toThrow(/symlink/i); + }, + ); + + it("chunks large transcript imports while preserving exact order", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ + stateDir, + sessionId: "long-meeting", + utteranceCount: 530, + }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = await store.readSession("long-meeting"); + const utterances = await store.readUtterancesForSession(session!); + expect(utterances).toHaveLength(530); + expect(utterances[0]).toMatchObject({ id: "u-1", text: "First line" }); + expect(utterances.at(-1)).toMatchObject({ id: "u-530", text: "Line 530" }); + }); + + it("preserves an existing empty markdown summary", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ + stateDir, + sessionId: "empty-summary", + emptyMarkdown: true, + omitSummaryJson: true, + }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = await store.readSession("empty-summary"); + await expect(store.readSummary(session!)).resolves.toEqual({ markdown: "" }); + }); + + it("resumes an interruption after the import commit", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const nativeSession = { + sessionId: "modified-before-interruption", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + await store.writeSession(nativeSession); + const nativeArtifacts = await store.materializeSessionArtifacts(nativeSession, "metadata"); + await fs.appendFile(nativeArtifacts.metadataPath, " "); + await seedLegacySession({ stateDir, sessionId: "interrupted-import" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const interrupted = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + testHooks: { + afterImport: () => { + throw new Error("interrupted"); + }, + }, + }); + expect(interrupted.warnings.join("\n")).toContain("interrupted"); + expect(interrupted.changes.join("\n")).toContain("modified meeting transcript export"); + + const pending = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(pending.pendingImportCount).toBe(1); + const resumed = await migrateLegacyMeetingTranscripts({ + detected: pending, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(resumed.warnings).toEqual([]); + expect(resumed.changes.join("\n")).toContain("Resumed and archived"); + }); + + it("refuses to archive a new legacy session added after a pending import", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ stateDir, sessionId: "pending-original" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + testHooks: { + afterImport: () => { + throw new Error("interrupted"); + }, + }, + }); + const lateDir = await seedLegacySession({ stateDir, sessionId: "pending-late" }); + + const pending = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + const resumed = await migrateLegacyMeetingTranscripts({ + detected: pending, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(resumed.changes).toEqual([]); + expect(resumed.warnings.join("\n")).toContain("session tree changed before archive"); + await expect(fs.stat(lateDir)).resolves.toBeDefined(); + }); + + it("finalizes receipts after interruption following the archive move", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const nativeSession = { + sessionId: "native-export-during-resume", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + await store.writeSession(nativeSession); + await store.materializeSessionArtifacts(nativeSession, "metadata"); + await seedLegacySession({ stateDir, sessionId: "interrupted-archive" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + + const interrupted = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + testHooks: { + afterArchive: () => { + throw new Error("interrupted"); + }, + }, + }); + expect(interrupted.warnings.join("\n")).toContain("interrupted"); + + const pending = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(pending).toMatchObject({ hasLegacy: true, pendingImportCount: 1 }); + const resumed = await migrateLegacyMeetingTranscripts({ + detected: pending, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(resumed.warnings).toEqual([]); + expect(resumed.changes.join("\n")).toContain("Finalized interrupted"); + await expect(fs.stat(store.sessionDir(nativeSession))).resolves.toBeDefined(); + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + expect( + database + .prepare("SELECT status, removed_source FROM migration_sources WHERE migration_kind = ?") + .get("meeting-transcripts-files-v1"), + ).toEqual({ status: "archived", removed_source: 1 }); + }); + + it("does not reimport or archive explicit canonical exports", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + await seedLegacySession({ stateDir, sessionId: "design-review" }); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + await migrateLegacyMeetingTranscripts({ detected, env: databaseEnv(stateDir), stateDir }); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = await store.readSession("design-review"); + await store.materializeSessionArtifacts(session!, "all"); + + const rerunDetected = detectLegacyMeetingTranscripts({ + stateDir, + doctorOnlyStateMigrations: true, + }); + const rerun = await migrateLegacyMeetingTranscripts({ + detected: rerunDetected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(rerun).toEqual({ changes: [], warnings: [] }); + await expect( + fs.stat(path.join(stateDir, "transcripts", "2026-07-01", "design-review")), + ).resolves.toBeDefined(); + }); + + it("does not classify an interrupted export of an advancing transcript as legacy", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = { + sessionId: "advancing-export", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + await store.writeSession(session); + await store.appendUtteranceForSession(session, { text: "first" }); + await store.materializeSessionArtifacts(session, "transcript"); + await store.appendUtteranceForSession(session, { text: "second" }); + + const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db; + database + .prepare( + "UPDATE meeting_transcript_sessions SET export_pending_json = ? WHERE session_id = ?", + ) + .run('["transcript.jsonl"]', session.sessionId); + await fs.writeFile(path.join(store.sessionDir(session), "transcript.jsonl"), '{"text":'); + + const detected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: false }); + await expect( + migrateLegacyMeetingTranscripts({ detected, env: databaseEnv(stateDir), stateDir }), + ).resolves.toEqual({ changes: [], warnings: [] }); + + await store.materializeSessionArtifacts(session, "transcript"); + const exported = await fs.readFile( + path.join(store.sessionDir(session), "transcript.jsonl"), + "utf8", + ); + expect( + exported + .trim() + .split("\n") + .map((line) => JSON.parse(line).text), + ).toEqual(["first", "second"]); + }); + + it("migrates legacy sessions while preserving coexisting SQLite exports", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const nativeSession = { + sessionId: "native-export", + title: "Native export", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + const nativeUtterance = { text: "SQLite-native line" }; + const nativeUtterances = [nativeUtterance]; + await store.writeSession(nativeSession); + await store.appendUtteranceForSession(nativeSession, nativeUtterance); + await store.writeSummary( + summarizeTranscripts({ session: nativeSession, utterances: nativeUtterances }), + nativeSession, + ); + await store.materializeSessionArtifacts(nativeSession, "all"); + const nativeExport = path.join( + stateDir, + "transcripts", + "2026-07-02", + "native-export", + "metadata.json", + ); + const nativeTranscriptExport = path.join(path.dirname(nativeExport), "transcript.jsonl"); + await fs.rm(nativeExport); + expect( + detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }), + ).toMatchObject({ hasLegacy: false }); + await seedLegacySession({ stateDir, sessionId: "legacy-alongside-export" }); + + const detected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + const result = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(result.warnings).toEqual([]); + await expect(fs.stat(nativeExport)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.stat(nativeTranscriptExport)).resolves.toBeDefined(); + await expect(store.readSession("native-export")).resolves.toMatchObject({ + sessionId: "native-export", + }); + await expect(store.readSession("legacy-alongside-export")).resolves.toMatchObject({ + sessionId: "legacy-alongside-export", + }); + }); + + it("does not trust a DB selector when exported bytes diverge", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = { + sessionId: "modified-export", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + await store.writeSession(session); + await store.materializeSessionArtifacts(session, "metadata"); + await fs.appendFile( + path.join(stateDir, "transcripts", "2026-07-02", "modified-export", "metadata.json"), + " ", + ); + + const detected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(detected).toMatchObject({ hasLegacy: true }); + + const recovered = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + + expect(recovered.warnings).toEqual([]); + expect(recovered.changes.join("\n")).toContain("modified meeting transcript export"); + const metadata = await fs.readFile( + path.join(stateDir, "transcripts", "2026-07-02", "modified-export", "metadata.json"), + "utf8", + ); + expect(metadata.endsWith("\n")).toBe(true); + expect( + (await fs.readdir(stateDir)).some((entry) => + entry.startsWith("transcripts.exports-recovered-"), + ), + ).toBe(true); + }); + + it("resolves a case-renamed export directory by metadata identity", async () => { + const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-"); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: databaseEnv(stateDir), + }); + const session = { + sessionId: "Capital", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-02T10:00:00.000Z", + }; + await store.writeSession(session); + await store.appendUtteranceForSession(session, { text: "case-stable transcript" }); + const artifacts = await store.materializeSessionArtifacts(session, "transcript"); + const renamedDir = path.join(stateDir, "transcripts", "2026-07-02", "capital"); + await fs.rename(artifacts.sessionDir, renamedDir); + await fs.rename(path.join(renamedDir, "metadata.json"), path.join(renamedDir, "METADATA.JSON")); + + expect( + detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }), + ).toMatchObject({ hasLegacy: false }); + await fs.rm(path.join(renamedDir, "METADATA.JSON")); + const metadataLessDetected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(metadataLessDetected.hasLegacy).toBe(!fsSync.existsSync(artifacts.sessionDir)); + await fs.writeFile( + path.join(renamedDir, "METADATA.JSON"), + `${JSON.stringify(session, null, 2)}\n `, + ); + const detected = detectLegacyMeetingTranscripts({ + stateDir, + env: databaseEnv(stateDir), + doctorOnlyStateMigrations: true, + }); + expect(detected.hasLegacy).toBe(true); + + const recovered = await migrateLegacyMeetingTranscripts({ + detected, + env: databaseEnv(stateDir), + stateDir, + }); + expect(recovered.warnings).toEqual([]); + expect(recovered.changes.join("\n")).toContain("modified meeting transcript export"); + await expect(store.readSession("2026-07-02/Capital")).resolves.toMatchObject({ + sessionId: "Capital", + }); + }); +}); diff --git a/src/infra/state-migrations.meeting-transcripts.ts b/src/infra/state-migrations.meeting-transcripts.ts new file mode 100644 index 000000000000..f5662eba17f6 --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts.ts @@ -0,0 +1,631 @@ +import { randomUUID } from "node:crypto"; +// Doctor-only import for the retired meeting-capture JSON/JSONL store. +import fsSync from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { ensureMeetingTranscriptsSchema } from "../transcripts/sqlite-schema.js"; +import { TranscriptsStore } from "../transcripts/store.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; +import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync } from "./kysely-sync.js"; +import { migrationDb } from "./state-migrations.meeting-transcripts-database.js"; +import { + readMeetingTranscriptMigrationDetectionState, + resolveMeetingTranscriptExportOwnership, +} from "./state-migrations.meeting-transcripts-detection.js"; +import { + archiveLegacyMeetingTranscriptSnapshots, + archiveDivergentMeetingTranscriptExport, + archivePartialMeetingTranscriptArtifacts, + isRecordedCanonicalTranscriptExport, + LegacyMeetingTranscriptArchiveMovedError, + listLegacyMeetingTranscriptArtifactDirs, + listLegacyMeetingTranscriptSessionDirs, + openLegacyMeetingTranscriptStage, + rehashLegacyMeetingTranscriptSnapshots, + restoreCanonicalMeetingTranscriptExports, + snapshotLegacyMeetingTranscriptSession, + validateMeetingTranscriptRoot, + type LegacyMeetingTranscriptSnapshot, +} from "./state-migrations.meeting-transcripts-files.js"; +import { insertMeetingTranscriptSnapshots } from "./state-migrations.meeting-transcripts-insert.js"; +import { verifyImportedMeetingTranscriptSnapshots } from "./state-migrations.meeting-transcripts-verify.js"; +import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; + +export { detectLegacyMeetingTranscripts } from "./state-migrations.meeting-transcripts-detection.js"; + +function resolveArchiveRoot(sourceRoot: string, now: number): string { + const base = `${sourceRoot}.migrated-${new Date(now).toISOString().replace(/[:.]/g, "-")}`; + return fsSync.existsSync(base) ? `${base}-${randomUUID()}` : base; +} + +function rollbackImportedSnapshots(params: { + snapshots: LegacyMeetingTranscriptSnapshot[]; + runId: string; + env: NodeJS.ProcessEnv; + stateDir: string; +}): void { + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = migrationDb(database); + for (const snapshot of params.snapshots) { + executeSqliteQuerySync( + database, + db + .deleteFrom("meeting_transcript_sessions") + .where("session_id", "=", snapshot.session.sessionId) + .where("started_at", "=", snapshot.session.startedAt), + ); + } + executeSqliteQuerySync( + database, + db.deleteFrom("migration_sources").where("last_run_id", "=", params.runId), + ); + executeSqliteQuerySync( + database, + db.deleteFrom("migration_runs").where("id", "=", params.runId), + ); + }, + { env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } }, + { operationLabel: "meeting-transcripts.legacy-import.rollback" }, + ); +} + +function finishPendingMigration(params: { + runId: string; + archiveRoot: string; + now: number; + env: NodeJS.ProcessEnv; + stateDir: string; +}): void { + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = migrationDb(database); + executeSqliteQuerySync( + database, + db + .updateTable("migration_sources") + .set({ status: "archived", removed_source: 1 }) + .where("last_run_id", "=", params.runId) + .where("migration_kind", "=", "meeting-transcripts-files-v1"), + ); + const run = executeSqliteQueryTakeFirstSync( + database, + db.selectFrom("migration_runs").select("report_json").where("id", "=", params.runId), + ); + const report = run ? (JSON.parse(run.report_json) as Record) : {}; + executeSqliteQuerySync( + database, + db + .updateTable("migration_runs") + .set({ + finished_at: params.now, + status: "completed", + report_json: JSON.stringify({ ...report, archiveRoot: params.archiveRoot }), + }) + .where("id", "=", params.runId), + ); + }, + { env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } }, + { operationLabel: "meeting-transcripts.legacy-import.finish" }, + ); +} + +type PendingImportRun = { + runId: string; + archiveRoot: string; + canonicalRelativeDirs: string[]; + sources: Array<{ sourcePath: string; sourceHash: string }>; +}; + +function isStrictRelativePathWithinRoot(root: string, relativePath: string): boolean { + if (!relativePath || relativePath === "." || path.isAbsolute(relativePath)) { + return false; + } + const resolvedRoot = path.resolve(root); + const resolvedPath = path.resolve(resolvedRoot, relativePath); + const relative = path.relative(resolvedRoot, resolvedPath); + return Boolean(relative && !relative.startsWith("..") && !path.isAbsolute(relative)); +} + +async function snapshotPendingImportRun(params: { + run: PendingImportRun; + snapshotRoot: string; + sourceRoot: string; + stageDatabase: DatabaseSync; +}): Promise<{ snapshots: LegacyMeetingTranscriptSnapshot[]; hashesMatch: boolean }> { + const snapshots: LegacyMeetingTranscriptSnapshot[] = []; + let hashesMatch = true; + for (const source of params.run.sources) { + const relativeDir = path.relative(params.sourceRoot, source.sourcePath) || "."; + if (relativeDir.startsWith("..") || path.isAbsolute(relativeDir)) { + throw new Error(`pending meeting transcript source escaped its root: ${source.sourcePath}`); + } + const snapshot = await snapshotLegacyMeetingTranscriptSession({ + rootDir: params.snapshotRoot, + relativeDir, + stageDatabase: params.stageDatabase, + }); + snapshots.push(snapshot); + hashesMatch &&= snapshot.sourceHash === source.sourceHash; + } + return { snapshots, hashesMatch }; +} + +function readPendingImportRuns(params: { + env: NodeJS.ProcessEnv; + stateDir: string; + sourceRoot: string; +}): PendingImportRun[] { + const database = openOpenClawStateDatabase({ + env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir }, + }); + const db = migrationDb(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("migration_sources as source") + .innerJoin("migration_runs as run", "run.id", "source.last_run_id") + .select([ + "source.last_run_id as run_id", + "source.source_path", + "source.source_sha256", + "run.report_json as run_report_json", + ]) + .where("source.migration_kind", "=", "meeting-transcripts-files-v1") + .where("source.status", "=", "imported") + .where("source.removed_source", "=", 0) + .orderBy("source.source_path", "asc"), + ).rows; + const runs = new Map(); + for (const row of rows) { + const report = JSON.parse(row.run_report_json) as Record; + const format = report.format; + const archiveRoot = report.archiveRoot; + const canonicalRelativeDirs = report.canonicalRelativeDirs; + if ( + typeof archiveRoot !== "string" || + format !== "meeting-transcripts-files-v1" || + !archiveRoot.startsWith(`${params.sourceRoot}.migrated-`) || + !Array.isArray(canonicalRelativeDirs) || + !canonicalRelativeDirs.every( + (relativeDir) => + typeof relativeDir === "string" && + isStrictRelativePathWithinRoot(params.sourceRoot, relativeDir), + ) || + typeof row.source_sha256 !== "string" + ) { + throw new Error(`invalid pending meeting transcript migration receipt: ${row.run_id}`); + } + const run: PendingImportRun = runs.get(row.run_id) ?? { + runId: row.run_id, + archiveRoot, + canonicalRelativeDirs, + sources: [], + }; + if ( + run.archiveRoot !== archiveRoot || + JSON.stringify(run.canonicalRelativeDirs) !== JSON.stringify(canonicalRelativeDirs) + ) { + throw new Error(`conflicting meeting transcript archive receipts: ${row.run_id}`); + } + run.sources.push({ sourcePath: row.source_path, sourceHash: row.source_sha256 }); + runs.set(row.run_id, run); + } + return [...runs.values()]; +} + +async function listCanonicalMeetingTranscriptExportDirs(params: { + rootDir: string; + env: NodeJS.ProcessEnv; +}): Promise { + const state = readMeetingTranscriptMigrationDetectionState({ env: params.env }); + const relativeDirs = await listLegacyMeetingTranscriptArtifactDirs(params.rootDir); + return relativeDirs.filter((relativeDir) => { + const selector = relativeDir.split(path.sep).join("/"); + const sessionDir = path.join(params.rootDir, relativeDir); + const ownership = resolveMeetingTranscriptExportOwnership({ + state, + selector, + sessionDir, + sourceRoot: params.rootDir, + }); + return Boolean( + ownership && + isRecordedCanonicalTranscriptExport({ + sessionDir, + manifest: ownership.manifest, + pending: ownership.pending, + }), + ); + }); +} + +async function resumePendingImports(params: { + env: NodeJS.ProcessEnv; + stateDir: string; + sourceRoot: string; + store: TranscriptsStore; + stageDatabase: DatabaseSync; +}): Promise { + const runs = readPendingImportRuns(params); + if (runs.length === 0) { + return undefined; + } + const changes: string[] = []; + const warnings: string[] = []; + for (const run of runs) { + if (fsSync.existsSync(run.archiveRoot)) { + try { + const archived = await snapshotPendingImportRun({ + run, + snapshotRoot: run.archiveRoot, + sourceRoot: params.sourceRoot, + stageDatabase: params.stageDatabase, + }); + if (!archived.hashesMatch) { + throw new Error("archived source hashes do not match migration receipts"); + } + const database = openOpenClawStateDatabase({ + env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir }, + }); + await verifyImportedMeetingTranscriptSnapshots({ + store: params.store, + snapshots: archived.snapshots, + stageDatabase: params.stageDatabase, + database: database.db, + }); + await restoreCanonicalMeetingTranscriptExports({ + sourceRoot: params.sourceRoot, + archiveRoot: run.archiveRoot, + migratedSourcePaths: run.sources.map((source) => source.sourcePath), + canonicalRelativeDirs: run.canonicalRelativeDirs, + }); + finishPendingMigration({ + runId: run.runId, + archiveRoot: run.archiveRoot, + now: Date.now(), + env: params.env, + stateDir: params.stateDir, + }); + changes.push(`Finalized interrupted meeting transcript archive → ${run.archiveRoot}`); + } catch (error) { + warnings.push( + `Pending meeting transcript migration ${run.runId} archive could not be verified or restored; left its rows and files for recovery: ${String(error)}`, + ); + } + continue; + } + if (!fsSync.existsSync(params.sourceRoot)) { + warnings.push( + `Pending meeting transcript migration ${run.runId} has neither source tree nor archive`, + ); + continue; + } + const canonicalRelativeDirs = [ + ...new Set([ + ...run.canonicalRelativeDirs, + ...(await listCanonicalMeetingTranscriptExportDirs({ + rootDir: params.sourceRoot, + env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir }, + })), + ]), + ]; + const expectedRelativeDirs = [ + ...new Set([ + ...run.sources.map((source) => path.relative(params.sourceRoot, source.sourcePath) || "."), + ...canonicalRelativeDirs, + ]), + ].toSorted((a, b) => a.localeCompare(b)); + const pending = await snapshotPendingImportRun({ + run, + snapshotRoot: params.sourceRoot, + sourceRoot: params.sourceRoot, + stageDatabase: params.stageDatabase, + }); + if (!pending.hashesMatch) { + warnings.push( + `Pending meeting transcript migration ${run.runId} source tree changed; left its imported rows and files for manual recovery`, + ); + continue; + } + const database = openOpenClawStateDatabase({ + env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir }, + }); + await verifyImportedMeetingTranscriptSnapshots({ + store: params.store, + snapshots: pending.snapshots, + stageDatabase: params.stageDatabase, + database: database.db, + }); + await archiveLegacyMeetingTranscriptSnapshots({ + sourceRoot: params.sourceRoot, + snapshots: pending.snapshots, + expectedRelativeDirs, + canonicalRelativeDirs, + archiveRoot: run.archiveRoot, + }); + finishPendingMigration({ + runId: run.runId, + archiveRoot: run.archiveRoot, + now: Date.now(), + env: params.env, + stateDir: params.stateDir, + }); + changes.push(`Resumed and archived meeting transcript migration → ${run.archiveRoot}`); + } + return { changes, warnings }; +} + +export async function migrateLegacyMeetingTranscripts(params: { + detected?: LegacyMeetingTranscriptsDetection; + env?: NodeJS.ProcessEnv; + stateDir: string; + now?: () => number; + testHooks?: { + afterImport?: () => void; + afterArchive?: () => void; + }; +}): Promise { + const detected = params.detected; + if (!detected?.hasLegacy) { + return { changes: [], warnings: [] }; + } + const env = params.env ?? process.env; + let lock: Awaited>; + try { + lock = await acquireGatewayLock({ + allowInTests: true, + env: { ...env, OPENCLAW_STATE_DIR: params.stateDir }, + role: "sqlite-maintenance", + timeoutMs: 5_000, + }); + } catch (error) { + return { + changes: [], + warnings: [ + `Skipped meeting transcript migration because exclusive state ownership is unavailable: ${String(error)}`, + ], + }; + } + if (!lock) { + return { + changes: [], + warnings: [ + "Skipped meeting transcript migration because exclusive state ownership is unavailable", + ], + }; + } + + let stageDatabase: DatabaseSync | undefined; + let stagePath: string | undefined; + const recoveryChanges: string[] = []; + try { + fsSync.mkdirSync(params.stateDir, { recursive: true }); + stagePath = path.join(params.stateDir, `.meeting-transcripts-migration-${randomUUID()}.sqlite`); + const stage = openLegacyMeetingTranscriptStage(stagePath); + stageDatabase = stage; + await validateMeetingTranscriptRoot(detected.sourceDir, { allowMissing: true }); + const databaseOptions = { env: { ...env, OPENCLAW_STATE_DIR: params.stateDir } }; + ensureMeetingTranscriptsSchema(databaseOptions); + const store = new TranscriptsStore(detected.sourceDir, databaseOptions); + const resumed = await resumePendingImports({ + env, + stateDir: params.stateDir, + sourceRoot: detected.sourceDir, + store, + stageDatabase: stage, + }); + if (resumed) { + return resumed; + } + const now = params.now?.() ?? Date.now(); + const relativeDirs = await listLegacyMeetingTranscriptArtifactDirs(detected.sourceDir); + const sessionRelativeDirs = await listLegacyMeetingTranscriptSessionDirs(detected.sourceDir); + const sessionRelativeDirSet = new Set(sessionRelativeDirs); + const detectionState = readMeetingTranscriptMigrationDetectionState({ + env: { ...env, OPENCLAW_STATE_DIR: params.stateDir }, + }); + const legacyRelativeDirs: string[] = []; + const partialRelativeDirs: string[] = []; + const divergentExportDirs: Array<{ relativeDir: string; ownerSelector: string }> = []; + for (const relativeDir of relativeDirs) { + const selector = relativeDir.split(path.sep).join("/"); + const ownership = resolveMeetingTranscriptExportOwnership({ + state: detectionState, + selector, + sessionDir: path.join(detected.sourceDir, relativeDir), + sourceRoot: detected.sourceDir, + }); + if ( + ownership && + isRecordedCanonicalTranscriptExport({ + sessionDir: path.join(detected.sourceDir, relativeDir), + manifest: ownership.manifest, + pending: ownership.pending, + }) + ) { + continue; + } + if (ownership) { + divergentExportDirs.push({ relativeDir, ownerSelector: ownership.selector }); + } else if (!sessionRelativeDirSet.has(relativeDir)) { + partialRelativeDirs.push(relativeDir); + } else { + legacyRelativeDirs.push(relativeDir); + } + } + const snapshots: LegacyMeetingTranscriptSnapshot[] = []; + for (const relativeDir of legacyRelativeDirs) { + snapshots.push( + await snapshotLegacyMeetingTranscriptSession({ + rootDir: detected.sourceDir, + relativeDir, + stageDatabase: stage, + }), + ); + } + const plans: LegacyMeetingTranscriptSnapshot[] = []; + for (const snapshot of snapshots) { + const database = openOpenClawStateDatabase(databaseOptions); + const existing = executeSqliteQueryTakeFirstSync( + database.db, + migrationDb(database.db) + .selectFrom("meeting_transcript_sessions") + .select("session_id") + .where("session_id", "=", snapshot.session.sessionId) + .where("started_at", "=", snapshot.session.startedAt), + ); + if (existing) { + throw new Error( + `legacy transcript conflicts with canonical SQLite state: ${snapshot.relativeDir}`, + ); + } + plans.push(snapshot); + } + if (divergentExportDirs.length > 0) { + const recoveryRoot = `${detected.sourceDir}.exports-recovered-${new Date(now) + .toISOString() + .replace(/[:.]/g, "-")}`; + for (const { relativeDir, ownerSelector } of divergentExportDirs) { + const session = await store.readSession(ownerSelector); + if (!session) { + throw new Error(`divergent transcript export has no SQLite owner: ${relativeDir}`); + } + await archiveDivergentMeetingTranscriptExport({ + sourceRoot: detected.sourceDir, + relativeDir, + recoveryRoot, + }); + recoveryChanges.push( + `Archived modified meeting transcript export ${relativeDir} → ${recoveryRoot}`, + ); + await store.materializeSessionArtifacts(session, "all"); + } + } + if (plans.length === 0 && partialRelativeDirs.length > 0) { + const recoveryRoot = `${detected.sourceDir}.partials-recovered-${new Date(now) + .toISOString() + .replace(/[:.]/g, "-")}`; + await archivePartialMeetingTranscriptArtifacts({ + sourceRoot: detected.sourceDir, + relativeDirs: partialRelativeDirs, + recoveryRoot, + }); + recoveryChanges.push( + `Archived ${partialRelativeDirs.length} incomplete meeting transcript director${partialRelativeDirs.length === 1 ? "y" : "ies"} → ${recoveryRoot}`, + ); + } + const expectedArchiveRelativeDirs = await listLegacyMeetingTranscriptSessionDirs( + detected.sourceDir, + ); + if (plans.length === 0) { + return { changes: recoveryChanges, warnings: [] }; + } + + const runId = randomUUID(); + const archiveRoot = resolveArchiveRoot(detected.sourceDir, now); + const canonicalRelativeDirs = await listCanonicalMeetingTranscriptExportDirs({ + rootDir: detected.sourceDir, + env: { ...env, OPENCLAW_STATE_DIR: params.stateDir }, + }); + insertMeetingTranscriptSnapshots({ + snapshots: plans, + runId, + now, + archiveRoot, + canonicalRelativeDirs, + stageDatabase: stage, + env, + stateDir: params.stateDir, + }); + try { + const database = openOpenClawStateDatabase(databaseOptions); + await verifyImportedMeetingTranscriptSnapshots({ + store, + snapshots: plans, + stageDatabase: stage, + database: database.db, + }); + if (!(await rehashLegacyMeetingTranscriptSnapshots(plans))) { + rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir }); + return { + changes: recoveryChanges, + warnings: [ + "Legacy meeting transcript files changed after import; rolled back SQLite rows and left every source in place for a Doctor retry", + ], + }; + } + } catch (error) { + rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir }); + throw error; + } + params.testHooks?.afterImport?.(); + let archiveRootAfterMove: string; + try { + archiveRootAfterMove = await archiveLegacyMeetingTranscriptSnapshots({ + sourceRoot: detected.sourceDir, + snapshots: plans, + expectedRelativeDirs: expectedArchiveRelativeDirs, + canonicalRelativeDirs, + archiveRoot, + }); + } catch (error) { + if (error instanceof LegacyMeetingTranscriptArchiveMovedError) { + return { + changes: [ + ...recoveryChanges, + `Imported ${plans.length} meeting transcript session${plans.length === 1 ? "" : "s"} into shared SQLite state`, + ], + warnings: [ + `Meeting transcript archive needs Doctor resume after moving the source tree: ${String(error)}`, + ], + }; + } + rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir }); + return { + changes: recoveryChanges, + warnings: [ + `Failed archiving verified legacy meeting transcripts; rolled back SQLite rows and left every source in place for Doctor retry: ${String(error)}`, + ], + }; + } + params.testHooks?.afterArchive?.(); + finishPendingMigration({ + runId, + archiveRoot: archiveRootAfterMove, + now, + env, + stateDir: params.stateDir, + }); + const utteranceCount = plans.reduce((total, snapshot) => total + snapshot.utteranceCount, 0); + return { + changes: [ + ...recoveryChanges, + `Migrated ${plans.length} meeting transcript session${plans.length === 1 ? "" : "s"} and ${utteranceCount} utterance${utteranceCount === 1 ? "" : "s"} to shared SQLite state`, + `Archived legacy meeting transcript files → ${archiveRootAfterMove}`, + ], + warnings: [], + }; + } catch (error) { + return { + changes: recoveryChanges, + warnings: [`Failed migrating meeting transcripts: ${String(error)}`], + }; + } finally { + try { + stageDatabase?.close(); + if (stagePath) { + fsSync.rmSync(stagePath, { force: true }); + fsSync.rmSync(`${stagePath}-shm`, { force: true }); + fsSync.rmSync(`${stagePath}-wal`, { force: true }); + } + } finally { + await lock.release(); + } + } +} diff --git a/src/infra/state-migrations.meeting-transcripts.types.ts b/src/infra/state-migrations.meeting-transcripts.types.ts new file mode 100644 index 000000000000..03750c2bf504 --- /dev/null +++ b/src/infra/state-migrations.meeting-transcripts.types.ts @@ -0,0 +1,5 @@ +export type LegacyMeetingTranscriptsDetection = { + sourceDir: string; + hasLegacy: boolean; + pendingImportCount: number; +}; diff --git a/src/infra/state-migrations.types.ts b/src/infra/state-migrations.types.ts index 047bcd0b92cf..c3e7ed6e1427 100644 --- a/src/infra/state-migrations.types.ts +++ b/src/infra/state-migrations.types.ts @@ -5,6 +5,7 @@ import type { LegacyAuditLogsDetection } from "./state-migrations.audit-logs.typ import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js"; import type { LegacyDeviceIdentityDetection } from "./state-migrations.device-identity.types.js"; import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js"; +import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js"; import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js"; import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js"; @@ -126,6 +127,7 @@ export type LegacyStateDetection = { }; deviceIdentity: LegacyDeviceIdentityDetection; mcpOauth: LegacyMcpOAuthDetection; + meetingTranscripts?: LegacyMeetingTranscriptsDetection; restartSentinel?: LegacyRestartSentinelDetection; workspace: LegacyWorkspaceStateDetection; webPush: { diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 04db6e086d33..2a86d355515e 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -783,6 +783,47 @@ export interface MediaBlobs { updated_at: number; } +export interface MeetingTranscriptSessions { + created_at_ms: number; + export_key: string; + export_manifest_json: Generated; + export_pending_json: Generated; + metadata_json: string | null; + next_utterance_seq: Generated; + provider_id: string; + selector: string; + session_id: string; + session_slug: string; + source_json: string; + started_at: string; + stopped_at: string | null; + title: string | null; + updated_at_ms: number; +} + +export interface MeetingTranscriptSummaries { + generated_at: string | null; + markdown: string | null; + session_id: string; + session_started_at: string; + summary_json: string | null; + utterance_count: number; +} + +export interface MeetingTranscriptUtterances { + ended_at: string | null; + final: number | null; + metadata_json: string | null; + sequence: number; + session_id: string; + session_started_at: string; + speaker_id: string | null; + speaker_label: string | null; + started_at: string | null; + text: string; + utterance_id: string | null; +} + export interface MigrationRuns { finished_at: number | null; id: string; @@ -1481,6 +1522,9 @@ export interface DB { managed_outgoing_image_records: ManagedOutgoingImageRecords; mcp_oauth_stores: McpOauthStores; media_blobs: MediaBlobs; + meeting_transcript_sessions: MeetingTranscriptSessions; + meeting_transcript_summaries: MeetingTranscriptSummaries; + meeting_transcript_utterances: MeetingTranscriptUtterances; migration_runs: MigrationRuns; migration_sources: MigrationSources; model_capability_cache: ModelCapabilityCache; diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index ee1c25a388a4..ff6ca169248d 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -1579,6 +1579,71 @@ CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at); +-- Durable meeting-capture sessions are gateway-global rather than agent-session +-- transcripts. JSON/JSONL files are doctor import inputs or explicit CLI exports. +CREATE TABLE IF NOT EXISTS meeting_transcript_sessions ( + session_id TEXT NOT NULL, + started_at TEXT NOT NULL, + selector TEXT NOT NULL UNIQUE, + export_key TEXT NOT NULL, + session_slug TEXT NOT NULL, + provider_id TEXT NOT NULL, + title TEXT, + source_json TEXT NOT NULL, + stopped_at TEXT, + metadata_json TEXT, + export_manifest_json TEXT NOT NULL DEFAULT '{}', + export_pending_json TEXT NOT NULL DEFAULT '[]', + next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0), + created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0), + updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0), + PRIMARY KEY (session_id, started_at) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started + ON meeting_transcript_sessions(started_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id + ON meeting_transcript_sessions(session_id, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug + ON meeting_transcript_sessions(session_slug, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key + ON meeting_transcript_sessions(export_key); + +CREATE TABLE IF NOT EXISTS meeting_transcript_utterances ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + utterance_id TEXT, + started_at TEXT, + ended_at TEXT, + speaker_id TEXT, + speaker_label TEXT, + text TEXT NOT NULL, + final INTEGER CHECK (final IN (0, 1)), + metadata_json TEXT, + PRIMARY KEY (session_id, session_started_at, sequence), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE +) STRICT; + +CREATE TABLE IF NOT EXISTS meeting_transcript_summaries ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + generated_at TEXT, + summary_json TEXT, + markdown TEXT, + utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0), + PRIMARY KEY (session_id, session_started_at), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE, + CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL) +) STRICT; + CREATE TABLE IF NOT EXISTS migration_runs ( id TEXT NOT NULL PRIMARY KEY, started_at INTEGER NOT NULL, diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index f97da94c3b6e..8ca7bd97700e 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -1574,6 +1574,71 @@ CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at); +-- Durable meeting-capture sessions are gateway-global rather than agent-session +-- transcripts. JSON/JSONL files are doctor import inputs or explicit CLI exports. +CREATE TABLE IF NOT EXISTS meeting_transcript_sessions ( + session_id TEXT NOT NULL, + started_at TEXT NOT NULL, + selector TEXT NOT NULL UNIQUE, + export_key TEXT NOT NULL, + session_slug TEXT NOT NULL, + provider_id TEXT NOT NULL, + title TEXT, + source_json TEXT NOT NULL, + stopped_at TEXT, + metadata_json TEXT, + export_manifest_json TEXT NOT NULL DEFAULT '{}', + export_pending_json TEXT NOT NULL DEFAULT '[]', + next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0), + created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0), + updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0), + PRIMARY KEY (session_id, started_at) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started + ON meeting_transcript_sessions(started_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id + ON meeting_transcript_sessions(session_id, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug + ON meeting_transcript_sessions(session_slug, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key + ON meeting_transcript_sessions(export_key); + +CREATE TABLE IF NOT EXISTS meeting_transcript_utterances ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + utterance_id TEXT, + started_at TEXT, + ended_at TEXT, + speaker_id TEXT, + speaker_label TEXT, + text TEXT NOT NULL, + final INTEGER CHECK (final IN (0, 1)), + metadata_json TEXT, + PRIMARY KEY (session_id, session_started_at, sequence), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE +) STRICT; + +CREATE TABLE IF NOT EXISTS meeting_transcript_summaries ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + generated_at TEXT, + summary_json TEXT, + markdown TEXT, + utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0), + PRIMARY KEY (session_id, session_started_at), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE, + CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL) +) STRICT; + CREATE TABLE IF NOT EXISTS migration_runs ( id TEXT NOT NULL PRIMARY KEY, started_at INTEGER NOT NULL, diff --git a/src/transcripts/sqlite-schema.ts b/src/transcripts/sqlite-schema.ts new file mode 100644 index 000000000000..81bbd2d43776 --- /dev/null +++ b/src/transcripts/sqlite-schema.ts @@ -0,0 +1,84 @@ +// Additive meeting-transcript schema used by the feature's one-time lazy ensure. +import type { DatabaseSync } from "node:sqlite"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; + +const ensuredDatabases = new WeakSet(); +const MEETING_TRANSCRIPTS_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS meeting_transcript_sessions ( + session_id TEXT NOT NULL, + started_at TEXT NOT NULL, + selector TEXT NOT NULL UNIQUE, + export_key TEXT NOT NULL, + session_slug TEXT NOT NULL, + provider_id TEXT NOT NULL, + title TEXT, + source_json TEXT NOT NULL, + stopped_at TEXT, + metadata_json TEXT, + export_manifest_json TEXT NOT NULL DEFAULT '{}', + export_pending_json TEXT NOT NULL DEFAULT '[]', + next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0), + created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0), + updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0), + PRIMARY KEY (session_id, started_at) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started + ON meeting_transcript_sessions(started_at DESC, session_id); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id + ON meeting_transcript_sessions(session_id, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug + ON meeting_transcript_sessions(session_slug, started_at DESC); + +CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key + ON meeting_transcript_sessions(export_key); + +CREATE TABLE IF NOT EXISTS meeting_transcript_utterances ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + utterance_id TEXT, + started_at TEXT, + ended_at TEXT, + speaker_id TEXT, + speaker_label TEXT, + text TEXT NOT NULL, + final INTEGER CHECK (final IN (0, 1)), + metadata_json TEXT, + PRIMARY KEY (session_id, session_started_at, sequence), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE +) STRICT; + +CREATE TABLE IF NOT EXISTS meeting_transcript_summaries ( + session_id TEXT NOT NULL, + session_started_at TEXT NOT NULL, + generated_at TEXT, + summary_json TEXT, + markdown TEXT, + utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0), + PRIMARY KEY (session_id, session_started_at), + FOREIGN KEY (session_id, session_started_at) + REFERENCES meeting_transcript_sessions(session_id, started_at) + ON DELETE CASCADE, + CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL) +) STRICT; +`; + +export function ensureMeetingTranscriptsSchema(options: OpenClawStateDatabaseOptions = {}): void { + const database = openOpenClawStateDatabase(options); + if (ensuredDatabases.has(database.db)) { + return; + } + runOpenClawStateWriteTransaction(({ db }) => db.exec(MEETING_TRANSCRIPTS_SCHEMA_SQL), options, { + operationLabel: "meeting-transcripts.schema.ensure", + }); + ensuredDatabases.add(database.db); +} diff --git a/src/transcripts/store-artifacts.ts b/src/transcripts/store-artifacts.ts new file mode 100644 index 000000000000..7d8cda7b9bc3 --- /dev/null +++ b/src/transcripts/store-artifacts.ts @@ -0,0 +1,107 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sha256Hex } from "../infra/crypto-digest.js"; +import { writeExternalFileWithinRoot } from "../infra/fs-safe.js"; +import type { TranscriptSessionDescriptor } from "./provider-types.js"; + +export const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([ + "metadata.json", + "summary.json", + "summary.md", + "transcript.jsonl", +]); + +export function safeTranscriptPathSegment(value: string): string { + const segment = value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); + if (segment === ".") { + return "%2E"; + } + if (segment === "..") { + return "%2E%2E"; + } + if (!segment) { + return "session"; + } + if (segment.endsWith(".") || /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/iu.test(segment)) { + return Buffer.from(segment, "utf8") + .toString("hex") + .match(/.{2}/gu)! + .map((byte) => `%${byte.toUpperCase()}`) + .join(""); + } + return segment; +} + +function legacyTranscriptPathSegment(value: string): string { + return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session"; +} + +function dateSegment(value: string | undefined): string { + const isoDate = value?.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1]; + return isoDate ?? new Date().toISOString().slice(0, 10); +} + +export function transcriptSessionSelector(session: TranscriptSessionDescriptor): string { + return `${dateSegment(session.startedAt)}/${safeTranscriptPathSegment(session.sessionId)}`; +} + +export function legacyTranscriptSessionSelector(session: TranscriptSessionDescriptor): string { + const date = dateSegment(session.startedAt); + const segment = legacyTranscriptPathSegment(session.sessionId); + // The shipped sanitizer allowed dot components: `.` collapsed to the date + // directory and `..` collapsed to the transcript root. + if (segment === ".") { + return date; + } + if (segment === "..") { + return "."; + } + return `${date}/${segment}`; +} + +export function transcriptSessionExportKey(session: TranscriptSessionDescriptor): string { + return transcriptSessionSelector(session).toLowerCase(); +} + +export function normalizeExportText(value: string): string { + return value.endsWith("\n") ? value : `${value}\n`; +} + +export async function writeTranscriptArtifact( + rootDir: string, + fileName: string, + content: string, +): Promise { + await writeExternalFileWithinRoot({ + rootDir, + path: fileName, + write: async (tempPath) => await fs.writeFile(tempPath, content, { mode: 0o600 }), + }); + return sha256Hex(content); +} + +export async function removeTranscriptArtifact(rootDir: string, fileName: string): Promise { + await fs.rm(path.join(rootDir, fileName), { force: true }); +} + +export async function isCaseSensitiveDirectory(directory: string): Promise { + const probeName = `.openclaw-case-probe-${randomUUID().toLowerCase()}`; + const probePath = path.join(directory, probeName); + const alternatePath = path.join(directory, probeName.toUpperCase()); + const handle = await fs.open(probePath, "wx", 0o600); + await handle.close(); + try { + try { + await fs.access(alternatePath); + return false; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return true; + } + throw error; + } + } finally { + await fs.rm(probePath, { force: true }); + } +} diff --git a/src/transcripts/store-export-jsonl.ts b/src/transcripts/store-export-jsonl.ts new file mode 100644 index 000000000000..adc439317766 --- /dev/null +++ b/src/transcripts/store-export-jsonl.ts @@ -0,0 +1,70 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import { writeExternalFileWithinRoot } from "../infra/fs-safe.js"; +import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync } from "../infra/kysely-sync.js"; +import { + openOpenClawStateDatabase, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import type { TranscriptSessionDescriptor } from "./provider-types.js"; +import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js"; +import { meetingTranscriptDb, utteranceFromRow } from "./store-sqlite.js"; + +const TRANSCRIPT_EXPORT_ROW_BATCH_SIZE = 64; + +export async function writeTranscriptJsonlArtifact(params: { + sessionDir: string; + session: TranscriptSessionDescriptor; + databaseOptions: OpenClawStateDatabaseOptions; +}): Promise { + ensureMeetingTranscriptsSchema(params.databaseOptions); + const database = openOpenClawStateDatabase(params.databaseOptions); + const sequenceHead = executeSqliteQueryTakeFirstSync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_sessions") + .select("next_utterance_seq") + .where("session_id", "=", params.session.sessionId) + .where("started_at", "=", params.session.startedAt), + )?.next_utterance_seq; + if (sequenceHead === undefined) { + throw new Error(`transcripts session not found: ${params.session.sessionId}`); + } + const digest = createHash("sha256"); + await writeExternalFileWithinRoot({ + rootDir: params.sessionDir, + path: "transcript.jsonl", + write: async (tempPath) => { + const handle = await fs.open(tempPath, "w", 0o600); + try { + let nextSequence = 0; + while (nextSequence < sequenceHead) { + const rows = executeSqliteQuerySync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_utterances") + .selectAll() + .where("session_id", "=", params.session.sessionId) + .where("session_started_at", "=", params.session.startedAt) + .where("sequence", ">=", nextSequence) + .where("sequence", "<", sequenceHead) + .orderBy("sequence", "asc") + .limit(TRANSCRIPT_EXPORT_ROW_BATCH_SIZE), + ).rows; + if (rows.length === 0) { + break; + } + nextSequence = rows.at(-1)!.sequence + 1; + const lines = rows.map((row) => `${JSON.stringify(utteranceFromRow(row))}\n`); + for (const line of lines) { + await handle.writeFile(line); + digest.update(line); + } + } + } finally { + await handle.close(); + } + }, + }); + return digest.digest("hex"); +} diff --git a/src/transcripts/store-export-ownership.ts b/src/transcripts/store-export-ownership.ts new file mode 100644 index 000000000000..933082976f66 --- /dev/null +++ b/src/transcripts/store-export-ownership.ts @@ -0,0 +1,221 @@ +import { constants as fsConstants } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sha256File } from "../infra/crypto-digest.js"; +import { ensureAbsoluteDirectory } from "../infra/fs-safe.js"; +import { executeSqliteQuerySync } from "../infra/kysely-sync.js"; +import { + openOpenClawStateDatabase, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import type { TranscriptSessionDescriptor } from "./provider-types.js"; +import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js"; +import { + isCaseSensitiveDirectory, + transcriptSessionExportKey, + transcriptSessionSelector, +} from "./store-artifacts.js"; +import { meetingTranscriptDb } from "./store-sqlite.js"; + +type ExportOwnershipParams = { + session: TranscriptSessionDescriptor; + exportRootDir: string; + databaseOptions: OpenClawStateDatabaseOptions; +}; + +const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([ + "metadata.json", + "summary.json", + "summary.md", + "transcript.jsonl", +]); + +function database(options: OpenClawStateDatabaseOptions) { + ensureMeetingTranscriptsSchema(options); + return openOpenClawStateDatabase(options); +} + +export async function assertTranscriptExportPathAvailable( + params: ExportOwnershipParams, +): Promise { + const stateDatabase = database(params.databaseOptions); + const collisions = executeSqliteQuerySync( + stateDatabase.db, + meetingTranscriptDb(stateDatabase.db) + .selectFrom("meeting_transcript_sessions") + .select(["session_id", "started_at", "selector", "export_pending_json"]) + .where("export_key", "=", transcriptSessionExportKey(params.session)) + .orderBy("selector", "asc"), + ).rows; + if (collisions.length <= 1) { + return; + } + const ensured = await ensureAbsoluteDirectory(params.exportRootDir, { + mode: 0o700, + scopeLabel: "transcript export root", + }); + if (!ensured.ok) { + throw ensured.error; + } + if (await isCaseSensitiveDirectory(params.exportRootDir)) { + return; + } + let ownerSelector: string | undefined; + try { + const metadata = JSON.parse( + await fs.readFile( + path.join(params.exportRootDir, collisions[0]!.selector, "metadata.json"), + "utf8", + ), + ) as { sessionId?: unknown; startedAt?: unknown }; + ownerSelector = collisions.find( + (row) => row.session_id === metadata.sessionId && row.started_at === metadata.startedAt, + )?.selector; + } catch (error) { + if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) { + if (!(error instanceof SyntaxError)) { + throw error; + } + } + } + if (!ownerSelector) { + const pendingOwners = collisions.filter((row) => + (JSON.parse(row.export_pending_json) as string[]).includes("metadata.json"), + ); + if (pendingOwners.length === 1) { + ownerSelector = pendingOwners[0]?.selector; + } + } + ownerSelector ??= transcriptSessionSelector(params.session); + if (ownerSelector !== transcriptSessionSelector(params.session)) { + throw new Error( + `transcript export path collides case-insensitively with another session: ${path.join(params.exportRootDir, transcriptSessionSelector(params.session))}`, + ); + } +} + +export async function hasAliasedCanonicalTranscriptExportPathOwner( + params: ExportOwnershipParams, +): Promise { + const stateDatabase = database(params.databaseOptions); + const owners = executeSqliteQuerySync( + stateDatabase.db, + meetingTranscriptDb(stateDatabase.db) + .selectFrom("meeting_transcript_sessions") + .select(["session_id", "started_at", "export_manifest_json", "export_pending_json"]) + .where("export_key", "=", transcriptSessionExportKey(params.session)) + .orderBy("selector", "asc"), + ).rows; + if (owners.length === 0) { + return false; + } + try { + await fs.access(params.exportRootDir); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return false; + } + throw error; + } + if (await isCaseSensitiveDirectory(params.exportRootDir)) { + return false; + } + const sessionDir = path.join(params.exportRootDir, transcriptSessionSelector(params.session)); + let entries; + try { + entries = await fs.readdir(sessionDir, { withFileTypes: true }); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return true; + } + throw error; + } + const artifactCaseSensitive = await isCaseSensitiveDirectory(sessionDir); + const artifacts = entries.flatMap((entry) => { + const canonicalName = artifactCaseSensitive ? entry.name : entry.name.toLowerCase(); + return TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName) ? [{ entry, canonicalName }] : []; + }); + if (artifacts.length === 0) { + return true; + } + let owner; + let identityVerified = false; + const metadataArtifact = artifacts.find(({ canonicalName }) => canonicalName === "metadata.json"); + if (metadataArtifact) { + const metadataPath = path.join(sessionDir, metadataArtifact.entry.name); + const metadataStat = await fs.lstat(metadataPath); + if (metadataStat.isSymbolicLink() || !metadataStat.isFile()) { + return false; + } + let handle; + try { + handle = await fs.open(metadataPath, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0)); + const metadata = JSON.parse(await handle.readFile("utf8")) as { + sessionId?: unknown; + startedAt?: unknown; + }; + owner = owners.find( + (row) => row.session_id === metadata.sessionId && row.started_at === metadata.startedAt, + ); + identityVerified = owner !== undefined; + } catch { + return false; + } finally { + await handle?.close(); + } + } + if (!owner && !metadataArtifact) { + const manifestMatches = []; + for (const candidate of owners) { + const candidateManifest = JSON.parse(candidate.export_manifest_json) as Record< + string, + string + >; + const candidatePending = new Set(JSON.parse(candidate.export_pending_json) as string[]); + let verified = 0; + let matches = true; + for (const { entry, canonicalName } of artifacts) { + const artifactPath = path.join(sessionDir, entry.name); + const stat = await fs.lstat(artifactPath); + const expectedHash = candidateManifest[canonicalName]; + if ( + stat.isSymbolicLink() || + !stat.isFile() || + candidatePending.has(canonicalName) || + !expectedHash || + (await sha256File(artifactPath)) !== expectedHash + ) { + matches = false; + break; + } + verified += 1; + } + if (matches && verified > 0) { + manifestMatches.push(candidate); + } + } + owner = manifestMatches.length === 1 ? manifestMatches[0] : undefined; + } + if (!owner) { + return false; + } + const manifest = JSON.parse(owner.export_manifest_json) as Record; + const pending = new Set(JSON.parse(owner.export_pending_json) as string[]); + let verifiedArtifactCount = 0; + for (const { entry, canonicalName } of artifacts) { + const artifactPath = path.join(sessionDir, entry.name); + const stat = await fs.lstat(artifactPath); + if (stat.isSymbolicLink() || !stat.isFile()) { + return false; + } + if (pending.has(canonicalName)) { + return false; + } + const expectedHash = manifest[canonicalName]; + if (!expectedHash || (await sha256File(artifactPath)) !== expectedHash) { + return false; + } + verifiedArtifactCount += 1; + } + return identityVerified || verifiedArtifactCount > 0; +} diff --git a/src/transcripts/store-sqlite.ts b/src/transcripts/store-sqlite.ts new file mode 100644 index 000000000000..b9389064da88 --- /dev/null +++ b/src/transcripts/store-sqlite.ts @@ -0,0 +1,76 @@ +import type { DatabaseSync } from "node:sqlite"; +import type { Selectable } from "kysely"; +import { getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import type { TranscriptSessionDescriptor, TranscriptUtterance } from "./provider-types.js"; +import type { TranscriptsSummary } from "./summary.js"; + +type MeetingTranscriptsDatabase = Pick< + OpenClawStateKyselyDatabase, + "meeting_transcript_sessions" | "meeting_transcript_summaries" | "meeting_transcript_utterances" +>; + +export type MeetingTranscriptSessionRow = Selectable< + OpenClawStateKyselyDatabase["meeting_transcript_sessions"] +>; +type MeetingTranscriptSummaryRow = Selectable< + OpenClawStateKyselyDatabase["meeting_transcript_summaries"] +>; +type MeetingTranscriptUtteranceRow = Selectable< + OpenClawStateKyselyDatabase["meeting_transcript_utterances"] +>; + +export function meetingTranscriptDb(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +function parseOptionalJsonRecord(value: string | null): Record | undefined { + if (!value) { + return undefined; + } + const parsed = JSON.parse(value) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : undefined; +} + +export function sessionFromRow(row: MeetingTranscriptSessionRow): TranscriptSessionDescriptor { + const source = parseOptionalJsonRecord(row.source_json); + const metadata = parseOptionalJsonRecord(row.metadata_json); + if (!source || typeof source.providerId !== "string") { + throw new Error(`invalid meeting transcript source for ${row.session_id}`); + } + return { + sessionId: row.session_id, + source: source as TranscriptSessionDescriptor["source"], + startedAt: row.started_at, + ...(row.title !== null ? { title: row.title } : {}), + ...(row.stopped_at !== null ? { stoppedAt: row.stopped_at } : {}), + ...(metadata ? { metadata } : {}), + }; +} + +export function utteranceFromRow(row: MeetingTranscriptUtteranceRow): TranscriptUtterance { + const speaker = + row.speaker_label !== null + ? { + label: row.speaker_label, + ...(row.speaker_id !== null ? { id: row.speaker_id } : {}), + } + : undefined; + const metadata = parseOptionalJsonRecord(row.metadata_json); + return { + sessionId: row.session_id, + text: row.text, + ...(row.utterance_id !== null ? { id: row.utterance_id } : {}), + ...(row.started_at !== null ? { startedAt: row.started_at } : {}), + ...(row.ended_at !== null ? { endedAt: row.ended_at } : {}), + ...(speaker ? { speaker } : {}), + ...(row.final === null ? {} : { final: row.final === 1 }), + ...(metadata ? { metadata } : {}), + }; +} + +export function summaryFromRow(row: MeetingTranscriptSummaryRow): TranscriptsSummary | undefined { + return row.summary_json ? (JSON.parse(row.summary_json) as TranscriptsSummary) : undefined; +} diff --git a/src/transcripts/store-types.ts b/src/transcripts/store-types.ts new file mode 100644 index 000000000000..ef9022fc934a --- /dev/null +++ b/src/transcripts/store-types.ts @@ -0,0 +1,20 @@ +import type { TranscriptSessionDescriptor } from "./provider-types.js"; + +export type TranscriptsSessionEntry = { + session: TranscriptSessionDescriptor; + sessionDir: string; + selector: string; + summaryPath: string; + hasSummary: boolean; +}; + +export type TranscriptArtifactKind = "all" | "metadata" | "summary" | "transcript"; + +export type MaterializedTranscriptArtifacts = { + sessionDir: string; + metadataPath: string; + transcriptPath: string; + summaryJsonPath: string; + summaryPath: string; + hasSummary: boolean; +}; diff --git a/src/transcripts/store.test.ts b/src/transcripts/store.test.ts index 1d1f314669e4..378a03fe2fd9 100644 --- a/src/transcripts/store.test.ts +++ b/src/transcripts/store.test.ts @@ -1,172 +1,367 @@ import fs from "node:fs"; import path from "node:path"; -// Tests TranscriptsStore stream cleanup and transcript reading behavior. -import { PassThrough } from "node:stream"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -const createReadStreamMock = vi.hoisted(() => vi.fn()); -vi.mock("node:fs", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - createReadStream: (...args: Parameters) => - createReadStreamMock(...args) ?? actual.createReadStream(...args), - }; -}); -import { listOpenFileDescriptorsForPath } from "../../src/infra/open-file-descriptors.test-support.js"; -import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; -import { TranscriptsStore } from "./store.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import type { TranscriptSessionDescriptor } from "./provider-types.js"; +import { safeTranscriptPathSegment, TranscriptsStore } from "./store.js"; import { summarizeTranscripts } from "./summary.js"; -const tempRoots: string[] = []; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); -describe("TranscriptsStore.readUtterancesFromSessionDir", () => { - afterEach(() => { - cleanupTempDirs(tempRoots); +afterEach(() => closeOpenClawStateDatabaseForTest()); + +function createStore(): { stateDir: string; store: TranscriptsStore } { + const stateDir = tempDirs.make("openclaw-transcript-test-"); + return { + stateDir, + store: new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + }; +} + +function session( + sessionId = "session-1", + startedAt = "2026-07-01T10:00:00.000Z", +): TranscriptSessionDescriptor { + return { + sessionId, + source: { providerId: "manual-transcript" }, + startedAt, + }; +} + +describe("TranscriptsStore", () => { + it("encodes portable slugs for Windows-reserved and trailing-dot IDs", () => { + expect(safeTranscriptPathSegment("CON")).toBe("%43%4F%4E"); + expect(safeTranscriptPathSegment("foo.")).toBe("%66%6F%6F%2E"); + expect(safeTranscriptPathSegment("foo")).toBe("foo"); }); - it("returns an empty array when transcript.jsonl is missing", () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "missing"); - fs.mkdirSync(sessionDir, { recursive: true }); + it("persists sessions and utterances only in SQLite until export", async () => { + const { stateDir, store } = createStore(); + const target = session(); - const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }); + await store.writeSession(target); + await store.appendUtteranceForSession(target, { text: "hello", final: true }); + await store.appendUtteranceForSession(target, { text: "world", final: true }); - return expect(result).resolves.toEqual([]); - }); - - it("reads utterances from transcript.jsonl", () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "session-1"); - fs.mkdirSync(sessionDir, { recursive: true }); - fs.writeFileSync( - path.join(sessionDir, "transcript.jsonl"), - [ - JSON.stringify({ text: "hello", sessionId: "session-1" }), - JSON.stringify({ text: "world", sessionId: "session-1" }), - ].join("\n") + "\n", - ); - - const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }); - - return expect(result).resolves.toEqual([ - expect.objectContaining({ text: "hello" }), - expect.objectContaining({ text: "world" }), + expect(await store.readSession(target.sessionId)).toEqual(target); + expect(await store.readUtterancesForSession(target)).toEqual([ + { sessionId: target.sessionId, text: "hello", final: true }, + { sessionId: target.sessionId, text: "world", final: true }, ]); + expect(fs.existsSync(path.join(stateDir, "transcripts"))).toBe(false); + expect(fs.existsSync(path.join(stateDir, "state", "openclaw.sqlite"))).toBe(true); }); - it("keeps only the tail when utterances exceed maxUtterances", () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "session-1"); - fs.mkdirSync(sessionDir, { recursive: true }); - const lines = Array.from({ length: 5 }, (_, i) => - JSON.stringify({ text: `line-${i}`, sessionId: "session-1" }), - ); - fs.writeFileSync(path.join(sessionDir, "transcript.jsonl"), lines.join("\n") + "\n"); + it("returns the requested ordered utterance tail", async () => { + const { store } = createStore(); + const target = session(); + await store.writeSession(target); + for (let index = 0; index < 5; index += 1) { + await store.appendUtteranceForSession(target, { text: `line-${index}` }); + } - const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 2 }); - - return expect(result).resolves.toEqual([ + await expect(store.readUtterancesForSession(target, { maxUtterances: 2 })).resolves.toEqual([ expect.objectContaining({ text: "line-3" }), expect.objectContaining({ text: "line-4" }), ]); }); - it.runIf(process.platform === "linux")( - "does not leak file descriptors when JSON.parse throws", - async () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "session-1"); - fs.mkdirSync(sessionDir, { recursive: true }); - const transcriptPath = path.join(sessionDir, "transcript.jsonl"); - fs.writeFileSync(transcriptPath, "not valid json\n"); + it("requires date-qualified selectors for repeated ids", async () => { + const { store } = createStore(); + await store.writeSession(session("standup", "2026-07-01T10:00:00.000Z")); + await store.writeSession(session("standup", "2026-07-02T10:00:00.000Z")); - const fdsBefore = listOpenFileDescriptorsForPath(sessionDir); - await expect( - store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }), - ).rejects.toThrow(); - const fdsAfter = listOpenFileDescriptorsForPath(sessionDir); - - const leaked = fdsAfter.filter((p) => !fdsBefore.includes(p)); - expect(leaked).toHaveLength(0); - }, - ); - - it.runIf(process.platform === "linux")( - "does not leak file descriptors in the happy path", - async () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "session-1"); - fs.mkdirSync(sessionDir, { recursive: true }); - const transcriptPath = path.join(sessionDir, "transcript.jsonl"); - fs.writeFileSync( - transcriptPath, - JSON.stringify({ text: "hello", sessionId: "session-1" }) + "\n", - ); - - const fdsBefore = listOpenFileDescriptorsForPath(sessionDir); - await store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }); - const fdsAfter = listOpenFileDescriptorsForPath(sessionDir); - - const leaked = fdsAfter.filter((p) => !fdsBefore.includes(p)); - expect(leaked).toHaveLength(0); - }, - ); - - it("rejects non-ENOENT read stream errors", async () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const sessionDir = path.join(tmpDir, "2026-07-01", "session-1"); - fs.mkdirSync(sessionDir, { recursive: true }); - fs.writeFileSync(path.join(sessionDir, "transcript.jsonl"), ""); - - createReadStreamMock.mockImplementation(() => { - const stream = new PassThrough(); - setTimeout(() => { - stream.write(JSON.stringify({ text: "hello", sessionId: "session-1" }) + "\n"); - stream.destroy(new Error("read failed")); - }, 10); - return stream; + await expect(store.readSession("standup")).rejects.toThrow( + "multiple transcripts sessions match standup", + ); + await expect(store.readSession("2026-07-01/standup")).resolves.toMatchObject({ + startedAt: "2026-07-01T10:00:00.000Z", }); + }); + + it("matches bare selector slugs literally and case-sensitively", async () => { + const { store } = createStore(); + await store.writeSession(session("fooXbar")); + await store.writeSession(session("Capital", "2026-07-02T10:00:00.000Z")); + await store.writeSession(session("foo@bar", "2026-07-03T10:00:00.000Z")); + await store.writeSession(session("foo-bar", "2026-07-04T10:00:00.000Z")); + + await expect(store.readSession("foo_bar")).resolves.toBeUndefined(); + await expect(store.readSession("capital")).resolves.toBeUndefined(); + await expect(store.readSession("foo#bar")).resolves.toBeUndefined(); + await expect(store.readSession("foo@bar")).resolves.toMatchObject({ sessionId: "foo@bar" }); + await expect(store.readSession("foo-bar")).rejects.toThrow( + "multiple transcripts sessions match foo-bar", + ); + }); + + it("round-trips empty nullable text values", async () => { + const { store } = createStore(); + const target = { ...session("empty-values"), title: "" }; + await store.writeSession(target); + await store.appendUtteranceForSession(target, { + id: "", + speaker: { id: "", label: "" }, + text: "", + }); + + await expect(store.readSession("empty-values")).resolves.toEqual(target); + await expect(store.readUtterancesForSession(target)).resolves.toEqual([ + { id: "", sessionId: "empty-values", speaker: { id: "", label: "" }, text: "" }, + ]); + }); + + it("rejects two session identities that map to one shipped selector", async () => { + const { store } = createStore(); + await store.writeSession(session("standup", "2026-07-01T10:00:00.000Z")); await expect( - store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }), - ).rejects.toThrow("read failed"); - }); -}); - -describe("TranscriptsStore.writeSummary", () => { - afterEach(() => { - cleanupTempDirs(tempRoots); + store.writeSession(session("standup", "2026-07-01T11:00:00.000Z")), + ).rejects.toThrow(); }); - it("stores the summary in the existing session directory without a session descriptor", async () => { - const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-"); - const store = new TranscriptsStore(tmpDir); - const session = { - sessionId: "ansi-\u001b[31mprovider\u001b[0m", - title: "ANSI import", - source: { providerId: "manual-transcript" }, - startedAt: "2026-05-22T10:00:00.000Z", - }; - await store.writeSession(session); - const summary = summarizeTranscripts({ - session, - utterances: [{ text: "We decided to ship the CLI.", speaker: { label: "Sam" } }], + it("stores case-distinct sessions and rejects only unsafe export collisions", async () => { + const { store } = createStore(); + const upper = session("Capital", "2026-07-01T10:00:00.000Z"); + const lower = session("capital", "2026-07-01T11:00:00.000Z"); + await store.writeSession(lower); + await store.materializeSessionArtifacts(lower, "metadata"); + await expect(store.writeSession(upper)).resolves.toBeUndefined(); + + if (fs.existsSync(store.sessionDir(upper))) { + await expect(store.materializeSessionArtifacts(lower, "metadata")).resolves.toMatchObject({ + metadataPath: path.join(store.sessionDir(lower), "metadata.json"), + }); + await expect(store.materializeSessionArtifacts(upper, "metadata")).rejects.toThrow( + "collides case-insensitively", + ); + } else { + await expect(store.materializeSessionArtifacts(upper, "metadata")).resolves.toMatchObject({ + metadataPath: path.join(store.sessionDir(upper), "metadata.json"), + }); + } + }); + + it("uses remaining manifest artifacts when aliased export metadata is absent", async () => { + const { store } = createStore(); + const upper = session("Capital", "2026-07-01T10:00:00.000Z"); + const lower = session("capital", "2026-07-01T11:00:00.000Z"); + await store.writeSession(upper); + await store.appendUtteranceForSession(upper, { text: "owned transcript" }); + const artifacts = await store.materializeSessionArtifacts(upper, "transcript"); + fs.rmSync(artifacts.metadataPath); + + await expect(store.writeSession(lower)).resolves.toBeUndefined(); + }); + + it("does not let a case-distinct SQLite owner mask a legacy directory", async () => { + const { stateDir, store } = createStore(); + const upper = session("Capital", "2026-07-01T10:00:00.000Z"); + const lower = session("capital", "2026-07-01T11:00:00.000Z"); + await store.writeSession(upper); + openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }) + .db.prepare( + "UPDATE meeting_transcript_sessions SET export_pending_json = ? WHERE session_id = ?", + ) + .run('["metadata.json","transcript.jsonl"]', upper.sessionId); + fs.mkdirSync(store.sessionDir(lower), { recursive: true }); + fs.writeFileSync(path.join(store.sessionDir(lower), "transcript.jsonl"), "legacy\n"); + + await expect(store.writeSession(lower)).rejects.toThrow("run openclaw doctor --fix"); + await expect(store.readSession(lower.sessionId)).resolves.toBeUndefined(); + }); + + it("recognizes case-variant artifact names only when the filesystem aliases them", async () => { + const { store } = createStore(); + const upper = session("Capital", "2026-07-01T10:00:00.000Z"); + const lower = session("capital", "2026-07-01T11:00:00.000Z"); + await store.writeSession(upper); + expect(fs.existsSync(store.sessionDir(upper))).toBe(false); + fs.mkdirSync(store.sessionDir(lower), { recursive: true }); + fs.rmSync(path.join(store.sessionDir(lower), "transcript.jsonl"), { force: true }); + fs.writeFileSync(path.join(store.sessionDir(lower), "TRANSCRIPT.JSONL"), "legacy\n"); + expect(fs.readdirSync(store.sessionDir(lower))).toContain("TRANSCRIPT.JSONL"); + + if (fs.existsSync(path.join(store.sessionDir(lower), "transcript.jsonl"))) { + await expect(store.writeSession(lower)).rejects.toThrow("run openclaw doctor --fix"); + } else { + await expect(store.writeSession(lower)).resolves.toBeUndefined(); + } + }); + + it("refuses to overwrite an unclaimed legacy export directory", async () => { + const { store } = createStore(); + const target = session("legacy-collision"); + const sessionDir = store.sessionDir(target); + fs.mkdirSync(sessionDir, { recursive: true }); + const transcriptPath = path.join(sessionDir, "transcript.jsonl"); + fs.writeFileSync(transcriptPath, '{"text":"legacy line"}\n'); + + await expect(store.writeSession(target)).rejects.toThrow("run openclaw doctor --fix"); + expect(fs.readFileSync(transcriptPath, "utf8")).toContain("legacy line"); + }); + + it.runIf(process.platform !== "win32")( + "checks the shipped slug path before inserting a portable encoded session", + async () => { + const { stateDir, store } = createStore(); + const target = session("trailing-dot."); + const legacyDir = path.join(stateDir, "transcripts", "2026-07-01", "trailing-dot."); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "transcript.jsonl"), "legacy\n"); + + await expect(store.writeSession(target)).rejects.toThrow("run openclaw doctor --fix"); + await expect(store.readSession(target.sessionId)).resolves.toBeUndefined(); + }, + ); + + it("does not let a dot session mask the shipped dot-dot root layout", async () => { + const { stateDir, store } = createStore(); + await store.writeSession(session(".")); + const transcriptRoot = path.join(stateDir, "transcripts"); + fs.mkdirSync(transcriptRoot, { recursive: true }); + fs.writeFileSync(path.join(transcriptRoot, "transcript.jsonl"), "legacy root transcript\n"); + + await expect(store.writeSession(session(".."))).rejects.toThrow("run openclaw doctor --fix"); + await expect(store.readSession("..")).resolves.toBeUndefined(); + }); + + it("does not let a modified export block canonical session updates", async () => { + const { store } = createStore(); + const target = session("mutable-export"); + await store.writeSession(target); + await store.appendUtteranceForSession(target, { text: "canonical" }); + const artifacts = await store.materializeSessionArtifacts(target, "transcript"); + fs.appendFileSync(artifacts.transcriptPath, '{"text":"external edit"}\n'); + + await expect(store.updateStopped(target.sessionId, "2026-07-01T11:00:00.000Z")).resolves.toBe( + undefined, + ); + await expect(store.readSession(target.sessionId)).resolves.toMatchObject({ + stoppedAt: "2026-07-01T11:00:00.000Z", }); + const summary = summarizeTranscripts({ session: target, utterances: [{ text: "canonical" }] }); + await expect(store.writeSummary(summary, target)).resolves.toBe( + path.join(store.sessionDir(target), "summary.md"), + ); + await expect(store.readSummary(target)).resolves.toMatchObject({ + summary: { sessionId: target.sessionId }, + }); + await expect(store.materializeSessionArtifacts(target, "transcript")).rejects.toThrow( + "run openclaw doctor --fix", + ); + }); - const markdownPath = await store.writeSummary(summary); + it("resolves descriptor exports through canonical SQLite identity", async () => { + const { store } = createStore(); + const target = { ...session("canonical-export"), title: "Canonical title" }; + await store.writeSession(target); - const sessionDir = store.sessionDir(session); - expect(markdownPath).toBe(path.join(sessionDir, "summary.md")); - const stored = JSON.parse(fs.readFileSync(path.join(sessionDir, "summary.json"), "utf8")) as { - sessionId: string; + const artifacts = await store.materializeSessionArtifacts( + { ...target, title: "Stale title" }, + "metadata", + ); + + expect(fs.readFileSync(artifacts.metadataPath, "utf8")).toContain("Canonical title"); + await expect( + store.materializeSessionArtifacts(session("phantom-export"), "metadata"), + ).rejects.toThrow("transcripts session not found"); + expect(fs.existsSync(store.sessionDir(session("phantom-export")))).toBe(false); + }); + + it("stores summaries in SQLite and materializes explicit artifacts", async () => { + const { stateDir, store } = createStore(); + const target = { + ...session("ansi-\u001b[31mprovider\u001b[0m", "2026-05-22T10:00:00.000Z"), + title: "ANSI import", }; - expect(stored.sessionId).toBe(session.sessionId); + await store.writeSession(target); + const utterance = { + text: "We decided to ship the CLI.", + speaker: { label: "Sam" }, + }; + const utterances = [utterance]; + await store.appendUtteranceForSession(target, utterance); + const summary = summarizeTranscripts({ session: target, utterances }); + + const markdownPath = await store.writeSummary(summary, target); + const artifacts = await store.materializeSessionArtifacts(target, "all"); + + expect(markdownPath).toBe(path.join(store.sessionDir(target), "summary.md")); + expect(JSON.parse(fs.readFileSync(artifacts.summaryJsonPath, "utf8"))).toMatchObject({ + sessionId: target.sessionId, + }); + expect(fs.readFileSync(artifacts.transcriptPath, "utf8")).toContain( + '"text":"We decided to ship the CLI."', + ); + closeOpenClawStateDatabaseForTest(); + const reopened = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + await expect(reopened.readSummary(target)).resolves.toMatchObject({ + summary: { sessionId: target.sessionId }, + }); + }); + + it("removes stale summary exports when canonical state has no summary", async () => { + const { stateDir, store } = createStore(); + const target = session("no-summary"); + await store.writeSession(target); + await store.writeSummary( + summarizeTranscripts({ session: target, utterances: [{ text: "stale" }] }), + target, + ); + openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }) + .db.prepare("DELETE FROM meeting_transcript_summaries WHERE session_id = ?") + .run(target.sessionId); + + const artifacts = await store.materializeSessionArtifacts(target, "summary"); + + expect(artifacts.hasSummary).toBe(false); + expect(fs.existsSync(artifacts.summaryJsonPath)).toBe(false); + expect(fs.existsSync(artifacts.summaryPath)).toBe(false); + }); + + it("repairs an interrupted manifest update and serializes concurrent exports", async () => { + const { stateDir, store } = createStore(); + const target = session("recover-export"); + await store.writeSession(target); + await store.appendUtteranceForSession(target, { text: "recover me" }); + await store.writeSummary( + summarizeTranscripts({ session: target, utterances: [{ text: "recover me" }] }), + target, + ); + openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }) + .db.prepare( + "UPDATE meeting_transcript_sessions SET export_manifest_json = '{}' WHERE session_id = ?", + ) + .run(target.sessionId); + + await expect( + Promise.all([ + store.materializeSessionArtifacts(target, "summary"), + store.materializeSessionArtifacts(target, "transcript"), + ]), + ).resolves.toHaveLength(2); + + const manifest = openOpenClawStateDatabase({ + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }) + .db.prepare( + "SELECT export_manifest_json FROM meeting_transcript_sessions WHERE session_id = ?", + ) + .get(target.sessionId) as { export_manifest_json: string }; + expect(JSON.parse(manifest.export_manifest_json)).toMatchObject({ + "metadata.json": expect.any(String), + "summary.md": expect.any(String), + "transcript.jsonl": expect.any(String), + }); }); }); diff --git a/src/transcripts/store.ts b/src/transcripts/store.ts index 3e5bf389837f..5d755b56b2c5 100644 --- a/src/transcripts/store.ts +++ b/src/transcripts/store.ts @@ -1,313 +1,719 @@ -// Stores and streams transcript files for later summary and replay. -import { createReadStream } from "node:fs"; -import type { Dirent } from "node:fs"; +// Stores meeting-capture transcripts in the shared SQLite state database. +import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; -import { createInterface } from "node:readline"; import { resolveOptionalIntegerOption } from "@openclaw/normalization-core/number-coercion"; +import { sha256File, sha256Hex } from "../infra/crypto-digest.js"; +import { ensureAbsoluteDirectory } from "../infra/fs-safe.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + iterateSqliteQuerySync, +} from "../infra/kysely-sync.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabase, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { withOpenClawStateLease } from "../state/openclaw-state-lease.js"; import type { TranscriptSessionDescriptor, TranscriptUtterance } from "./provider-types.js"; +import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js"; +import { + isCaseSensitiveDirectory, + legacyTranscriptSessionSelector, + normalizeExportText, + removeTranscriptArtifact, + safeTranscriptPathSegment, + TRANSCRIPT_EXPORT_FILE_NAMES, + transcriptSessionExportKey, + transcriptSessionSelector, + writeTranscriptArtifact, +} from "./store-artifacts.js"; +import { writeTranscriptJsonlArtifact } from "./store-export-jsonl.js"; +import { + assertTranscriptExportPathAvailable, + hasAliasedCanonicalTranscriptExportPathOwner, +} from "./store-export-ownership.js"; +import { + meetingTranscriptDb, + type MeetingTranscriptSessionRow, + sessionFromRow, + summaryFromRow, + utteranceFromRow, +} from "./store-sqlite.js"; +import type * as StoreTypes from "./store-types.js"; import type { TranscriptsSummary } from "./summary.js"; import { renderTranscriptsMarkdown } from "./summary.js"; -/** - * File-backed transcript session store. - * - * Sessions are stored by date/session id with metadata JSON, append-only - * utterance JSONL, and rendered summary artifacts. - */ -/** Stored session metadata plus the resolved session directory. */ -export type TranscriptsSessionEntry = { - session: TranscriptSessionDescriptor; - sessionDir: string; -}; +export type * from "./store-types.js"; +export { safeTranscriptPathSegment, transcriptSessionExportKey, transcriptSessionSelector }; -function safeSegment(value: string): string { - // Session ids can come from external providers; path segments stay conservative. - return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session"; -} - -function dateSegment(value: string | undefined): string { - const isoDate = value?.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1]; - return isoDate ?? new Date().toISOString().slice(0, 10); -} - -async function readJsonFile(filePath: string): Promise { - try { - return JSON.parse(await fs.readFile(filePath, "utf8")) as T; - } catch (err) { - if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") { - return undefined; - } - throw err; - } -} - -function sameSessionIdentity( - left: TranscriptSessionDescriptor, - right: TranscriptSessionDescriptor, -): boolean { - return left.sessionId === right.sessionId && left.startedAt === right.startedAt; -} - -/** Durable transcript store rooted at a caller-provided directory. */ +/** Canonical meeting-capture transcript store. Files are explicit exports only. */ export class TranscriptsStore { - constructor(private readonly rootDir: string) {} + constructor( + private readonly exportRootDir: string, + private readonly databaseOptions: OpenClawStateDatabaseOptions = {}, + ) {} + + private database() { + ensureMeetingTranscriptsSchema(this.databaseOptions); + return openOpenClawStateDatabase(this.databaseOptions); + } - /** Resolve the dated directory for a transcript session. */ sessionDir(session: TranscriptSessionDescriptor): string { - return path.join(this.rootDir, dateSegment(session.startedAt), safeSegment(session.sessionId)); + return path.join(this.exportRootDir, transcriptSessionSelector(session)); } - private async hasSessionMetadata(dir: string): Promise { - return (await readJsonFile(path.join(dir, "metadata.json"))) !== undefined; + private entryFromRow( + row: MeetingTranscriptSessionRow, + summaryKeys: ReadonlySet, + ): StoreTypes.TranscriptsSessionEntry { + const session = sessionFromRow(row); + const sessionDir = this.sessionDir(session); + const key = `${session.sessionId}\0${session.startedAt}`; + return { + session, + sessionDir, + selector: row.selector, + summaryPath: path.join(sessionDir, "summary.md"), + hasSummary: summaryKeys.has(key), + }; } - private async findSessionDirForSession(session: TranscriptSessionDescriptor): Promise { - const datedDir = this.sessionDir(session); - const datedSession = await readJsonFile( - path.join(datedDir, "metadata.json"), + private readSummaryKeys(database: OpenClawStateDatabase): Set { + const rows = executeSqliteQuerySync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_summaries") + .select(["session_id", "session_started_at"]), + ).rows; + return new Set(rows.map((row) => `${row.session_id}\0${row.session_started_at}`)); + } + + private hasSummary(database: OpenClawStateDatabase, row: MeetingTranscriptSessionRow): boolean { + return Boolean( + executeSqliteQueryTakeFirstSync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_summaries") + .select("session_id") + .where("session_id", "=", row.session_id) + .where("session_started_at", "=", row.started_at) + .limit(1), + ), ); - if (datedSession && sameSessionIdentity(datedSession, session)) { - return datedDir; - } - return datedDir; } - private async findSessionDir(selector: string): Promise { - const qualified = selector.match(/^(\d{4}-\d{2}-\d{2})\/(.+)$/); - if (qualified?.[1] && qualified[2]) { - const directDir = path.join(this.rootDir, qualified[1], safeSegment(qualified[2])); - return (await this.hasSessionMetadata(directDir)) ? directDir : undefined; - } + private readExportOwnership(session: TranscriptSessionDescriptor): { + manifest: Record; + pending: Set; + } { + const database = this.database(); + const row = executeSqliteQueryTakeFirstSync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_sessions") + .select(["export_manifest_json", "export_pending_json"]) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + return row + ? { + manifest: JSON.parse(row.export_manifest_json) as Record, + pending: new Set(JSON.parse(row.export_pending_json) as string[]), + } + : { manifest: {}, pending: new Set() }; + } - const safeSessionId = safeSegment(selector); - const idDate = selector - .match(/^meeting-(\d{4})-(\d{2})-(\d{2})T/) - ?.slice(1, 4) - .join("-"); - if (idDate) { - const directDir = path.join(this.rootDir, idDate, safeSessionId); - return (await this.hasSessionMetadata(directDir)) ? directDir : undefined; + private readSessionByIdentity( + session: TranscriptSessionDescriptor, + ): TranscriptSessionDescriptor | undefined { + const database = this.database(); + const row = executeSqliteQueryTakeFirstSync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_sessions") + .selectAll() + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + return row ? sessionFromRow(row) : undefined; + } + + private transcriptRows(session: TranscriptSessionDescriptor) { + const database = this.database(); + return { + database, + query: meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_utterances") + .selectAll() + .where("session_id", "=", session.sessionId) + .where("session_started_at", "=", session.startedAt) + .orderBy("sequence", "asc"), + }; + } + + private transcriptJsonlDigest(session: TranscriptSessionDescriptor): string { + const { database, query } = this.transcriptRows(session); + const digest = createHash("sha256"); + for (const row of iterateSqliteQuerySync(database.db, query)) { + digest.update(`${JSON.stringify(utteranceFromRow(row))}\n`); } - let entries: Dirent[]; + return digest.digest("hex"); + } + + private async expectedExportHashes( + session: TranscriptSessionDescriptor, + ): Promise> { + const storedSession = this.readSessionByIdentity(session); + if (!storedSession) { + return {}; + } + const hashes: Record = { + "metadata.json": sha256Hex(`${JSON.stringify(storedSession, null, 2)}\n`), + }; + hashes["transcript.jsonl"] = this.transcriptJsonlDigest(storedSession); + const summary = await this.readSummary(storedSession); + if (summary.summary) { + hashes["summary.json"] = sha256Hex(`${JSON.stringify(summary.summary, null, 2)}\n`); + } + if (summary.markdown !== undefined) { + hashes["summary.md"] = sha256Hex(normalizeExportText(summary.markdown)); + } + return hashes; + } + + private updateExportManifest( + session: TranscriptSessionDescriptor, + exportedHashes: Readonly>, + removedExports: ReadonlySet = new Set(), + ): void { + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = meetingTranscriptDb(database); + const stored = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("meeting_transcript_sessions") + .select(["export_manifest_json", "export_pending_json"]) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + const manifest = stored + ? (JSON.parse(stored.export_manifest_json) as Record) + : {}; + const pending = new Set(stored ? (JSON.parse(stored.export_pending_json) as string[]) : []); + for (const fileName of removedExports) { + delete manifest[fileName]; + } + for (const fileName of [...Object.keys(exportedHashes), ...removedExports]) { + pending.delete(fileName); + } + executeSqliteQuerySync( + database, + db + .updateTable("meeting_transcript_sessions") + .set({ + export_manifest_json: JSON.stringify({ ...manifest, ...exportedHashes }), + export_pending_json: JSON.stringify([...pending].toSorted()), + }) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + }, + this.databaseOptions, + { operationLabel: "meeting-transcripts.export.record" }, + ); + } + + private markPendingExports(session: TranscriptSessionDescriptor, fileNames: string[]): void { + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = meetingTranscriptDb(database); + const stored = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("meeting_transcript_sessions") + .select("export_pending_json") + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + if (!stored) { + throw new Error(`transcripts session not found: ${session.sessionId}`); + } + const pending = new Set(JSON.parse(stored.export_pending_json) as string[]); + for (const fileName of fileNames) { + pending.add(fileName); + } + executeSqliteQuerySync( + database, + db + .updateTable("meeting_transcript_sessions") + .set({ export_pending_json: JSON.stringify([...pending].toSorted()) }) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + }, + this.databaseOptions, + { operationLabel: "meeting-transcripts.export.pending" }, + ); + } + + private async assertExportDestinationOwned( + session: TranscriptSessionDescriptor, + sessionDir = this.sessionDir(session), + ): Promise { + let entries; try { - entries = await fs.readdir(this.rootDir, { withFileTypes: true }); - } catch (err) { - if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") { - return undefined; + entries = await fs.readdir(sessionDir, { withFileTypes: true }); + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return; } - throw err; + throw error; } - const datedEntries = entries - .filter((entry) => entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name)) - .toSorted((left, right) => right.name.localeCompare(left.name)); - const matches: string[] = []; - for (const entry of datedEntries) { - const candidate = path.join(this.rootDir, entry.name, safeSessionId); - const session = await readJsonFile( - path.join(candidate, "metadata.json"), - ); - if (session?.sessionId === selector) { - matches.push(candidate); + const ownership = this.readExportOwnership(session); + const caseSensitive = await isCaseSensitiveDirectory(sessionDir); + let expectedHashes: Record | undefined; + const repairedHashes: Record = {}; + for (const entry of entries) { + const canonicalName = caseSensitive ? entry.name : entry.name.toLowerCase(); + if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName)) { + continue; } + const filePath = path.join(sessionDir, entry.name); + const stat = await fs.lstat(filePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error( + `legacy transcript artifacts require migration before writing ${sessionDir}; run openclaw doctor --fix`, + ); + } + const actualHash = await sha256File(filePath); + if ( + ownership.manifest[canonicalName] === actualHash || + ownership.pending.has(canonicalName) + ) { + continue; + } + expectedHashes ??= await this.expectedExportHashes(session); + if (expectedHashes[canonicalName] !== actualHash) { + throw new Error( + `legacy transcript artifacts require migration before writing ${sessionDir}; run openclaw doctor --fix`, + ); + } + repairedHashes[canonicalName] = actualHash; } - if (matches.length > 1) { - // Ambiguous bare ids require an explicit date prefix to avoid reading the wrong session. - throw new Error( - `multiple transcripts sessions match ${selector}; use a YYYY-MM-DD/${selector} selector`, - ); + if (Object.keys(repairedHashes).length > 0) { + this.updateExportManifest(session, repairedHashes); } - return matches[0]; } - /** Persist transcript session metadata. */ + async listSessionEntries(): Promise { + const database = this.database(); + const rows = executeSqliteQuerySync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_sessions") + .selectAll() + .orderBy("started_at", "desc") + .orderBy("session_id", "asc"), + ).rows; + const summaryKeys = this.readSummaryKeys(database); + return rows.map((row) => this.entryFromRow(row, summaryKeys)); + } + async writeSession(session: TranscriptSessionDescriptor): Promise { - const dir = this.sessionDir(session); - await fs.mkdir(dir, { recursive: true }); - await fs.writeFile(path.join(dir, "metadata.json"), `${JSON.stringify(session, null, 2)}\n`); + ensureMeetingTranscriptsSchema(this.databaseOptions); + if ( + !this.readSessionByIdentity(session) && + !(await hasAliasedCanonicalTranscriptExportPathOwner({ + session, + exportRootDir: this.exportRootDir, + databaseOptions: this.databaseOptions, + })) + ) { + await this.assertExportDestinationOwned(session); + const legacySessionDir = path.join( + this.exportRootDir, + legacyTranscriptSessionSelector(session), + ); + const legacyOwner = await this.readSession(legacyTranscriptSessionSelector(session)); + const legacyPathIsCanonical = + legacyOwner !== undefined && + path.resolve(this.sessionDir(legacyOwner)) === path.resolve(legacySessionDir); + if ( + path.resolve(legacySessionDir) !== path.resolve(this.sessionDir(session)) && + !legacyPathIsCanonical + ) { + await this.assertExportDestinationOwned(session, legacySessionDir); + } + } + const selector = transcriptSessionSelector(session); + const sourceJson = JSON.stringify(session.source); + const metadataJson = session.metadata ? JSON.stringify(session.metadata) : null; + const now = Date.now(); + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = meetingTranscriptDb(database); + executeSqliteQuerySync( + database, + db + .insertInto("meeting_transcript_sessions") + .values({ + session_id: session.sessionId, + started_at: session.startedAt, + selector, + export_key: transcriptSessionExportKey(session), + session_slug: safeTranscriptPathSegment(session.sessionId), + provider_id: session.source.providerId, + title: session.title ?? null, + source_json: sourceJson, + stopped_at: session.stoppedAt ?? null, + metadata_json: metadataJson, + export_manifest_json: "{}", + export_pending_json: "[]", + next_utterance_seq: 0, + created_at_ms: now, + updated_at_ms: now, + }) + .onConflict((conflict) => + conflict.columns(["session_id", "started_at"]).doUpdateSet({ + selector, + export_key: transcriptSessionExportKey(session), + session_slug: safeTranscriptPathSegment(session.sessionId), + provider_id: session.source.providerId, + title: session.title ?? null, + source_json: sourceJson, + stopped_at: session.stoppedAt ?? null, + metadata_json: metadataJson, + updated_at_ms: now, + }), + ), + ); + }, + this.databaseOptions, + { operationLabel: "meeting-transcripts.session.write" }, + ); } - /** Read one session descriptor by session id or qualified date/id selector. */ - async readSession(sessionId: string): Promise { - return (await this.readSessionEntry(sessionId))?.session; + async readSession(sessionSelector: string): Promise { + return (await this.readSessionEntry(sessionSelector))?.session; } - /** Read one session descriptor plus its directory. */ - async readSessionEntry(sessionId: string): Promise { - const dir = await this.findSessionDir(sessionId); - if (!dir) { + async readSessionEntry( + sessionSelector: string, + ): Promise { + const database = this.database(); + const db = meetingTranscriptDb(database.db); + const qualified = /^\d{4}-\d{2}-\d{2}\//u.test(sessionSelector); + const exactRows = qualified + ? executeSqliteQuerySync( + database.db, + db + .selectFrom("meeting_transcript_sessions") + .selectAll() + .where("selector", "=", sessionSelector), + ).rows + : executeSqliteQuerySync( + database.db, + db + .selectFrom("meeting_transcript_sessions") + .selectAll() + .where("session_id", "=", sessionSelector) + .orderBy("started_at", "desc") + .limit(2), + ).rows; + const slugRows = qualified + ? [] + : executeSqliteQuerySync( + database.db, + db + .selectFrom("meeting_transcript_sessions") + .selectAll() + .where("session_slug", "=", sessionSelector) + .orderBy("started_at", "desc") + .limit(2), + ).rows; + const rows = [ + ...new Map( + [...exactRows, ...slugRows].map((row) => [`${row.session_id}\0${row.started_at}`, row]), + ).values(), + ]; + if (rows.length > 1) { + throw new Error( + `multiple transcripts sessions match ${sessionSelector}; use one of: ${rows + .map((row) => row.selector) + .join(", ")}`, + ); + } + const row = rows[0]; + if (!row) { return undefined; } - const session = await readJsonFile( - path.join(dir, "metadata.json"), - ); - return session ? { session, sessionDir: dir } : undefined; + const summaryKeys = this.hasSummary(database, row) + ? new Set([`${row.session_id}\0${row.started_at}`]) + : new Set(); + return this.entryFromRow(row, summaryKeys); } - /** Append an utterance for an exact session descriptor. */ async appendUtteranceForSession( session: TranscriptSessionDescriptor, utterance: TranscriptUtterance, ): Promise { - const dir = await this.findSessionDirForSession(session); - await this.appendUtteranceToDir(dir, session.sessionId, utterance); - } - - private async appendUtteranceToDir( - dir: string, - sessionId: string, - utterance: TranscriptUtterance, - ): Promise { - await fs.mkdir(dir, { recursive: true }); - await fs.appendFile( - path.join(dir, "transcript.jsonl"), - `${JSON.stringify({ ...utterance, sessionId })}\n`, + const metadataJson = utterance.metadata ? JSON.stringify(utterance.metadata) : null; + const now = Date.now(); + ensureMeetingTranscriptsSchema(this.databaseOptions); + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = meetingTranscriptDb(database); + const stored = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("meeting_transcript_sessions") + .select("next_utterance_seq") + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + if (!stored) { + throw new Error(`transcripts session not found: ${session.sessionId}`); + } + const sequence = stored.next_utterance_seq; + executeSqliteQuerySync( + database, + db.insertInto("meeting_transcript_utterances").values({ + session_id: session.sessionId, + session_started_at: session.startedAt, + sequence, + utterance_id: utterance.id ?? null, + started_at: utterance.startedAt ?? null, + ended_at: utterance.endedAt ?? null, + speaker_id: utterance.speaker?.id ?? null, + speaker_label: utterance.speaker?.label ?? null, + text: utterance.text, + final: utterance.final === undefined ? null : utterance.final ? 1 : 0, + metadata_json: metadataJson, + }), + ); + executeSqliteQuerySync( + database, + db + .updateTable("meeting_transcript_sessions") + .set({ next_utterance_seq: sequence + 1, updated_at_ms: now }) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + }, + this.databaseOptions, + { operationLabel: "meeting-transcripts.utterance.append" }, ); } - /** Read utterances for an exact session descriptor. */ async readUtterancesForSession( session: TranscriptSessionDescriptor, options: { maxUtterances?: number } = {}, ): Promise { - return await this.readUtterancesFromDir(await this.findSessionDirForSession(session), options); - } - - /** Read utterances directly from a known session directory. */ - async readUtterancesFromSessionDir( - sessionDir: string, - options: { maxUtterances?: number } = {}, - ): Promise { - return await this.readUtterancesFromDir(sessionDir, options); - } - - private async readUtterancesFromDir( - dir: string, - options: { maxUtterances?: number } = {}, - ): Promise { - const transcriptPath = path.join(dir, "transcript.jsonl"); + const database = this.database(); const maxUtterances = resolveOptionalIntegerOption(options.maxUtterances, { min: 1 }); - if (maxUtterances !== undefined) { - return await new Promise((resolve, reject) => { - const utterances: TranscriptUtterance[] = []; - const stream = createReadStream(transcriptPath, { encoding: "utf8" }); - const lines = createInterface({ - input: stream, - crlfDelay: Infinity, - }); - let settled = false; - let emptyForENOENT = false; - let pendingError: Error | undefined; - - const settle = () => { - if (settled) { - return; - } - settled = true; - lines.close(); - stream.destroy(); - if (pendingError) { - reject(pendingError); - } else if (emptyForENOENT) { - resolve([]); - } else { - resolve(utterances); - } - }; - const setError = (err: unknown) => { - if (!pendingError) { - pendingError = err instanceof Error ? err : new Error(String(err)); - } - }; - - stream.on("close", settle); - stream.on("error", (err) => { - if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") { - emptyForENOENT = true; - return; - } - setError(err); - stream.destroy(); - }); - lines.on("error", (err) => { - if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") { - emptyForENOENT = true; - return; - } - setError(err); - stream.destroy(); - }); - lines.on("line", (line) => { - if (!line) { - return; - } - try { - utterances.push(JSON.parse(line) as TranscriptUtterance); - } catch (err) { - setError(err); - stream.destroy(); - return; - } - if (utterances.length > maxUtterances) { - // Stream and keep only the tail so large transcripts do not require full-file memory. - utterances.shift(); - } - }); - }); + const query = meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_utterances") + .selectAll() + .where("session_id", "=", session.sessionId) + .where("session_started_at", "=", session.startedAt); + if (maxUtterances === undefined) { + return executeSqliteQuerySync(database.db, query.orderBy("sequence", "asc")).rows.map( + utteranceFromRow, + ); } - let raw: string; - try { - raw = await fs.readFile(transcriptPath, "utf8"); - } catch (err) { - if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") { - return []; - } - throw err; - } - return raw - .split(/\r?\n/) - .filter(Boolean) - .map((line) => JSON.parse(line) as TranscriptUtterance); + return executeSqliteQuerySync( + database.db, + query.orderBy("sequence", "desc").limit(maxUtterances), + ) + .rows.toReversed() + .map(utteranceFromRow); } - /** Mark a transcript session as stopped when metadata exists. */ - async updateStopped(sessionId: string, stoppedAt: string): Promise { - const dir = await this.findSessionDir(sessionId); - if (!dir) { + async updateStopped(sessionSelector: string, stoppedAt: string): Promise { + const entry = await this.readSessionEntry(sessionSelector); + if (!entry) { return; } - const session = await readJsonFile( - path.join(dir, "metadata.json"), - ); - if (!session) { - return; - } - await fs.writeFile( - path.join(dir, "metadata.json"), - `${JSON.stringify({ ...session, stoppedAt }, null, 2)}\n`, - ); + await this.writeSession({ ...entry.session, stoppedAt }); } - /** Write summary artifacts for a session and return the markdown path. */ async writeSummary( summary: TranscriptsSummary, session?: TranscriptSessionDescriptor, ): Promise { - const dir = - session !== undefined - ? await this.findSessionDirForSession(session) - : ((await this.findSessionDir(summary.sessionId)) ?? - path.join(this.rootDir, dateSegment(summary.sessionId), safeSegment(summary.sessionId))); - return await this.writeSummaryToDir(summary, dir); + const resolved = session ?? (await this.readSession(summary.sessionId)); + if (!resolved) { + throw new Error(`transcripts session not found: ${summary.sessionId}`); + } + const summaryJson = JSON.stringify(summary); + const markdown = renderTranscriptsMarkdown(summary); + ensureMeetingTranscriptsSchema(this.databaseOptions); + runOpenClawStateWriteTransaction( + ({ db: database }) => { + const db = meetingTranscriptDb(database); + executeSqliteQuerySync( + database, + db + .insertInto("meeting_transcript_summaries") + .values({ + session_id: resolved.sessionId, + session_started_at: resolved.startedAt, + generated_at: summary.generatedAt, + summary_json: summaryJson, + markdown, + utterance_count: summary.utteranceCount, + }) + .onConflict((conflict) => + conflict.columns(["session_id", "session_started_at"]).doUpdateSet({ + generated_at: summary.generatedAt, + summary_json: summaryJson, + markdown, + utterance_count: summary.utteranceCount, + }), + ), + ); + }, + this.databaseOptions, + { operationLabel: "meeting-transcripts.summary.write" }, + ); + return path.join(this.sessionDir(resolved), "summary.md"); } - /** Write summary JSON and markdown to a known directory. */ - async writeSummaryToDir(summary: TranscriptsSummary, dir: string): Promise { - await fs.mkdir(dir, { recursive: true }); - await fs.writeFile(path.join(dir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`); - const markdown = renderTranscriptsMarkdown(summary); - const markdownPath = path.join(dir, "summary.md"); - await fs.writeFile(markdownPath, `${markdown}\n`); - return markdownPath; + async readSummary( + session: TranscriptSessionDescriptor, + ): Promise<{ summary?: TranscriptsSummary; markdown?: string }> { + const database = this.database(); + const row = executeSqliteQueryTakeFirstSync( + database.db, + meetingTranscriptDb(database.db) + .selectFrom("meeting_transcript_summaries") + .selectAll() + .where("session_id", "=", session.sessionId) + .where("session_started_at", "=", session.startedAt), + ); + if (!row) { + return {}; + } + const summary = summaryFromRow(row); + return { + ...(summary ? { summary } : {}), + ...(row.markdown !== null ? { markdown: row.markdown } : {}), + }; + } + + async materializeSessionArtifacts( + sessionOrSelector: TranscriptSessionDescriptor | string, + kind: StoreTypes.TranscriptArtifactKind, + ): Promise { + const session = + typeof sessionOrSelector === "string" + ? await this.readSession(sessionOrSelector) + : this.readSessionByIdentity(sessionOrSelector); + if (!session) { + const selector = + typeof sessionOrSelector === "string" ? sessionOrSelector : sessionOrSelector.sessionId; + throw new Error(`transcripts session not found: ${selector}`); + } + return await withOpenClawStateLease( + { + scope: "meeting-transcript.export", + key: transcriptSessionExportKey(session), + database: { scope: "shared", options: this.databaseOptions }, + leaseMs: 60_000, + waitMs: 10_000, + leaseLabel: "meeting transcript export lease", + operationLabel: "meeting-transcripts.export.lease", + }, + async () => await this.materializeSessionArtifactsOwned(session, kind), + ); + } + + private async materializeSessionArtifactsOwned( + session: TranscriptSessionDescriptor, + kind: StoreTypes.TranscriptArtifactKind, + ): Promise { + const sessionDir = this.sessionDir(session); + const metadataPath = path.join(sessionDir, "metadata.json"); + const transcriptPath = path.join(sessionDir, "transcript.jsonl"); + const summaryJsonPath = path.join(sessionDir, "summary.json"); + const summaryPath = path.join(sessionDir, "summary.md"); + // Every export starts with identity metadata, so even an interrupted partial + // materialization remains inspectable by Doctor without guessing its owner. + const includeMetadata = true; + const includeTranscript = kind === "all" || kind === "transcript"; + const includeSummary = kind === "all" || kind === "summary"; + const storedSummary = includeSummary ? await this.readSummary(session) : {}; + const exportedHashes: Record = {}; + const removedExports = new Set(); + await assertTranscriptExportPathAvailable({ + session, + exportRootDir: this.exportRootDir, + databaseOptions: this.databaseOptions, + }); + await this.assertExportDestinationOwned(session); + const pendingFiles = [ + "metadata.json", + ...(includeTranscript ? ["transcript.jsonl"] : []), + ...(includeSummary ? ["summary.json", "summary.md"] : []), + ]; + this.markPendingExports(session, pendingFiles); + const ensured = await ensureAbsoluteDirectory(sessionDir, { + mode: 0o700, + scopeLabel: "transcript export directory", + }); + if (!ensured.ok) { + throw ensured.error; + } + if (includeMetadata) { + exportedHashes["metadata.json"] = await writeTranscriptArtifact( + sessionDir, + "metadata.json", + `${JSON.stringify(session, null, 2)}\n`, + ); + } + if (includeTranscript) { + exportedHashes["transcript.jsonl"] = await writeTranscriptJsonlArtifact({ + sessionDir, + session, + databaseOptions: this.databaseOptions, + }); + } + if (includeSummary) { + if (storedSummary.summary) { + exportedHashes["summary.json"] = await writeTranscriptArtifact( + sessionDir, + "summary.json", + `${JSON.stringify(storedSummary.summary, null, 2)}\n`, + ); + } else { + await removeTranscriptArtifact(sessionDir, "summary.json"); + removedExports.add("summary.json"); + } + if (storedSummary.markdown !== undefined) { + exportedHashes["summary.md"] = await writeTranscriptArtifact( + sessionDir, + "summary.md", + normalizeExportText(storedSummary.markdown), + ); + } else { + await removeTranscriptArtifact(sessionDir, "summary.md"); + removedExports.add("summary.md"); + } + } + if (Object.keys(exportedHashes).length > 0 || removedExports.size > 0) { + this.updateExportManifest(session, exportedHashes, removedExports); + } + return { + sessionDir, + metadataPath, + transcriptPath, + summaryJsonPath, + summaryPath, + hasSummary: storedSummary.summary !== undefined || storedSummary.markdown !== undefined, + }; } } diff --git a/test/scripts/docker-all-scheduler.test.ts b/test/scripts/docker-all-scheduler.test.ts index 01dc6aee4b44..17df3c04c8ca 100644 --- a/test/scripts/docker-all-scheduler.test.ts +++ b/test/scripts/docker-all-scheduler.test.ts @@ -403,7 +403,7 @@ describe("scripts/test-docker-all scheduler", () => { const summary = JSON.parse(readFileSync(path.join(logDir, "summary.json"), "utf8")); expect(summary.status).toBe("passed"); expect(summary.lanes).toEqual([]); - expect(summary.omittedUnsupportedLanes).toHaveLength(10); + expect(summary.omittedUnsupportedLanes).toHaveLength(11); expect(summary.omittedUnsupportedLanes).toContain("published-upgrade-survivor"); expect(summary.omittedUnsupportedLanes).toContain( "published-upgrade-survivor-versioned-runtime-deps", diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index 988edf208a59..6847c86a538c 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -793,6 +793,7 @@ describe("scripts/lib/docker-e2e-plan", () => { "published-upgrade-survivor-2026.4.29-configured-plugin-installs", "published-upgrade-survivor-2026.4.29-stale-source-plugin-shadow", "published-upgrade-survivor-2026.4.29-tilde-log-path", + "published-upgrade-survivor-2026.4.29-meeting-transcripts-sqlite", "published-upgrade-survivor-2026.4.29-versioned-runtime-deps", ]); }); @@ -830,6 +831,7 @@ describe("scripts/lib/docker-e2e-plan", () => { ]); expect(plan.omittedUnsupportedLanes).toEqual([ "published-upgrade-survivor-2026.6.11-acpx-openclaw-tools-bridge", + "published-upgrade-survivor-2026.6.11-meeting-transcripts-sqlite", ]); }); @@ -867,6 +869,7 @@ describe("scripts/lib/docker-e2e-plan", () => { expect(plan.omittedUnsupportedLanes).toEqual([ "published-upgrade-survivor-2026.6.11-acpx-openclaw-tools-bridge", + "published-upgrade-survivor-2026.6.11-meeting-transcripts-sqlite", ]); }); @@ -882,7 +885,7 @@ describe("scripts/lib/docker-e2e-plan", () => { }); expect(plan.lanes).toEqual([]); - expect(plan.omittedUnsupportedLanes).toHaveLength(10); + expect(plan.omittedUnsupportedLanes).toHaveLength(11); expect(plan.omittedUnsupportedLanes).toContain("published-upgrade-survivor-2026.6.11"); expect(plan.omittedUnsupportedLanes).toContain( "published-upgrade-survivor-2026.6.11-versioned-runtime-deps", @@ -927,7 +930,7 @@ describe("scripts/lib/docker-e2e-plan", () => { }); expect(plan.lanes.map((lane) => lane.name)).toEqual(["plugin-binding-command-escape"]); - expect(plan.omittedUnsupportedLanes).toHaveLength(10); + expect(plan.omittedUnsupportedLanes).toHaveLength(11); expect(plan.omittedUnsupportedLanes).toContain("published-upgrade-survivor"); expect(plan.omittedUnsupportedLanes).toContain( "published-upgrade-survivor-versioned-runtime-deps", @@ -1061,6 +1064,7 @@ describe("scripts/lib/docker-e2e-plan", () => { "published-upgrade-survivor-2026.4.29-configured-plugin-installs", "published-upgrade-survivor-2026.4.29-stale-source-plugin-shadow", "published-upgrade-survivor-2026.4.29-tilde-log-path", + "published-upgrade-survivor-2026.4.29-meeting-transcripts-sqlite", "published-upgrade-survivor-2026.4.29-versioned-runtime-deps", "published-upgrade-survivor-2026.4.22", "published-upgrade-survivor-2026.4.22-acpx-openclaw-tools-bridge", @@ -1070,6 +1074,7 @@ describe("scripts/lib/docker-e2e-plan", () => { "published-upgrade-survivor-2026.4.22-configured-plugin-installs", "published-upgrade-survivor-2026.4.22-stale-source-plugin-shadow", "published-upgrade-survivor-2026.4.22-tilde-log-path", + "published-upgrade-survivor-2026.4.22-meeting-transcripts-sqlite", "published-upgrade-survivor-2026.4.22-versioned-runtime-deps", "published-upgrade-survivor-2026.4.21", "published-upgrade-survivor-2026.4.21-feishu-channel", @@ -1078,6 +1083,7 @@ describe("scripts/lib/docker-e2e-plan", () => { "published-upgrade-survivor-2026.4.21-configured-plugin-installs", "published-upgrade-survivor-2026.4.21-stale-source-plugin-shadow", "published-upgrade-survivor-2026.4.21-tilde-log-path", + "published-upgrade-survivor-2026.4.21-meeting-transcripts-sqlite", "published-upgrade-survivor-2026.4.21-versioned-runtime-deps", "published-upgrade-survivor-2026.3.13", "published-upgrade-survivor-2026.3.13-feishu-channel", @@ -1086,6 +1092,7 @@ describe("scripts/lib/docker-e2e-plan", () => { "published-upgrade-survivor-2026.3.13-configured-plugin-installs", "published-upgrade-survivor-2026.3.13-stale-source-plugin-shadow", "published-upgrade-survivor-2026.3.13-tilde-log-path", + "published-upgrade-survivor-2026.3.13-meeting-transcripts-sqlite", "published-upgrade-survivor-2026.3.13-versioned-runtime-deps", ]); });