From 129f86bbf5f6ec5ceb811eb8e3d6fd720af1cec4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 23 Jul 2026 11:41:08 -0700 Subject: [PATCH] feat: auto-collect durable meeting transcripts (#113122) * feat(meetings): auto-collect durable transcripts * chore(plugin-sdk): refresh meeting transcript API baseline * fix(plugin-sdk): keep meeting transcript options implicit * fix(meetings): satisfy transcript bridge typecheck * fix(google-meet): preserve optional caption policy context * fix(google-meet): retain caption mode callback type * refactor(meetings): split transcript lifecycle helpers * chore(plugin-sdk): refresh split meeting runtime baseline * refactor(google-meet): split session helpers * fix(meetings): decouple transcript retries from leave * fix(meetings): isolate provider subscribers * fix(meetings): harden capture lifecycle * fix(meetings): preserve capture during storage outages * fix(meetings): preserve pending transcript ordering * fix(meetings): retry capture initialization * test(agents): reflect default transcript registration --- .../.generated/plugin-sdk-api-baseline.sha256 | 2 +- docs/cli/transcripts.md | 23 +- docs/plugins/architecture.md | 34 +- docs/plugins/google-meet.md | 9 +- docs/plugins/meeting-plugins.md | 14 +- docs/plugins/sdk-overview.md | 40 +- docs/plugins/sdk-subpaths.md | 2 +- docs/plugins/teams-meetings.md | 12 +- docs/plugins/zoom-meetings.md | 12 +- .../google-meet/index.transcripts.test.ts | 32 ++ extensions/google-meet/index.ts | 9 + extensions/google-meet/openclaw.plugin.json | 3 +- extensions/google-meet/src/runtime-session.ts | 38 ++ extensions/google-meet/src/runtime.ts | 59 +-- .../google-meet/src/transports/chrome.test.ts | 10 +- .../google-meet/src/transports/chrome.ts | 15 + .../google-meet-platform-adapter.test.ts | 10 + .../google-meet-platform-adapter.ts | 4 +- extensions/teams-meetings/index.test.ts | 10 + extensions/teams-meetings/index.ts | 5 + .../teams-meetings/openclaw.plugin.json | 3 +- extensions/teams-meetings/src/runtime.ts | 18 + .../teams-meetings-caption-capture.test.ts | 6 +- .../teams-meetings-platform-adapter.ts | 4 +- extensions/zoom-meetings/index.test.ts | 10 + extensions/zoom-meetings/index.ts | 5 + extensions/zoom-meetings/openclaw.plugin.json | 3 +- extensions/zoom-meetings/src/runtime.ts | 18 + .../zoom-meetings-platform-adapter.test.ts | 5 +- .../zoom-meetings-platform-adapter.ts | 4 +- src/agents/openclaw-tools.ts | 4 +- src/agents/openclaw-tools.update-plan.test.ts | 10 +- src/agents/tools/transcripts-tool-runtime.ts | 16 +- src/agents/tools/transcripts-tool.test.ts | 86 ++- src/agents/tools/transcripts-tool.ts | 56 +- src/config/schema.help.automation.ts | 4 +- src/meeting-bot/browser-controller.test.ts | 6 +- src/meeting-bot/browser-controller.ts | 10 +- src/meeting-bot/chrome-transport.test.ts | 7 +- src/meeting-bot/chrome-transport.ts | 13 + src/meeting-bot/platform-adapter-contract.ts | 1 + src/meeting-bot/plugin-entry.ts | 48 +- .../session-durable-transcripts.ts | 184 +++++++ src/meeting-bot/session-runtime.test.ts | 236 ++++++++- src/meeting-bot/session-runtime.ts | 118 ++--- src/meeting-bot/session-speech-readiness.ts | 73 +++ .../session-transcript-store.test.ts | 263 ++++++++++ src/meeting-bot/session-transcript-store.ts | 279 +++++++++- src/meeting-bot/transcripts-bridge.runtime.ts | 494 ++++++++++++++++++ src/meeting-bot/transcripts-bridge.test.ts | 408 +++++++++++++++ src/meeting-bot/transcripts-bridge.ts | 52 ++ src/plugin-sdk/transcripts.ts | 5 + src/transcripts/config.test.ts | 10 + src/transcripts/config.ts | 2 +- src/transcripts/source-locator.test.ts | 16 + src/transcripts/source-locator.ts | 17 + src/transcripts/store-sqlite.ts | 95 +++- src/transcripts/store.test.ts | 22 + src/transcripts/store.ts | 42 +- 59 files changed, 2738 insertions(+), 258 deletions(-) create mode 100644 extensions/google-meet/index.transcripts.test.ts create mode 100644 extensions/google-meet/src/runtime-session.ts create mode 100644 extensions/google-meet/src/transports/google-meet-platform-adapter.test.ts create mode 100644 src/meeting-bot/session-durable-transcripts.ts create mode 100644 src/meeting-bot/session-speech-readiness.ts create mode 100644 src/meeting-bot/transcripts-bridge.runtime.ts create mode 100644 src/meeting-bot/transcripts-bridge.test.ts create mode 100644 src/meeting-bot/transcripts-bridge.ts create mode 100644 src/transcripts/config.test.ts create mode 100644 src/transcripts/source-locator.test.ts create mode 100644 src/transcripts/source-locator.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 29217cd6ff62..5d93a9d39310 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -82,7 +82,7 @@ a5f59c9acbcaa3f82247bf806eb5ba08032373fb853719f0ec9457690f16fc70 module/media-m 7a5a1de743fff9d5a86515b501ffd88e7fa49970c9f46a5153816e2ebaef7bb0 module/media-store c5e3eb1a584f4b8126d9d6c177a840ec9103671e8d1242634ee67db9b5b9e573 module/media-understanding c0ffaed532578cf33493992e1ff806b2268b8e3774a92edbaede5cf5bda162a6 module/media-understanding-runtime -8e318232d62a5a75e5c39e38d924083efae2ff6f9a7dea25cabd5b84593083f8 module/meeting-runtime +bebd2931dc51d67c063ff19fa1c278f8dcfe00ab23cfbd480d47329ea8e5088e module/meeting-runtime d09ddb38c9d20a41fcf4cedaf6a2c28eb5333a9fba2287e34b458fa5caba751c module/memory-core-host-engine-foundation 646773d8282a2ac6a89101685c406200935ddb3459a2830e820da132fa433b3c module/memory-host-core 87b7a3206346c0d4b294fb3a2395cbaabc3e73ff8b1b9ea925bc3aade3e52687 module/messaging-targets diff --git a/docs/cli/transcripts.md b/docs/cli/transcripts.md index 85d7a22c83aa..70ae100a54f6 100644 --- a/docs/cli/transcripts.md +++ b/docs/cli/transcripts.md @@ -9,8 +9,9 @@ title: "Transcripts CLI" # `openclaw transcripts` -Inspector and export command for transcripts written by the `transcripts` agent -tool. Capture, import, and summarization run through that tool, not this CLI. +Inspector and export command for durable meeting transcripts. Google Meet, +Microsoft Teams, and Zoom browser participants capture notes automatically; +the `transcripts` agent tool also supports provider capture and manual import. Canonical transcript state lives in the shared SQLite database at `$OPENCLAW_STATE_DIR/state/openclaw.sqlite`. `show` and `path` explicitly @@ -75,6 +76,8 @@ The selector is the safest value to pass back to `show` or `path`. `list --json` returns objects with `sessionId`, `selector`, `date`, `title`, `startedAt`, `stoppedAt`, `source`, `path`, `summaryPath`, `hasSummary`. +Stored meeting source URLs contain only the origin and path; query strings, +fragments, and embedded credentials are removed before persistence. `show --json` returns the stored session metadata, selector, session directory, summary path, and summary Markdown text. @@ -126,18 +129,21 @@ sessions and any exports you rely on. ## Configuration -Capture is opt-in (live sources can join and record meeting audio). Enable it -with: +Meeting transcript capture is enabled by default. To opt out globally: ```json { "transcripts": { - "enabled": true + "enabled": false } } ``` -- `enabled` (default `false`): turn the tool on. +- `enabled` (default `true`): enable automatic meeting notes, the transcripts + tool, and configured auto-start sources. Set it to `false` when meeting + notes should not be persisted on the host. An explicitly requested meeting + `transcribe` mode keeps its existing bounded live-caption tail, but does not + write durable rows while this setting is false. Configure auto-start sources with `transcripts.autoStart`. Each entry is enabled by being present; omit an entry to disable that source. `discord-voice` is the bundled auto-start-capable source and requires `guildId` and @@ -157,3 +163,8 @@ with: } } ``` + +The meeting provider ids are `google-meet`, `teams`, and `zoom`. Their aliases +are `googlemeet`/`meet`, `teams-meetings`/`microsoft-teams`/`msteams`, and +`zoom-meetings`, respectively. Meeting providers attach to an already-active +meeting bot session; normal meeting joins do not need an `autoStart` entry. diff --git a/docs/plugins/architecture.md b/docs/plugins/architecture.md index c2937628ceba..d271bbe3bf6e 100644 --- a/docs/plugins/architecture.md +++ b/docs/plugins/architecture.md @@ -33,23 +33,23 @@ This is the **deep architecture reference** for the OpenClaw plugin system. For Capabilities are the public **native plugin** model inside OpenClaw. Every native OpenClaw plugin registers against one or more capability types: -| Capability | Registration method | Example plugins | -| ---------------------- | ------------------------------------------------ | ------------------------------ | -| Text inference | `api.registerProvider(...)` | `anthropic`, `openai` | -| CLI inference backend | `api.registerCliBackend(...)` | `anthropic`, `openai` | -| Embeddings | `api.registerEmbeddingProvider(...)` | Provider-owned vector plugins | -| Speech | `api.registerSpeechProvider(...)` | `elevenlabs`, `microsoft` | -| Realtime transcription | `api.registerRealtimeTranscriptionProvider(...)` | `openai` | -| Realtime voice | `api.registerRealtimeVoiceProvider(...)` | `google`, `openai` | -| Media understanding | `api.registerMediaUnderstandingProvider(...)` | `google`, `openai` | -| Transcripts source | `api.registerTranscriptSourceProvider(...)` | `discord` | -| Image generation | `api.registerImageGenerationProvider(...)` | `fal`, `google`, `openai` | -| Music generation | `api.registerMusicGenerationProvider(...)` | `fal`, `google`, `minimax` | -| Video generation | `api.registerVideoGenerationProvider(...)` | `fal`, `google`, `qwen` | -| Web fetch | `api.registerWebFetchProvider(...)` | `firecrawl` | -| Web search | `api.registerWebSearchProvider(...)` | `brave`, `firecrawl`, `google` | -| Channel / messaging | `api.registerChannel(...)` | `matrix`, `msteams` | -| Gateway discovery | `api.registerGatewayDiscoveryService(...)` | `bonjour` | +| Capability | Registration method | Example plugins | +| ---------------------- | ------------------------------------------------ | ----------------------------------------------------------- | +| Text inference | `api.registerProvider(...)` | `anthropic`, `openai` | +| CLI inference backend | `api.registerCliBackend(...)` | `anthropic`, `openai` | +| Embeddings | `api.registerEmbeddingProvider(...)` | Provider-owned vector plugins | +| Speech | `api.registerSpeechProvider(...)` | `elevenlabs`, `microsoft` | +| Realtime transcription | `api.registerRealtimeTranscriptionProvider(...)` | `openai` | +| Realtime voice | `api.registerRealtimeVoiceProvider(...)` | `google`, `openai` | +| Media understanding | `api.registerMediaUnderstandingProvider(...)` | `google`, `openai` | +| Transcripts source | `api.registerTranscriptSourceProvider(...)` | `discord`, `google-meet`, `teams-meetings`, `zoom-meetings` | +| Image generation | `api.registerImageGenerationProvider(...)` | `fal`, `google`, `openai` | +| Music generation | `api.registerMusicGenerationProvider(...)` | `fal`, `google`, `minimax` | +| Video generation | `api.registerVideoGenerationProvider(...)` | `fal`, `google`, `qwen` | +| Web fetch | `api.registerWebFetchProvider(...)` | `firecrawl` | +| Web search | `api.registerWebSearchProvider(...)` | `brave`, `firecrawl`, `google` | +| Channel / messaging | `api.registerChannel(...)` | `matrix`, `msteams` | +| Gateway discovery | `api.registerGatewayDiscoveryService(...)` | `bonjour` | A plugin that registers zero capabilities but provides hooks, tools, discovery services, or background services is a **legacy hook-only** plugin. That pattern is still fully supported. diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index 1521acbcab76..9b90207fa692 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -121,7 +121,7 @@ If the browser fallback hits a Google login or Meet permission blocker, the tool ### Observe-only join -Set `"mode": "transcribe"` to skip the duplex realtime bridge (no BlackHole/SoX requirement, no talk-back). Transcribe-mode Chrome joins also skip OpenClaw's microphone/camera permission grant and the Meet **Use microphone** path; if Meet shows the audio-choice interstitial, automation tries **Continue without microphone** first. Managed Chrome transports in this mode install a best-effort Meet caption observer. `googlemeet status --json` and `googlemeet doctor` report `captioning`, `captionsEnabledAttempted`, `transcriptLines`, `lastCaptionAt`, `lastCaptionSpeaker`, `lastCaptionText`, and a `recentTranscript` tail. +Set `"mode": "transcribe"` to skip the duplex realtime bridge (no BlackHole/SoX requirement, no talk-back). Transcribe-mode Chrome joins also skip OpenClaw's microphone/camera permission grant and the Meet **Use microphone** path; if Meet shows the audio-choice interstitial, automation tries **Continue without microphone** first. Managed Chrome transports install a best-effort Meet caption observer in every mode so durable notes are available without changing the live agent-consult path. `googlemeet status --json` and `googlemeet doctor` report `captioning`, `captionsEnabledAttempted`, `transcriptLines`, `lastCaptionAt`, `lastCaptionSpeaker`, `lastCaptionText`, and a `recentTranscript` tail. For the bounded session transcript, read the exact tracked Meet tab: @@ -130,7 +130,12 @@ openclaw googlemeet transcript openclaw googlemeet transcript --since --json ``` -The observer keeps at most 2,000 completed caption lines in the Meet page. Visible progressive text stays in the status health tail until the caption row completes, so saving `nextIndex` cannot skip a later text expansion; leaving finalizes visible rows before the snapshot. `droppedLines` reports lines lost from the head when the cap is exceeded. The four most recently ended session transcripts remain readable until the gateway restarts. Older ended transcripts return `evicted: true`. This is intentionally runtime memory, not durable meeting-history storage: restarting the gateway, closing the tab before a snapshot, or exceeding the documented caps can lose captions. +The observer keeps at most 2,000 completed caption lines in the Meet page. Visible progressive text stays in the status health tail until the caption row completes, so saving `nextIndex` cannot skip a later text expansion; leaving finalizes visible rows before the snapshot. `droppedLines` reports lines lost from the head when the cap is exceeded. The bounded `googlemeet transcript` tail still keeps only the four most recently ended sessions and resets with the Gateway. Separately, OpenClaw appends completed caption rows to the shared state database throughout the meeting and writes a derived summary on leave. Use [`openclaw transcripts`](/cli/transcripts) to inspect or export those durable notes. + +Automatic notes are enabled by default. Set `transcripts.enabled: false` to +disable durable notes globally; explicit `transcribe` mode still exposes only +its bounded live tail. Twilio joins do not have the browser caption stream and +are not captured by this path. For a yes/no listen probe: diff --git a/docs/plugins/meeting-plugins.md b/docs/plugins/meeting-plugins.md index a1d1e3be8afc..bac88b9fe2ae 100644 --- a/docs/plugins/meeting-plugins.md +++ b/docs/plugins/meeting-plugins.md @@ -19,7 +19,7 @@ These plugins participate in meetings. They are separate from messaging channels | Microsoft Teams | [`teams-meetings`](/plugins/teams-meetings) | Work links under `teams.microsoft.com/l/meetup-join/...` and consumer links under `teams.live.com/meet/...` | Included; enabled by default | Local Chrome or Chrome on a paired node | Guest join for work and consumer meetings | | Zoom | [`zoom-meetings`](/plugins/zoom-meetings) | `zoom.us/j/...` and account subdomains such as `example.zoom.us/j/...` | Included; enabled by default | Local Chrome or Chrome on a paired node | Guest join through the Zoom Web App | -Choose Google Meet when you need meeting creation, Google API artifacts, or a Twilio phone path. Choose Teams or Zoom for direct browser guest participation on those platforms. The Teams and Zoom plugins do not create meetings, dial in, call the vendor API, or record meetings. +Choose Google Meet when you need meeting creation, Google API artifacts, or a Twilio phone path. Choose Teams or Zoom for direct browser guest participation on those platforms. The Teams and Zoom plugins do not create meetings, dial in, call the vendor API, or capture audio/video recordings. ## Choose a mode @@ -33,7 +33,17 @@ The three plugins share the same modes: Use `transcribe` when the agent only needs meeting text. Use `agent` for normal OpenClaw reasoning and tools. Use `bidi` when low-latency direct voice is more important than routing each turn through the regular agent. -Caption transcripts are session-scoped runtime data, not durable meeting recordings. Caption availability still depends on the meeting platform, account, language, and host policy. See the platform guide for its transcript limits and status fields. +The bounded live transcript remains available only in `transcribe` mode. In all +three modes, browser joins also persist completed caption rows and a derived +summary to the shared state database. Leaving the meeting finalizes visible +captions and writes the summary; use [`openclaw transcripts`](/cli/transcripts) +to list, inspect, or export it. This durable notes path does not change the live +agent-consult transcript or create an audio/video recording. + +Automatic notes are on by default. Set `transcripts.enabled: false` to disable +durable notes globally. An explicitly selected `transcribe` session retains its +bounded live-caption tail without writing durable rows. Caption availability +still depends on the meeting platform, account, language, and host policy. ## Prepare Chrome and audio diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index 24eedc85905b..153657279670 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -96,26 +96,26 @@ and external URLs. Registering another provider replaces the current provider. ### Capability registration -| Method | What it registers | -| ------------------------------------------------ | --------------------------------------------------------------------------------- | -| `api.registerProvider(...)` | Text inference (LLM) | -| `api.registerWorkerProvider(...)` | Cloud-worker lifecycle leases | -| `api.registerModelCatalogProvider(...)` | Model catalog rows for text and media generation | -| `api.registerAgentHarness(...)` | [Experimental](/plugins/sdk-agent-harness) native agent executor (Codex, Copilot) | -| `api.registerCliBackend(...)` | Local CLI inference backend | -| `api.registerChannel(...)` | Messaging channel | -| `api.registerEmbeddingProvider(...)` | Reusable vector embedding provider | -| `api.registerSpeechProvider(...)` | Text-to-speech / STT synthesis | -| `api.registerRealtimeTranscriptionProvider(...)` | Streaming realtime transcription | -| `api.registerRealtimeVoiceProvider(...)` | Duplex realtime voice sessions | -| `api.registerMediaUnderstandingProvider(...)` | Image/audio/video analysis | -| `api.registerTranscriptSourceProvider(...)` | Live or imported meeting transcript source | -| `api.registerImageGenerationProvider(...)` | Image generation | -| `api.registerMusicGenerationProvider(...)` | Music generation | -| `api.registerVideoGenerationProvider(...)` | Video generation | -| `api.registerWebFetchProvider(...)` | Web fetch / scrape provider | -| `api.registerWebSearchProvider(...)` | Web search | -| `api.registerCompactionProvider(...)` | Pluggable transcript-compaction backend | +| Method | What it registers | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `api.registerProvider(...)` | Text inference (LLM) | +| `api.registerWorkerProvider(...)` | Cloud-worker lifecycle leases | +| `api.registerModelCatalogProvider(...)` | Model catalog rows for text and media generation | +| `api.registerAgentHarness(...)` | [Experimental](/plugins/sdk-agent-harness) native agent executor (Codex, Copilot) | +| `api.registerCliBackend(...)` | Local CLI inference backend | +| `api.registerChannel(...)` | Messaging channel | +| `api.registerEmbeddingProvider(...)` | Reusable vector embedding provider | +| `api.registerSpeechProvider(...)` | Text-to-speech / STT synthesis | +| `api.registerRealtimeTranscriptionProvider(...)` | Streaming realtime transcription | +| `api.registerRealtimeVoiceProvider(...)` | Duplex realtime voice sessions | +| `api.registerMediaUnderstandingProvider(...)` | Image/audio/video analysis | +| `api.registerTranscriptSourceProvider(...)` | Live or imported meeting transcript source; meeting plugins can use `createMeetingTranscriptSourceProvider` from `plugin-sdk/transcripts` | +| `api.registerImageGenerationProvider(...)` | Image generation | +| `api.registerMusicGenerationProvider(...)` | Music generation | +| `api.registerVideoGenerationProvider(...)` | Video generation | +| `api.registerWebFetchProvider(...)` | Web fetch / scrape provider | +| `api.registerWebSearchProvider(...)` | Web search | +| `api.registerCompactionProvider(...)` | Pluggable transcript-compaction backend | Worker providers must also declare their id in `contracts.workerProviders`. Core persists durable intent before `provision(profile, operationId)`. Providers validate settings before external allocation and throw `WorkerProviderError` for permanent profile rejection. `provision` must adopt the same lease when the operation id repeats. diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index b931b52e8669..15d845d1e272 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -335,7 +335,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/music-generation` | Private-local after July 2026; Music generation provider/request/result types | | `plugin-sdk/video-generation` | Private-local after July 2026; Video generation provider/request/result types | | `plugin-sdk/video-generation-core` | Private-local after July 2026; Shared video-generation types, failover helpers, provider lookup, and model-ref parsing | - | `plugin-sdk/transcripts` | Private-local after July 2026; Shared transcripts source provider types, registry helpers, session descriptors, and utterance metadata | + | `plugin-sdk/transcripts` | Private-local after July 2026; Shared transcript source provider types, registry helpers, meeting-provider bridge factory, session descriptors, and utterance metadata | | `plugin-sdk/webhook-targets` | Private-local after July 2026; Webhook target registry and route-install helpers | | `plugin-sdk/web-media` | Shared remote/local media loading helpers | | `plugin-sdk/zod` | Deprecated compatibility re-export; import `zod` from `zod` directly | diff --git a/docs/plugins/teams-meetings.md b/docs/plugins/teams-meetings.md index d7e79d0f6d57..61b59e16dc0a 100644 --- a/docs/plugins/teams-meetings.md +++ b/docs/plugins/teams-meetings.md @@ -6,7 +6,7 @@ read_when: title: "Microsoft Teams meetings plugin" --- -The `teams-meetings` plugin joins Microsoft Teams links as a guest in the OpenClaw Chrome profile. It accepts work links under `teams.microsoft.com/l/meetup-join/...` and consumer links under `teams.live.com/meet/...`. It does not create meetings, dial in, call Microsoft Graph, or record meetings. +The `teams-meetings` plugin joins Microsoft Teams links as a guest in the OpenClaw Chrome profile. It accepts work links under `teams.microsoft.com/l/meetup-join/...` and consumer links under `teams.live.com/meet/...`. It does not create meetings, dial in, call Microsoft Graph, or capture audio/video recordings. ## Setup @@ -53,7 +53,15 @@ Use `chromeNode.node` to run Chrome, BlackHole, and SoX on a paired macOS node. | `bidi` | A realtime voice model listens and replies directly. | | `transcribe` | Observe-only join with live-caption transcript snapshots. | -Transcribe mode enables Teams live captions after admission and captures speaker-attributed caption rows. The `transcript` action returns the bounded caption buffer for the active OpenClaw meeting session. +Teams live captions are enabled after admission in every mode so OpenClaw can +persist speaker-attributed notes. The `transcript` action still returns the +bounded live buffer only for `transcribe` sessions. On leave, OpenClaw stores +the durable transcript and derived summary in the shared state database; list +or export them with [`openclaw transcripts`](/cli/transcripts). + +Automatic notes are enabled by default. Set `transcripts.enabled: false` to +disable durable notes globally; explicit `transcribe` mode still exposes only +its bounded live tail. ## Guest join limits diff --git a/docs/plugins/zoom-meetings.md b/docs/plugins/zoom-meetings.md index c5ad2593a31d..4605fce7c276 100644 --- a/docs/plugins/zoom-meetings.md +++ b/docs/plugins/zoom-meetings.md @@ -6,7 +6,7 @@ read_when: title: "Zoom meetings plugin" --- -The `zoom-meetings` plugin joins Zoom meeting links as a guest through the Zoom Web App in the OpenClaw Chrome profile. It accepts meeting links under `zoom.us/j/...` and account subdomains such as `example.zoom.us/j/...`. It does not create meetings, dial in, use the Zoom Meeting SDK, or record meetings. +The `zoom-meetings` plugin joins Zoom meeting links as a guest through the Zoom Web App in the OpenClaw Chrome profile. It accepts meeting links under `zoom.us/j/...` and account subdomains such as `example.zoom.us/j/...`. It does not create meetings, dial in, use the Zoom Meeting SDK, or capture audio/video recordings. ## Setup @@ -53,7 +53,15 @@ Use `chromeNode.node` to run Chrome, BlackHole, and SoX on a paired macOS node. | `bidi` | A realtime voice model listens and replies directly. | | `transcribe` | Observe-only join with live-caption transcript snapshots. | -Transcribe mode enables Zoom live captions after admission and captures the bounded caption display. The `transcript` action returns the caption buffer for the active OpenClaw meeting session. +Zoom live captions are enabled after admission in every mode so OpenClaw can +persist meeting notes. The `transcript` action still returns the bounded live +buffer only for `transcribe` sessions. On leave, OpenClaw stores the durable +transcript and derived summary in the shared state database; list or export +them with [`openclaw transcripts`](/cli/transcripts). + +Automatic notes are enabled by default. Set `transcripts.enabled: false` to +disable durable notes globally; explicit `transcribe` mode still exposes only +its bounded live tail. ## Guest join limits diff --git a/extensions/google-meet/index.transcripts.test.ts b/extensions/google-meet/index.transcripts.test.ts new file mode 100644 index 000000000000..3b78a296e14c --- /dev/null +++ b/extensions/google-meet/index.transcripts.test.ts @@ -0,0 +1,32 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import type { TranscriptSourceProvider } from "openclaw/plugin-sdk/transcripts"; +import { describe, expect, it } from "vitest"; +import plugin from "./index.js"; + +describe("Google Meet transcript source registration", () => { + it("registers the canonical provider and aliases", () => { + const providers: TranscriptSourceProvider[] = []; + const api = createTestPluginApi({ + id: "google-meet", + name: "Google Meet", + description: "test", + version: "0", + source: "test", + config: {}, + pluginConfig: {}, + runtime: {} as OpenClawPluginApi["runtime"], + registerTranscriptSourceProvider: (provider) => providers.push(provider), + }); + + plugin.register(api); + + expect(providers).toEqual([ + expect.objectContaining({ + id: "google-meet", + aliases: ["googlemeet", "meet"], + sourceKinds: ["live-caption"], + }), + ]); + }); +}); diff --git a/extensions/google-meet/index.ts b/extensions/google-meet/index.ts index 02d237a58e76..8f89550a8022 100644 --- a/extensions/google-meet/index.ts +++ b/extensions/google-meet/index.ts @@ -5,6 +5,7 @@ import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/p import { normalizeAgentId, parseAgentSessionKey } from "openclaw/plugin-sdk/routing"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { jsonResult as json } from "openclaw/plugin-sdk/tool-results"; +import { createMeetingTranscriptSourceProvider } from "openclaw/plugin-sdk/transcripts"; import { buildGoogleMeetCalendarDayWindow, listGoogleMeetCalendarEvents } from "./src/calendar.js"; import { buildGoogleMeetPreflightReport, @@ -55,6 +56,14 @@ export default definePluginEntry({ register(api: OpenClawPluginApi) { const config = googleMeetConfigSchema.parse(api.pluginConfig); const ensureRuntime = createGoogleMeetRuntimeAccessor({ api, config }); + api.registerTranscriptSourceProvider( + createMeetingTranscriptSourceProvider({ + id: "google-meet", + aliases: ["googlemeet", "meet"], + name: "Google Meet", + runtime: async () => (await ensureRuntime()).transcriptSourceRuntime(), + }), + ); api.registerGatewayMethod( "googlemeet.join", diff --git a/extensions/google-meet/openclaw.plugin.json b/extensions/google-meet/openclaw.plugin.json index 05b977db9031..3365d45e6833 100644 --- a/extensions/google-meet/openclaw.plugin.json +++ b/extensions/google-meet/openclaw.plugin.json @@ -11,7 +11,8 @@ "onCapabilities": ["tool"] }, "contracts": { - "tools": ["google_meet"] + "tools": ["google_meet"], + "transcriptSourceProviders": ["google-meet"] }, "uiHints": { "defaults.meeting": { diff --git a/extensions/google-meet/src/runtime-session.ts b/extensions/google-meet/src/runtime-session.ts new file mode 100644 index 000000000000..ba0016b8811c --- /dev/null +++ b/extensions/google-meet/src/runtime-session.ts @@ -0,0 +1,38 @@ +import type { + GoogleMeetConfig, + GoogleMeetMode, + GoogleMeetModeInput, + GoogleMeetTransport, +} from "./config.js"; +import type { GoogleMeetSession } from "./transports/types.js"; + +export function resolveTransport( + input: GoogleMeetTransport | undefined, + config: GoogleMeetConfig, +): GoogleMeetTransport { + return input ?? config.defaultTransport; +} + +export function resolveMode( + input: GoogleMeetModeInput | undefined, + config: GoogleMeetConfig, +): GoogleMeetMode { + return input === "realtime" ? "agent" : (input ?? config.defaultMode); +} + +export function withSessionAgentConfig( + config: GoogleMeetConfig, + agentId: string, +): GoogleMeetConfig { + return config.realtime.agentId === agentId + ? config + : { ...config, realtime: { ...config.realtime, agentId } }; +} + +export function isBrowserTransport(transport: GoogleMeetTransport): boolean { + return transport === "chrome" || transport === "chrome-node"; +} + +export function noteSession(session: GoogleMeetSession, note: string): void { + session.notes = [...session.notes.filter((item) => item !== note), note]; +} diff --git a/extensions/google-meet/src/runtime.ts b/extensions/google-meet/src/runtime.ts index e64d66e03917..85b535f02834 100644 --- a/extensions/google-meet/src/runtime.ts +++ b/extensions/google-meet/src/runtime.ts @@ -23,6 +23,13 @@ import { testGoogleMeetSpeech, type GoogleMeetRuntimeProbeContext, } from "./runtime-probes.js"; +import { + isBrowserTransport, + noteSession, + resolveMode, + resolveTransport, + withSessionAgentConfig, +} from "./runtime-session.js"; import { getGoogleMeetRuntimeSetupStatus } from "./runtime-setup.js"; import { launchChromeMeet, @@ -34,7 +41,10 @@ import { recoverCurrentMeetTab, recoverCurrentMeetTabOnNode, } from "./transports/chrome.js"; -import { GOOGLE_MEET_PLATFORM_ADAPTER } from "./transports/google-meet-platform-adapter.js"; +import { + GOOGLE_MEET_PLATFORM_ADAPTER, + isGoogleMeetTalkBackMode, +} from "./transports/google-meet-platform-adapter.js"; import type { GoogleMeetBrowserTab, GoogleMeetChromeHealth, @@ -79,39 +89,7 @@ type GoogleMeetJoinContext = MeetingSessionRuntimeJoinContext< GoogleMeetBrowserTab >; -function nowIso(): string { - return new Date().toISOString(); -} - -function buildTwilioVoiceCallSessionKey(meetingSessionId: string): string { - return `voice:google-meet:${meetingSessionId}`; -} - -function resolveTransport(input: GoogleMeetTransport | undefined, config: GoogleMeetConfig) { - return input ?? config.defaultTransport; -} - -function resolveMode(input: GoogleMeetModeInput | undefined, config: GoogleMeetConfig) { - return input === "realtime" ? "agent" : (input ?? config.defaultMode); -} - -function withSessionAgentConfig(config: GoogleMeetConfig, agentId: string): GoogleMeetConfig { - return config.realtime.agentId === agentId - ? config - : { ...config, realtime: { ...config.realtime, agentId } }; -} - -function isGoogleMeetTalkBackMode(mode: GoogleMeetMode): boolean { - return mode === "agent" || mode === "bidi"; -} - -function isBrowserTransport(transport: GoogleMeetTransport): boolean { - return transport === "chrome" || transport === "chrome-node"; -} - -function noteSession(session: GoogleMeetSession, note: string): void { - session.notes = [...session.notes.filter((item) => item !== note), note]; -} +const nowIso = () => new Date().toISOString(); export class GoogleMeetRuntime { readonly #createdBrowserTabs = new Map(); @@ -218,6 +196,11 @@ export class GoogleMeetRuntime { await this.#captureTranscript(session, options), speakViaTransport: async (session, instructions) => await this.#speakViaTransport(session, instructions), + durableTranscripts: { + config: params.fullConfig.transcripts, + providerId: "google-meet", + providerName: "Google Meet", + }, }); } @@ -233,6 +216,8 @@ export class GoogleMeetRuntime { return await this.#sessions.transcript(sessionId, options); } + transcriptSourceRuntime = () => this.#sessions; + async setupStatus( options: { transport?: GoogleMeetTransport; @@ -271,11 +256,13 @@ export class GoogleMeetRuntime { ? await recoverCurrentMeetTabOnNode({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, url, }) : await recoverCurrentMeetTab({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, url, }); } @@ -425,7 +412,7 @@ export class GoogleMeetRuntime { agentId: delegatedAgentId, sessionKey: delegatedAgentId ? `agent:${delegatedAgentId}:google-meet:${session.id}` - : buildTwilioVoiceCallSessionKey(session.id), + : `voice:google-meet:${session.id}`, message: isGoogleMeetTalkBackMode(session.mode) ? (request.message ?? this.params.config.voiceCall.introMessage ?? @@ -539,6 +526,7 @@ export class GoogleMeetRuntime { ? await recoverCurrentMeetTabOnNode({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, mode: session.mode, readOnly: options.readOnly, trackedMeetingUrl: session.url, @@ -548,6 +536,7 @@ export class GoogleMeetRuntime { : await recoverCurrentMeetTab({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, mode: session.mode, readOnly: options.readOnly, trackedMeetingUrl: session.url, diff --git a/extensions/google-meet/src/transports/chrome.test.ts b/extensions/google-meet/src/transports/chrome.test.ts index 6e23fa0afab0..0b13ba00150d 100644 --- a/extensions/google-meet/src/transports/chrome.test.ts +++ b/extensions/google-meet/src/transports/chrome.test.ts @@ -32,6 +32,7 @@ function browserRuntime(request: TestGatewayRequest): PluginRuntime { describe("google meet chrome transport", () => { it("prefers a meeting tab over a login fallback during untargeted recovery", async () => { + const statusScripts: string[] = []; const gatewayRequest = vi.fn(async (_method, params) => { if (params.path === "/tabs") { return { @@ -53,6 +54,10 @@ describe("google meet chrome transport", () => { return { ok: true }; } if (params.path === "/act") { + const fn = (params.body as { fn?: unknown } | undefined)?.fn; + if (typeof fn === "string") { + statusScripts.push(fn); + } return { result: JSON.stringify({ inCall: true, @@ -67,11 +72,14 @@ describe("google meet chrome transport", () => { const recovered = await recoverCurrentMeetTab({ runtime: browserRuntime(gatewayRequest), config: resolveGoogleMeetConfig({}), - mode: "transcribe", + fullConfig: { transcripts: { enabled: false } }, + mode: "agent", readOnly: true, }); expect(recovered).toMatchObject({ found: true, targetId: "meet-tab" }); + expect(statusScripts).toHaveLength(1); + expect(statusScripts[0]).toContain("const captureCaptions = false"); }); it("prefers the tracked target for an unchanged Google Meet URL", async () => { diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index c0eda936a555..c7edcb3859e5 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -16,6 +16,7 @@ import { import { addTimerTimeoutGraceMs } from "openclaw/plugin-sdk/number-runtime"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; import type { RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import { resolveTranscriptsConfig } from "openclaw/plugin-sdk/transcripts"; import type { GoogleMeetConfig, GoogleMeetMode } from "../config.js"; import { GOOGLE_MEET_SYSTEM_PROFILER_COMMAND, @@ -48,6 +49,12 @@ type ChromeNodeRealtimeAudioBridgeHandle = MeetingRealtimeAudioEngineHandle & { bridgeId: string; }; +function shouldCaptureCaptions(mode: GoogleMeetMode, fullConfig?: OpenClawConfig): boolean { + return ( + mode === "transcribe" || !fullConfig || resolveTranscriptsConfig(fullConfig.transcripts).enabled + ); +} + export async function assertBlackHole2chAvailable(params: { runtime: PluginRuntime; timeoutMs: number; @@ -210,6 +217,7 @@ export async function launchChromeMeet(params: { callBrowser: await resolveLocalMeetingBrowserRequest(params.runtime), config: params.config.chrome, session: { + captureCaptions: shouldCaptureCaptions(params.mode, params.fullConfig), meetingSessionId: params.meetingSessionId, mode: params.mode, url: params.url, @@ -354,6 +362,7 @@ async function openMeetWithBrowserProxy(params: { runtime: PluginRuntime; nodeId: string; config: GoogleMeetConfig; + captureCaptions: boolean; mode: GoogleMeetMode; meetingSessionId: string; url: string; @@ -371,6 +380,7 @@ async function openMeetWithBrowserProxy(params: { }), config: params.config.chrome, session: { + captureCaptions: params.captureCaptions, mode: params.mode, meetingSessionId: params.meetingSessionId, url: params.url, @@ -381,6 +391,7 @@ async function openMeetWithBrowserProxy(params: { export async function recoverCurrentMeetTab(params: { runtime: PluginRuntime; config: GoogleMeetConfig; + fullConfig?: OpenClawConfig; mode?: GoogleMeetMode; readOnly?: boolean; trackedMeetingUrl?: string; @@ -400,6 +411,7 @@ export async function recoverCurrentMeetTab(params: { ...(await recoverMeetingBrowserTab({ adapter: GOOGLE_MEET_PLATFORM_ADAPTER, callBrowser: await resolveLocalMeetingBrowserRequest(params.runtime), + captureCaptions: shouldCaptureCaptions(params.mode ?? "bidi", params.fullConfig), config: params.config.chrome, locationLabel: "in local Chrome", mode: params.mode ?? "bidi", @@ -414,6 +426,7 @@ export async function recoverCurrentMeetTab(params: { export async function recoverCurrentMeetTabOnNode(params: { runtime: PluginRuntime; config: GoogleMeetConfig; + fullConfig?: OpenClawConfig; mode?: GoogleMeetMode; readOnly?: boolean; trackedMeetingUrl?: string; @@ -446,6 +459,7 @@ export async function recoverCurrentMeetTabOnNode(params: { body: request.body, timeoutMs: request.timeoutMs, }), + captureCaptions: shouldCaptureCaptions(params.mode ?? "bidi", params.fullConfig), config: params.config.chrome, locationLabel: "on the selected Chrome node", mode: params.mode ?? "bidi", @@ -501,6 +515,7 @@ export async function launchChromeMeetOnNode(params: { runtime: params.runtime, nodeId, config: params.config, + captureCaptions: shouldCaptureCaptions(params.mode, params.fullConfig), mode: params.mode, meetingSessionId: params.meetingSessionId, url: params.url, diff --git a/extensions/google-meet/src/transports/google-meet-platform-adapter.test.ts b/extensions/google-meet/src/transports/google-meet-platform-adapter.test.ts new file mode 100644 index 000000000000..0a44ba60aaa7 --- /dev/null +++ b/extensions/google-meet/src/transports/google-meet-platform-adapter.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { GOOGLE_MEET_PLATFORM_ADAPTER } from "./google-meet-platform-adapter.js"; + +describe("GOOGLE_MEET_PLATFORM_ADAPTER captions", () => { + it("enables caption capture for durable notes in every browser mode", () => { + expect(GOOGLE_MEET_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(true); + expect(GOOGLE_MEET_PLATFORM_ADAPTER.browser.captions.enabled("bidi")).toBe(true); + expect(GOOGLE_MEET_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(true); + }); +}); diff --git a/extensions/google-meet/src/transports/google-meet-platform-adapter.ts b/extensions/google-meet/src/transports/google-meet-platform-adapter.ts index 801ccfd83636..80c42c04147d 100644 --- a/extensions/google-meet/src/transports/google-meet-platform-adapter.ts +++ b/extensions/google-meet/src/transports/google-meet-platform-adapter.ts @@ -307,7 +307,9 @@ export const GOOGLE_MEET_PLATFORM_ADAPTER = { buildLeaveScript: meetLeaveScript, parseLeaveResult: parseMeetLeaveResult, captions: { - enabled: (mode) => mode === "transcribe", + // Durable notes observe the caption stream in every mode; live transcript + // visibility remains gated by MeetingSessionRuntime. + enabled: (_mode) => true, buildTranscriptScript: ({ finalize, meetingSessionId, meetingUrl }) => meetTranscriptScript(meetingUrl, meetingSessionId, finalize), parseTranscript: parseMeetTranscriptSnapshot, diff --git a/extensions/teams-meetings/index.test.ts b/extensions/teams-meetings/index.test.ts index f005a63405ab..7df4f5d68b0f 100644 --- a/extensions/teams-meetings/index.test.ts +++ b/extensions/teams-meetings/index.test.ts @@ -1,6 +1,7 @@ import { ErrorCodes } from "openclaw/plugin-sdk/gateway-runtime"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import type { TranscriptSourceProvider } from "openclaw/plugin-sdk/transcripts"; import { describe, expect, it, vi } from "vitest"; import plugin from "./index.js"; @@ -117,6 +118,7 @@ describe("Microsoft Teams meetings plugin surface", () => { const cli: unknown[] = []; const nodeCommands: unknown[] = []; const policies: unknown[] = []; + const transcriptProviders: TranscriptSourceProvider[] = []; const api = createTestPluginApi({ id: "teams-meetings", name: "Microsoft Teams meetings", @@ -140,6 +142,7 @@ describe("Microsoft Teams meetings plugin surface", () => { registerCli: (_registrar: unknown, options: unknown) => cli.push(options), registerNodeHostCommand: (command: unknown) => nodeCommands.push(command), registerNodeInvokePolicy: (policy: unknown) => policies.push(policy), + registerTranscriptSourceProvider: (provider) => transcriptProviders.push(provider), }); plugin.register(api); @@ -162,6 +165,13 @@ describe("Microsoft Teams meetings plugin surface", () => { expect.objectContaining({ command: "teamsmeetings.chrome", cap: "teams-meetings" }), ]); expect(policies).toHaveLength(1); + expect(transcriptProviders).toEqual([ + expect.objectContaining({ + id: "teams", + aliases: ["teams-meetings", "microsoft-teams", "msteams"], + sourceKinds: ["live-caption"], + }), + ]); }); it("scopes trusted tool session operations to the invoking agent", async () => { diff --git a/extensions/teams-meetings/index.ts b/extensions/teams-meetings/index.ts index 50654f5f7b22..c5515bd6fd15 100644 --- a/extensions/teams-meetings/index.ts +++ b/extensions/teams-meetings/index.ts @@ -95,6 +95,11 @@ export default definePluginEntry( toolDescription: "Join and manage Microsoft Teams meeting browser guests. Guest admission, tenant sign-in, and media permissions may require manual action in the OpenClaw Chrome profile.", toolParameters: TeamsMeetingsToolSchema, + transcriptSource: { + id: "teams", + aliases: ["teams-meetings", "microsoft-teams", "msteams"], + name: "Microsoft Teams meetings", + }, createRuntime: ({ api, config }) => new TeamsMeetingsRuntime({ config, diff --git a/extensions/teams-meetings/openclaw.plugin.json b/extensions/teams-meetings/openclaw.plugin.json index 2bc7a79c0269..62412c1c185c 100644 --- a/extensions/teams-meetings/openclaw.plugin.json +++ b/extensions/teams-meetings/openclaw.plugin.json @@ -11,7 +11,8 @@ "onCapabilities": ["tool"] }, "contracts": { - "tools": ["teams_meetings"] + "tools": ["teams_meetings"], + "transcriptSourceProviders": ["teams"] }, "uiHints": { "defaultMode": { diff --git a/extensions/teams-meetings/src/runtime.ts b/extensions/teams-meetings/src/runtime.ts index 5525895422ac..ad2dd158d2ae 100644 --- a/extensions/teams-meetings/src/runtime.ts +++ b/extensions/teams-meetings/src/runtime.ts @@ -9,6 +9,10 @@ import { } from "openclaw/plugin-sdk/meeting-runtime"; import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import type { + TranscriptStartRequest, + TranscriptStopRequest, +} from "openclaw/plugin-sdk/transcripts"; import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; import { testTeamsMeetingListening, @@ -191,6 +195,11 @@ export class TeamsMeetingsRuntime { captureTranscript: async (session, options) => await this.#captureTranscript(session, options), speakViaTransport: async () => undefined, + durableTranscripts: { + config: params.fullConfig.transcripts, + providerId: "teams", + providerName: "Microsoft Teams", + }, }); } @@ -198,6 +207,14 @@ export class TeamsMeetingsRuntime { return this.#sessions.list(); } + async startTranscriptSource(request: TranscriptStartRequest) { + return await this.#sessions.startTranscriptSource(request); + } + + async stopTranscriptSource(request: TranscriptStopRequest) { + return await this.#sessions.stopTranscriptSource(request); + } + ownsSession(agentId: string, sessionId: string): boolean { return this.list().some((session) => session.id === sessionId && session.agentId === agentId); } @@ -424,6 +441,7 @@ export class TeamsMeetingsRuntime { const result = await recoverCurrentTeamsMeetingTab({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, meetingSessionId: session.id, mode: session.mode, nodeId: session.chrome?.nodeId, diff --git a/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts index 468526357061..53ab117fcc3f 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts @@ -894,9 +894,9 @@ describe("Microsoft Teams meeting captions and permissions", () => { expect(captions.lines).toMatchObject([{ text: "Final caption" }]); }); - it("enables caption capture only for transcribe mode and parses snapshots", () => { - expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(false); - expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("bidi")).toBe(false); + it("enables caption capture for durable notes in every mode and parses snapshots", () => { + expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(true); + expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("bidi")).toBe(true); expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(true); expect( TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.parseTranscript({ diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts index a14df42f461f..c1dba018369f 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts @@ -145,7 +145,9 @@ export const TEAMS_MEETINGS_PLATFORM_ADAPTER = MeetingPlatformAdapter.create< }), buildSessionLeaveScript: teamsMeetingLeaveScript, captions: { - enabled: (mode) => mode === "transcribe", + // Durable notes observe the caption stream in every mode; live transcript + // visibility remains gated by MeetingSessionRuntime. + enabled: () => true, buildTranscriptScript: ({ finalize, meetingSessionId, meetingUrl }) => teamsMeetingTranscriptScript(meetingUrl, meetingSessionId, finalize), }, diff --git a/extensions/zoom-meetings/index.test.ts b/extensions/zoom-meetings/index.test.ts index 557d7b3d8cba..d04ac1797bbd 100644 --- a/extensions/zoom-meetings/index.test.ts +++ b/extensions/zoom-meetings/index.test.ts @@ -1,6 +1,7 @@ import { ErrorCodes } from "openclaw/plugin-sdk/gateway-runtime"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import type { TranscriptSourceProvider } from "openclaw/plugin-sdk/transcripts"; import { describe, expect, it, vi } from "vitest"; import plugin from "./index.js"; @@ -116,6 +117,7 @@ describe("Zoom meetings plugin surface", () => { const cli: unknown[] = []; const nodeCommands: unknown[] = []; const policies: unknown[] = []; + const transcriptProviders: TranscriptSourceProvider[] = []; const api = createTestPluginApi({ id: "zoom-meetings", name: "Zoom meetings", @@ -139,6 +141,7 @@ describe("Zoom meetings plugin surface", () => { registerCli: (_registrar: unknown, options: unknown) => cli.push(options), registerNodeHostCommand: (command: unknown) => nodeCommands.push(command), registerNodeInvokePolicy: (policy: unknown) => policies.push(policy), + registerTranscriptSourceProvider: (provider) => transcriptProviders.push(provider), }); plugin.register(api); @@ -161,6 +164,13 @@ describe("Zoom meetings plugin surface", () => { expect.objectContaining({ command: "zoommeetings.chrome", cap: "zoom-meetings" }), ]); expect(policies).toHaveLength(1); + expect(transcriptProviders).toEqual([ + expect.objectContaining({ + id: "zoom", + aliases: ["zoom-meetings"], + sourceKinds: ["live-caption"], + }), + ]); }); it("does not expose the dangerous node surface when disabled", () => { diff --git a/extensions/zoom-meetings/index.ts b/extensions/zoom-meetings/index.ts index a02473f082d4..e7a7f8b9f8d2 100644 --- a/extensions/zoom-meetings/index.ts +++ b/extensions/zoom-meetings/index.ts @@ -90,6 +90,11 @@ export default definePluginEntry( toolDescription: "Join and manage Zoom meeting browser guests. Guest admission, tenant sign-in, and media permissions may require manual action in the OpenClaw Chrome profile.", toolParameters: ZoomMeetingsToolSchema, + transcriptSource: { + id: "zoom", + aliases: ["zoom-meetings"], + name: "Zoom meetings", + }, createRuntime: ({ api, config }) => new ZoomMeetingsRuntime({ config, diff --git a/extensions/zoom-meetings/openclaw.plugin.json b/extensions/zoom-meetings/openclaw.plugin.json index 8f2d0e2d9863..fdc1142ee2ab 100644 --- a/extensions/zoom-meetings/openclaw.plugin.json +++ b/extensions/zoom-meetings/openclaw.plugin.json @@ -11,7 +11,8 @@ "onCapabilities": ["tool"] }, "contracts": { - "tools": ["zoom_meetings"] + "tools": ["zoom_meetings"], + "transcriptSourceProviders": ["zoom"] }, "uiHints": { "defaultMode": { diff --git a/extensions/zoom-meetings/src/runtime.ts b/extensions/zoom-meetings/src/runtime.ts index 19ce7438b933..7aaba16bf255 100644 --- a/extensions/zoom-meetings/src/runtime.ts +++ b/extensions/zoom-meetings/src/runtime.ts @@ -9,6 +9,10 @@ import { } from "openclaw/plugin-sdk/meeting-runtime"; import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import type { + TranscriptStartRequest, + TranscriptStopRequest, +} from "openclaw/plugin-sdk/transcripts"; import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js"; import { testZoomMeetingListening, @@ -245,6 +249,11 @@ export class ZoomMeetingsRuntime { captureTranscript: async (session, options) => await this.#captureTranscript(session, options), speakViaTransport: async () => undefined, + durableTranscripts: { + config: params.fullConfig.transcripts, + providerId: "zoom", + providerName: "Zoom", + }, }); } @@ -252,6 +261,14 @@ export class ZoomMeetingsRuntime { return this.#sessions.list(); } + async startTranscriptSource(request: TranscriptStartRequest) { + return await this.#sessions.startTranscriptSource(request); + } + + async stopTranscriptSource(request: TranscriptStopRequest) { + return await this.#sessions.stopTranscriptSource(request); + } + ownsSession(agentId: string, sessionId: string): boolean { return this.list().some((session) => session.id === sessionId && session.agentId === agentId); } @@ -488,6 +505,7 @@ export class ZoomMeetingsRuntime { const result = await recoverCurrentZoomMeetingTab({ runtime: this.params.runtime, config: this.params.config, + fullConfig: this.params.fullConfig, meetingSessionId: session.id, mode: session.mode, nodeId: session.chrome?.nodeId, diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts index 3f7710e41b0b..fa33347f74d2 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts @@ -231,9 +231,10 @@ describe("Zoom meeting platform adapter", () => { expect(script).toContain("BlackHole"); }); - it("enables caption snapshots only in transcribe mode", () => { + it("enables caption snapshots for durable notes in every mode", () => { expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(true); - expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(false); + expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(true); + expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("bidi")).toBe(true); }); it("requires verified bidirectional audio before realtime startup", () => { diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts index f1ebe1116685..e00e96b222ab 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts @@ -140,7 +140,9 @@ export const ZOOM_MEETINGS_PLATFORM_ADAPTER = MeetingPlatformAdapter.create< }), buildSessionLeaveScript: zoomMeetingLeaveScript, captions: { - enabled: (mode) => mode === "transcribe", + // Durable notes observe the caption stream in every mode; live transcript + // visibility remains gated by MeetingSessionRuntime. + enabled: () => true, buildTranscriptScript: ({ finalize, meetingSessionId, meetingUrl }) => zoomMeetingTranscriptScript(meetingUrl, meetingSessionId, finalize), }, diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 3a0ef652d751..19e91dee081e 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -548,7 +548,9 @@ export function createOpenClawTools( agentId: sessionAgentId, agentAccountId: options?.agentAccountId, }), - ...(includeTranscriptsTool ? [createTranscriptsTool({ config: resolvedConfig })] : []), + ...(includeTranscriptsTool + ? [createTranscriptsTool({ agentId: sessionAgentId, config: resolvedConfig })] + : []), ...collectPresentOpenClawTools([imageGenerateTool, musicGenerateTool, videoGenerateTool]), ...(embedded ? [] diff --git a/src/agents/openclaw-tools.update-plan.test.ts b/src/agents/openclaw-tools.update-plan.test.ts index 894710978664..ef7c28d1af89 100644 --- a/src/agents/openclaw-tools.update-plan.test.ts +++ b/src/agents/openclaw-tools.update-plan.test.ts @@ -181,16 +181,16 @@ describe("openclaw-tools update_plan gating", () => { expect(embedded).not.toContain("openclaw"); }); - it("requires explicit transcripts enablement before registering the transcripts tool", () => { + it("registers transcripts by default with an explicit global opt-out", () => { const defaultTools = createFastToolNames({ config: {} as OpenClawConfig, }); - const enabledTools = createFastToolNames({ - config: { transcripts: { enabled: true } } as OpenClawConfig, + const disabledTools = createFastToolNames({ + config: { transcripts: { enabled: false } } as OpenClawConfig, }); - expect(defaultTools).not.toContain("transcripts"); - expect(enabledTools).toContain("transcripts"); + expect(defaultTools).toContain("transcripts"); + expect(disabledTools).not.toContain("transcripts"); }); it("registers task suggestions only for sessions with an actionable gateway sink", () => { diff --git a/src/agents/tools/transcripts-tool-runtime.ts b/src/agents/tools/transcripts-tool-runtime.ts index 79238fb76116..a91f7c25e64b 100644 --- a/src/agents/tools/transcripts-tool-runtime.ts +++ b/src/agents/tools/transcripts-tool-runtime.ts @@ -8,6 +8,7 @@ import type { TranscriptSourceProvider, TranscriptsStartResult, } from "../../transcripts/provider-types.js"; +import { sanitizeTranscriptSourceLocator } from "../../transcripts/source-locator.js"; import type { TranscriptsStore } from "../../transcripts/store.js"; export type TranscriptsLogger = { @@ -15,6 +16,7 @@ export type TranscriptsLogger = { }; export type TranscriptsRuntimeContext = { + agentId?: string; config?: OpenClawConfig; stateDir: string; logger: TranscriptsLogger; @@ -146,16 +148,20 @@ export async function startTranscripts(params: { if (params.abortSignal?.aborted) { throw new Error("transcripts start aborted"); } - const source = sourceFromParams(params.rawParams); - const provider = resolveSourceProvider(source.providerId, params.ctx); + const providerSource = { + ...sourceFromParams(params.rawParams), + ...(params.ctx.agentId ? { agentId: params.ctx.agentId } : {}), + }; + const provider = resolveSourceProvider(providerSource.providerId, params.ctx); if (!provider?.start) { - throw new Error(`transcripts provider ${source.providerId} cannot start live capture`); + throw new Error(`transcripts provider ${providerSource.providerId} cannot start live capture`); } const session: TranscriptSessionDescriptor = { sessionId: readStringParam(params.rawParams, "sessionId", { trim: true }) ?? createSessionId(), title: readStringParam(params.rawParams, "title", { trim: true }), - source, + source: sanitizeTranscriptSourceLocator(providerSource), startedAt: new Date().toISOString(), + ...(params.ctx.agentId ? { metadata: { agentId: params.ctx.agentId } } : {}), }; if (activeSessions.has(session.sessionId) || startingSessionIds.has(session.sessionId)) { throw new Error(`transcripts session already active: ${session.sessionId}`); @@ -169,7 +175,7 @@ export async function startTranscripts(params: { try { result = await provider.start({ cfg: params.ctx.config, - session, + session: { ...session, source: providerSource }, abortSignal: startupAbort.signal, startupWaitMs: params.startupWaitMs, onUtterance: async (utterance) => { diff --git a/src/agents/tools/transcripts-tool.test.ts b/src/agents/tools/transcripts-tool.test.ts index 61db842bffbe..b7dd149816bf 100644 --- a/src/agents/tools/transcripts-tool.test.ts +++ b/src/agents/tools/transcripts-tool.test.ts @@ -29,13 +29,22 @@ function currentDateDir(): string { return new Date().toISOString().slice(0, 10); } -async function createHarness(stateDir: string, pluginConfig: Record = {}) { +async function createHarness( + stateDir: string, + pluginConfig: Record = {}, + agentId?: string, +) { const config = { transcripts: { enabled: true, ...pluginConfig } }; const logger = { warn: vi.fn() }; return { logger, service: createTranscriptsAutoStartService({ config, stateDir, logger }), - tool: createTranscriptsTool({ config, stateDir, logger }), + tool: createTranscriptsTool({ + config, + stateDir, + logger, + ...(agentId ? { agentId } : {}), + }), }; } @@ -59,6 +68,79 @@ describe("transcripts tool", () => { expect(tool.name).toBe("transcripts"); }); + it("adds the trusted tool agent to live source ownership metadata", async () => { + const stateDir = await makeStateDir(); + const start = vi.fn(async (request) => { + expect(request.session).toMatchObject({ + source: { + agentId: "research", + meetingUrl: "https://zoom.us/j/1234567890?context=opaque-value#fragment", + providerId: "zoom", + }, + metadata: { agentId: "research" }, + }); + return { ok: false as const, error: "ownership checked" }; + }); + getTranscriptSourceProviderMock.mockReturnValue({ + id: "proof-live", + name: "Proof Live", + sourceKinds: ["live-caption"], + start, + }); + const { tool } = await createHarness(stateDir, {}, "research"); + + await expect( + tool.execute( + "call-1", + { + action: "start", + meetingUrl: "https://zoom.us/j/1234567890?context=opaque-value#fragment", + providerId: "zoom", + sessionId: "owned-meeting", + }, + undefined, + vi.fn(), + ), + ).rejects.toThrow("ownership checked"); + + expect(start).toHaveBeenCalledOnce(); + await expect(storeFor(stateDir).readSession("owned-meeting")).resolves.toMatchObject({ + source: { meetingUrl: "https://zoom.us/j/1234567890" }, + }); + }); + + it("keeps ownerless shipped sessions visible only to the main agent", async () => { + const stateDir = await makeStateDir(); + const store = storeFor(stateDir); + const legacySession = { + sessionId: "legacy-ownerless", + source: { providerId: "manual-transcript" }, + startedAt: "2026-07-01T12:00:00.000Z", + stoppedAt: "2026-07-01T12:05:00.000Z", + }; + await store.writeSession(legacySession); + await store.appendUtteranceForSession(legacySession, { text: "legacy notes" }); + const { tool: mainTool } = await createHarness(stateDir, {}, "main"); + const { tool: researchTool } = await createHarness(stateDir, {}, "research"); + + await expect( + mainTool.execute( + "call-main", + { action: "summarize", sessionId: legacySession.sessionId }, + undefined, + vi.fn(), + ), + ).resolves.toMatchObject({ details: { sessionId: legacySession.sessionId } }); + await expect( + researchTool.execute( + "call-research", + { action: "summarize", sessionId: legacySession.sessionId }, + undefined, + vi.fn(), + ), + ).rejects.toThrow(`transcripts session not found: ${legacySession.sessionId}`); + }); + it("requires explicit enablement before execution", async () => { const stateDir = await makeStateDir(); const { tool } = await createHarness(stateDir, { enabled: false }); diff --git a/src/agents/tools/transcripts-tool.ts b/src/agents/tools/transcripts-tool.ts index 1d59d3db573f..3704ecc89716 100644 --- a/src/agents/tools/transcripts-tool.ts +++ b/src/agents/tools/transcripts-tool.ts @@ -15,6 +15,7 @@ import { import { manualTranscriptSourceProvider } from "../../transcripts/manual-source.js"; import { listTranscriptSourceProviders } from "../../transcripts/provider-registry.js"; import type { TranscriptSessionDescriptor } from "../../transcripts/provider-types.js"; +import { sanitizeTranscriptSourceLocator } from "../../transcripts/source-locator.js"; import { TranscriptsStore, type TranscriptsSessionEntry } from "../../transcripts/store.js"; import { summarizeTranscripts } from "../../transcripts/summary.js"; import type { AnyAgentTool } from "./common.js"; @@ -42,6 +43,22 @@ function sameSessionIdentity( return left.sessionId === right.sessionId && left.startedAt === right.startedAt; } +function ownsTranscriptSession( + ctx: TranscriptsRuntimeContext, + session: TranscriptSessionDescriptor, +): boolean { + if (!ctx.agentId) { + return true; + } + const ownerAgentId = session.metadata?.agentId; + if (typeof ownerAgentId === "string") { + return ownerAgentId === ctx.agentId; + } + // Shipped rows predate agent attribution. Treat them as operator-owned legacy + // state: main can curate them, but isolated agents cannot claim them. + return ctx.agentId === "main"; +} + function asParamsRecord(params: unknown): Record { return params && typeof params === "object" && !Array.isArray(params) ? (params as Record) @@ -136,7 +153,7 @@ async function stopTranscripts(params: { sameSessionIdentity(activeCandidate.session, resolvedSession); const selectedActive = directActive ?? (activeMatchesResolved ? activeCandidate : undefined); const session = selectedActive?.session ?? resolvedSession; - if (!session) { + if (!session || !ownsTranscriptSession(params.ctx, session)) { throw new Error(`transcripts session not found: ${sessionSelector}`); } const sessionId = session.sessionId; @@ -210,17 +227,21 @@ async function importTranscripts(params: { store: TranscriptsStore; rawParams: Record; }) { - const source = sourceFromParams(params.rawParams); - const provider = resolveSourceProvider(source.providerId, params.ctx); + const providerSource = { + ...sourceFromParams(params.rawParams), + ...(params.ctx.agentId ? { agentId: params.ctx.agentId } : {}), + }; + const provider = resolveSourceProvider(providerSource.providerId, params.ctx); if (!provider?.importTranscript) { - throw new Error(`transcripts provider ${source.providerId} cannot import transcripts`); + throw new Error(`transcripts provider ${providerSource.providerId} cannot import transcripts`); } const session: TranscriptSessionDescriptor = { sessionId: readStringParam(params.rawParams, "sessionId", { trim: true }) ?? createSessionId(), title: readStringParam(params.rawParams, "title", { trim: true }), - source, + source: sanitizeTranscriptSourceLocator(providerSource), startedAt: new Date().toISOString(), stoppedAt: new Date().toISOString(), + ...(params.ctx.agentId ? { metadata: { agentId: params.ctx.agentId } } : {}), }; const transcript = readStringParam(params.rawParams, "transcript", { required: true, @@ -229,7 +250,7 @@ async function importTranscripts(params: { await params.store.writeSession(session); const utterances = await provider.importTranscript({ cfg: params.ctx.config, - session, + session: { ...session, source: providerSource }, text: transcript, speakerLabel: readStringParam(params.rawParams, "speakerLabel", { trim: true }), }); @@ -257,6 +278,7 @@ async function importTranscripts(params: { async function summarizeExisting(params: { config: ReturnType; + ctx: TranscriptsRuntimeContext; store: TranscriptsStore; rawParams: Record; }) { @@ -265,7 +287,7 @@ async function summarizeExisting(params: { trim: true, }); const entry = await params.store.readSessionEntry(sessionId); - if (!entry) { + if (!entry || !ownsTranscriptSession(params.ctx, entry.session)) { throw new Error(`transcripts session not found: ${sessionId}`); } const { summaryPath, intendedSummaryPath, summary, summaryExportError } = @@ -292,13 +314,15 @@ async function statusTranscripts(ctx: TranscriptsRuntimeContext) { ...listTranscriptSourceProviders(ctx.config).map((provider) => provider.id), ]; const uniqueProviders = uniqueStrings(providers); - const active = [...activeSessions.values()].map((entry) => ({ - sessionId: entry.session.sessionId, - providerId: entry.providerId, - title: entry.session.title, - source: entry.session.source, - cleanupPending: entry.cleanupPending === true, - })); + const active = [...activeSessions.values()] + .filter((entry) => ownsTranscriptSession(ctx, entry.session)) + .map((entry) => ({ + sessionId: entry.session.sessionId, + providerId: entry.providerId, + title: entry.session.title, + source: entry.session.source, + cleanupPending: entry.cleanupPending === true, + })); return toolText( [ `Transcripts providers: ${uniqueProviders.length ? uniqueProviders.join(", ") : "none"}`, @@ -310,6 +334,7 @@ async function statusTranscripts(ctx: TranscriptsRuntimeContext) { /** Create the agent-facing transcripts tool. */ export function createTranscriptsTool(options?: { + agentId?: string; config?: OpenClawConfig; stateDir?: string; logger?: TranscriptsLogger; @@ -318,6 +343,7 @@ export function createTranscriptsTool(options?: { config: options?.config, stateDir: options?.stateDir ?? resolveStateDir(), logger: options?.logger ?? console, + ...(options?.agentId ? { agentId: options.agentId } : {}), }; return { name: "transcripts", @@ -341,7 +367,7 @@ export function createTranscriptsTool(options?: { case "import": return await importTranscripts({ ctx, store, rawParams: params }); case "summarize": - return await summarizeExisting({ config, store, rawParams: params }); + return await summarizeExisting({ config, ctx, store, rawParams: params }); case "status": return await statusTranscripts(ctx); default: diff --git a/src/config/schema.help.automation.ts b/src/config/schema.help.automation.ts index 55bc4952b3d6..0e3c52750525 100644 --- a/src/config/schema.help.automation.ts +++ b/src/config/schema.help.automation.ts @@ -92,9 +92,9 @@ export const AUTOMATION_FIELD_HELP: Record = { "cron.sessionRetention": "Controls how long completed cron run sessions are kept before pruning (`24h`, `7d`, `1h30m`, or `false` to disable pruning; default: `24h`). Use shorter retention to reduce storage growth on high-frequency schedules.", transcripts: - "Core transcript capture settings for recording-capable agent tools and configured live meeting auto-start sources. Keep disabled unless operators explicitly want agents to capture or import meeting transcripts.", + "Core transcript capture settings for meeting notes, recording-capable agent tools, and configured live meeting auto-start sources. Meeting plugins capture durable notes by default; set enabled to false to opt out globally.", "transcripts.enabled": - "Enables the recording-capable transcripts agent tool and configured auto-start sources. Default: false. Enable only on hosts where operators have reviewed meeting capture policy and provider permissions.", + "Enables durable automatic meeting notes, the transcripts agent tool, and configured auto-start sources. Default: true. Set false to disable persistence and the tool; explicit meeting transcribe mode retains its bounded live tail.", "transcripts.autoStart": "Live transcript sources started automatically when the gateway starts. Each entry is enabled by being present; remove an entry to disable that source.", "transcripts.autoStart[].providerId": diff --git a/src/meeting-bot/browser-controller.test.ts b/src/meeting-bot/browser-controller.test.ts index 0b54b4cdba13..ab0474fba663 100644 --- a/src/meeting-bot/browser-controller.test.ts +++ b/src/meeting-bot/browser-controller.test.ts @@ -18,6 +18,7 @@ describe("meeting browser navigation errors", () => { describe("meeting browser join readiness", () => { it("retries a platform-owned transient in-call status", async () => { const adoptionAttempts: boolean[] = []; + const captionCaptureAttempts: boolean[] = []; let evaluationAttempts = 0; const result = await openMeetingWithBrowser({ adapter: { @@ -42,11 +43,12 @@ describe("meeting browser join readiness", () => { buildLeaveScript: () => "", buildStatusJoinScript: (params) => { adoptionAttempts.push(params.allowSessionAdoption); + captionCaptureAttempts.push(params.captureCaptions); return "() => '{}'"; }, captions: { buildTranscriptScript: () => "", - enabled: () => false, + enabled: () => true, parseTranscript: () => ({ droppedLines: 0, lines: [] }), }, classifyManualAction: (health) => @@ -86,6 +88,7 @@ describe("meeting browser join readiness", () => { waitForInCallMs: 1_000, }, session: { + captureCaptions: false, meetingSessionId: "session-1", mode: "agent", url: "https://meet.test/meeting", @@ -94,6 +97,7 @@ describe("meeting browser join readiness", () => { expect(evaluationAttempts).toBe(2); expect(adoptionAttempts).toEqual([true, false]); + expect(captionCaptureAttempts).toEqual([false, false]); expect(result.browser).toMatchObject({ inCall: true, manualActionRequired: false, diff --git a/src/meeting-bot/browser-controller.ts b/src/meeting-bot/browser-controller.ts index 521bbe069388..eddda4896c64 100644 --- a/src/meeting-bot/browser-controller.ts +++ b/src/meeting-bot/browser-controller.ts @@ -247,7 +247,9 @@ export async function openMeetingWithBrowser< ...params.session, allowSessionAdoption: adoptSession, autoJoin: params.config.autoJoin, - captureCaptions: params.adapter.browser.captions.enabled(params.session.mode), + captureCaptions: + params.session.captureCaptions ?? + params.adapter.browser.captions.enabled(params.session.mode), guestName: params.config.guestName, waitForInCallMs: params.config.waitForInCallMs, }), @@ -361,6 +363,7 @@ async function inspectRecoverableTab< allowSessionAdoption?: boolean; autoJoin?: boolean; callBrowser: MeetingBrowserRequestCaller; + captureCaptions?: boolean; config: MeetingBrowserControllerConfig; meetingSessionId?: string; mode: Mode; @@ -437,7 +440,8 @@ async function inspectRecoverableTab< mode: params.mode, url: params.requestedMeetingUrl ?? params.tab.url ?? "", autoJoin: params.autoJoin ?? false, - captureCaptions: params.adapter.browser.captions.enabled(params.mode), + captureCaptions: + params.captureCaptions ?? params.adapter.browser.captions.enabled(params.mode), guestName: params.config.guestName, readOnly: params.readOnly, waitForInCallMs: params.config.waitForInCallMs, @@ -504,6 +508,7 @@ export async function recoverMeetingBrowserTab< allowSessionAdoption?: boolean; autoJoin?: boolean; callBrowser: MeetingBrowserRequestCaller; + captureCaptions?: boolean; config: MeetingBrowserControllerConfig; locationLabel: string; meetingSessionId?: string; @@ -580,6 +585,7 @@ export async function recoverMeetingBrowserTab< allowSessionAdoption: params.allowSessionAdoption, autoJoin: params.autoJoin, callBrowser: params.callBrowser, + captureCaptions: params.captureCaptions, config: params.config, ...(deadline === undefined ? {} : { deadline }), meetingSessionId: params.meetingSessionId, diff --git a/src/meeting-bot/chrome-transport.test.ts b/src/meeting-bot/chrome-transport.test.ts index ba871f6c2c3c..b2523a9ffe69 100644 --- a/src/meeting-bot/chrome-transport.test.ts +++ b/src/meeting-bot/chrome-transport.test.ts @@ -185,7 +185,7 @@ describe.each(cases)("$name Chrome transport parity", (testCase) => { await expect( transport.launchInChrome({ config, - fullConfig: {} as OpenClawConfig, + fullConfig: { transcripts: { enabled: false } } as OpenClawConfig, logger, meetingSessionId: "session-1", mode: "agent", @@ -196,6 +196,11 @@ describe.each(cases)("$name Chrome transport parity", (testCase) => { ).rejects.toThrow("realtime startup failed"); expect(dispose).toHaveBeenCalledOnce(); + expect(browserMocks.open).toHaveBeenCalledWith( + expect.objectContaining({ + session: expect.objectContaining({ captureCaptions: false, mode: "agent" }), + }), + ); expect(browserMocks.leave).toHaveBeenCalledTimes(testCase.expectedLeaves); }); }); diff --git a/src/meeting-bot/chrome-transport.ts b/src/meeting-bot/chrome-transport.ts index 4470e6f50ea1..217d52cdda92 100644 --- a/src/meeting-bot/chrome-transport.ts +++ b/src/meeting-bot/chrome-transport.ts @@ -1,6 +1,7 @@ import { addTimerTimeoutGraceMs } from "@openclaw/normalization-core/number-coercion"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginRuntime, RuntimeLogger } from "../plugins/runtime/types.js"; +import { resolveTranscriptsConfig } from "../transcripts/config.js"; import type { createMeetingRealtimeEngineBindings } from "./agent-consult.js"; import { openMeetingWithBrowser, recoverMeetingBrowserTab } from "./browser-controller.js"; import { callMeetingBrowserProxyOnNode, resolveMeetingBrowserNode } from "./browser-node.js"; @@ -102,18 +103,23 @@ export function createMeetingChromeTransport< async function openOrRecoverMeeting(params: { callBrowser: MeetingBrowserRequestCaller; config: Config; + fullConfig: OpenClawConfig; meetingSessionId: string; mode: Mode; trackedTargetId?: string; url: string; locationLabel: string; }) { + const captureCaptions = + params.mode === "transcribe" || + resolveTranscriptsConfig(params.fullConfig.transcripts).enabled; if (params.config.chrome.launch) { return await openMeetingWithBrowser({ adapter: options.platform, callBrowser: params.callBrowser, config: params.config.chrome, session: { + captureCaptions, meetingSessionId: params.meetingSessionId, mode: params.mode, url: params.url, @@ -125,6 +131,7 @@ export function createMeetingChromeTransport< allowSessionAdoption: true, autoJoin: params.config.chrome.autoJoin, callBrowser: params.callBrowser, + captureCaptions, config: params.config.chrome, locationLabel: params.locationLabel, meetingSessionId: params.meetingSessionId, @@ -283,6 +290,7 @@ export function createMeetingChromeTransport< const result = await openOrRecoverMeeting({ callBrowser, config: params.config, + fullConfig: params.fullConfig, locationLabel: "in local Chrome", meetingSessionId: params.meetingSessionId, mode: params.mode, @@ -398,6 +406,7 @@ export function createMeetingChromeTransport< const browser = await openOrRecoverMeeting({ callBrowser, config: params.config, + fullConfig: params.fullConfig, locationLabel: "on the selected Chrome node", meetingSessionId: params.meetingSessionId, mode: params.mode, @@ -528,6 +537,7 @@ export function createMeetingChromeTransport< async function recoverCurrentTab(params: { runtime: PluginRuntime; config: Config; + fullConfig?: OpenClawConfig; meetingSessionId?: string; mode: Mode; nodeId?: string; @@ -562,6 +572,9 @@ export function createMeetingChromeTransport< timeoutMs: request.timeoutMs, }) : await resolveLocalMeetingBrowserRequest(params.runtime), + captureCaptions: + params.mode === "transcribe" || + resolveTranscriptsConfig(params.fullConfig?.transcripts).enabled, config: params.config.chrome, locationLabel: nodeId ? "on the selected Chrome node" : "in local Chrome", meetingSessionId: params.meetingSessionId, diff --git a/src/meeting-bot/platform-adapter-contract.ts b/src/meeting-bot/platform-adapter-contract.ts index 8f23524983a5..8d446f0f22ff 100644 --- a/src/meeting-bot/platform-adapter-contract.ts +++ b/src/meeting-bot/platform-adapter-contract.ts @@ -29,6 +29,7 @@ export type MeetingBrowserRequestParams = { export type MeetingBrowserRequestCaller = (params: MeetingBrowserRequestParams) => Promise; export type MeetingBrowserJoinSession = { + captureCaptions?: boolean; meetingSessionId: string; mode: Mode; url: string; diff --git a/src/meeting-bot/plugin-entry.ts b/src/meeting-bot/plugin-entry.ts index 146eb53e9d52..f02bdb2a7c21 100644 --- a/src/meeting-bot/plugin-entry.ts +++ b/src/meeting-bot/plugin-entry.ts @@ -10,6 +10,10 @@ import type { OpenClawPluginApi } from "../plugins/plugin-api.types.js"; import type { OpenClawPluginConfigSchema } from "../plugins/plugin-config-schema.types.js"; import type { OpenClawPluginNodeInvokePolicy } from "../plugins/plugin-registration.types.js"; import { parseAgentSessionKey } from "../sessions/session-key-utils.js"; +import { + createMeetingTranscriptSourceProvider, + type MeetingTranscriptSourceRuntime, +} from "./transcripts-bridge.js"; type MeetingToolAction = "join" | "leave" | "status" | "transcript" | "speak"; type MeetingMode = "agent" | "bidi" | "transcribe"; @@ -30,18 +34,19 @@ type MeetingPluginConfig = { chromeNode: { node?: string }; }; -type MeetingPluginRuntime = { - join(request: Request): Promise; - leave(sessionId: string): Promise; - ownsSession(agentId: string, sessionId: string): boolean; - setupStatus(params: { mode?: MeetingMode; transport?: MeetingTransport }): Promise; - speak(sessionId: string, message?: string): Promise; - status(sessionId?: string): Promise; - statusForAgent(agentId: string, sessionId?: string): Promise; - testListen(request: Request): Promise; - testSpeech(request: Request): Promise; - transcript(sessionId: string, options: { sinceIndex?: number }): Promise; -}; +type MeetingPluginRuntime = + Partial & { + join(request: Request): Promise; + leave(sessionId: string): Promise; + ownsSession(agentId: string, sessionId: string): boolean; + setupStatus(params: { mode?: MeetingMode; transport?: MeetingTransport }): Promise; + speak(sessionId: string, message?: string): Promise; + status(sessionId?: string): Promise; + statusForAgent(agentId: string, sessionId?: string): Promise; + testListen(request: Request): Promise; + testSpeech(request: Request): Promise; + transcript(sessionId: string, options: { sinceIndex?: number }): Promise; + }; type MeetingPluginEntryOptions< Config extends MeetingPluginConfig, @@ -75,6 +80,11 @@ type MeetingPluginEntryOptions< toolLabel: string; toolName: string; toolParameters: TObject; + transcriptSource?: { + id: string; + aliases?: readonly string[]; + name: string; + }; unknownActionMessage: string; }; @@ -217,6 +227,20 @@ export function createMeetingPluginEntryOptions< runtime ??= options.createRuntime({ api, config }); return runtime; }; + if (options.transcriptSource) { + api.registerTranscriptSourceProvider( + createMeetingTranscriptSourceProvider({ + ...options.transcriptSource, + runtime: async () => { + const resolved = await ensureRuntime(); + if (!resolved.startTranscriptSource || !resolved.stopTranscriptSource) { + throw new Error(`${options.name} transcript source runtime is unavailable`); + } + return resolved as MeetingTranscriptSourceRuntime; + }, + }), + ); + } const sendError = ( respond: GatewayRequestHandlerOptions["respond"], error: unknown, diff --git a/src/meeting-bot/session-durable-transcripts.ts b/src/meeting-bot/session-durable-transcripts.ts new file mode 100644 index 000000000000..1a350ced0430 --- /dev/null +++ b/src/meeting-bot/session-durable-transcripts.ts @@ -0,0 +1,184 @@ +import type { RuntimeLogger } from "../plugins/runtime/types.js"; +import type { + TranscriptSourceLocator, + TranscriptStartRequest, + TranscriptsStartResult, + TranscriptStopRequest, + TranscriptsStopResult, +} from "../transcripts/provider-types.js"; +import { MeetingSessionTranscriptStore } from "./session-transcript-store.js"; +import type { MeetingSessionRecord, MeetingTranscriptLine } from "./session-types.js"; +import type { + MeetingDurableTranscriptBridge, + MeetingDurableTranscriptsOptions, +} from "./transcripts-bridge.js"; + +const STOP_RETRY_DELAY_MS = 1_000; +const STOP_RETRY_MAX_DELAY_MS = 60_000; + +export class MeetingSessionDurableTranscripts { + #bridge?: Promise | undefined>; + readonly #stopRetries = new Map(); + + constructor( + private readonly options: { + config: MeetingDurableTranscriptsOptions | undefined; + formatError(error: unknown): string; + isBrowserSession(session: TSession): boolean; + isTranscribeSession(session: TSession): boolean; + listSessions(): TSession[]; + logger: RuntimeLogger; + logScope: string; + sameMeetingUrl(left: string | undefined, right: string | undefined): boolean; + transcriptStore: MeetingSessionTranscriptStore; + }, + ) {} + + async ingest(session: TSession, lines: MeetingTranscriptLine[]): Promise { + await (await this.#getBridge())?.ingest(session, lines); + } + + async start(session: TSession): Promise { + if (!this.options.isBrowserSession(session)) { + return; + } + const bridge = await this.#getBridge(); + await bridge?.start( + session, + async () => await this.options.transcriptStore.captureNotes(session), + ); + } + + async stop(session: TSession, options: { allowFallback: boolean }): Promise { + const finalCapture = async () => + await this.options.transcriptStore.captureNotes(session, { finalize: true }); + const bridge = await this.#getBridge(); + if (bridge?.enabled) { + try { + if (await bridge.stop(session, finalCapture)) { + this.#stopRetries.delete(session.id); + return true; + } + } catch (error) { + this.options.logger.warn( + `${this.options.logScope} durable transcript finalization queued for retry: ${this.options.formatError(error)}`, + ); + this.#scheduleStopRetry(session, bridge); + return false; + } + } + if (options.allowFallback && this.options.isTranscribeSession(session)) { + await finalCapture().catch((error: unknown) => { + this.options.logger.debug?.( + `${this.options.logScope} final transcript snapshot ignored: ${this.options.formatError(error)}`, + ); + }); + } + return true; + } + + async startSource(request: TranscriptStartRequest): Promise { + const bridge = await this.#getBridge(); + if (!bridge?.enabled) { + return { ok: false, error: "meeting transcripts are disabled" }; + } + const session = this.#findSourceSession(request.session.source); + if (!session) { + return { ok: false, error: "No active meeting session matches the transcript source." }; + } + if (request.session.source.agentId !== session.agentId) { + return { ok: false, error: "meeting transcript source belongs to another agent" }; + } + return await bridge.attach(session, request); + } + + async stopSource(request: TranscriptStopRequest): Promise { + const bridge = await this.#getBridge(); + return bridge + ? await bridge.detach(request) + : { ok: true, sessionId: request.sessionId, stoppedAt: new Date().toISOString() }; + } + + #findSourceSession(source: TranscriptSourceLocator): TSession | undefined { + const agentId = source.agentId?.trim(); + const meetingUrl = source.meetingUrl?.trim(); + const sessionId = source.channelId?.trim(); + if (!agentId || (!meetingUrl && !sessionId)) { + return undefined; + } + return this.options + .listSessions() + .filter((session) => session.state === "active") + .toSorted((left, right) => left.createdAt.localeCompare(right.createdAt)) + .find( + (session) => + session.agentId === agentId && + (!sessionId || session.id === sessionId) && + (!meetingUrl || this.options.sameMeetingUrl(session.url, meetingUrl)), + ); + } + + async #getBridge(): Promise | undefined> { + if (!this.options.config) { + return undefined; + } + this.#bridge ??= import("./transcripts-bridge.runtime.js") + .then(({ createMeetingDurableTranscriptBridge }) => + createMeetingDurableTranscriptBridge({ + logger: this.options.logger, + options: this.options.config!, + }), + ) + .catch((error: unknown) => { + this.options.logger.warn( + `${this.options.logScope} durable transcripts unavailable: ${this.options.formatError(error)}`, + ); + return undefined; + }); + return await this.#bridge; + } + + #scheduleStopRetry(session: TSession, bridge: MeetingDurableTranscriptBridge): void { + if (this.#stopRetries.has(session.id)) { + return; + } + const token = Symbol(session.id); + this.#stopRetries.set(session.id, { attempt: 1, token }); + const retry = () => { + const attempt = this.#stopRetries.get(session.id)?.attempt ?? 1; + const delayMs = Math.min( + STOP_RETRY_DELAY_MS * 2 ** Math.min(attempt - 1, 16), + STOP_RETRY_MAX_DELAY_MS, + ); + const timer = setTimeout(() => void run(), delayMs); + timer.unref?.(); + }; + const run = async () => { + const state = this.#stopRetries.get(session.id); + if (!state || state.token !== token) { + return; + } + try { + const handled = await bridge.stop( + session, + async () => await this.options.transcriptStore.flushPending(session), + ); + if (!handled) { + throw new Error("durable transcript capture is no longer active"); + } + if (this.#stopRetries.get(session.id)?.token !== token) { + return; + } + this.options.transcriptStore.retire(session.id); + this.#stopRetries.delete(session.id); + } catch { + if (this.#stopRetries.get(session.id)?.token !== token) { + return; + } + this.#stopRetries.set(session.id, { attempt: state.attempt + 1, token }); + retry(); + } + }; + retry(); + } +} diff --git a/src/meeting-bot/session-runtime.test.ts b/src/meeting-bot/session-runtime.test.ts index 9a3d16508b1d..611266ff3e3e 100644 --- a/src/meeting-bot/session-runtime.test.ts +++ b/src/meeting-bot/session-runtime.test.ts @@ -1,4 +1,8 @@ -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TranscriptsStore } from "../transcripts/store.js"; import { createMeetingSession } from "./session-factory.js"; import { MeetingSessionRuntime, type MeetingSessionRuntimeJoinContext } from "./session-runtime.js"; import type { @@ -93,7 +97,17 @@ describe("createMeetingSession", () => { }); function createTestRuntime(params: { + captureTranscript?: (options?: { finalize?: boolean }) => Promise< + | { + droppedLines: number; + epoch?: string; + lines: Array<{ at?: string; speaker?: string; text: string }>; + } + | undefined + >; + durableTranscripts?: { stateDir: string }; talkBack?: boolean; + transcribe?: boolean; refreshReusableSession?( session: TestSession, request: TestRequest, @@ -166,7 +180,7 @@ function createTestRuntime(params: { resolveSpeechInstructions: () => undefined, isBrowserTransport: () => true, isTalkBackMode: () => params.talkBack === true, - isTranscribeMode: () => false, + isTranscribeMode: () => params.transcribe === true, sameMeetingUrl: (left, right) => left === right, normalizeMeetingUrlForReuse: (url) => url, getBrowser: (session) => @@ -195,12 +209,228 @@ function createTestRuntime(params: { refreshReusableSession: async (session, request, resolved) => await params.refreshReusableSession?.(session, request, resolved), ensureRealtimeBridge: async () => undefined, - captureTranscript: async () => undefined, + captureTranscript: async (_session, options) => await params.captureTranscript?.(options), speakViaTransport: async () => undefined, + ...(params.durableTranscripts + ? { + durableTranscripts: { + providerId: "test-meeting", + providerName: "Test Meeting", + stateDir: params.durableTranscripts.stateDir, + }, + } + : {}), }); return { createdSessions, runtime }; } +const tempDirs: string[] = []; + +afterEach(async () => { + vi.useRealTimers(); + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true }))); +}); + +describe("MeetingSessionRuntime durable transcripts", () => { + it("persists joined agent-mode captions and writes summary rows on leave", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-meeting-notes-")); + tempDirs.push(stateDir); + const snapshots = [ + { + droppedLines: 0, + epoch: "page-1", + lines: [ + { + at: "2026-07-23T12:00:00.000Z", + speaker: "Avery", + text: "We decided to ship the durable notes bridge.", + }, + ], + }, + { + droppedLines: 0, + epoch: "page-1", + lines: [ + { + at: "2026-07-23T12:00:00.000Z", + speaker: "Avery", + text: "We decided to ship the durable notes bridge.", + }, + { + at: "2026-07-23T12:00:05.000Z", + speaker: "Blake", + text: "Action: follow up with the docs.", + }, + ], + }, + ]; + const { runtime } = createTestRuntime({ + captureTranscript: async () => snapshots.shift(), + durableTranscripts: { stateDir }, + releaseBrowserTab: async () => true, + joinTransport: async ({ session }) => { + session.browser = { + launched: true, + tab: { targetId: "notes-tab", openedByPlugin: true }, + }; + return {}; + }, + }); + + const { session } = await runtime.join({ + url: "https://meeting.example/notes?context=opaque-value", + agentId: "notes-agent", + }); + await expect( + runtime.startTranscriptSource({ + session: { + sessionId: "external-mismatch", + source: { + providerId: "test-meeting", + agentId: "notes-agent", + channelId: "another-session", + meetingUrl: session.url, + }, + startedAt: session.createdAt, + }, + onUtterance: vi.fn(), + }), + ).resolves.toMatchObject({ ok: false }); + await expect( + runtime.startTranscriptSource({ + session: { + sessionId: "external-agent-mismatch", + source: { + providerId: "test-meeting", + agentId: "another-agent", + meetingUrl: session.url, + }, + startedAt: session.createdAt, + }, + onUtterance: vi.fn(), + }), + ).resolves.toMatchObject({ + ok: false, + error: "No active meeting session matches the transcript source.", + }); + await runtime.leave(session.id); + + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const storedSession = await store.readSession(session.id); + expect(storedSession).toMatchObject({ + sessionId: session.id, + source: { providerId: "test-meeting", meetingUrl: "https://meeting.example/notes" }, + metadata: { agentId: "notes-agent", meetingSessionId: session.id, mode: "agent" }, + stoppedAt: expect.any(String), + }); + expect(await store.readUtterancesForSession(storedSession!)).toMatchObject([ + { speaker: { label: "Avery" }, text: "We decided to ship the durable notes bridge." }, + { speaker: { label: "Blake" }, text: "Action: follow up with the docs." }, + ]); + expect(await store.readSummary(storedSession!)).toMatchObject({ + summary: { + actionItems: [ + "Avery: We decided to ship the durable notes bridge.", + "Blake: Action: follow up with the docs.", + ], + decisions: ["Avery: We decided to ship the durable notes bridge."], + utteranceCount: 2, + }, + }); + }); + + it("keeps transcribe finalization when durable session startup fails", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-meeting-notes-")); + tempDirs.push(tempDir); + const blockedStateDir = path.join(tempDir, "not-a-directory"); + await fs.writeFile(blockedStateDir, "blocked", "utf8"); + const captureTranscript = vi.fn(async () => ({ droppedLines: 0, lines: [] })); + const { runtime } = createTestRuntime({ + captureTranscript, + durableTranscripts: { stateDir: blockedStateDir }, + transcribe: true, + releaseBrowserTab: async () => true, + joinTransport: async ({ session }) => { + session.browser = { + launched: true, + tab: { targetId: "notes-tab", openedByPlugin: true }, + }; + return {}; + }, + }); + + const { session } = await runtime.join({ + url: "https://meeting.example/notes", + agentId: "notes-agent", + }); + await runtime.leave(session.id); + + expect(captureTranscript).toHaveBeenCalledTimes(2); + expect(captureTranscript).toHaveBeenCalledWith({ finalize: true }); + }); + + it("does not let subscriber delivery failure block meeting leave", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-meeting-notes-")); + tempDirs.push(stateDir); + const empty = { droppedLines: 0, epoch: "page-1", lines: [] }; + const final = { + droppedLines: 0, + epoch: "page-1", + lines: [{ speaker: "Avery", text: "Final decision" }], + }; + const snapshots = [empty, final]; + const releaseBrowserTab = vi.fn(async () => true); + const { runtime } = createTestRuntime({ + captureTranscript: async () => snapshots.shift(), + durableTranscripts: { stateDir }, + transcribe: true, + releaseBrowserTab, + joinTransport: async ({ session }) => { + session.browser = { + launched: true, + tab: { targetId: "notes-tab", openedByPlugin: true }, + }; + return {}; + }, + }); + const { session } = await runtime.join({ + url: "https://meeting.example/notes", + agentId: "notes-agent", + }); + const onUtterance = vi.fn(async () => { + throw new Error("subscriber unavailable"); + }); + await runtime.startTranscriptSource({ + session: { + sessionId: "external-final", + source: { + providerId: "test-meeting", + agentId: "notes-agent", + meetingUrl: session.url, + }, + startedAt: session.createdAt, + }, + onUtterance, + }); + + await expect(runtime.leave(session.id)).resolves.toMatchObject({ found: true }); + expect(session.state).toBe("ended"); + expect(releaseBrowserTab).toHaveBeenCalledOnce(); + + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const stored = await store.readSession(session.id); + expect(await store.readUtterancesForSession(stored!)).toHaveLength(1); + expect(await store.readSummary(stored!)).toMatchObject({ + summary: { utteranceCount: 1 }, + }); + expect(onUtterance).toHaveBeenCalledOnce(); + }); +}); + describe("MeetingSessionRuntime failed joins", () => { it("cleans an externally ended reusable session before replacing it", async () => { const stop = vi.fn(async () => {}); diff --git a/src/meeting-bot/session-runtime.ts b/src/meeting-bot/session-runtime.ts index 52e010310054..845b9c4e4bab 100644 --- a/src/meeting-bot/session-runtime.ts +++ b/src/meeting-bot/session-runtime.ts @@ -1,11 +1,19 @@ import type { RuntimeLogger } from "../plugins/runtime/types.js"; +import type { + TranscriptStartRequest, + TranscriptsStartResult, + TranscriptStopRequest, + TranscriptsStopResult, +} from "../transcripts/provider-types.js"; import { MeetingSessionCleanupTracker } from "./session-cleanup-tracker.js"; +import { MeetingSessionDurableTranscripts } from "./session-durable-transcripts.js"; import { MeetingSessionJoinLock } from "./session-join-lock.js"; import type { MeetingBrowserSessionView, MeetingSessionRuntimeHandles, MeetingSessionRuntimeJoinContext, } from "./session-runtime-types.js"; +import { evaluateMeetingSpeechReadiness } from "./session-speech-readiness.js"; import { MeetingSessionTranscriptStore } from "./session-transcript-store.js"; import type { MeetingBrowserHealth, @@ -14,6 +22,7 @@ import type { MeetingSessionRecord, MeetingTranscriptSnapshot, } from "./session-types.js"; +import type { MeetingDurableTranscriptsOptions } from "./transcripts-bridge.js"; export type { MeetingBrowserSessionView, MeetingSessionRuntimeHandles, @@ -100,6 +109,7 @@ export type MeetingSessionRuntimeOptions< instructions?: string, ): Promise<{ handled: boolean; spoken: boolean } | undefined>; defaultSpeechInstructions?: string; + durableTranscripts?: MeetingDurableTranscriptsOptions; }; export type MeetingSessionLeaveResult = { @@ -128,6 +138,7 @@ export class MeetingSessionRuntime< readonly #sessionStops = new Map Promise>(); readonly #sessionSpeakers = new Map void>(); readonly #sessionHealth = new Map Partial>(); + readonly #durableTranscripts: MeetingSessionDurableTranscripts; readonly #transcriptStore: MeetingSessionTranscriptStore; constructor( @@ -149,6 +160,18 @@ export class MeetingSessionRuntime< hasBrowserTab: (session) => Boolean(this.options.getBrowser(session)?.tab), capture: async (session, captureOptions) => await this.options.captureTranscript(session, captureOptions), + onLines: async (session, lines) => await this.#durableTranscripts.ingest(session, lines), + }); + this.#durableTranscripts = new MeetingSessionDurableTranscripts({ + config: options.durableTranscripts, + formatError: (error) => options.formatError(error), + isBrowserSession: (session) => options.isBrowserTransport(session.transport), + isTranscribeSession: (session) => options.isTranscribeMode(session.mode), + listSessions: () => [...this.#sessions.values()], + logger: options.logger, + logScope: options.logScope, + sameMeetingUrl: (left, right) => options.sameMeetingUrl(left, right), + transcriptStore: this.#transcriptStore, }); } @@ -185,6 +208,14 @@ export class MeetingSessionRuntime< return await this.#transcriptStore.read(sessionId, options); } + async startTranscriptSource(request: TranscriptStartRequest): Promise { + return await this.#durableTranscripts.startSource(request); + } + + async stopTranscriptSource(request: TranscriptStopRequest): Promise { + return await this.#durableTranscripts.stopSource(request); + } + isReusableSession(session: TSession, resolved: MeetingResolvedJoin): boolean { return ( session.state === "active" && @@ -213,6 +244,8 @@ export class MeetingSessionRuntime< if (!session) { return { found: false }; } + // The meeting lock fences joins and leaves before terminal transcript work; + // #sessionLeaves then coalesces retries owned by the same session. return await this.#meetingLock.run( this.#meetingKey(session.transport, session.url), async () => await this.#leaveUnlocked(sessionId, options), @@ -427,6 +460,7 @@ export class MeetingSessionRuntime< } const speechInstructions = this.options.resolveSpeechInstructions(request); if (reusable) { + await this.#durableTranscripts.start(reusable); await this.refreshBrowserHealth(reusable); this.#noteSession(reusable, this.options.messages.reusedSessionNote); reusable.updatedAt = nowIso(); @@ -471,6 +505,7 @@ export class MeetingSessionRuntime< } this.#sessions.set(session.id, session); + await this.#durableTranscripts.start(session); const spoken = delegatedSpoken ? true : this.options.isTalkBackMode(resolved.mode) && speechInstructions @@ -514,20 +549,22 @@ export class MeetingSessionRuntime< options?: { keepBrowserTab?: boolean }, ): Promise> { const firstAttempt = this.#sessionCleanup.begin(session.id, session.browserLeft); - if (firstAttempt && this.options.isTranscribeMode(session.mode)) { + const transcribe = this.options.isTranscribeMode(session.mode); + let transcriptStopped = false; + if (transcribe) { + // Fence new live reads before final capture; the store's capture chain drains + // reads already admitted before this terminal boundary. this.#transcriptStore.startFinalizing(session.id); - await this.#transcriptStore.capture(session, { finalize: true }).catch((error: unknown) => { - this.options.logger.debug?.( - `${this.options.logScope} final transcript snapshot ignored: ${this.options.formatError(error)}`, - ); - }); } - session.state = "ended"; - session.updatedAt = nowIso(); - this.#sessionSpeakers.delete(session.id); - this.#sessionHealth.delete(session.id); - const stop = this.#sessionStops.get(session.id); try { + transcriptStopped = await this.#durableTranscripts.stop(session, { + allowFallback: firstAttempt, + }); + session.state = "ended"; + session.updatedAt = nowIso(); + this.#sessionSpeakers.delete(session.id); + this.#sessionHealth.delete(session.id); + const stop = this.#sessionStops.get(session.id); const cleanup = await this.#sessionCleanup.cleanup({ sessionId: session.id, stop, @@ -561,8 +598,10 @@ export class MeetingSessionRuntime< ...(cleanup.browserLeft === undefined ? {} : { browserLeft: cleanup.browserLeft }), }; } finally { - if (firstAttempt) { + if (transcriptStopped) { this.#transcriptStore.retire(session.id); + } + if (transcribe) { this.#transcriptStore.finishFinalizing(session.id); } } @@ -695,55 +734,12 @@ export class MeetingSessionRuntime< reason?: TSpeechBlockedReason; message?: string; } { - const speech = this.options.messages.speech; - const browser = this.options.getBrowser(session); - if (!this.options.isTalkBackMode(session.mode) || !browser) { - return { ready: true }; - } - if (!this.#isManagedBrowserSession(session)) { - return browser.hasAudioBridge - ? { ready: true } - : { - ready: false, - reason: speech.audioBridgeUnavailableReason, - message: speech.audioBridgeUnavailable, - }; - } - const health = browser.health; - if (health?.manualActionRequired) { - return { - ready: false, - reason: (health.manualActionReason ?? - speech.browserUnverifiedReason) as TSpeechBlockedReason, - message: health.manualActionMessage ?? speech.manualActionFallback, - }; - } - if (health?.inCall === true) { - if (health.micMuted !== false) { - const muted = health.micMuted === true; - // Unknown is transiently blocked: omitted mic controls cannot prove talk-back readiness. - return { - ready: false, - reason: muted ? speech.microphoneMutedReason : speech.browserUnverifiedReason, - message: muted ? speech.microphoneMuted : speech.browserUnverified, - }; - } - return browser.hasAudioBridge - ? { ready: true } - : { - ready: false, - reason: speech.audioBridgeUnavailableReason, - message: speech.audioBridgeUnavailable, - }; - } - if (health?.inCall === false) { - return { ready: false, reason: speech.notInCallReason, message: speech.notInCall }; - } - return { - ready: false, - reason: speech.browserUnverifiedReason, - message: speech.browserUnverified, - }; + return evaluateMeetingSpeechReadiness({ + browser: this.options.getBrowser(session), + managedBrowser: this.#isManagedBrowserSession(session), + speech: this.options.messages.speech, + talkBack: this.options.isTalkBackMode(session.mode), + }); } #noteSession(session: TSession, note: string): void { diff --git a/src/meeting-bot/session-speech-readiness.ts b/src/meeting-bot/session-speech-readiness.ts new file mode 100644 index 000000000000..f2b389e4c620 --- /dev/null +++ b/src/meeting-bot/session-speech-readiness.ts @@ -0,0 +1,73 @@ +import type { MeetingBrowserHealth } from "./session-types.js"; + +type SpeechMessages = { + audioBridgeUnavailable: string; + audioBridgeUnavailableReason: TReason; + browserUnverified: string; + browserUnverifiedReason: TReason; + manualActionFallback: string; + microphoneMuted: string; + microphoneMutedReason: TReason; + notInCall: string; + notInCallReason: TReason; +}; + +export function evaluateMeetingSpeechReadiness(params: { + browser: + | { + hasAudioBridge: boolean; + health?: MeetingBrowserHealth; + } + | undefined; + managedBrowser: boolean; + speech: SpeechMessages; + talkBack: boolean; +}): { ready: boolean; reason?: TReason; message?: string } { + const { browser, speech } = params; + if (!params.talkBack || !browser) { + return { ready: true }; + } + if (!params.managedBrowser) { + return browser.hasAudioBridge + ? { ready: true } + : { + ready: false, + reason: speech.audioBridgeUnavailableReason, + message: speech.audioBridgeUnavailable, + }; + } + const health = browser.health; + if (health?.manualActionRequired) { + return { + ready: false, + reason: (health.manualActionReason ?? speech.browserUnverifiedReason) as TReason, + message: health.manualActionMessage ?? speech.manualActionFallback, + }; + } + if (health?.inCall === true) { + if (health.micMuted !== false) { + const muted = health.micMuted === true; + // Unknown is transiently blocked: omitted mic controls cannot prove talk-back readiness. + return { + ready: false, + reason: muted ? speech.microphoneMutedReason : speech.browserUnverifiedReason, + message: muted ? speech.microphoneMuted : speech.browserUnverified, + }; + } + return browser.hasAudioBridge + ? { ready: true } + : { + ready: false, + reason: speech.audioBridgeUnavailableReason, + message: speech.audioBridgeUnavailable, + }; + } + if (health?.inCall === false) { + return { ready: false, reason: speech.notInCallReason, message: speech.notInCall }; + } + return { + ready: false, + reason: speech.browserUnverifiedReason, + message: speech.browserUnverified, + }; +} diff --git a/src/meeting-bot/session-transcript-store.test.ts b/src/meeting-bot/session-transcript-store.test.ts index b86f0b9bb664..2f4bf1c95e90 100644 --- a/src/meeting-bot/session-transcript-store.test.ts +++ b/src/meeting-bot/session-transcript-store.test.ts @@ -21,6 +21,7 @@ function createSession(): MeetingSessionRecord<"chrome", "transcribe"> { function createStore( session: MeetingSessionRecord<"chrome", "transcribe">, snapshots: MeetingTranscriptSnapshot[], + onLines?: (lines: MeetingTranscriptSnapshot["lines"]) => Promise, ) { return new MeetingSessionTranscriptStore({ getSession: (sessionId) => (sessionId === session.id ? session : undefined), @@ -28,6 +29,7 @@ function createStore( isTranscribeSession: () => true, hasBrowserTab: () => true, capture: async () => snapshots.shift(), + ...(onLines ? { onLines: async (_session, lines) => await onLines(lines) } : {}), }); } @@ -106,4 +108,265 @@ describe("MeetingSessionTranscriptStore", () => { expect(afterAppend).toMatchObject({ startIndex: 7, nextIndex: 8, droppedLines: 5 }); expect(afterAppend.lines?.map((line) => line.text)).toEqual(["new-5"]); }); + + it("retries durable delivery when a snapshot consumer fails", async () => { + const session = createSession(); + const snapshot = { + droppedLines: 0, + epoch: "page-1", + lines: [{ text: "retry me" }], + }; + let attempts = 0; + const delivered: string[][] = []; + const store = createStore(session, [snapshot, snapshot], async (lines) => { + attempts += 1; + if (attempts === 1) { + throw new Error("temporary store failure"); + } + delivered.push(lines.map((line) => line.text)); + }); + + await expect(store.read(session.id)).resolves.toMatchObject({ found: true }); + await expect(store.read(session.id)).resolves.toMatchObject({ found: true }); + + expect(delivered).toEqual([["retry me"]]); + }); + + it("acknowledges a durable snapshot one line at a time", async () => { + const session = createSession(); + const snapshot = { + droppedLines: 0, + epoch: "page-1", + lines: [{ text: "first" }, { text: "second" }], + }; + let failedSecond = false; + const delivered: string[] = []; + const store = createStore(session, [snapshot, snapshot], async (lines) => { + const text = lines[0]?.text; + if (text === "second" && !failedSecond) { + failedSecond = true; + throw new Error("temporary second-line failure"); + } + if (text) { + delivered.push(text); + } + }); + + await expect(store.read(session.id)).resolves.toMatchObject({ found: true }); + await expect(store.read(session.id)).resolves.toMatchObject({ found: true }); + + expect(delivered).toEqual(["first", "second"]); + }); + + it("treats a backward cursor without an epoch as a reset stream", async () => { + const session = createSession(); + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "old-0" }, { text: "old-1" }] }, + { droppedLines: 0, lines: [{ text: "new-0" }] }, + ], + async (lines) => { + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await store.read(session.id); + await store.read(session.id); + + expect(delivered).toEqual(["old-0", "old-1", "new-0"]); + }); + + it("detects a larger no-epoch reset after an empty snapshot", async () => { + const session = createSession(); + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "old-0" }] }, + { droppedLines: 0, lines: [] }, + { droppedLines: 0, lines: [{ text: "new-0" }, { text: "new-1" }] }, + ], + async (lines) => { + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await store.read(session.id); + await store.read(session.id); + await store.read(session.id); + + expect(delivered).toEqual(["old-0", "new-0", "new-1"]); + }); + + it("strips the maximal suffix-prefix overlap from a no-epoch reset", async () => { + const session = createSession(); + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "A" }, { text: "B" }] }, + { droppedLines: 0, lines: [{ text: "B" }, { text: "C" }] }, + ], + async (lines) => { + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await store.read(session.id); + await store.read(session.id); + + expect(delivered).toEqual(["A", "B", "C"]); + }); + + it("keeps repeated text after a non-overlapping dropped prefix", async () => { + const session = createSession(); + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "A" }, { text: "B" }] }, + { droppedLines: 2, lines: [{ text: "B" }, { text: "C" }] }, + ], + async (lines) => { + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await store.read(session.id); + await store.read(session.id); + + expect(delivered).toEqual(["A", "B", "B", "C"]); + }); + + it("commits an empty no-epoch reset before repeated rows return", async () => { + const session = createSession(); + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "A" }] }, + { droppedLines: 0, lines: [] }, + { droppedLines: 0, lines: [{ text: "A" }, { text: "B" }] }, + ], + async (lines) => { + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await store.read(session.id); + await store.read(session.id); + await store.read(session.id); + + expect(delivered).toEqual(["A", "A", "B"]); + }); + + it("queues newer final rows behind an unavailable pending batch", async () => { + const session = createSession(); + let unavailable = true; + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, epoch: "page-1", lines: [{ text: "A" }] }, + { droppedLines: 0, epoch: "page-1", lines: [{ text: "A" }, { text: "B" }] }, + ], + async (lines) => { + if (unavailable) { + throw new Error("store unavailable"); + } + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + await expect(store.captureNotes(session, { finalize: true })).rejects.toThrow( + "store unavailable", + ); + unavailable = false; + await store.flushPending(session); + + expect(delivered).toEqual(["A", "B"]); + }); + + it("keeps polling snapshots while durable delivery is pending", async () => { + const session = createSession(); + let unavailable = true; + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, epoch: "page-1", lines: [{ text: "A" }] }, + { droppedLines: 0, epoch: "page-1", lines: [{ text: "A" }, { text: "B" }] }, + ], + async (lines) => { + if (unavailable) { + throw new Error("store unavailable"); + } + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + unavailable = false; + await store.flushPending(session); + + expect(delivered).toEqual(["A", "B"]); + }); + + it("preserves pending delivery when the final browser snapshot also fails", async () => { + const session = createSession(); + let captureCount = 0; + const store = new MeetingSessionTranscriptStore({ + getSession: () => session, + isBrowserSession: () => true, + isTranscribeSession: () => true, + hasBrowserTab: () => true, + capture: async () => { + captureCount += 1; + if (captureCount === 1) { + return { droppedLines: 0, lines: [{ text: "pending" }] }; + } + throw new Error("browser snapshot unavailable"); + }, + onLines: async () => { + throw new Error("store unavailable"); + }, + }); + + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + await expect(store.captureNotes(session, { finalize: true })).rejects.toMatchObject({ + finalCaptureError: "browser snapshot unavailable", + }); + }); + + it("queues an empty reset marker while delivery is pending", async () => { + const session = createSession(); + let unavailable = true; + const delivered: string[] = []; + const store = createStore( + session, + [ + { droppedLines: 0, lines: [{ text: "A" }] }, + { droppedLines: 0, lines: [] }, + { droppedLines: 0, lines: [{ text: "A" }, { text: "B" }] }, + ], + async (lines) => { + if (unavailable) { + throw new Error("store unavailable"); + } + delivered.push(...lines.map((line) => line.text)); + }, + ); + + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + await expect(store.captureNotes(session)).rejects.toThrow("store unavailable"); + unavailable = false; + await store.flushPending(session); + + expect(delivered).toEqual(["A", "A", "B"]); + }); }); diff --git a/src/meeting-bot/session-transcript-store.ts b/src/meeting-bot/session-transcript-store.ts index 5a86e6bfab0a..2a9eb35e81a4 100644 --- a/src/meeting-bot/session-transcript-store.ts +++ b/src/meeting-bot/session-transcript-store.ts @@ -9,14 +9,71 @@ type RetainedTranscriptSnapshot = MeetingTranscriptSnapshot & { pageNextIndex: number; }; +type TranscriptStreamCursor = { + pageEpoch?: string; + pageNextIndex: number; + tailKeys: string[]; +}; + +type PendingTranscriptLine = { + cursor: TranscriptStreamCursor; + line?: MeetingTranscriptLine; +}; + +type TranscriptSnapshotDelta = { + commitEmpty: boolean; + lines: MeetingTranscriptLine[]; + prefixKeys: string[]; + startIndex: number; +}; + const ENDED_TRANSCRIPTS_MAX = 4; +const TRANSCRIPT_CURSOR_TAIL = 64; const TRANSCRIPT_MAX_LINES = 2_000; +function transcriptLineKey(line: MeetingTranscriptLine): string { + return JSON.stringify([line.at ?? "", line.speaker ?? "", line.text]); +} + +function maximalTranscriptOverlap(previousKeys: string[], currentKeys: string[]): number { + const limit = Math.min(previousKeys.length, currentKeys.length); + for (let length = limit; length > 0; length -= 1) { + const previousStart = previousKeys.length - length; + if ( + currentKeys + .slice(0, length) + .every((key, index) => key === previousKeys[previousStart + index]) + ) { + return length; + } + } + return 0; +} + +export class MeetingTranscriptDeliveryError extends Error { + readonly finalCaptureError?: string; + + constructor(cause: unknown, finalCaptureError?: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = "MeetingTranscriptDeliveryError"; + if (finalCaptureError !== undefined) { + this.finalCaptureError = + finalCaptureError instanceof Error + ? finalCaptureError.message + : typeof finalCaptureError === "string" + ? finalCaptureError + : "unknown final capture error"; + } + } +} + export class MeetingSessionTranscriptStore { readonly #transcripts = new Map(); readonly #captures = new Map>(); readonly #finalizing = new Set(); + readonly #pendingLines = new Map(); readonly #retired = new Set(); + readonly #streamCursors = new Map(); constructor( private readonly options: { @@ -28,6 +85,7 @@ export class MeetingSessionTranscriptStore; + onLines?(session: TSession, lines: MeetingTranscriptLine[]): Promise; }, ) {} @@ -79,20 +137,109 @@ export class MeetingSessionTranscriptStore { + try { + await this.#capture(session, options, true); + } catch (error) { + if (!(error instanceof MeetingTranscriptDeliveryError)) { + throw error; + } + } + } + + async captureNotes(session: TSession, options: { finalize?: boolean } = {}): Promise { + await this.#capture(session, options, false); + } + + async flushPending(session: TSession): Promise { + await this.#capture(session, {}, false, true); + } + + async #capture( + session: TSession, + options: { finalize?: boolean }, + requireTranscribeMode: boolean, + pendingOnly = false, + ): Promise { + // Live reads, periodic notes, and finalization share this per-session chain. + // Keep cursor reads inside it so overlapping snapshots cannot deliver twice. const previous = this.#captures.get(session.id) ?? Promise.resolve(); const capture = previous .catch(() => {}) .then(async () => { - if ( - !this.options.isBrowserSession(session) || - !this.options.isTranscribeSession(session) || - !this.options.hasBrowserTab(session) - ) { + let pendingError: MeetingTranscriptDeliveryError | undefined; + try { + await this.#flushPending(session); + } catch (error) { + if (pendingOnly) { + throw error; + } + pendingError = error as MeetingTranscriptDeliveryError; + } + if (pendingOnly) { return; } - const snapshot = await this.options.capture(session, options); + if ( + !this.options.isBrowserSession(session) || + (requireTranscribeMode && !this.options.isTranscribeSession(session)) || + !this.options.hasBrowserTab(session) + ) { + if (pendingError) { + throw pendingError; + } + return; + } + let snapshot: MeetingTranscriptSnapshot | undefined; + try { + snapshot = await this.options.capture(session, options); + } catch (error) { + if (pendingError) { + throw new MeetingTranscriptDeliveryError(pendingError.cause ?? pendingError, error); + } + throw error; + } if (snapshot) { - this.#merge(session.id, snapshot); + if (this.options.isTranscribeSession(session)) { + this.#merge(session.id, snapshot); + } + const pending = this.#pendingLines.get(session.id); + const cursor = pending?.at(-1)?.cursor ?? this.#streamCursors.get(session.id); + const delta = this.#snapshotDelta(cursor, snapshot); + if (pendingError) { + if (delta.lines.length > 0) { + this.#queuePending(session.id, snapshot, delta, 0, delta.prefixKeys, true); + } else if (delta.commitEmpty) { + this.#queuePendingCursor(session.id, { + pageEpoch: snapshot.epoch, + pageNextIndex: snapshot.droppedLines + snapshot.lines.length, + tailKeys: delta.prefixKeys, + }); + } + throw pendingError; + } + let tailKeys = [...delta.prefixKeys]; + for (const [index, line] of delta.lines.entries()) { + try { + await this.options.onLines?.(session, [line]); + } catch (error) { + this.#queuePending(session.id, snapshot, delta, index, tailKeys); + throw new MeetingTranscriptDeliveryError(error); + } + tailKeys = [...tailKeys, transcriptLineKey(line)].slice(-TRANSCRIPT_CURSOR_TAIL); + this.#streamCursors.set(session.id, { + pageEpoch: snapshot.epoch, + pageNextIndex: delta.startIndex + index + 1, + tailKeys, + }); + } + if (delta.lines.length === 0 && delta.commitEmpty) { + this.#streamCursors.set(session.id, { + pageEpoch: snapshot.epoch, + pageNextIndex: snapshot.droppedLines + snapshot.lines.length, + tailKeys: delta.prefixKeys, + }); + } + } else if (pendingError) { + throw pendingError; } }); this.#captures.set(session.id, capture); @@ -105,6 +252,60 @@ export class MeetingSessionTranscriptStore { + const pending = this.#pendingLines.get(session.id); + for (;;) { + const next = pending?.[0]; + if (!next) { + break; + } + if (next.line) { + try { + await this.options.onLines?.(session, [next.line]); + } catch (error) { + throw new MeetingTranscriptDeliveryError(error); + } + } + this.#streamCursors.set(session.id, next.cursor); + pending.shift(); + } + if (pending?.length === 0) { + this.#pendingLines.delete(session.id); + } + } + + #queuePending( + sessionId: string, + snapshot: MeetingTranscriptSnapshot, + delta: TranscriptSnapshotDelta, + failedIndex: number, + prefixKeys: string[], + append = false, + ): void { + let tailKeys = [...prefixKeys]; + const pending = delta.lines.slice(failedIndex).map((line, offset) => { + tailKeys = [...tailKeys, transcriptLineKey(line)].slice(-TRANSCRIPT_CURSOR_TAIL); + return { + line, + cursor: { + pageEpoch: snapshot.epoch, + pageNextIndex: delta.startIndex + failedIndex + offset + 1, + tailKeys, + }, + }; + }); + const combined = [...(append ? (this.#pendingLines.get(sessionId) ?? []) : []), ...pending]; + // Undelivered rows are the only copy once the browser rolls its caption buffer. + // Preserve them until durable delivery succeeds; retirement clears the queue. + this.#pendingLines.set(sessionId, combined); + } + + #queuePendingCursor(sessionId: string, cursor: TranscriptStreamCursor): void { + const pending = this.#pendingLines.get(sessionId) ?? []; + pending.push({ cursor }); + this.#pendingLines.set(sessionId, pending); + } + retire(sessionId: string): void { const snapshot = this.#transcripts.get(sessionId); if (snapshot) { @@ -128,6 +329,70 @@ export class MeetingSessionTranscriptStore= previous.pageNextIndex) { + return { + commitEmpty: true, + lines: snapshot.lines, + prefixKeys: [], + startIndex: snapshot.droppedLines, + }; + } + if (snapshot.epoch === undefined && previous.pageEpoch === undefined) { + const previousStartIndex = previous.pageNextIndex - previous.tailKeys.length; + const overlapStart = Math.max(previousStartIndex, snapshot.droppedLines); + const overlapEnd = Math.min(previous.pageNextIndex, pageNextIndex); + const continuation = + overlapStart < overlapEnd && + Array.from( + { length: overlapEnd - overlapStart }, + (_, offset) => overlapStart + offset, + ).every( + (absoluteIndex) => + previous.tailKeys[absoluteIndex - previousStartIndex] === + transcriptLineKey(snapshot.lines[absoluteIndex - snapshot.droppedLines]!), + ); + if ((!continuation && snapshot.lines.length > 0) || pageNextIndex < previous.pageNextIndex) { + const currentKeys = snapshot.lines.map(transcriptLineKey); + const overlap = maximalTranscriptOverlap(previous.tailKeys, currentKeys); + return { + commitEmpty: true, + lines: snapshot.lines.slice(overlap), + prefixKeys: previous.tailKeys.slice(previous.tailKeys.length - overlap), + startIndex: snapshot.droppedLines + overlap, + }; + } + } + if (pageNextIndex <= previous.pageNextIndex) { + return { + commitEmpty: false, + lines: [], + prefixKeys: previous.tailKeys, + startIndex: pageNextIndex, + }; + } + return { + commitEmpty: false, + lines: snapshot.lines.slice(previous.pageNextIndex - snapshot.droppedLines), + prefixKeys: previous.tailKeys, + startIndex: previous.pageNextIndex, + }; } #merge(sessionId: string, snapshot: MeetingTranscriptSnapshot): void { diff --git a/src/meeting-bot/transcripts-bridge.runtime.ts b/src/meeting-bot/transcripts-bridge.runtime.ts new file mode 100644 index 000000000000..0703013debaf --- /dev/null +++ b/src/meeting-bot/transcripts-bridge.runtime.ts @@ -0,0 +1,494 @@ +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { resolveTranscriptsConfig } from "../transcripts/config.js"; +import type { + TranscriptSessionDescriptor, + TranscriptSourceStatus, + TranscriptStartRequest, + TranscriptsStartResult, + TranscriptsStopResult, + TranscriptUtterance, +} from "../transcripts/provider-types.js"; +import { sanitizeTranscriptSourceLocator } from "../transcripts/source-locator.js"; +import { TranscriptsStore } from "../transcripts/store.js"; +import { summarizeTranscripts } from "../transcripts/summary.js"; +import { MeetingTranscriptDeliveryError } from "./session-transcript-store.js"; +import type { MeetingSessionRecord, MeetingTranscriptLine } from "./session-types.js"; +import type { + MeetingDurableTranscriptBridge, + MeetingDurableTranscriptsOptions, + MeetingTranscriptBridgeLogger, +} from "./transcripts-bridge.js"; + +const CAPTURE_INTERVAL_MS = 5_000; + +type ActiveCapture = { + closing: boolean; + descriptor: TranscriptSessionDescriptor; + finalCaptureError?: string; + finalCaptureFailedAt?: string; + initialized: boolean; + initializationWarned: boolean; + polling: boolean; + runCapture(task: () => Promise): Promise; + session: TSession; + timer?: ReturnType; + utteranceCount: number; +}; + +type Subscriber = { + agentId: string; + deliveredUtteranceIds: Set; + meetingSessionId: string; + onStatus?: TranscriptStartRequest["onStatus"]; + onUtterance: TranscriptStartRequest["onUtterance"]; +}; + +function descriptorForSession( + session: MeetingSessionRecord, + options: MeetingDurableTranscriptsOptions, +): TranscriptSessionDescriptor { + return { + sessionId: session.id, + title: `${options.providerName} meeting`, + source: sanitizeTranscriptSourceLocator({ + providerId: options.providerId, + kind: "live-caption", + meetingUrl: session.url, + }), + startedAt: session.createdAt, + metadata: { + agentId: session.agentId, + meetingSessionId: session.id, + mode: session.mode, + participantIdentity: session.participantIdentity, + }, + }; +} + +function utteranceFromLine(params: { + line: MeetingTranscriptLine; + session: MeetingSessionRecord; + sequence: number; +}): TranscriptUtterance { + return { + id: `${params.session.id}:${params.sequence}`, + sessionId: params.session.id, + startedAt: params.line.at, + speaker: params.line.speaker ? { label: params.line.speaker } : undefined, + text: params.line.text, + final: true, + metadata: { + agentId: params.session.agentId, + meetingSessionId: params.session.id, + }, + }; +} + +export function createMeetingDurableTranscriptBridge< + TSession extends MeetingSessionRecord, +>(params: { + logger: MeetingTranscriptBridgeLogger; + options: MeetingDurableTranscriptsOptions; +}): MeetingDurableTranscriptBridge { + const config = resolveTranscriptsConfig(params.options.config); + const stateDir = params.options.stateDir ?? resolveStateDir(); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const captures = new Map>(); + const pendingSubscribers = new Map(); + const subscribers = new Map(); + const lifecycleTasks = new Map>(); + const tasks = new Map>(); + + const runSerial = async (sessionId: string, task: () => Promise): Promise => { + const previous = tasks.get(sessionId) ?? Promise.resolve(); + const result = previous.catch(() => {}).then(task); + const settled = result.then( + () => undefined, + () => undefined, + ); + tasks.set(sessionId, settled); + try { + return await result; + } finally { + if (tasks.get(sessionId) === settled) { + tasks.delete(sessionId); + } + } + }; + + const runLifecycle = async (sessionId: string, task: () => Promise): Promise => { + const previous = lifecycleTasks.get(sessionId) ?? Promise.resolve(); + const result = previous.catch(() => {}).then(task); + const settled = result.then( + () => undefined, + () => undefined, + ); + lifecycleTasks.set(sessionId, settled); + try { + return await result; + } finally { + if (lifecycleTasks.get(sessionId) === settled) { + lifecycleTasks.delete(sessionId); + } + } + }; + + const reportCaptureError = (sessionId: string, error: unknown) => { + params.logger.debug?.( + `[meeting-transcripts] capture ignored session=${sessionId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }; + + const notifySubscriberStatus = (subscriber: Subscriber, status: TranscriptSourceStatus) => { + if (!subscriber.onStatus) { + return; + } + try { + void Promise.resolve(subscriber.onStatus(status)).catch((error: unknown) => { + params.logger.warn( + `[meeting-transcripts] subscriber status failed session=${status.sessionId ?? "unknown"}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + } catch (error) { + params.logger.warn( + `[meeting-transcripts] subscriber status failed session=${status.sessionId ?? "unknown"}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + }; + + return { + enabled: config.enabled, + async start(session, capture) { + await runLifecycle(session.id, async () => { + if (!config.enabled || captures.has(session.id)) { + return; + } + const descriptor = descriptorForSession(session, params.options); + let captureQueue = Promise.resolve(); + const runCapture = async (task: () => Promise) => { + const result = captureQueue.catch(() => {}).then(task); + captureQueue = result.then( + () => undefined, + () => undefined, + ); + return await result; + }; + const active: ActiveCapture = { + closing: false, + descriptor, + initialized: false, + initializationWarned: false, + polling: false, + runCapture, + session, + utteranceCount: 0, + }; + captures.set(session.id, active); + // Start and stop share runLifecycle(session.id), so teardown cannot mark + // this published capture closing while initialization awaits. + const initialize = async () => { + if (active.initialized) { + return; + } + try { + await store.writeSession(descriptor); + active.initialized = true; + active.initializationWarned = false; + } catch (error) { + if (!active.initializationWarned) { + params.logger.warn( + `[meeting-transcripts] durable capture initialization pending session=${session.id}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + active.initializationWarned = true; + } + } + }; + const timer = setInterval(() => { + // polling covers both initialize() and capture, so session writes are + // single-flight too. A skipped tick is followed within CAPTURE_INTERVAL_MS. + if (active.polling || active.closing) { + return; + } + active.polling = true; + void initialize() + .then(async () => await active.runCapture(capture)) + .catch((error: unknown) => reportCaptureError(session.id, error)) + .finally(() => { + active.polling = false; + }); + }, CAPTURE_INTERVAL_MS); + timer.unref?.(); + active.timer = timer; + active.polling = true; + try { + await initialize(); + await active + .runCapture(capture) + .catch((error: unknown) => reportCaptureError(session.id, error)); + } finally { + active.polling = false; + } + }); + }, + async ingest(session, lines) { + const active = captures.get(session.id); + if (!active || lines.length === 0) { + return; + } + await runSerial(session.id, async () => { + for (const line of lines) { + const sequence = active.utteranceCount; + const utterance = utteranceFromLine({ + line, + session, + sequence, + }); + await store.appendUtteranceForSession(active.descriptor, utterance); + for (const [subscriberSessionId, subscriber] of subscribers) { + if ( + subscriber.meetingSessionId !== session.id || + (utterance.id && subscriber.deliveredUtteranceIds.has(utterance.id)) + ) { + continue; + } + const subscriberUtterance = { + ...utterance, + id: `${subscriberSessionId}:${utterance.id ?? sequence}`, + sessionId: subscriberSessionId, + }; + try { + await subscriber.onUtterance(subscriberUtterance); + if (utterance.id) { + subscriber.deliveredUtteranceIds.add(utterance.id); + } + } catch (error) { + subscribers.delete(subscriberSessionId); + params.logger.warn( + `[meeting-transcripts] detached failing subscriber session=${subscriberSessionId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + notifySubscriberStatus(subscriber, { + sessionId: subscriberSessionId, + active: false, + message: "Detached after transcript delivery failed.", + source: active.descriptor.source, + }); + } + } + active.utteranceCount += 1; + } + }); + }, + async stop(session, finalCapture) { + const active = await runLifecycle(session.id, async () => { + const current = captures.get(session.id); + if (!current) { + return undefined; + } + current.closing = true; + if (current.timer) { + clearInterval(current.timer); + delete current.timer; + } + return current; + }); + if (!active) { + return false; + } + let initializationError: Error | undefined; + if (!active.initialized) { + try { + await store.writeSession(active.descriptor); + active.initialized = true; + } catch (error) { + initializationError = + error instanceof Error + ? error + : new Error("could not initialize durable transcript session", { cause: error }); + } + } + let deliveryError: MeetingTranscriptDeliveryError | undefined; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + await active.runCapture(finalCapture); + deliveryError = undefined; + break; + } catch (error) { + if (!(error instanceof MeetingTranscriptDeliveryError)) { + reportCaptureError(session.id, error); + active.finalCaptureError = error instanceof Error ? error.message : String(error); + active.finalCaptureFailedAt ??= new Date().toISOString(); + deliveryError = undefined; + break; + } + if (error.finalCaptureError !== undefined) { + active.finalCaptureError = error.finalCaptureError; + active.finalCaptureFailedAt ??= new Date().toISOString(); + } + deliveryError = error; + } + } + if (deliveryError) { + throw deliveryError; + } + if (initializationError !== undefined) { + throw initializationError; + } + const finalCaptureError = active.finalCaptureError; + const stoppedAt = new Date().toISOString(); + const stopped = { + ...active.descriptor, + stoppedAt, + ...(finalCaptureError !== undefined + ? { + metadata: { + ...active.descriptor.metadata, + finalCaptureError, + finalCaptureFailedAt: active.finalCaptureFailedAt, + }, + } + : {}), + }; + try { + await runSerial(session.id, async () => { + await store.writeSession(stopped); + const utterances = await store.readUtterancesForSession(stopped, { + maxUtterances: config.maxUtterances, + }); + await store.writeSummary(summarizeTranscripts({ session: stopped, utterances }), stopped); + for (const [subscriberSessionId, subscriber] of subscribers) { + if (subscriber.meetingSessionId !== session.id) { + continue; + } + notifySubscriberStatus(subscriber, { + sessionId: subscriberSessionId, + active: false, + message: `${params.options.providerName} meeting capture ended.`, + source: stopped.source, + }); + subscribers.delete(subscriberSessionId); + } + }); + } catch (error) { + params.logger.warn( + `[meeting-transcripts] could not finalize durable capture session=${session.id}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + throw error; + } + captures.delete(session.id); + return true; + }, + async attach(session, request): Promise { + const active = captures.get(session.id); + if (!config.enabled || !active || active.closing) { + return { + ok: false, + error: `${params.options.providerName} meeting capture is not active.`, + }; + } + if ( + subscribers.has(request.session.sessionId) || + pendingSubscribers.has(request.session.sessionId) + ) { + return { + ok: false, + error: `transcripts session already attached: ${request.session.sessionId}`, + }; + } + let attached = false; + pendingSubscribers.set(request.session.sessionId, { + agentId: session.agentId, + meetingSessionId: session.id, + }); + try { + await runSerial(session.id, async () => { + if (captures.get(session.id) !== active || active.closing) { + return; + } + const utterances = await store.readUtterancesForSession(active.descriptor); + const deliveredUtteranceIds = new Set(); + for (const utterance of utterances) { + await request.onUtterance({ + ...utterance, + id: `${request.session.sessionId}:${utterance.id ?? "replay"}`, + sessionId: request.session.sessionId, + }); + if (utterance.id) { + deliveredUtteranceIds.add(utterance.id); + } + } + subscribers.set(request.session.sessionId, { + agentId: session.agentId, + deliveredUtteranceIds, + meetingSessionId: session.id, + onStatus: request.onStatus, + onUtterance: request.onUtterance, + }); + try { + await request.onStatus?.({ + sessionId: request.session.sessionId, + active: true, + message: `Attached to active ${params.options.providerName} meeting capture.`, + source: active.descriptor.source, + }); + attached = true; + } catch (error) { + subscribers.delete(request.session.sessionId); + throw error; + } + }); + } finally { + pendingSubscribers.delete(request.session.sessionId); + } + return attached + ? { ok: true, session: request.session } + : { ok: false, error: `${params.options.providerName} meeting capture is ending.` }; + }, + async detach(request): Promise { + const subscriber = subscribers.get(request.sessionId); + const pending = pendingSubscribers.get(request.sessionId); + const owner = subscriber ?? pending; + if (!owner) { + return { ok: true, sessionId: request.sessionId, stoppedAt: new Date().toISOString() }; + } + if (request.source.agentId !== owner.agentId) { + return { ok: false, error: "transcripts session belongs to another agent" }; + } + return await runSerial(owner.meetingSessionId, async () => { + const current = subscribers.get(request.sessionId); + if (!current) { + return { ok: true, sessionId: request.sessionId, stoppedAt: new Date().toISOString() }; + } + if (request.source.agentId !== current.agentId) { + return { ok: false as const, error: "transcripts session belongs to another agent" }; + } + notifySubscriberStatus(current, { + sessionId: request.sessionId, + active: false, + message: `Detached from ${params.options.providerName} meeting capture.`, + source: request.source, + }); + subscribers.delete(request.sessionId); + return { + ok: true as const, + sessionId: request.sessionId, + stoppedAt: new Date().toISOString(), + }; + }); + }, + }; +} diff --git a/src/meeting-bot/transcripts-bridge.test.ts b/src/meeting-bot/transcripts-bridge.test.ts new file mode 100644 index 000000000000..8e59efc8504d --- /dev/null +++ b/src/meeting-bot/transcripts-bridge.test.ts @@ -0,0 +1,408 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TranscriptsStore } from "../transcripts/store.js"; +import { MeetingTranscriptDeliveryError } from "./session-transcript-store.js"; +import type { MeetingSessionRecord } from "./session-types.js"; +import { createMeetingDurableTranscriptBridge } from "./transcripts-bridge.runtime.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + vi.useRealTimers(); + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true }))); +}); + +function session(): MeetingSessionRecord<"chrome", "agent"> { + return { + id: "meeting-1", + url: "https://meeting.example/room", + transport: "chrome", + mode: "agent", + agentId: "research", + state: "active", + createdAt: "2026-07-23T12:00:00.000Z", + updatedAt: "2026-07-23T12:00:00.000Z", + participantIdentity: "OpenClaw browser guest", + realtime: { enabled: true, toolPolicy: "safe-read-only" }, + notes: [], + }; +} + +describe("MeetingDurableTranscriptBridge", () => { + it("replays stored lines to an attached provider and streams new lines in order", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + const onUtterance = vi.fn(); + const transcriptSession = { + sessionId: "external-1", + source: { providerId: "google-meet", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }; + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { + providerId: "google-meet", + providerName: "Google Meet", + stateDir, + }, + }); + + await bridge.start(current, async () => {}); + await bridge.ingest(current, [{ speaker: "Avery", text: "First line" }]); + await expect( + bridge.attach(current, { + session: transcriptSession, + onUtterance, + }), + ).resolves.toMatchObject({ ok: true }); + await bridge.ingest(current, [{ speaker: "Blake", text: "Second line" }]); + await expect( + bridge.detach({ + sessionId: "external-1", + source: { providerId: "google-meet", agentId: "another-agent" }, + }), + ).resolves.toMatchObject({ ok: false }); + await expect( + bridge.detach({ + sessionId: "external-1", + source: { providerId: "google-meet", agentId: "research" }, + }), + ).resolves.toMatchObject({ ok: true }); + const replayed = vi.fn(); + await expect( + bridge.attach(current, { session: transcriptSession, onUtterance: replayed }), + ).resolves.toMatchObject({ ok: true }); + await bridge.stop(current, async () => {}); + + expect(onUtterance.mock.calls.map(([utterance]) => utterance)).toMatchObject([ + { sessionId: "external-1", speaker: { label: "Avery" }, text: "First line" }, + { sessionId: "external-1", speaker: { label: "Blake" }, text: "Second line" }, + ]); + expect(replayed.mock.calls.map(([utterance]) => utterance.id)).toEqual( + onUtterance.mock.calls.map(([utterance]) => utterance.id), + ); + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const stored = await store.readSession(current.id); + expect(await store.readSummary(stored!)).toMatchObject({ + summary: { utteranceCount: 2 }, + }); + }); + + it("honors the existing global transcripts opt-out", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { + config: { enabled: false }, + providerId: "zoom", + providerName: "Zoom", + stateDir, + }, + }); + + await bridge.start(session(), async () => {}); + + expect(bridge.enabled).toBe(false); + await expect( + bridge.attach(session(), { + session: { + sessionId: "external-disabled", + source: { providerId: "zoom", agentId: "research" }, + startedAt: "2026-07-23T12:00:00.000Z", + }, + onUtterance: vi.fn(), + }), + ).resolves.toMatchObject({ ok: false }); + }); + + it("rolls back an attachment when its start status callback fails", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { + providerId: "teams", + providerName: "Microsoft Teams", + stateDir, + }, + }); + const transcriptSession = { + sessionId: "external-status", + source: { providerId: "teams", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }; + await bridge.start(current, async () => {}); + + await expect( + bridge.attach(current, { + session: transcriptSession, + onStatus: async () => { + throw new Error("status delivery failed"); + }, + onUtterance: vi.fn(), + }), + ).rejects.toThrow("status delivery failed"); + await expect( + bridge.attach(current, { + session: transcriptSession, + onUtterance: vi.fn(), + }), + ).resolves.toMatchObject({ ok: true }); + + await bridge.stop(current, async () => {}); + }); + + it("drains an in-flight periodic capture before the final capture", async () => { + vi.useFakeTimers(); + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + let releasePeriodic!: () => void; + const periodic = new Promise((resolve) => { + releasePeriodic = resolve; + }); + let captureCount = 0; + const capture = vi.fn(async () => { + captureCount += 1; + if (captureCount === 2) { + await periodic; + } + }); + const finalCapture = vi.fn(async () => {}); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "zoom", providerName: "Zoom", stateDir }, + }); + await bridge.start(current, capture); + await vi.advanceTimersByTimeAsync(5_000); + + const stopping = bridge.stop(current, finalCapture); + expect(finalCapture).not.toHaveBeenCalled(); + releasePeriodic(); + await expect(stopping).resolves.toBe(true); + + expect(finalCapture).toHaveBeenCalledOnce(); + }); + + it("retries final durable delivery before completing the capture", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + const finalCapture = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new MeetingTranscriptDeliveryError(new Error("write failed"))) + .mockRejectedValueOnce(new MeetingTranscriptDeliveryError(new Error("write failed"))) + .mockResolvedValueOnce(); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "teams", providerName: "Microsoft Teams", stateDir }, + }); + await bridge.start(current, async () => {}); + + await expect(bridge.stop(current, finalCapture)).resolves.toBe(true); + + expect(finalCapture).toHaveBeenCalledTimes(3); + }); + + it("detaches a failing subscriber without blocking durable rows", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + const onUtterance = vi.fn(async () => { + throw new Error("subscriber store unavailable"); + }); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "google-meet", providerName: "Google Meet", stateDir }, + }); + await bridge.start(current, async () => {}); + await bridge.attach(current, { + session: { + sessionId: "external-retry", + source: { providerId: "google-meet", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }, + onUtterance, + }); + + await expect(bridge.ingest(current, [{ text: "first" }])).resolves.toBeUndefined(); + await expect(bridge.ingest(current, [{ text: "second" }])).resolves.toBeUndefined(); + + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + const stored = await store.readSession(current.id); + expect(await store.readUtterancesForSession(stored!)).toHaveLength(2); + expect(onUtterance).toHaveBeenCalledOnce(); + await bridge.stop(current, async () => {}); + }); + + it("rejects attachments once finalization begins", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + let releaseFinal!: () => void; + const finalizing = new Promise((resolve) => { + releaseFinal = resolve; + }); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "zoom", providerName: "Zoom", stateDir }, + }); + await bridge.start(current, async () => {}); + + const stopping = bridge.stop(current, async () => await finalizing); + await expect( + bridge.attach(current, { + session: { + sessionId: "external-late", + source: { providerId: "zoom", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }, + onUtterance: vi.fn(), + }), + ).resolves.toMatchObject({ ok: false }); + releaseFinal(); + await expect(stopping).resolves.toBe(true); + }); + + it("drains subscriber delivery before detaching", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + let releaseDelivery!: () => void; + const delivery = new Promise((resolve) => { + releaseDelivery = resolve; + }); + const onStatus = vi.fn(async () => {}); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "teams", providerName: "Microsoft Teams", stateDir }, + }); + await bridge.start(current, async () => {}); + await bridge.attach(current, { + session: { + sessionId: "external-drain", + source: { providerId: "teams", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }, + onStatus, + onUtterance: async () => await delivery, + }); + + const ingesting = bridge.ingest(current, [{ text: "pending delivery" }]); + const detaching = bridge.detach({ + sessionId: "external-drain", + source: { providerId: "teams", agentId: "research" }, + }); + await Promise.resolve(); + expect(onStatus).toHaveBeenCalledTimes(1); + releaseDelivery(); + await ingesting; + await expect(detaching).resolves.toMatchObject({ ok: true }); + + expect(onStatus).toHaveBeenCalledTimes(2); + await bridge.stop(current, async () => {}); + }); + + it("does not let terminal subscriber notification block finalization", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + let statusCalls = 0; + const onStatus = vi.fn(() => { + statusCalls += 1; + if (statusCalls === 2) { + throw new Error("terminal status unavailable"); + } + }); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "zoom", providerName: "Zoom", stateDir }, + }); + await bridge.start(current, async () => {}); + await bridge.attach(current, { + session: { + sessionId: "external-terminal", + source: { providerId: "zoom", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }, + onStatus, + onUtterance: vi.fn(), + }); + + await expect(bridge.stop(current, async () => {})).resolves.toBe(true); + + expect(onStatus).toHaveBeenCalledTimes(2); + }); + + it("queues detach behind a pending attachment replay", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + let releaseReplay!: () => void; + const replay = new Promise((resolve) => { + releaseReplay = resolve; + }); + const onUtterance = vi.fn(async () => await replay); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "google-meet", providerName: "Google Meet", stateDir }, + }); + await bridge.start(current, async () => {}); + await bridge.ingest(current, [{ text: "existing row" }]); + const transcriptSession = { + sessionId: "external-pending", + source: { providerId: "google-meet", agentId: "research", meetingUrl: current.url }, + startedAt: current.createdAt, + }; + + const attaching = bridge.attach(current, { session: transcriptSession, onUtterance }); + const detaching = bridge.detach({ + sessionId: transcriptSession.sessionId, + source: transcriptSession.source, + }); + releaseReplay(); + await expect(attaching).resolves.toMatchObject({ ok: true }); + await expect(detaching).resolves.toMatchObject({ ok: true }); + await bridge.ingest(current, [{ text: "after detach" }]); + + expect(onUtterance).toHaveBeenCalledOnce(); + await bridge.stop(current, async () => {}); + }); + + it("records a non-blocking final browser snapshot failure", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcript-bridge-")); + tempDirs.push(stateDir); + const current = session(); + const bridge = createMeetingDurableTranscriptBridge({ + logger: { warn: vi.fn() }, + options: { providerId: "zoom", providerName: "Zoom", stateDir }, + }); + await bridge.start(current, async () => {}); + + await expect( + bridge.stop(current, async () => { + throw new Error(""); + }), + ).resolves.toBe(true); + + const store = new TranscriptsStore(path.join(stateDir, "transcripts"), { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); + await expect(store.readSession(current.id)).resolves.toMatchObject({ + metadata: { + finalCaptureError: "", + finalCaptureFailedAt: expect.any(String), + }, + stoppedAt: expect.any(String), + }); + }); +}); diff --git a/src/meeting-bot/transcripts-bridge.ts b/src/meeting-bot/transcripts-bridge.ts new file mode 100644 index 000000000000..bffa45a09e66 --- /dev/null +++ b/src/meeting-bot/transcripts-bridge.ts @@ -0,0 +1,52 @@ +import type { + TranscriptSourceProvider, + TranscriptStartRequest, + TranscriptsStartResult, + TranscriptStopRequest, + TranscriptsStopResult, +} from "../transcripts/provider-types.js"; +import type { MeetingSessionRecord, MeetingTranscriptLine } from "./session-types.js"; + +export type MeetingDurableTranscriptsOptions = { + config?: unknown; + providerId: string; + providerName: string; + stateDir?: string; +}; + +export type MeetingTranscriptBridgeLogger = { + debug?: (message: string) => void; + warn: (message: string) => void; +}; + +export type MeetingTranscriptSourceRuntime = { + startTranscriptSource(request: TranscriptStartRequest): Promise; + stopTranscriptSource(request: TranscriptStopRequest): Promise; +}; + +export type MeetingDurableTranscriptBridge< + TSession extends MeetingSessionRecord = MeetingSessionRecord, +> = { + readonly enabled: boolean; + start(session: TSession, capture: () => Promise): Promise; + ingest(session: TSession, lines: MeetingTranscriptLine[]): Promise; + stop(session: TSession, finalCapture: () => Promise): Promise; + attach(session: TSession, request: TranscriptStartRequest): Promise; + detach(request: TranscriptStopRequest): Promise; +}; + +export function createMeetingTranscriptSourceProvider(params: { + id: string; + aliases?: readonly string[]; + name: string; + runtime: () => Promise; +}): TranscriptSourceProvider { + return { + id: params.id, + aliases: params.aliases, + name: params.name, + sourceKinds: ["live-caption"], + start: async (request) => await (await params.runtime()).startTranscriptSource(request), + stop: async (request) => await (await params.runtime()).stopTranscriptSource(request), + }; +} diff --git a/src/plugin-sdk/transcripts.ts b/src/plugin-sdk/transcripts.ts index 64bf00aba212..13d7eb649429 100644 --- a/src/plugin-sdk/transcripts.ts +++ b/src/plugin-sdk/transcripts.ts @@ -20,3 +20,8 @@ export { listTranscriptSourceProviders, normalizeTranscriptSourceProviderId, } from "../transcripts/provider-registry.js"; +export { resolveTranscriptsConfig } from "../transcripts/config.js"; +export { + createMeetingTranscriptSourceProvider, + type MeetingTranscriptSourceRuntime, +} from "../meeting-bot/transcripts-bridge.js"; diff --git a/src/transcripts/config.test.ts b/src/transcripts/config.test.ts new file mode 100644 index 000000000000..c097f471323d --- /dev/null +++ b/src/transcripts/config.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { resolveTranscriptsConfig } from "./config.js"; + +describe("resolveTranscriptsConfig", () => { + it("enables meeting transcripts by default with an explicit global opt-out", () => { + expect(resolveTranscriptsConfig(undefined).enabled).toBe(true); + expect(resolveTranscriptsConfig({}).enabled).toBe(true); + expect(resolveTranscriptsConfig({ enabled: false }).enabled).toBe(false); + }); +}); diff --git a/src/transcripts/config.ts b/src/transcripts/config.ts index 03ac4fb212ee..8c5602df3209 100644 --- a/src/transcripts/config.ts +++ b/src/transcripts/config.ts @@ -72,7 +72,7 @@ function resolveAutoStart(raw: unknown): ResolvedTranscriptsAutoStartConfig[] { export function resolveTranscriptsConfig(raw: unknown): ResolvedTranscriptsConfig { const config = raw && typeof raw === "object" ? (raw as Record) : {}; return { - enabled: config.enabled === true, + enabled: config.enabled !== false, maxUtterances: DEFAULT_TRANSCRIPTS_MAX_UTTERANCES, autoStart: resolveAutoStart(config.autoStart), }; diff --git a/src/transcripts/source-locator.test.ts b/src/transcripts/source-locator.test.ts new file mode 100644 index 000000000000..3ad9c10b1917 --- /dev/null +++ b/src/transcripts/source-locator.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { sanitizeTranscriptSourceLocator } from "./source-locator.js"; + +describe("sanitizeTranscriptSourceLocator", () => { + it("removes meeting invitation credentials and drops invalid locators", () => { + expect( + sanitizeTranscriptSourceLocator({ + providerId: "zoom", + meetingUrl: "https://zoom.us/j/123?context=opaque-value#fragment", + }), + ).toEqual({ providerId: "zoom", meetingUrl: "https://zoom.us/j/123" }); + expect( + sanitizeTranscriptSourceLocator({ providerId: "zoom", meetingUrl: "not a URL" }), + ).toEqual({ providerId: "zoom" }); + }); +}); diff --git a/src/transcripts/source-locator.ts b/src/transcripts/source-locator.ts new file mode 100644 index 000000000000..acebb011f392 --- /dev/null +++ b/src/transcripts/source-locator.ts @@ -0,0 +1,17 @@ +import type { TranscriptSourceLocator } from "./provider-types.js"; + +/** Strip invitation credentials from meeting locators before persistence/provider handoff. */ +export function sanitizeTranscriptSourceLocator( + source: TranscriptSourceLocator, +): TranscriptSourceLocator { + if (!source.meetingUrl) { + return source; + } + const { meetingUrl: _meetingUrl, ...rest } = source; + try { + const url = new URL(source.meetingUrl); + return { ...rest, meetingUrl: `${url.origin}${url.pathname}` }; + } catch { + return rest; + } +} diff --git a/src/transcripts/store-sqlite.ts b/src/transcripts/store-sqlite.ts index b9389064da88..9c743656914c 100644 --- a/src/transcripts/store-sqlite.ts +++ b/src/transcripts/store-sqlite.ts @@ -1,6 +1,10 @@ import type { DatabaseSync } from "node:sqlite"; import type { Selectable } from "kysely"; -import { getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import type { TranscriptSessionDescriptor, TranscriptUtterance } from "./provider-types.js"; import type { TranscriptsSummary } from "./summary.js"; @@ -24,6 +28,95 @@ export function meetingTranscriptDb(db: DatabaseSync) { return getNodeSqliteKysely(db); } +function hasExactMeetingTranscriptUtterance(params: { + database: DatabaseSync; + metadataJson: string | null; + sessionId: string; + sessionStartedAt: string; + utterance: TranscriptUtterance & { id: string }; +}): boolean { + const db = meetingTranscriptDb(params.database); + const rows = executeSqliteQuerySync( + params.database, + db + .selectFrom("meeting_transcript_utterances") + .selectAll() + .where("session_id", "=", params.sessionId) + .where("session_started_at", "=", params.sessionStartedAt) + .where("utterance_id", "=", params.utterance.id), + ).rows; + const utterance = params.utterance; + return rows.some( + (row) => + row.started_at === (utterance.startedAt ?? null) && + row.ended_at === (utterance.endedAt ?? null) && + row.speaker_id === (utterance.speaker?.id ?? null) && + row.speaker_label === (utterance.speaker?.label ?? null) && + row.text === utterance.text && + row.final === (utterance.final === undefined ? null : utterance.final ? 1 : 0) && + row.metadata_json === params.metadataJson, + ); +} + +export function appendMeetingTranscriptUtterance(params: { + database: DatabaseSync; + metadataJson: string | null; + now: number; + session: TranscriptSessionDescriptor; + utterance: TranscriptUtterance; +}): void { + const { database, session, utterance } = params; + const db = meetingTranscriptDb(database); + if ( + utterance.id && + hasExactMeetingTranscriptUtterance({ + database, + metadataJson: params.metadataJson, + sessionId: session.sessionId, + sessionStartedAt: session.startedAt, + utterance: { ...utterance, id: utterance.id }, + }) + ) { + return; + } + const stored = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("meeting_transcript_sessions") + .select("next_utterance_seq") + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); + if (!stored) { + throw new Error(`transcripts session not found: ${session.sessionId}`); + } + const sequence = stored.next_utterance_seq; + executeSqliteQuerySync( + database, + db.insertInto("meeting_transcript_utterances").values({ + session_id: session.sessionId, + session_started_at: session.startedAt, + sequence, + utterance_id: utterance.id ?? null, + started_at: utterance.startedAt ?? null, + ended_at: utterance.endedAt ?? null, + speaker_id: utterance.speaker?.id ?? null, + speaker_label: utterance.speaker?.label ?? null, + text: utterance.text, + final: utterance.final === undefined ? null : utterance.final ? 1 : 0, + metadata_json: params.metadataJson, + }), + ); + executeSqliteQuerySync( + database, + db + .updateTable("meeting_transcript_sessions") + .set({ next_utterance_seq: sequence + 1, updated_at_ms: params.now }) + .where("session_id", "=", session.sessionId) + .where("started_at", "=", session.startedAt), + ); +} + function parseOptionalJsonRecord(value: string | null): Record | undefined { if (!value) { return undefined; diff --git a/src/transcripts/store.test.ts b/src/transcripts/store.test.ts index 378a03fe2fd9..766a24da7b07 100644 --- a/src/transcripts/store.test.ts +++ b/src/transcripts/store.test.ts @@ -73,6 +73,28 @@ describe("TranscriptsStore", () => { ]); }); + it("deduplicates exact retries but preserves same-id revisions", async () => { + const { store } = createStore(); + const target = session(); + const interim = { id: "utterance-1", text: "draft", final: false }; + await store.writeSession(target); + await store.appendUtteranceForSession(target, interim); + await store.appendUtteranceForSession(target, interim); + const final = { + id: "utterance-1", + text: "final text", + final: true, + }; + await store.appendUtteranceForSession(target, final); + await store.appendUtteranceForSession(target, interim); + await store.appendUtteranceForSession(target, final); + + await expect(store.readUtterancesForSession(target)).resolves.toMatchObject([ + interim, + { id: "utterance-1", text: "final text", final: true }, + ]); + }); + it("requires date-qualified selectors for repeated ids", async () => { const { store } = createStore(); await store.writeSession(session("standup", "2026-07-01T10:00:00.000Z")); diff --git a/src/transcripts/store.ts b/src/transcripts/store.ts index 5d755b56b2c5..d9b9c065eb01 100644 --- a/src/transcripts/store.ts +++ b/src/transcripts/store.ts @@ -36,6 +36,7 @@ import { hasAliasedCanonicalTranscriptExportPathOwner, } from "./store-export-ownership.js"; import { + appendMeetingTranscriptUtterance, meetingTranscriptDb, type MeetingTranscriptSessionRow, sessionFromRow, @@ -465,45 +466,8 @@ export class TranscriptsStore { const now = Date.now(); ensureMeetingTranscriptsSchema(this.databaseOptions); runOpenClawStateWriteTransaction( - ({ db: database }) => { - const db = meetingTranscriptDb(database); - const stored = executeSqliteQueryTakeFirstSync( - database, - db - .selectFrom("meeting_transcript_sessions") - .select("next_utterance_seq") - .where("session_id", "=", session.sessionId) - .where("started_at", "=", session.startedAt), - ); - if (!stored) { - throw new Error(`transcripts session not found: ${session.sessionId}`); - } - const sequence = stored.next_utterance_seq; - executeSqliteQuerySync( - database, - db.insertInto("meeting_transcript_utterances").values({ - session_id: session.sessionId, - session_started_at: session.startedAt, - sequence, - utterance_id: utterance.id ?? null, - started_at: utterance.startedAt ?? null, - ended_at: utterance.endedAt ?? null, - speaker_id: utterance.speaker?.id ?? null, - speaker_label: utterance.speaker?.label ?? null, - text: utterance.text, - final: utterance.final === undefined ? null : utterance.final ? 1 : 0, - metadata_json: metadataJson, - }), - ); - executeSqliteQuerySync( - database, - db - .updateTable("meeting_transcript_sessions") - .set({ next_utterance_seq: sequence + 1, updated_at_ms: now }) - .where("session_id", "=", session.sessionId) - .where("started_at", "=", session.startedAt), - ); - }, + ({ db: database }) => + appendMeetingTranscriptUtterance({ database, metadataJson, now, session, utterance }), this.databaseOptions, { operationLabel: "meeting-transcripts.utterance.append" }, );