From e717d24c2e7b3a72d39fa9c7cbbb96af2a90d646 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 26 Jul 2026 23:25:19 -0400 Subject: [PATCH] refactor(meetings): close manual action state (#114247) --- docs/plugins/google-meet.md | 47 ++++----- docs/plugins/meeting-plugins.md | 2 +- extensions/google-meet/index.create.test.ts | 45 +++++---- extensions/google-meet/index.test.ts | 98 +++++++++++-------- .../google-meet/src/cli-runtime.test.ts | 4 +- extensions/google-meet/src/cli-shared.ts | 8 +- extensions/google-meet/src/cli.test.ts | 7 +- extensions/google-meet/src/runtime.ts | 6 +- .../src/test-support/plugin-harness.ts | 3 +- .../src/transports/chrome-create.ts | 52 +++++----- .../google-meet/src/transports/chrome.test.ts | 6 +- .../google-meet/src/transports/chrome.ts | 4 +- .../transports/google-meet-page-scripts.ts | 27 ++--- .../google-meet/src/transports/types.ts | 3 - .../teams-meetings/src/runtime-node.test.ts | 7 +- .../teams-meetings/src/runtime-probes.test.ts | 7 +- extensions/teams-meetings/src/runtime.test.ts | 7 +- extensions/teams-meetings/src/runtime.ts | 4 +- .../teams-meetings-audio-routing.test.ts | 12 +-- .../teams-meetings-caption-capture.test.ts | 8 +- ...-meetings-platform-adapter.test-helpers.ts | 6 +- .../teams-meetings-platform-adapter.test.ts | 40 ++++---- .../teams-meetings-platform-adapter.ts | 4 +- .../teams-meetings-status-prejoin-source.ts | 52 ++++------ .../zoom-meetings/src/runtime-node.test.ts | 7 +- .../zoom-meetings/src/runtime-probes.test.ts | 7 +- extensions/zoom-meetings/src/runtime.test.ts | 31 +++--- extensions/zoom-meetings/src/runtime.ts | 19 ++-- .../zoom-meetings-platform-adapter.test.ts | 31 +++--- .../zoom-meetings-platform-adapter.ts | 4 +- .../zoom-meetings-status-access-source.ts | 6 +- .../zoom-meetings-status-prejoin-source.ts | 45 +++------ src/meeting-bot/browser-controller.test.ts | 10 +- src/meeting-bot/browser-controller.ts | 16 +-- src/meeting-bot/platform-adapter.ts | 32 +++--- src/meeting-bot/runtime-probes.ts | 16 ++- src/meeting-bot/session-runtime.test.ts | 10 +- src/meeting-bot/session-runtime.ts | 7 +- src/meeting-bot/session-speech-readiness.ts | 7 +- src/meeting-bot/session-types.ts | 4 +- src/meeting-bot/status-call-source.ts | 12 +-- src/meeting-bot/status-prejoin-source.ts | 5 +- 42 files changed, 333 insertions(+), 395 deletions(-) diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index 9b90207fa692..165d497747d1 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -117,7 +117,7 @@ openclaw googlemeet create --access-type OPEN --transport chrome-node --mode age This only applies to API-created rooms, so OAuth must be configured. If you authenticated before this option existed, rerun `openclaw googlemeet auth login --json` after adding the `meetings.space.settings` scope to your OAuth consent screen. -If the browser fallback hits a Google login or Meet permission blocker, the tool returns `manualActionRequired: true` with `manualActionReason`, `manualActionMessage`, and the `browser.nodeId`/`browser.targetId`/`browserUrl`. Report that message and stop opening new Meet tabs until the operator finishes the browser step. +If the browser fallback hits a Google login or Meet permission blocker, the tool returns `manualAction: { reason, message }` with the `browser.nodeId`/`browser.targetId`/`browserUrl`. Report that message and stop opening new Meet tabs until the operator finishes the browser step. ### Observe-only join @@ -147,7 +147,7 @@ It joins in transcribe mode, waits for fresh caption/transcript movement, and re ### Realtime session health -During talk-back sessions, `google_meet` status reports Chrome/audio bridge health: `inCall`, `manualActionRequired`, `providerConnected`, `realtimeReady`, `audioInputActive`, `audioOutputActive`, last input/output timestamps, byte counters, and bridge-closed state. Managed Chrome sessions only speak the intro/test phrase after health reports `inCall: true`; otherwise `speechReady: false` and the speech attempt is blocked rather than silently no-opping. +During talk-back sessions, `google_meet` status reports Chrome/audio bridge health: `inCall`, `manualAction`, `providerConnected`, `realtimeReady`, `audioInputActive`, `audioOutputActive`, last input/output timestamps, byte counters, and bridge-closed state. Managed Chrome sessions only speak the intro/test phrase after health reports `inCall: true`; otherwise `speechReady: false` and the speech attempt is blocked rather than silently no-opping. Local Chrome joins through the signed-in OpenClaw browser profile and needs `BlackHole 2ch` for the mic/speaker path. A single BlackHole device is enough for a first smoke test but can echo; use separate virtual devices or a Loopback-style graph for clean duplex audio. @@ -242,7 +242,7 @@ For a one-command smoke test that creates or reuses a session, speaks a known ph openclaw googlemeet test-speech https://meet.google.com/abc-defg-hij ``` -During realtime join, browser automation fills the guest name, clicks Join/Ask to join, and accepts Meet's first-run "Use microphone" prompt when it appears (or "Continue without microphone" during observe-only join and browser-only meeting creation). If the profile is signed out, Meet is waiting for host admission, Chrome needs mic/camera permission, or Meet is stuck on an unresolved prompt, the result reports `manualActionRequired: true` with `manualActionReason` and `manualActionMessage`. Stop retrying, report that message plus `browserUrl`/`browserTitle`, and retry only after the manual action completes. +During realtime join, browser automation fills the guest name, clicks Join/Ask to join, and accepts Meet's first-run "Use microphone" prompt when it appears (or "Continue without microphone" during observe-only join and browser-only meeting creation). If the profile is signed out, Meet is waiting for host admission, Chrome needs mic/camera permission, or Meet is stuck on an unresolved prompt, the result includes `manualAction: { reason, message }`. Stop retrying, report that message plus `browserUrl`/`browserTitle`, and retry only after the manual action completes. If `chromeNode.node` is omitted, OpenClaw auto-selects only when exactly one connected node advertises both `googlemeet.chrome` and browser control; pin `chromeNode.node` (node id, display name, or remote IP) when several capable nodes are connected. @@ -621,9 +621,10 @@ If the browser fallback hits Google login or a Meet permission blocker first, `g { "source": "browser", "error": "google-login-required: Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", - "manualActionRequired": true, - "manualActionReason": "google-login-required", - "manualActionMessage": "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + "manualAction": { + "reason": "google-login-required", + "message": "Sign in to Google in the OpenClaw browser profile, then retry meeting creation." + }, "browser": { "nodeId": "ba0f4e4bc...", "targetId": "tab-1", @@ -654,7 +655,7 @@ API create JSON: } ``` -Creating joins by default, but Chrome/Chrome-node still needs a signed-in Google profile to join through the browser; if signed out, OpenClaw reports `manualActionRequired: true` or a browser fallback error and asks the operator to finish Google login before retrying. +Creating joins by default, but Chrome/Chrome-node still needs a signed-in Google profile to join through the browser; if signed out, OpenClaw returns `manualAction` or a browser fallback error and asks the operator to finish Google login before retrying. Set `preview.enrollmentAcknowledged: true` only after confirming your Cloud project, OAuth principal, and meeting participants are enrolled in the Google Workspace Developer Preview Program for Meet media APIs. @@ -908,20 +909,20 @@ Speaking on demand: `status` includes Chrome health when available: -| Field | Meaning | -| --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -| `inCall` | Chrome appears to be inside the Meet call | -| `micMuted` | Best-effort Meet microphone state | -| `manualActionRequired` / `manualActionReason` / `manualActionMessage` | Browser profile needs manual login, Meet host admission, permissions, or browser-control repair before speech can work | -| `speechReady` / `speechBlockedReason` / `speechBlockedMessage` | Whether managed Chrome speech is allowed now; `speechReady: false` means OpenClaw did not send the intro/test phrase | -| `providerConnected` / `realtimeReady` | Realtime voice bridge state | -| `lastInputAt` / `lastOutputAt` | Last audio seen from/sent to the bridge | -| `audioOutputRouted` / `audioOutputDeviceLabel` | Whether the Meet tab's media output was actively routed to the bridge's BlackHole device | -| `lastOutputLoopbackAt` / `outputLoopbackSignalBytes` | Fresh output whose waveform fingerprint was correlated on the BlackHole microphone capture path | -| `lastOutputLoopbackCorrelation` | Correlation score tying the captured signal to the current assistant-output generation | -| `outputGeneration` / `verifiedOutputGeneration` | Monotonic ids; equality means the current output, rather than an older utterance, passed loopback proof | -| `lastOutputLoopbackRms` / `lastOutputLoopbackPeak` | Audio-energy diagnostics for the latest verified loopback capture chunk | -| `lastSuppressedInputAt` / `suppressedInputBytes` | Loopback input ignored while assistant playback is active | +| Field | Meaning | +| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `inCall` | Chrome appears to be inside the Meet call | +| `micMuted` | Best-effort Meet microphone state | +| `manualAction.reason` / `manualAction.message` | Browser profile needs manual login, Meet host admission, permissions, or browser-control repair before speech can work | +| `speechReady` / `speechBlockedReason` / `speechBlockedMessage` | Whether managed Chrome speech is allowed now; `speechReady: false` means OpenClaw did not send the intro/test phrase | +| `providerConnected` / `realtimeReady` | Realtime voice bridge state | +| `lastInputAt` / `lastOutputAt` | Last audio seen from/sent to the bridge | +| `audioOutputRouted` / `audioOutputDeviceLabel` | Whether the Meet tab's media output was actively routed to the bridge's BlackHole device | +| `lastOutputLoopbackAt` / `outputLoopbackSignalBytes` | Fresh output whose waveform fingerprint was correlated on the BlackHole microphone capture path | +| `lastOutputLoopbackCorrelation` | Correlation score tying the captured signal to the current assistant-output generation | +| `outputGeneration` / `verifiedOutputGeneration` | Monotonic ids; equality means the current output, rather than an older utterance, passed loopback proof | +| `lastOutputLoopbackRms` / `lastOutputLoopbackPeak` | Audio-energy diagnostics for the latest verified loopback capture chunk | +| `lastSuppressedInputAt` / `suppressedInputBytes` | Loopback input ignored while assistant playback is active | ## Agent and bidi modes @@ -1073,7 +1074,7 @@ openclaw nodes status --connected ### Browser opens but agent cannot join -Run `googlemeet test-listen` for observe-only joins or `googlemeet test-speech` for realtime joins, then inspect the returned Chrome health. If either reports `manualActionRequired: true`, show `manualActionMessage` to the operator and stop retrying until the browser action is complete. +Run `googlemeet test-listen` for observe-only joins or `googlemeet test-speech` for realtime joins, then inspect the returned Chrome health. If either includes `manualAction`, show `manualAction.message` to the operator and stop retrying until the browser action is complete. Common manual actions: sign in to the Chrome profile; admit the guest from the Meet host account; grant Chrome microphone/camera permissions when the native prompt appears; close or repair a stuck Meet permission dialog. @@ -1086,7 +1087,7 @@ Do not report "not signed in" just because Meet asks "Do you want people to hear - **API creation**: `oauth.clientId` and `oauth.refreshToken` (or matching `OPENCLAW_GOOGLE_MEET_*` env vars) are present, and the refresh token was minted after create support was added; older tokens may lack `meetings.space.created`, so rerun `openclaw googlemeet auth login --json`. - **Browser fallback**: `defaultTransport: "chrome-node"` and `chromeNode.node` point at a connected node with `browser.proxy` and `googlemeet.chrome`; the OpenClaw Chrome profile on that node is signed in and can open `https://meet.google.com/new`. - **Browser fallback retries**: reuse an existing `.../new` or Google account prompt tab before opening a new one; retry the tool call rather than manually opening another tab. -- **Manual action**: if the tool returns `manualActionRequired: true`, use `browser.nodeId`, `browser.targetId`, `browserUrl`, and `manualActionMessage` to guide the operator; do not retry in a loop. +- **Manual action**: if the tool returns `manualAction`, use `browser.nodeId`, `browser.targetId`, `browserUrl`, and `manualAction.message` to guide the operator; do not retry in a loop. - **Audio-choice interstitial**: if Meet shows "Do you want people to hear you in the meeting?", leave the tab open. OpenClaw should click **Use microphone** or (create-only) **Continue without microphone** and keep waiting for the generated URL; if it cannot, the error should mention `meet-audio-choice-required`, not `google-login-required`. ### Agent joins but does not talk diff --git a/docs/plugins/meeting-plugins.md b/docs/plugins/meeting-plugins.md index bac88b9fe2ae..1e05541f6936 100644 --- a/docs/plugins/meeting-plugins.md +++ b/docs/plugins/meeting-plugins.md @@ -105,7 +105,7 @@ Browser automation handles the normal guest-name, prejoin camera and microphone, - Microsoft Teams may require tenant sign-in, email verification, or organizer admission. - Zoom may require authentication, email verification, a passcode, CAPTCHA completion, or host admission; an account can also disable browser join. -When a join or status result reports `manualActionRequired`, complete the reported step in the same OpenClaw Chrome profile before retrying. Repeatedly opening new tabs does not resolve an account, tenant, lobby, or CAPTCHA gate. +When a join or status result includes `manualAction`, complete its reported step in the same OpenClaw Chrome profile before retrying. Repeatedly opening new tabs does not resolve an account, tenant, lobby, or CAPTCHA gate. Only join meetings where the operator is authorized to add an agent. Tell participants when local policy or consent rules require disclosure of automated participation, transcription, or synthesized speech. diff --git a/extensions/google-meet/index.create.test.ts b/extensions/google-meet/index.create.test.ts index 74ac6dfd8070..ee753a99fe58 100644 --- a/extensions/google-meet/index.create.test.ts +++ b/extensions/google-meet/index.create.test.ts @@ -96,7 +96,7 @@ async function runCreateMeetBrowserScript(params: { buttonText: string }) { vi.stubGlobal("location", location); type BrowserScriptResult = { meetingUri?: string; - manualActionReason?: string; + manualAction?: { reason: string; message: string }; notes?: string[]; retryAfterMs?: number; }; @@ -138,8 +138,10 @@ async function runCreateMeetBrowserScript(params: { buttonText: string }) { ok: true, targetId: "create-script-tab", result: { - manualActionReason: "meet-permission-required", - manualAction: "Stop after exercising the browser script.", + manualAction: { + reason: "meet-permission-required", + message: "Stop after exercising the browser script.", + }, browserUrl: location.href, browserTitle: document.title, }, @@ -420,9 +422,11 @@ describe("google-meet create flow", () => { ok: true, targetId: "login-tab", result: { - manualActionReason: "google-login-required", - manualAction: - "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + manualAction: { + reason: "google-login-required", + message: + "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + }, browserUrl: "https://accounts.google.com/signin", browserTitle: "Sign in - Google Accounts", notes: ["Sign-in page detected."], @@ -450,11 +454,10 @@ describe("google-meet create flow", () => { expect(payload.error).toBe( "google-login-required: Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", ); - expect(payload.manualActionRequired).toBe(true); - expect(payload.manualActionReason).toBe("google-login-required"); - expect(payload.manualActionMessage).toBe( - "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", - ); + expect(payload.manualAction).toEqual({ + reason: "google-login-required", + message: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + }); const browser = requireRecord(payload.browser, "browser payload"); expect(browser.nodeId).toBe("node-1"); expect(browser.targetId).toBe("login-tab"); @@ -594,9 +597,11 @@ describe("google-meet create flow", () => { ok: true, targetId: "permission-tab", result: { - manualActionReason: "meet-permission-required", - manualAction: - "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + manualAction: { + reason: "meet-permission-required", + message: + "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + }, browserUrl: "https://meet.google.com/new", browserTitle: "Meet", }, @@ -615,11 +620,11 @@ describe("google-meet create flow", () => { const result = await tool.execute("id", { action: "create" }); expect(result.details.source).toBe("browser"); - expect(result.details.manualActionRequired).toBe(true); - expect(result.details.manualActionReason).toBe("meet-permission-required"); - expect(result.details.manualActionMessage).toBe( - "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", - ); + expect(result.details.manualAction).toEqual({ + reason: "meet-permission-required", + message: + "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + }); const browser = requireRecord(result.details.browser, "browser details"); expect(browser.nodeId).toBe("node-1"); expect(browser.targetId).toBe("permission-tab"); @@ -835,7 +840,7 @@ describe("google-meet create flow", () => { expect(result.notes).toEqual([note]); expect(button.click).toHaveBeenCalledTimes(1); expect(result.meetingUri).toBeUndefined(); - expect(result.manualActionReason).toBeUndefined(); + expect(result.manualAction).toBeUndefined(); }, ); }); diff --git a/extensions/google-meet/index.test.ts b/extensions/google-meet/index.test.ts index f64bd7d5a3b4..9e2b71de8822 100644 --- a/extensions/google-meet/index.test.ts +++ b/extensions/google-meet/index.test.ts @@ -722,8 +722,10 @@ async function captureMeetStatusScript(params: { script = requireRecord(request.body, "Meet status request body").fn as string; return { result: JSON.stringify({ - manualActionRequired: true, - manualActionReason: "meet-admission-required", + manualAction: { + reason: "meet-admission-required", + message: "Waiting for admission", + }, }), }; } @@ -3764,9 +3766,10 @@ describe("google-meet plugin", () => { result: JSON.stringify({ inCall: false, lobbyWaiting: true, - manualActionRequired: true, - manualActionReason: "meet-admission-required", - manualActionMessage: "Admit the OpenClaw browser participant in Google Meet.", + manualAction: { + reason: "meet-admission-required", + message: "Admit the OpenClaw browser participant in Google Meet.", + }, title: "Meet", url: MEET_URL, }), @@ -3970,9 +3973,10 @@ describe("google-meet plugin", () => { }), inspect: () => ({ inCall: false, - manualActionRequired: true, - manualActionReason: "meet-audio-choice-required", - manualActionMessage: "Choose the Meet microphone path manually.", + manualAction: { + reason: "meet-audio-choice-required", + message: "Choose the Meet microphone path manually.", + }, title: "Meet", url: MEET_URL, }), @@ -3989,9 +3993,12 @@ describe("google-meet plugin", () => { const status = (await invokeGoogleMeetGatewayMethodForTest(methods, "googlemeet.status", { sessionId: join.session.id, - })) as { session?: { chrome?: { health?: { manualActionRequired?: boolean } } } }; + })) as { session?: { chrome?: { health?: { manualAction?: unknown } } } }; - expect(status.session?.chrome?.health?.manualActionRequired).toBe(true); + expect(status.session?.chrome?.health?.manualAction).toEqual({ + reason: "meet-audio-choice-required", + message: "Choose the Meet microphone path manually.", + }); const actCall = nodesInvoke.mock.calls.find(([rawCall]) => { const call = requireRecord(rawCall, "node invoke"); const params = requireRecord(call.params, "node invoke params"); @@ -4288,15 +4295,12 @@ describe("google-meet plugin", () => { const result = JSON.parse(await inspect()) as { inCall?: boolean; - manualActionRequired?: boolean; - manualActionReason?: string; - manualActionMessage?: string; + manualAction?: { reason: string; message: string }; }; expect(result.inCall).toBe(true); - expect(result.manualActionRequired).toBe(true); - expect(result.manualActionReason).toBe("meet-permission-required"); - expect(result.manualActionMessage).toContain("Allow microphone/camera/speaker permissions"); + expect(result.manualAction?.reason).toBe("meet-permission-required"); + expect(result.manualAction?.message).toContain("Allow microphone/camera/speaker permissions"); }); it("does not auto-join when Meet is already active elsewhere", async () => { @@ -4336,11 +4340,11 @@ describe("google-meet plugin", () => { const result = JSON.parse(await inspect()) as { clickedJoin?: boolean; - manualActionReason?: string; + manualAction?: { reason: string; message: string }; }; expect(result.clickedJoin).toBe(false); - expect(result.manualActionReason).toBe("meet-session-conflict"); + expect(result.manualAction?.reason).toBe("meet-session-conflict"); expect(joinElsewhere.click).not.toHaveBeenCalled(); }); @@ -4847,9 +4851,10 @@ describe("google-meet plugin", () => { expect(result.details.found).toBe(true); expect(result.details.targetId).toBe("existing-meet-tab"); const browser = requireRecord(result.details.browser, "recovered browser state"); - expect(browser.manualActionRequired).toBe(true); - expect(browser.manualActionReason).toBe("meet-locale-required"); - expect(browser.manualActionMessage).toContain("not pinned to English"); + expect(browser.manualAction).toMatchObject({ + reason: "meet-locale-required", + message: expect.stringContaining("not pinned to English"), + }); const focusCall = nodesInvoke.mock.calls.find(([rawCall]) => { const call = requireRecord(rawCall, "node invoke"); const params = requireRecord(call.params, "node invoke params"); @@ -4941,9 +4946,10 @@ describe("google-meet plugin", () => { focus: true, inspect: () => ({ inCall: false, - manualActionRequired: true, - manualActionReason: "google-login-required", - manualActionMessage: "Sign in to Google, then retry.", + manualAction: { + reason: "google-login-required", + message: "Sign in to Google, then retry.", + }, url: "https://accounts.google.com/signin", }), }), @@ -4954,7 +4960,10 @@ describe("google-meet plugin", () => { const result = await tool.execute("id", { action: "recover_current_tab" }); const browser = requireRecord(result.details.browser, "recovered browser state"); expect(result.details.targetId).toBe("google-sign-in-tab"); - expect(browser.manualActionReason).toBe("google-login-required"); + expect(browser.manualAction).toEqual({ + reason: "google-login-required", + message: "Sign in to Google, then retry.", + }); }); it("reports an ambiguous local Chrome Meet tab without reloading it", async () => { @@ -4990,9 +4999,10 @@ describe("google-meet plugin", () => { return { result: JSON.stringify({ inCall: false, - manualActionRequired: true, - manualActionReason: "meet-admission-required", - manualActionMessage: "Admit the OpenClaw browser participant in Google Meet.", + manualAction: { + reason: "meet-admission-required", + message: "Admit the OpenClaw browser participant in Google Meet.", + }, title: "Meet", url: "https://meet.google.com/abc-defg-hij?authuser=me%40example.com&hl=en", }), @@ -5020,9 +5030,10 @@ describe("google-meet plugin", () => { expect(result.details.found).toBe(true); expect(result.details.targetId).toBe("local-meet-tab"); const browser = requireRecord(result.details.browser, "recovered browser state"); - expect(browser.manualActionRequired).toBe(true); - expect(browser.manualActionReason).toBe("meet-locale-required"); - expect(browser.manualActionMessage).toContain("not pinned to English"); + expect(browser.manualAction).toMatchObject({ + reason: "meet-locale-required", + message: expect.stringContaining("not pinned to English"), + }); const focusCall = callGatewayFromCli.mock.calls.find( (call) => requireRecord(call[2], "browser request").path === "/tabs/focus", ); @@ -6155,10 +6166,10 @@ describe("google-meet plugin", () => { { browserActResult: { inCall: false, - manualActionRequired: true, - manualActionReason: "google-login-required", - manualActionMessage: - "Sign in to Google in the OpenClaw browser profile, then retry the Meet join.", + manualAction: { + reason: "google-login-required", + message: "Sign in to Google in the OpenClaw browser profile, then retry the Meet join.", + }, title: "Sign in - Google Accounts", url: "https://accounts.google.com/signin", }, @@ -6177,16 +6188,20 @@ describe("google-meet plugin", () => { message: "Say exactly: hello.", }); - expect(result.details.manualActionRequired).toBe(true); - expect(result.details.manualActionReason).toBe("google-login-required"); + expect(result.details.manualAction).toEqual({ + reason: "google-login-required", + message: "Sign in to Google in the OpenClaw browser profile, then retry the Meet join.", + }); expect(result.details.spoken).toBe(false); expect(result.details.speechReady).toBe(false); expect(result.details.speechBlockedReason).toBe("google-login-required"); const session = requireRecord(result.details.session, "manual action session"); const chrome = requireRecord(session.chrome, "manual action session chrome"); const health = requireRecord(chrome.health, "manual action chrome health"); - expect(health.manualActionRequired).toBe(true); - expect(health.manualActionReason).toBe("google-login-required"); + expect(health.manualAction).toEqual({ + reason: "google-login-required", + message: "Sign in to Google in the OpenClaw browser profile, then retry the Meet join.", + }); expect(health.speechReady).toBe(false); expect(health.speechBlockedReason).toBe("google-login-required"); }); @@ -6223,7 +6238,6 @@ describe("google-meet plugin", () => { ? { inCall: true, micMuted: false, - manualActionRequired: false, title: "Meet call", url: MEET_URL, } @@ -6254,7 +6268,7 @@ describe("google-meet plugin", () => { chrome?: { health?: { inCall?: boolean; - manualActionRequired?: boolean; + manualAction?: unknown; speechBlockedReason?: string; }; }; @@ -6267,7 +6281,7 @@ describe("google-meet plugin", () => { const retryChrome = requireRecord(retrySession.chrome, "retry session chrome"); const retryHealth = requireRecord(retryChrome.health, "retry chrome health"); expect(retryHealth.inCall).toBe(true); - expect(retryHealth.manualActionRequired).toBe(false); + expect(retryHealth.manualAction).toBeUndefined(); expect(retryHealth.speechBlockedReason).toBe("audio-bridge-unavailable"); const nodeStartCalls = nodesInvoke.mock.calls.filter(([rawCall]) => { const call = requireRecord(rawCall, "node invoke"); diff --git a/extensions/google-meet/src/cli-runtime.test.ts b/extensions/google-meet/src/cli-runtime.test.ts index 9854139612a0..af5e43053bfc 100644 --- a/extensions/google-meet/src/cli-runtime.test.ts +++ b/extensions/google-meet/src/cli-runtime.test.ts @@ -363,9 +363,7 @@ describe("google-meet CLI", () => { const testListen = vi.fn(async () => ({ createdSession: true, inCall: true, - manualActionRequired: false, - manualActionReason: undefined, - manualActionMessage: undefined, + manualAction: undefined, listenVerified: true, listenTimedOut: false, captioning: true, diff --git a/extensions/google-meet/src/cli-shared.ts b/extensions/google-meet/src/cli-shared.ts index a449bb0bcd10..13d93c50cdd2 100644 --- a/extensions/google-meet/src/cli-shared.ts +++ b/extensions/google-meet/src/cli-shared.ts @@ -371,10 +371,10 @@ export function writeDoctorStatus(status: Awaited { tab: { targetId: "tab-1", url: "https://meet.google.com/abc-defg-hij" }, browser: { inCall: false, - manualActionRequired: true, - manualActionReason: "meet-admission-required", - manualActionMessage: "Admit the OpenClaw browser participant in Google Meet.", + manualAction: { + reason: "meet-admission-required", + message: "Admit the OpenClaw browser participant in Google Meet.", + }, browserUrl: "https://meet.google.com/abc-defg-hij", }, message: "Admit the OpenClaw browser participant in Google Meet.", diff --git a/extensions/google-meet/src/runtime.ts b/extensions/google-meet/src/runtime.ts index f0f049ae7d9f..51135760e34c 100644 --- a/extensions/google-meet/src/runtime.ts +++ b/extensions/google-meet/src/runtime.ts @@ -67,7 +67,7 @@ type ChromeAudioBridgeResult = NonNullable< type ChromeLaunchResult = | Awaited> | Awaited>; -type GoogleMeetManualActionReason = NonNullable; +type GoogleMeetManualActionReason = NonNullable["reason"]; type GoogleMeetSpeechBlockedReason = NonNullable; type GoogleMeetSessionRuntime = MeetingSessionRuntime< GoogleMeetSession, @@ -130,8 +130,6 @@ export class GoogleMeetRuntime { speech: { audioBridgeUnavailable: "Realtime speech requires an active Chrome audio bridge.", browserUnverified: "Google Meet browser state has not been verified yet.", - manualActionFallback: - "Resolve the Google Meet browser prompt before asking OpenClaw to speak.", microphoneMuted: "Turn on the OpenClaw Google Meet microphone before asking OpenClaw to speak.", microphoneMutedReason: "meet-microphone-muted", @@ -476,7 +474,7 @@ export class GoogleMeetRuntime { session.chrome.audioBridge || session.chrome.health?.inCall !== true || session.chrome.health.micMuted !== false || - session.chrome.health.manualActionRequired === true + session.chrome.health.manualAction ) { return undefined; } diff --git a/extensions/google-meet/src/test-support/plugin-harness.ts b/extensions/google-meet/src/test-support/plugin-harness.ts index fb22660ff29a..c1bc2464c0e0 100644 --- a/extensions/google-meet/src/test-support/plugin-harness.ts +++ b/extensions/google-meet/src/test-support/plugin-harness.ts @@ -225,8 +225,7 @@ export function setupGoogleMeetPlugin( type GoogleMeetToolError = { error?: string; - manualActionRequired?: boolean; - manualActionReason?: string; + manualAction?: { reason: string; message: string }; }; type GoogleMeetToolDetails = { diff --git a/extensions/google-meet/src/transports/chrome-create.ts b/extensions/google-meet/src/transports/chrome-create.ts index 0756367da804..b3cb19fc38c0 100644 --- a/extensions/google-meet/src/transports/chrome-create.ts +++ b/extensions/google-meet/src/transports/chrome-create.ts @@ -18,12 +18,13 @@ const GOOGLE_MEET_BROWSER_STEP_TIMEOUT_MS = 10_000; const GOOGLE_MEET_BROWSER_NAVIGATION_RETRY_MS = 1_000; const GOOGLE_MEET_BROWSER_POLL_MS = 500; +type GoogleMeetBrowserManualActionState = NonNullable; + type BrowserCreateStepResult = { meetingUri?: string; browserUrl?: string; browserTitle?: string; - manualAction?: string; - manualActionReason?: GoogleMeetChromeHealth["manualActionReason"]; + manualAction?: GoogleMeetBrowserManualActionState; notes?: string[]; retryAfterMs?: number; }; @@ -42,9 +43,7 @@ type GoogleMeetBrowserCreateResult = { type GoogleMeetBrowserManualAction = { source: "browser"; error: string; - manualActionRequired: true; - manualActionReason?: GoogleMeetChromeHealth["manualActionReason"]; - manualActionMessage: string; + manualAction: GoogleMeetBrowserManualActionState; browser: { nodeId: string; targetId?: string; @@ -58,8 +57,7 @@ class GoogleMeetBrowserManualActionError extends Error { readonly payload: GoogleMeetBrowserManualAction; constructor(payload: Omit) { - const prefix = payload.manualActionReason ? `${payload.manualActionReason}: ` : ""; - super(`${prefix}${payload.manualActionMessage}`); + super(`${payload.manualAction.reason}: ${payload.manualAction.message}`); this.name = "GoogleMeetBrowserManualActionError"; this.payload = { source: "browser", @@ -142,6 +140,19 @@ function readStringArray(value: unknown): string[] | undefined { : undefined; } +function readBrowserManualAction(value: unknown): GoogleMeetBrowserManualActionState | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const action = value as Record; + return typeof action.reason === "string" && typeof action.message === "string" + ? { + reason: action.reason as GoogleMeetBrowserManualActionState["reason"], + message: action.message, + } + : undefined; +} + function readBrowserCreateResult(result: unknown): BrowserCreateStepResult { const record = result && typeof result === "object" ? (result as Record) : {}; const nested = @@ -152,11 +163,7 @@ function readBrowserCreateResult(result: unknown): BrowserCreateStepResult { meetingUri: typeof nested.meetingUri === "string" ? nested.meetingUri : undefined, browserUrl: typeof nested.browserUrl === "string" ? nested.browserUrl : undefined, browserTitle: typeof nested.browserTitle === "string" ? nested.browserTitle : undefined, - manualAction: typeof nested.manualAction === "string" ? nested.manualAction : undefined, - manualActionReason: - typeof nested.manualActionReason === "string" - ? (nested.manualActionReason as GoogleMeetChromeHealth["manualActionReason"]) - : undefined, + manualAction: readBrowserManualAction(nested.manualAction), notes: readStringArray(nested.notes), retryAfterMs: typeof nested.retryAfterMs === "number" && Number.isFinite(nested.retryAfterMs) @@ -169,6 +176,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { const meetUrlPattern = /^https:\\/\\/meet\\.google\\.com\\/[a-z]{3}-[a-z]{4}-[a-z]{3}(?:$|[/?#])/i; const text = (node) => (node?.innerText || node?.textContent || "").trim(); const current = () => location.href; + const manualActionFor = (reason, message) => ({ reason, message }); const notes = []; const findButton = (pattern) => [...document.querySelectorAll("button")].find((button) => { @@ -192,8 +200,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { }; if (!current().startsWith("https://meet.google.com/")) { return { - manualActionReason: "google-login-required", - manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + manualAction: manualActionFor("google-login-required", "Sign in to Google in the OpenClaw browser profile, then retry meeting creation."), browserUrl: current(), browserTitle: document.title, notes, @@ -219,8 +226,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { } if (/do you want people to hear you in the meeting/i.test(pageText)) { return { - manualActionReason: "meet-audio-choice-required", - manualAction: "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry meeting creation.", + manualAction: manualActionFor("meet-audio-choice-required", "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry meeting creation."), browserUrl: href, browserTitle: document.title, notes, @@ -228,8 +234,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { } if (/allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) { return { - manualActionReason: "meet-permission-required", - manualAction: "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + manualAction: manualActionFor("meet-permission-required", "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation."), browserUrl: href, browserTitle: document.title, notes, @@ -237,7 +242,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { } if (/couldn't create|unable to create/i.test(pageText)) { return { - manualAction: "Resolve the Google Meet page prompt in the OpenClaw browser profile, then retry meeting creation.", + manualAction: manualActionFor("browser-control-unavailable", "Resolve the Google Meet page prompt in the OpenClaw browser profile, then retry meeting creation."), browserUrl: href, browserTitle: document.title, notes, @@ -245,8 +250,7 @@ const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { } if (location.hostname.toLowerCase() === "accounts.google.com" || /use your google account|to continue to google meet|choose an account|sign in to (join|continue)/i.test(pageText)) { return { - manualActionReason: "google-login-required", - manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + manualAction: manualActionFor("google-login-required", "Sign in to Google in the OpenClaw browser profile, then retry meeting creation."), browserUrl: href, browserTitle: document.title, notes, @@ -363,9 +367,7 @@ export async function createMeetWithBrowserProxyOnNode(params: { } if (result.manualAction) { throw new GoogleMeetBrowserManualActionError({ - manualActionRequired: true, - manualActionReason: result.manualActionReason, - manualActionMessage: result.manualAction, + manualAction: result.manualAction, browser: { nodeId, targetId, @@ -385,7 +387,7 @@ export async function createMeetWithBrowserProxyOnNode(params: { } } throw new Error( - lastResult?.manualAction ?? + lastResult?.manualAction?.message ?? `Google Meet did not return a meeting URL from the browser create flow before timeout.${ lastError ? ` Last browser automation error: ${formatBrowserAutomationError(lastError)}` diff --git a/extensions/google-meet/src/transports/chrome.test.ts b/extensions/google-meet/src/transports/chrome.test.ts index 0b13ba00150d..06cbf5107612 100644 --- a/extensions/google-meet/src/transports/chrome.test.ts +++ b/extensions/google-meet/src/transports/chrome.test.ts @@ -231,8 +231,10 @@ describe("google meet chrome transport", () => { if (params.path === "/act") { return { result: JSON.stringify({ - manualActionRequired: true, - manualActionReason: "meet-admission-required", + manualAction: { + reason: "meet-admission-required", + message: "Waiting for admission", + }, }), }; } diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index b555e0128ebe..7729823f6868 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -225,7 +225,7 @@ export async function launchChromeMeet(params: { MeetingPlatformAdapter.isTalkBackMode(params.mode) && result.browser?.inCall === true && result.browser.micMuted === false && - result.browser.manualActionRequired !== true; + result.browser.manualAction === undefined; const audioBridge = shouldStartRealtimeBridge ? await startRealtimeAudioBridge() : undefined; return { ...result, audioBridge }; } @@ -525,7 +525,7 @@ export async function launchChromeMeetOnNode(params: { MeetingPlatformAdapter.isTalkBackMode(params.mode) && (browserControl.browser?.inCall !== true || browserControl.browser.micMuted !== false || - browserControl.browser.manualActionRequired === true) + browserControl.browser.manualAction) ) { return { nodeId, diff --git a/extensions/google-meet/src/transports/google-meet-page-scripts.ts b/extensions/google-meet/src/transports/google-meet-page-scripts.ts index 277b801eba99..beede8c9ff78 100644 --- a/extensions/google-meet/src/transports/google-meet-page-scripts.ts +++ b/extensions/google-meet/src/transports/google-meet-page-scripts.ts @@ -14,6 +14,7 @@ export function meetStatusScript(params: { }) { return `async () => { const text = (node) => (node?.innerText || node?.textContent || "").trim(); + const manualActionFor = (reason, message) => ({ reason, message }); const allowMicrophone = ${JSON.stringify(params.allowMicrophone)}; const captionSessionId = ${JSON.stringify(params.captionSessionId)}; const captureCaptions = ${JSON.stringify(params.captureCaptions)}; @@ -312,27 +313,17 @@ export function meetStatusScript(params: { const leaveReason = !inCall && /you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i.test(pageText) ? pageText.match(/you left the meeting|you.?ve left the meeting|removed from the meeting|you were removed|call ended|meeting ended/i)?.[0] : undefined; - let manualActionReason; - let manualActionMessage; + let manualAction; if (!inCall && (host === "accounts.google.com" || /use your google account|to continue to google meet|choose an account|sign in to (join|continue)/i.test(pageText))) { - manualActionReason = "google-login-required"; - manualActionMessage = "Sign in to Google in the OpenClaw browser profile, then retry the Meet join."; + manualAction = manualActionFor("google-login-required", "Sign in to Google in the OpenClaw browser profile, then retry the Meet join."); } else if (!inCall && joinElsewhere) { - manualActionReason = "meet-session-conflict"; - manualActionMessage = "Meet is already active in another tab or device. Leave that session or reuse an English-pinned tab before retrying."; + manualAction = manualActionFor("meet-session-conflict", "Meet is already active in another tab or device. Leave that session or reuse an English-pinned tab before retrying."); } else if (!inCall && /asking to be let in|you.?ll join when someone lets you in|waiting to be let in|ask to join/i.test(pageText)) { - manualActionReason = "meet-admission-required"; - manualActionMessage = "Admit the OpenClaw browser participant in Google Meet, then retry speech."; + manualAction = manualActionFor("meet-admission-required", "Admit the OpenClaw browser participant in Google Meet, then retry speech."); } else if (permissionNeeded) { - manualActionReason = "meet-permission-required"; - manualActionMessage = allowMicrophone - ? "Allow microphone/camera/speaker permissions for Meet in the OpenClaw browser profile, then retry." - : "Join without microphone/camera permissions in the OpenClaw browser profile, then retry."; + manualAction = manualActionFor("meet-permission-required", allowMicrophone ? "Allow microphone/camera/speaker permissions for Meet in the OpenClaw browser profile, then retry." : "Join without microphone/camera permissions in the OpenClaw browser profile, then retry."); } else if (!inCall && (allowMicrophone ? !microphoneChoice : !noMicrophoneChoice) && /do you want people to hear you in the meeting/i.test(pageText)) { - manualActionReason = "meet-audio-choice-required"; - manualActionMessage = allowMicrophone - ? "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry." - : "Meet is showing the microphone choice. Choose the no-microphone option in the OpenClaw browser profile, then retry."; + manualAction = manualActionFor("meet-audio-choice-required", allowMicrophone ? "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry." : "Meet is showing the microphone choice. Choose the no-microphone option in the OpenClaw browser profile, then retry."); } return JSON.stringify({ clickedJoin: Boolean(join), @@ -351,9 +342,7 @@ export function meetStatusScript(params: { audioOutputRouted, audioOutputDeviceLabel, audioOutputRouteError, - manualActionRequired: Boolean(manualActionReason), - manualActionReason, - manualActionMessage, + manualAction, title: document.title, url: pageUrl, notes diff --git a/extensions/google-meet/src/transports/types.ts b/extensions/google-meet/src/transports/types.ts index 55350a6383dd..ede458f2109c 100644 --- a/extensions/google-meet/src/transports/types.ts +++ b/extensions/google-meet/src/transports/types.ts @@ -87,9 +87,6 @@ export type GoogleMeetChromeHealth = MeetingBrowserHealth< timestamp: string; final?: boolean; }>; - manualActionRequired?: boolean; - manualActionReason?: GoogleMeetManualActionReason; - manualActionMessage?: string; speechReady?: boolean; speechBlockedReason?: GoogleMeetSpeechBlockedReason; speechBlockedMessage?: string; diff --git a/extensions/teams-meetings/src/runtime-node.test.ts b/extensions/teams-meetings/src/runtime-node.test.ts index bcd22582bbfb..f6038b502089 100644 --- a/extensions/teams-meetings/src/runtime-node.test.ts +++ b/extensions/teams-meetings/src/runtime-node.test.ts @@ -79,9 +79,10 @@ describe("Microsoft Teams meetings node realtime recovery", () => { } : { inCall: false, - manualActionMessage: "Waiting for admission", - manualActionReason: "teams-admission-required", - manualActionRequired: true, + manualAction: { + reason: "teams-admission-required", + message: "Waiting for admission", + }, url: URL, }, ), diff --git a/extensions/teams-meetings/src/runtime-probes.test.ts b/extensions/teams-meetings/src/runtime-probes.test.ts index d9378b9b646e..44007c808a46 100644 --- a/extensions/teams-meetings/src/runtime-probes.test.ts +++ b/extensions/teams-meetings/src/runtime-probes.test.ts @@ -139,7 +139,7 @@ describe("Microsoft Teams meeting runtime probes", () => { session.chrome!.health = { ...session.chrome!.health, lastCaptionText: "Caption already waiting", - manualActionRequired: true, + manualAction: { reason: "teams-admission-required", message: "Waiting" }, transcriptLines: 1, }; }, @@ -162,7 +162,10 @@ describe("Microsoft Teams meeting runtime probes", () => { }); expect(result.listenVerified).toBe(true); - expect(result.manualActionRequired).toBe(true); + expect(result.manualAction).toEqual({ + reason: "teams-admission-required", + message: "Waiting", + }); expect(refreshCaptionHealth).toHaveBeenCalledTimes(1); expect(Date.now()).toBe(0); expect(vi.getTimerCount()).toBe(0); diff --git a/extensions/teams-meetings/src/runtime.test.ts b/extensions/teams-meetings/src/runtime.test.ts index 94a8ea03df95..14c8d22a72c3 100644 --- a/extensions/teams-meetings/src/runtime.test.ts +++ b/extensions/teams-meetings/src/runtime.test.ts @@ -57,9 +57,10 @@ function runtimeHarness(options?: { tabOpen?: boolean }) { cameraOff: true, ...(sessionConflict && fn.includes("const allowSessionAdoption = false") ? { - manualActionRequired: true, - manualActionReason: "teams-session-conflict", - manualActionMessage: "This Teams tab is owned by another active meeting session.", + manualAction: { + reason: "teams-session-conflict", + message: "This Teams tab is owned by another active meeting session.", + }, } : {}), url: tabUrl, diff --git a/extensions/teams-meetings/src/runtime.ts b/extensions/teams-meetings/src/runtime.ts index 75127db2b50c..77da257323fe 100644 --- a/extensions/teams-meetings/src/runtime.ts +++ b/extensions/teams-meetings/src/runtime.ts @@ -37,7 +37,7 @@ import type { TeamsMeetingsSession, } from "./transports/types.js"; -type ManualActionReason = NonNullable; +type ManualActionReason = NonNullable["reason"]; type SpeechBlockedReason = NonNullable; type SessionRuntime = MeetingSessionRuntime< TeamsMeetingsSession, @@ -123,8 +123,6 @@ export class TeamsMeetingsRuntime { speech: { audioBridgeUnavailable: "Realtime speech requires an active Chrome audio bridge.", browserUnverified: "Microsoft Teams browser state has not been verified yet.", - manualActionFallback: - "Resolve the Microsoft Teams browser prompt before asking OpenClaw to speak.", microphoneMuted: "Turn on the OpenClaw Teams microphone before asking OpenClaw to speak.", microphoneMutedReason: "teams-microphone-muted", notInCall: "Microsoft Teams has not reported that the browser guest is in the call.", diff --git a/extensions/teams-meetings/src/transports/teams-meetings-audio-routing.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-audio-routing.test.ts index e038758f7669..7692c74007b7 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-audio-routing.test.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-audio-routing.test.ts @@ -50,7 +50,7 @@ describe("Microsoft Teams meeting audio routing", () => { audioInputRouted: true, audioOutputRouted: false, audioOutputRouteRetryable: true, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, }); }); @@ -89,7 +89,7 @@ describe("Microsoft Teams meeting audio routing", () => { expect(result).toMatchObject({ audioInputRouted: true, audioOutputRouted: false, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, }); expect(result.audioOutputRouteError).toBeUndefined(); expect(window["__openclawTeamsAudioOutputs"]).toEqual([ @@ -232,7 +232,7 @@ describe("Microsoft Teams meeting audio routing", () => { expect(source.sinkId).toBe("built-in-output"); expect(source.muted).toBe(true); - expect(conflict.result.manualActionReason).toBe("teams-session-conflict"); + expect(conflict.result.manualAction).toMatchObject({ reason: "teams-session-conflict" }); expect(conflict.window).not.toHaveProperty("__openclawTeamsAudioOutputs"); }); @@ -912,8 +912,8 @@ describe("Microsoft Teams meeting audio routing", () => { expect(result).toMatchObject({ audioInputRouted: true, audioOutputRouted: true, - manualActionRequired: false, }); + expect(result.manualAction).toBeUndefined(); }); it("keeps a loaded non-MediaStream AbortError retryable", async () => { @@ -948,7 +948,7 @@ describe("Microsoft Teams meeting audio routing", () => { audioOutputRouteError: "Cannot route loaded media.", audioOutputRouteRetryable: true, audioOutputRouted: false, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, }); }); @@ -1017,7 +1017,7 @@ describe("Microsoft Teams meeting audio routing", () => { expect(result).toMatchObject({ audioInputRouted: false, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, micMuted: true, }); expect(microphone.clicks).toBe(1); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts index 53ab117fcc3f..a33b13832776 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts @@ -783,8 +783,7 @@ describe("Microsoft Teams meeting captions and permissions", () => { }); expect(result).toMatchObject({ - manualActionReason: "teams-session-conflict", - manualActionRequired: true, + manualAction: { reason: "teams-session-conflict" }, }); expect(window[MEETING_STATE_KEY]).toBe(priorMeeting); expect(window["__openclawTeamsCaptions"]).toBe(priorCaptions); @@ -821,8 +820,7 @@ describe("Microsoft Teams meeting captions and permissions", () => { }); expect(result).toMatchObject({ - manualActionReason: "teams-session-conflict", - manualActionRequired: true, + manualAction: { reason: "teams-session-conflict" }, }); expect(window[MEETING_STATE_KEY]).toBe(priorMeeting); expect(window["__openclawTeamsCaptions"]).toBe(priorCaptions); @@ -851,7 +849,7 @@ describe("Microsoft Teams meeting captions and permissions", () => { }, }); - expect(result.manualActionRequired).toBe(false); + expect(result.manualAction).toBeUndefined(); expect(disconnects).toBe(1); expect(staleCaptions).toMatchObject({ finalized: true }); expect(window["__openclawTeamsCaptions"]).toMatchObject({ sessionId: "session-1" }); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test-helpers.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test-helpers.ts index 6cc0607536a0..c284b9631111 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test-helpers.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test-helpers.ts @@ -15,13 +15,11 @@ export function consumerLightMeetingUrl(meetingCode: string, passcode: string) { return `https://teams.live.com/light-meetings/launch?coords=${encodeURIComponent(coordinates)}`; } -export function status(manualActionReason: string, manualActionMessage = "manual action") { +export function status(reason: string, message = "manual action") { const health = TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ result: JSON.stringify({ inCall: false, - manualActionRequired: true, - manualActionReason, - manualActionMessage, + manualAction: { reason, message }, url: URL, }), }); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts index e7e8259011f8..1a6a3a5d3655 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts @@ -72,8 +72,7 @@ describe("Microsoft Teams meeting platform adapter", () => { expect( TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({ ...pending, - manualActionReason: "teams-session-conflict", - manualActionRequired: true, + manualAction: { reason: "teams-session-conflict", message: "Conflict" }, }), ).toBe(false); }); @@ -162,7 +161,8 @@ describe("Microsoft Teams meeting platform adapter", () => { }); expect(join.clicks).toBe(1); - expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false }); + expect(result.clickedJoin).toBe(true); + expect(result.manualAction).toBeUndefined(); }); it("preserves a newer owner for a different meeting identity", async () => { @@ -177,7 +177,7 @@ describe("Microsoft Teams meeting platform adapter", () => { priorMeeting, }); - expect(result).toMatchObject({ manualActionReason: "teams-session-conflict" }); + expect(result.manualAction).toMatchObject({ reason: "teams-session-conflict" }); expect(window[MEETING_STATE_KEY]).toBe(priorMeeting); }); @@ -192,7 +192,7 @@ describe("Microsoft Teams meeting platform adapter", () => { }, }); - expect(result).toMatchObject({ manualActionReason: "teams-session-conflict" }); + expect(result.manualAction).toMatchObject({ reason: "teams-session-conflict" }); }); it("does not unmute or join until BlackHole is visibly selected as the Teams input", async () => { @@ -211,7 +211,7 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ audioInputRouted: false, clickedJoin: false, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, micMuted: true, }); expect(microphone.clicks).toBe(0); @@ -229,8 +229,7 @@ describe("Microsoft Teams meeting platform adapter", () => { }); expect(result).toMatchObject({ - manualActionReason: "teams-microphone-required", - manualActionRequired: true, + manualAction: { reason: "teams-microphone-required" }, }); expect(join.clicks).toBe(0); }); @@ -255,7 +254,7 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ audioInputRouted: false, clickedJoin: false, - manualActionReason: "teams-audio-choice-required", + manualAction: { reason: "teams-audio-choice-required" }, }); expect(microphone.clicks).toBe(0); }); @@ -279,7 +278,7 @@ describe("Microsoft Teams meeting platform adapter", () => { meetingUrl: CONSUMER_URL, }); - expect(result.manualActionRequired).toBe(false); + expect(result.manualAction).toBeUndefined(); expect(window[MEETING_STATE_KEY]).toMatchObject({ identity: "teams-consumer:9326458712345:p:abc", sessionId: "session-1", @@ -295,7 +294,7 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ inCall: false, - manualActionReason: "teams-session-conflict", + manualAction: { reason: "teams-session-conflict" }, }); expect(window).not.toHaveProperty(MEETING_STATE_KEY); }); @@ -671,8 +670,8 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ inCall: true, - manualActionRequired: false, }); + expect(result.manualAction).toBeUndefined(); }); it("classifies a stable device permission prompt outside the call", async () => { @@ -683,8 +682,7 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ inCall: false, - manualActionReason: "teams-permission-required", - manualActionRequired: true, + manualAction: { reason: "teams-permission-required" }, }); }); @@ -696,7 +694,7 @@ describe("Microsoft Teams meeting platform adapter", () => { permissionPrompt: control({ label: "Device permission prompt" }), }); - expect(result).toMatchObject({ manualActionRequired: false }); + expect(result.manualAction).toBeUndefined(); expect(continueWithoutDevices.clicks).toBe(1); }); @@ -712,7 +710,8 @@ describe("Microsoft Teams meeting platform adapter", () => { microphoneDevice: control({ label: "BlackHole 2ch" }), }); - expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false }); + expect(result.clickedJoin).toBe(true); + expect(result.manualAction).toBeUndefined(); expect(join.clicks).toBe(1); }); @@ -730,7 +729,8 @@ describe("Microsoft Teams meeting platform adapter", () => { permissionPrompt: control({ label: "Camera permission warning" }), }); - expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false }); + expect(result.clickedJoin).toBe(true); + expect(result.manualAction).toBeUndefined(); expect(join.clicks).toBe(1); }); @@ -798,9 +798,9 @@ describe("Microsoft Teams meeting platform adapter", () => { audioInputRouted: true, audioOutputRouted: true, inCall: true, - manualActionRequired: false, micMuted: false, }); + expect(result.manualAction).toBeUndefined(); expect(media.sinkId).toBe("blackhole-output"); }); @@ -826,8 +826,8 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ audioInputRouted: true, audioOutputRouted: true, - manualActionRequired: false, }); + expect(result.manualAction).toBeUndefined(); }); it("routes a directly playable media element before its MediaStream is attached", async () => { @@ -972,8 +972,8 @@ describe("Microsoft Teams meeting platform adapter", () => { expect(result).toMatchObject({ audioInputRouted: true, audioOutputRouted: true, - manualActionRequired: false, }); + expect(result.manualAction).toBeUndefined(); expect(bridge.sinkId).toBe("blackhole-output"); expect(source.muted).toBe(true); expect(window).toHaveProperty("__openclawTeamsAudioOutputs"); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts index ff5f3dfd2ae8..59a223e1de7c 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts @@ -107,10 +107,10 @@ export const TEAMS_MEETINGS_PLATFORM_ADAPTER = MeetingPlatformAdapter.create< }), shouldRetryJoinStatus: (health) => health.inCall === true && - ((health.manualActionReason === "teams-audio-choice-required" && + ((health.manualAction?.reason === "teams-audio-choice-required" && health.audioInputRouted === true && health.audioOutputRouteRetryable === true) || - (health.manualActionRequired !== true && + (health.manualAction === undefined && health.captionCaptureRequested === true && health.captioning !== true)), browserControlUnavailable: () => ({ diff --git a/extensions/teams-meetings/src/transports/teams-meetings-status-prejoin-source.ts b/extensions/teams-meetings/src/transports/teams-meetings-status-prejoin-source.ts index 908b03f236c6..a2f9df3a415f 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-status-prejoin-source.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-status-prejoin-source.ts @@ -119,8 +119,7 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara let microphoneState = identityVerified ? toggleState(microphone, "microphone") : undefined; const camera = first(selectors.camera) || findTextButton(/camera|video/i); let cameraState = identityVerified ? toggleState(camera, "camera") : undefined; - let controlManualActionReason; - let controlManualActionMessage; + let controlManualAction; if (canMutateSession && identityVerified && !inCall && camera && cameraState === "on") { camera.click(); await waitForUi(); @@ -132,8 +131,7 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara } const join = first(selectors.join) || findTextButton(/^\\s*(join now|ask to join|join meeting)\\s*$/i); if (identityVerified && !inCall && join && cameraState !== "off") { - controlManualActionReason = "teams-camera-required"; - controlManualActionMessage = "Turn the Teams camera off and verify the camera control shows it is off, then retry joining."; + controlManualAction = manualActionFor("teams-camera-required", "Turn the Teams camera off and verify the camera control shows it is off, then retry joining."); } const isBlackHole = (value) => /^blackhole 2ch(?: \\(virtual\\))?$/i.test(String(value || "").replace(/\\s+/g, " ").trim()); @@ -244,8 +242,7 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); microphoneState = toggleState(currentMicrophone, "microphone"); } - controlManualActionReason = "teams-audio-choice-required"; - controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."; + controlManualAction = manualActionFor("teams-audio-choice-required", "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."); } else if (canMutateSession && microphoneState === "off") { microphone.click(); await waitForUi(); @@ -256,8 +253,7 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara } } if (audioInputRouted && microphoneState !== "on") { - controlManualActionReason = "teams-microphone-required"; - controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."; + controlManualAction = manualActionFor("teams-microphone-required", "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."); } } else if (canMutateSession && identityVerified && !inCall && !allowMicrophone && microphoneState === "on") { microphone.click(); @@ -292,19 +288,15 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara } } if (identityVerified && !inCall && join && !allowMicrophone && microphoneState !== "off") { - controlManualActionReason = "teams-microphone-required"; - controlManualActionMessage = "Mute the Teams microphone and verify the microphone control shows it is off, then retry joining."; + controlManualAction = manualActionFor("teams-microphone-required", "Mute the Teams microphone and verify the microphone control shows it is off, then retry joining."); } - if (identityVerified && !inCall && join && allowMicrophone && !controlManualActionReason) { + if (identityVerified && !inCall && join && allowMicrophone && !controlManualAction) { if (!microphone) { - controlManualActionReason = "teams-microphone-required"; - controlManualActionMessage = "Open Teams device settings and verify the microphone control before enabling talk-back."; + controlManualAction = manualActionFor("teams-microphone-required", "Open Teams device settings and verify the microphone control before enabling talk-back."); } else if (audioInputRouted !== true) { - controlManualActionReason = "teams-audio-choice-required"; - controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."; + controlManualAction = manualActionFor("teams-audio-choice-required", "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."); } else if (microphoneState !== "on") { - controlManualActionReason = "teams-microphone-required"; - controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."; + controlManualAction = manualActionFor("teams-microphone-required", "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."); } }`, manualActionSource: ` const pageText = text(document.body); @@ -332,30 +324,20 @@ export function teamsMeetingStatusPreludeSource(params: MeetingStatusPreludePara // A granted microphone plus the verified BlackHole input is sufficient for talk-back. const permissionRequired = devicePermissionPrompt && (!allowMicrophone || microphonePermissionState !== "granted"); - let manualActionReason; - let manualActionMessage; + let manualAction; if (committedOwnerConflict && !canMutateSession) { - manualActionReason = "teams-session-conflict"; - manualActionMessage = "This Teams tab is owned by another active meeting session."; + manualAction = manualActionFor("teams-session-conflict", "This Teams tab is owned by another active meeting session."); } else if (!inCall && loginRequired) { - manualActionReason = "teams-login-required"; - manualActionMessage = tenantLoginRequired - ? "This Teams tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry." - : "Sign in to Microsoft Teams in the OpenClaw browser profile, then retry the meeting join."; + manualAction = manualActionFor("teams-login-required", tenantLoginRequired ? "This Teams tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry." : "Sign in to Microsoft Teams in the OpenClaw browser profile, then retry the meeting join."); } else if (!inCall && lobbyWaiting) { - manualActionReason = "teams-admission-required"; - manualActionMessage = "Admit the OpenClaw guest from the Microsoft Teams lobby, then retry speech."; + manualAction = manualActionFor("teams-admission-required", "Admit the OpenClaw guest from the Microsoft Teams lobby, then retry speech."); } else if (!inCall && permissionRequired) { - manualActionReason = "teams-permission-required"; - manualActionMessage = allowMicrophone - ? "Allow microphone permission for Teams in the OpenClaw browser profile, then retry." - : "Dismiss the Teams device-permission prompt or continue without devices, then retry."; - } else if (!inCall && controlManualActionReason) { - manualActionReason = controlManualActionReason; - manualActionMessage = controlManualActionMessage; + manualAction = manualActionFor("teams-permission-required", allowMicrophone ? "Allow microphone permission for Teams in the OpenClaw browser profile, then retry." : "Dismiss the Teams device-permission prompt or continue without devices, then retry."); + } else if (!inCall && controlManualAction) { + manualAction = controlManualAction; } let clickedJoin = false; - if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) { + if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualAction) { join.click(); clickedJoin = true; notes.push("Clicked the Teams guest join button."); diff --git a/extensions/zoom-meetings/src/runtime-node.test.ts b/extensions/zoom-meetings/src/runtime-node.test.ts index 4eed19fd38a0..16a1773556b6 100644 --- a/extensions/zoom-meetings/src/runtime-node.test.ts +++ b/extensions/zoom-meetings/src/runtime-node.test.ts @@ -84,9 +84,10 @@ describe("Zoom meetings node realtime recovery", () => { } : { inCall: false, - manualActionMessage: "Waiting for admission", - manualActionReason: "zoom-admission-required", - manualActionRequired: true, + manualAction: { + reason: "zoom-admission-required", + message: "Waiting for admission", + }, url: URL, }, ), diff --git a/extensions/zoom-meetings/src/runtime-probes.test.ts b/extensions/zoom-meetings/src/runtime-probes.test.ts index 1be157511c1b..0ad0196ccac3 100644 --- a/extensions/zoom-meetings/src/runtime-probes.test.ts +++ b/extensions/zoom-meetings/src/runtime-probes.test.ts @@ -147,7 +147,7 @@ describe("Zoom meeting runtime probes", () => { session.chrome!.health = { ...session.chrome!.health, lastCaptionText: "Caption already waiting", - manualActionRequired: true, + manualAction: { reason: "zoom-admission-required", message: "Waiting" }, transcriptLines: 1, }; }, @@ -170,7 +170,10 @@ describe("Zoom meeting runtime probes", () => { }); expect(result.listenVerified).toBe(true); - expect(result.manualActionRequired).toBe(true); + expect(result.manualAction).toEqual({ + reason: "zoom-admission-required", + message: "Waiting", + }); expect(refreshCaptionHealth).toHaveBeenCalledTimes(1); expect(Date.now()).toBe(0); expect(vi.getTimerCount()).toBe(0); diff --git a/extensions/zoom-meetings/src/runtime.test.ts b/extensions/zoom-meetings/src/runtime.test.ts index ce257d57908e..0d573d37ff9a 100644 --- a/extensions/zoom-meetings/src/runtime.test.ts +++ b/extensions/zoom-meetings/src/runtime.test.ts @@ -78,22 +78,24 @@ function runtimeHarness(options?: { ...(!inCall ? { ...(pendingReason === "admission" ? { lobbyWaiting: true } : {}), - manualActionRequired: true, - manualActionReason: - pendingReason === "admission" - ? "zoom-admission-required" - : "zoom-passcode-required", - manualActionMessage: - pendingReason === "admission" - ? "Waiting for host admission." - : "Enter the meeting passcode.", + manualAction: { + reason: + pendingReason === "admission" + ? "zoom-admission-required" + : "zoom-passcode-required", + message: + pendingReason === "admission" + ? "Waiting for host admission." + : "Enter the meeting passcode.", + }, } : {}), ...(sessionConflict && fn.includes("const allowSessionAdoption = false") ? { - manualActionRequired: true, - manualActionReason: "zoom-session-conflict", - manualActionMessage: "This Zoom tab is owned by another active meeting session.", + manualAction: { + reason: "zoom-session-conflict", + message: "This Zoom tab is owned by another active meeting session.", + }, } : {}), url: tabUrl, @@ -215,7 +217,7 @@ describe("Zoom meeting session flow", () => { audioOutputRouted: false, captioning: false, inCall: false, - manualActionRequired: false, + manualAction: undefined, providerConnected: false, realtimeReady: false, }, @@ -401,8 +403,7 @@ describe("Zoom meeting session flow", () => { browserTab: undefined, health: { inCall: false, - manualActionReason: undefined, - manualActionRequired: false, + manualAction: undefined, status: "browser-tab-missing", }, }, diff --git a/extensions/zoom-meetings/src/runtime.ts b/extensions/zoom-meetings/src/runtime.ts index f271e952d29f..16be8a19ba64 100644 --- a/extensions/zoom-meetings/src/runtime.ts +++ b/extensions/zoom-meetings/src/runtime.ts @@ -38,7 +38,7 @@ import type { import { ZOOM_MEETINGS_PLATFORM_ADAPTER } from "./transports/zoom-meetings-platform-adapter.js"; import { hasSameZoomMeetingJoinCredential } from "./transports/zoom-meetings-urls.js"; -type ManualActionReason = NonNullable; +type ManualActionReason = NonNullable["reason"]; type SpeechBlockedReason = NonNullable; type SessionRuntime = MeetingSessionRuntime< ZoomMeetingsSession, @@ -86,7 +86,7 @@ function noteSession(session: ZoomMeetingsSession, note: string): void { function isAwaitingAdmission(session: ZoomMeetingsSession): boolean { return ( session.chrome?.health?.lobbyWaiting === true || - session.chrome?.health?.manualActionReason === "zoom-admission-required" + session.chrome?.health?.manualAction?.reason === "zoom-admission-required" ); } @@ -131,7 +131,6 @@ export class ZoomMeetingsRuntime { speech: { audioBridgeUnavailable: "Realtime speech requires an active Chrome audio bridge.", browserUnverified: "Zoom browser state has not been verified yet.", - manualActionFallback: "Resolve the Zoom browser prompt before asking OpenClaw to speak.", microphoneMuted: "Turn on the OpenClaw Zoom microphone before asking OpenClaw to speak.", microphoneMutedReason: "zoom-microphone-muted", notInCall: "Zoom has not reported that the browser guest is in the call.", @@ -218,12 +217,12 @@ export class ZoomMeetingsRuntime { const staleSession = !browser?.browserTab || health?.meetingEnded === true || - health?.manualActionReason === "zoom-session-conflict" || - health?.manualActionReason === "browser-control-unavailable" || + health?.manualAction?.reason === "zoom-session-conflict" || + health?.manualAction?.reason === "browser-control-unavailable" || health?.bridgeClosed === true; const replacePendingJoin = health?.inCall !== true && - health?.manualActionReason === "zoom-passcode-required" && + health?.manualAction?.reason === "zoom-passcode-required" && !hasSameZoomMeetingJoinCredential(session.url, request.url); if (staleSession || replacePendingJoin) { session.state = "ended"; @@ -536,9 +535,7 @@ export class ZoomMeetingsRuntime { captioning: false, audioInputRouted: false, audioOutputRouted: false, - manualActionRequired: true, - manualActionReason: "browser-control-unavailable", - manualActionMessage: result.message, + manualAction: { reason: "browser-control-unavailable", message: result.message }, status: "browser-tab-missing", notes: [ ...(session.chrome.health?.notes ?? []).filter((note) => note !== result.message), @@ -558,9 +555,7 @@ export class ZoomMeetingsRuntime { captioning: false, audioInputRouted: false, audioOutputRouted: false, - manualActionRequired: true, - manualActionReason: "browser-control-unavailable", - manualActionMessage: message, + manualAction: { reason: "browser-control-unavailable", message }, status: "browser-control", notes: [ ...(session.chrome.health?.notes ?? []).filter((note) => note !== message), diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts index e03c9fd88043..51dbcc0758e7 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.test.ts @@ -143,9 +143,7 @@ function status(reason: string) { const health = ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ result: JSON.stringify({ inCall: false, - manualActionRequired: true, - manualActionReason: reason, - manualActionMessage: "manual action", + manualAction: { reason, message: "manual action" }, url: URL, }), }); @@ -378,8 +376,7 @@ describe("Zoom meeting platform adapter", () => { expect(result).toMatchObject({ clickedJoin: false, - manualActionReason: reason, - manualActionRequired: true, + manualAction: { reason }, }); expect(join.click).not.toHaveBeenCalled(); }); @@ -400,7 +397,8 @@ describe("Zoom meeting platform adapter", () => { expect(guest.value).toBe("OpenClaw Agent"); expect(guest.dispatchEvent).toHaveBeenCalledTimes(2); expect(join.click).toHaveBeenCalledOnce(); - expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false }); + expect(result.clickedJoin).toBe(true); + expect(result.manualAction).toBeUndefined(); }); it("persists Zoom's confirmed no-device state for observe-only joins", async () => { @@ -424,16 +422,14 @@ describe("Zoom meeting platform adapter", () => { }); expect(devicePrompt.click).toHaveBeenCalled(); - expect(first.manualActionReason).toBeUndefined(); + expect(first.manualAction).toBeUndefined(); expect(first).toMatchObject({ cameraOff: true, clickedJoin: true, - manualActionRequired: false, micMuted: true, }); expect(second).toMatchObject({ cameraOff: true, - manualActionRequired: false, micMuted: true, }); expect(window["__openclawZoomMeeting"]).toMatchObject({ devicesDisabled: true }); @@ -456,7 +452,8 @@ describe("Zoom meeting platform adapter", () => { }), }); - expect(result).toMatchObject({ inCall: true, micMuted: true, manualActionRequired: false }); + expect(result).toMatchObject({ inCall: true, micMuted: true }); + expect(result.manualAction).toBeUndefined(); expect(microphone.click).toHaveBeenCalledOnce(); }); @@ -471,8 +468,7 @@ describe("Zoom meeting platform adapter", () => { expect(result).toMatchObject({ inCall: true, - manualActionReason: "zoom-microphone-required", - manualActionRequired: true, + manualAction: { reason: "zoom-microphone-required" }, }); }); @@ -493,7 +489,8 @@ describe("Zoom meeting platform adapter", () => { }), }); - expect(result).toMatchObject({ cameraOff: true, inCall: true, manualActionRequired: false }); + expect(result).toMatchObject({ cameraOff: true, inCall: true }); + expect(result.manualAction).toBeUndefined(); expect(camera.click).toHaveBeenCalledOnce(); }); @@ -508,8 +505,7 @@ describe("Zoom meeting platform adapter", () => { expect(result).toMatchObject({ inCall: true, - manualActionReason: "zoom-camera-required", - manualActionRequired: true, + manualAction: { reason: "zoom-camera-required" }, }); }); @@ -583,8 +579,7 @@ describe("Zoom meeting platform adapter", () => { expect(result).toMatchObject({ audioInputRouted: false, - manualActionReason: "zoom-audio-choice-required", - manualActionRequired: true, + manualAction: { reason: "zoom-audio-choice-required" }, }); expect(meetingState).not.toHaveProperty("audioInputDeviceId"); }); @@ -610,7 +605,7 @@ describe("Zoom meeting platform adapter", () => { expect(waiting).toMatchObject({ lobbyWaiting: true, - manualActionReason: "zoom-admission-required", + manualAction: { reason: "zoom-admission-required" }, }); expect(admitted).toMatchObject({ inCall: true, micMuted: true }); expect(window["__openclawZoomMeeting"]).toMatchObject({ diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts index 2a831ed61d6a..d04006e138ac 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-platform-adapter.ts @@ -102,10 +102,10 @@ export const ZOOM_MEETINGS_PLATFORM_ADAPTER = MeetingPlatformAdapter.create< }), shouldRetryJoinStatus: (health) => health.inCall === true && - ((health.manualActionReason === "zoom-audio-choice-required" && + ((health.manualAction?.reason === "zoom-audio-choice-required" && health.audioInputRouted === true && health.audioOutputRouteRetryable === true) || - (health.manualActionRequired !== true && + (health.manualAction === undefined && health.captionCaptureRequested === true && health.captioning !== true)), browserControlUnavailable: () => ({ diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-status-access-source.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-status-access-source.ts index d8627c6de766..b10cfa9a224d 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-status-access-source.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-status-access-source.ts @@ -7,11 +7,9 @@ export function zoomMeetingStatusAccessSource(): string { const captchaRequired = Boolean(firstRaw(selectors.captcha)) || /complete (?:the )?captcha|security check|verify (?:that )?you(?:'re| are) (?:a )?human/i.test(pageTextLower); if (identityVerified && !inCall && passcodeRequired) { - controlManualActionReason = "zoom-passcode-required"; - controlManualActionMessage = "Enter the Zoom meeting passcode in the OpenClaw browser profile, then retry joining."; + controlManualAction = manualActionFor("zoom-passcode-required", "Enter the Zoom meeting passcode in the OpenClaw browser profile, then retry joining."); } else if (identityVerified && !inCall && captchaRequired) { - controlManualActionReason = "zoom-captcha-required"; - controlManualActionMessage = "Complete Zoom's security check in the OpenClaw browser profile, then retry joining."; + controlManualAction = manualActionFor("zoom-captcha-required", "Complete Zoom's security check in the OpenClaw browser profile, then retry joining."); } `; } diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-status-prejoin-source.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-status-prejoin-source.ts index 495250d7db9b..5f254707f640 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-status-prejoin-source.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-status-prejoin-source.ts @@ -157,15 +157,14 @@ export function zoomMeetingStatusPreludeSource(params: MeetingStatusPreludeParam let microphoneState = identityVerified ? (toggleState(microphone, "microphone") || (devicesDisabled ? "off" : undefined)) : undefined; const camera = first(selectors.camera) || findTextButton(/camera|video/i); let cameraState = identityVerified ? (toggleState(camera, "camera") || (devicesDisabled ? "off" : undefined)) : undefined; - let controlManualActionReason; - let controlManualActionMessage; + let controlManualAction; ${zoomMeetingStatusAccessSource()} if ( canMutateSession && identityVerified && camera && cameraState === "on" && - !controlManualActionReason + !controlManualAction ) { camera.click(); await waitForUi(); @@ -186,12 +185,9 @@ export function zoomMeetingStatusPreludeSource(params: MeetingStatusPreludeParam identityVerified && (inCall || join) && cameraState !== "off" && - !controlManualActionReason + !controlManualAction ) { - controlManualActionReason = "zoom-camera-required"; - controlManualActionMessage = inCall - ? "Turn the Zoom camera off and verify the in-call camera control shows it is off." - : "Turn the Zoom camera off and verify the camera control shows it is off, then retry joining."; + controlManualAction = manualActionFor("zoom-camera-required", inCall ? "Turn the Zoom camera off and verify the in-call camera control shows it is off." : "Turn the Zoom camera off and verify the camera control shows it is off, then retry joining."); } const isBlackHole = (value) => /^blackhole 2ch(?: \\(virtual\\))?$/i.test(String(value || "").replace(/\\s+/g, " ").trim()); @@ -343,12 +339,9 @@ export function zoomMeetingStatusPreludeSource(params: MeetingStatusPreludeParam (inCall || join) && !allowMicrophone && microphoneState !== "off" && - !controlManualActionReason + !controlManualAction ) { - controlManualActionReason = "zoom-microphone-required"; - controlManualActionMessage = inCall - ? "Mute the Zoom microphone and verify it stays muted for observe-only mode." - : "Mute the Zoom microphone and verify the microphone control shows it is off, then retry joining."; + controlManualAction = manualActionFor("zoom-microphone-required", inCall ? "Mute the Zoom microphone and verify it stays muted for observe-only mode." : "Mute the Zoom microphone and verify the microphone control shows it is off, then retry joining."); }`, manualActionSource: ` const signInControl = first(selectors.signIn); const tenantLoginRequired = @@ -368,30 +361,20 @@ export function zoomMeetingStatusPreludeSource(params: MeetingStatusPreludeParam // A granted microphone plus the verified BlackHole input is sufficient for talk-back. const permissionRequired = devicePermissionPrompt && (!allowMicrophone || microphonePermissionState !== "granted"); - let manualActionReason; - let manualActionMessage; + let manualAction; if (committedOwnerConflict && !canMutateSession) { - manualActionReason = "zoom-session-conflict"; - manualActionMessage = "This Zoom tab is owned by another active meeting session."; + manualAction = manualActionFor("zoom-session-conflict", "This Zoom tab is owned by another active meeting session."); } else if (!inCall && loginRequired) { - manualActionReason = "zoom-login-required"; - manualActionMessage = tenantLoginRequired - ? "This Zoom tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry." - : "Sign in to Zoom in the OpenClaw browser profile, then retry the meeting join."; + manualAction = manualActionFor("zoom-login-required", tenantLoginRequired ? "This Zoom tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry." : "Sign in to Zoom in the OpenClaw browser profile, then retry the meeting join."); } else if (!inCall && lobbyWaiting) { - manualActionReason = "zoom-admission-required"; - manualActionMessage = "Admit the OpenClaw guest from the Zoom lobby, then retry speech."; + manualAction = manualActionFor("zoom-admission-required", "Admit the OpenClaw guest from the Zoom lobby, then retry speech."); } else if (!inCall && permissionRequired) { - manualActionReason = "zoom-permission-required"; - manualActionMessage = allowMicrophone - ? "Allow microphone permission for Zoom in the OpenClaw browser profile, then retry." - : "Dismiss the Zoom device-permission prompt or continue without devices, then retry."; - } else if (controlManualActionReason) { - manualActionReason = controlManualActionReason; - manualActionMessage = controlManualActionMessage; + manualAction = manualActionFor("zoom-permission-required", allowMicrophone ? "Allow microphone permission for Zoom in the OpenClaw browser profile, then retry." : "Dismiss the Zoom device-permission prompt or continue without devices, then retry."); + } else if (controlManualAction) { + manualAction = controlManualAction; } let clickedJoin = false; - if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) { + if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualAction) { join.click(); clickedJoin = true; notes.push("Clicked the Zoom guest join button."); diff --git a/src/meeting-bot/browser-controller.test.ts b/src/meeting-bot/browser-controller.test.ts index ab0474fba663..471f3f12ffb5 100644 --- a/src/meeting-bot/browser-controller.test.ts +++ b/src/meeting-bot/browser-controller.test.ts @@ -52,7 +52,7 @@ describe("meeting browser join readiness", () => { parseTranscript: () => ({ droppedLines: 0, lines: [] }), }, classifyManualAction: (health) => - health.manualActionRequired + health.manualAction ? { category: "audio-choice-required", reason: "audio-choice", message: "Wait." } : undefined, parseLeaveResult: () => ({ departed: false }), @@ -60,14 +60,13 @@ describe("meeting browser join readiness", () => { evaluationAttempts === 1 ? { inCall: true, - manualActionRequired: true, - manualActionReason: "audio-choice", + manualAction: { reason: "audio-choice", message: "Wait." }, micMuted: false, } - : { inCall: true, manualActionRequired: false, micMuted: false }, + : { inCall: true, micMuted: false }, permissions: () => undefined, permissionNotes: () => [], - shouldRetryJoinStatus: (health) => health.manualActionReason === "audio-choice", + shouldRetryJoinStatus: (health) => health.manualAction?.reason === "audio-choice", }, }, callBrowser: async (request) => { @@ -100,7 +99,6 @@ describe("meeting browser join readiness", () => { expect(captionCaptureAttempts).toEqual([false, false]); expect(result.browser).toMatchObject({ inCall: true, - manualActionRequired: false, micMuted: false, }); }); diff --git a/src/meeting-bot/browser-controller.ts b/src/meeting-bot/browser-controller.ts index eddda4896c64..3eca0b97d8de 100644 --- a/src/meeting-bot/browser-controller.ts +++ b/src/meeting-bot/browser-controller.ts @@ -54,9 +54,7 @@ function applyMeetingManualAction( return browser && manual ? { ...browser, - manualActionRequired: true, - manualActionReason: manual.reason, - manualActionMessage: manual.message, + manualAction: { reason: manual.reason, message: manual.message }, } : browser; } @@ -273,12 +271,12 @@ export async function openMeetingWithBrowser< if ( !shouldRetry && browser?.inCall === true && - browser.manualActionRequired !== true && + browser.manualAction === undefined && (!allowMicrophone || browser.micMuted !== true) ) { return { launched: true, browser, tab: tabIdentity }; } - if (!shouldRetry && browser?.manualActionRequired === true) { + if (!shouldRetry && browser?.manualAction) { return { launched: true, browser, tab: tabIdentity }; } } catch (error) { @@ -291,9 +289,7 @@ export async function openMeetingWithBrowser< browser = { ...browser, inCall: false, - manualActionRequired: true, - manualActionReason: manual.reason, - manualActionMessage: manual.message, + manualAction: { reason: manual.reason, message: manual.message }, notes: [ ...permissionNotes, `Browser control could not inspect or auto-join ${params.adapter.browserLabel}: ${ @@ -396,9 +392,7 @@ async function inspectRecoverableTab< status: "browser-control", browserUrl: params.tab.url, browserTitle: params.tab.title, - manualActionRequired: true, - manualActionReason: localeAction.reason, - manualActionMessage: localeAction.message, + manualAction: { reason: localeAction.reason, message: localeAction.message }, } as unknown as Health, message: localeAction.message, }; diff --git a/src/meeting-bot/platform-adapter.ts b/src/meeting-bot/platform-adapter.ts index 72d0d2b5d2d1..6b022e1a31d2 100644 --- a/src/meeting-bot/platform-adapter.ts +++ b/src/meeting-bot/platform-adapter.ts @@ -94,6 +94,17 @@ function browserResultString(result: unknown): string | undefined { return typeof value === "string" && value.trim() ? value : undefined; } +function parseMeetingManualAction(value: unknown): MeetingBrowserHealth["manualAction"] { + if (!value || typeof value !== "object") { + return undefined; + } + const action = value as Record; + if (typeof action.reason !== "string" || typeof action.message !== "string") { + return undefined; + } + return { reason: action.reason, message: action.message }; +} + function parseMeetingBrowserStatus( result: unknown, options: MeetingPlatformAdapterOptions< @@ -168,12 +179,7 @@ function parseMeetingBrowserStatus( typeof parsed.audioOutputRouteRetryable === "boolean" ? parsed.audioOutputRouteRetryable : undefined, - manualActionRequired: - typeof parsed.manualActionRequired === "boolean" ? parsed.manualActionRequired : undefined, - manualActionReason: - typeof parsed.manualActionReason === "string" ? parsed.manualActionReason : undefined, - manualActionMessage: - typeof parsed.manualActionMessage === "string" ? parsed.manualActionMessage : undefined, + manualAction: parseMeetingManualAction(parsed.manualAction), browserUrl: typeof parsed.url === "string" ? parsed.url : undefined, browserTitle: typeof parsed.title === "string" ? parsed.title : undefined, status: "browser-control", @@ -311,17 +317,13 @@ function createMeetingPlatformAdapter< ...browser, parseStatus: (result) => parseMeetingBrowserStatus(result, parsing), classifyManualAction: (health) => { - if ( - !health.manualActionRequired || - !health.manualActionReason || - !health.manualActionMessage - ) { + if (!health.manualAction) { return undefined; } return { - category: parsing.classifyManualActionReason(health.manualActionReason), - reason: health.manualActionReason, - message: health.manualActionMessage, + category: parsing.classifyManualActionReason(health.manualAction.reason), + reason: health.manualAction.reason, + message: health.manualAction.message, }; }, parseLeaveResult: parseMeetingLeaveResult, @@ -380,7 +382,7 @@ function isMeetingRealtimeRouteReady( health.micMuted === false && health.audioInputRouted === true && health.audioOutputRouted === true && - health.manualActionRequired !== true + health.manualAction === undefined ); } diff --git a/src/meeting-bot/runtime-probes.ts b/src/meeting-bot/runtime-probes.ts index 0df33a3cf9c8..d670f39109ff 100644 --- a/src/meeting-bot/runtime-probes.ts +++ b/src/meeting-bot/runtime-probes.ts @@ -145,7 +145,7 @@ export function createMeetingRuntimeProbes< health?.verifiedOutputGeneration === health?.outputGeneration; const shouldWait = result.spoken === true && - health?.manualActionRequired !== true && + health?.manualAction === undefined && context.hasHealthHandle(result.session.id); if (shouldWait && !verified()) { const deadline = @@ -162,9 +162,7 @@ export function createMeetingRuntimeProbes< return { createdSession: !before.has(result.session.id), inCall: health?.inCall, - manualActionRequired: health?.manualActionRequired, - manualActionReason: health?.manualActionReason, - manualActionMessage: health?.manualActionMessage, + manualAction: health?.manualAction, spoken: result.spoken ?? false, speechOutputVerified, speechOutputTimedOut: shouldWait && !speechOutputVerified, @@ -218,7 +216,7 @@ export function createMeetingRuntimeProbes< Boolean(health?.lastCaptionAt && health.lastCaptionAt !== start.at) || Boolean(health?.lastCaptionText && health.lastCaptionText !== start.text); const shouldWait = - health?.manualActionRequired !== true && options.shouldWaitForListening(result.session); + health?.manualAction === undefined && options.shouldWaitForListening(result.session); let listenVerified = advanced(); if (shouldWait && !listenVerified) { const deadline = @@ -254,7 +252,7 @@ export function createMeetingRuntimeProbes< if (advanced()) { listenVerified = true; } - if (listenVerified || health?.manualActionRequired) { + if (listenVerified || health?.manualAction) { break; } const retryDelayMs = deadline - Date.now(); @@ -267,11 +265,9 @@ export function createMeetingRuntimeProbes< return { createdSession: !before.has(result.session.id), inCall: health?.inCall, - manualActionRequired: health?.manualActionRequired, - manualActionReason: health?.manualActionReason, - manualActionMessage: health?.manualActionMessage, + manualAction: health?.manualAction, listenVerified, - listenTimedOut: shouldWait && !listenVerified && health?.manualActionRequired !== true, + listenTimedOut: shouldWait && !listenVerified && health?.manualAction === undefined, captioning: health?.captioning, captionsEnabledAttempted: health?.captionsEnabledAttempted, transcriptLines: health?.transcriptLines, diff --git a/src/meeting-bot/session-runtime.test.ts b/src/meeting-bot/session-runtime.test.ts index 611266ff3e3e..021138dec573 100644 --- a/src/meeting-bot/session-runtime.test.ts +++ b/src/meeting-bot/session-runtime.test.ts @@ -144,7 +144,6 @@ function createTestRuntime(params: { speech: { audioBridgeUnavailable: "bridge unavailable", browserUnverified: "browser unverified", - manualActionFallback: "manual action", microphoneMuted: "microphone muted", microphoneMutedReason: "microphone-muted", notInCall: "not in call", @@ -632,9 +631,7 @@ describe("MeetingSessionRuntime leave cleanup", () => { health: { inCall: true, micMuted: false, - manualActionMessage: "old action", - manualActionReason: "old-action", - manualActionRequired: true, + manualAction: { reason: "old-action", message: "old action" }, speechReady: true, speechBlockedMessage: "old speech block", speechBlockedReason: "old-speech-block", @@ -655,14 +652,13 @@ describe("MeetingSessionRuntime leave cleanup", () => { browser: { health: { inCall: false, - manualActionRequired: false, + manualAction: undefined, speechReady: false, }, }, }, }); - expect(session.browser?.health?.manualActionReason).toBeUndefined(); - expect(session.browser?.health?.manualActionMessage).toBeUndefined(); + expect(session.browser?.health?.manualAction).toBeUndefined(); expect(session.browser?.health?.micMuted).toBeUndefined(); expect(session.browser?.health?.speechBlockedReason).toBeUndefined(); expect(session.browser?.health?.speechBlockedMessage).toBeUndefined(); diff --git a/src/meeting-bot/session-runtime.ts b/src/meeting-bot/session-runtime.ts index 845b9c4e4bab..7f5d85072dd0 100644 --- a/src/meeting-bot/session-runtime.ts +++ b/src/meeting-bot/session-runtime.ts @@ -38,7 +38,6 @@ export type MeetingSessionRuntimeMessages = speech: { audioBridgeUnavailable: string; browserUnverified: string; - manualActionFallback: string; microphoneMuted: string; microphoneMutedReason: TSpeechBlockedReason; notInCall: string; @@ -322,7 +321,7 @@ export class MeetingSessionRuntime< return true; } const health = this.options.getBrowser(result.session as TSession)?.health; - if (health?.manualActionRequired || result.session?.state !== "active") { + if (health?.manualAction || result.session?.state !== "active") { return false; } const blocked = health?.speechBlockedReason; @@ -578,9 +577,7 @@ export class MeetingSessionRuntime< ...browser.health, inCall: false, micMuted: undefined, - manualActionRequired: false, - manualActionReason: undefined, - manualActionMessage: undefined, + manualAction: undefined, speechReady: false, speechBlockedReason: undefined, speechBlockedMessage: undefined, diff --git a/src/meeting-bot/session-speech-readiness.ts b/src/meeting-bot/session-speech-readiness.ts index f2b389e4c620..6fb8f855ed9a 100644 --- a/src/meeting-bot/session-speech-readiness.ts +++ b/src/meeting-bot/session-speech-readiness.ts @@ -5,7 +5,6 @@ type SpeechMessages = { audioBridgeUnavailableReason: TReason; browserUnverified: string; browserUnverifiedReason: TReason; - manualActionFallback: string; microphoneMuted: string; microphoneMutedReason: TReason; notInCall: string; @@ -37,11 +36,11 @@ export function evaluateMeetingSpeechReadiness(params: { }; } const health = browser.health; - if (health?.manualActionRequired) { + if (health?.manualAction) { return { ready: false, - reason: (health.manualActionReason ?? speech.browserUnverifiedReason) as TReason, - message: health.manualActionMessage ?? speech.manualActionFallback, + reason: health.manualAction.reason as TReason, + message: health.manualAction.message, }; } if (health?.inCall === true) { diff --git a/src/meeting-bot/session-types.ts b/src/meeting-bot/session-types.ts index a61a63f5fcdb..a6fbe1c71d3f 100644 --- a/src/meeting-bot/session-types.ts +++ b/src/meeting-bot/session-types.ts @@ -37,9 +37,7 @@ export type MeetingBrowserHealth< > = { inCall?: boolean; micMuted?: boolean; - manualActionRequired?: boolean; - manualActionReason?: TManualReason; - manualActionMessage?: string; + manualAction?: { reason: TManualReason; message: string }; speechReady?: boolean; speechBlockedReason?: TSpeechBlockedReason; speechBlockedMessage?: string; diff --git a/src/meeting-bot/status-call-source.ts b/src/meeting-bot/status-call-source.ts index 4a42b4206478..d2d30a7c26a1 100644 --- a/src/meeting-bot/status-call-source.ts +++ b/src/meeting-bot/status-call-source.ts @@ -609,13 +609,11 @@ export function createMeetingStatusCallSource(options: MeetingStatusCallSourceOp text: entry.text, })); } - if (inCall && allowMicrophone && !manualActionReason) { + if (inCall && allowMicrophone && !manualAction) { if (audioInputRouted !== true || audioOutputRouted !== true) { - manualActionReason = "${options.platform.manualActionReasonPrefix}-audio-choice-required"; - manualActionMessage = "Verify BlackHole 2ch is selected as both the ${options.platform.displayName} microphone and speaker before starting talk-back."; + manualAction = manualActionFor("${options.platform.manualActionReasonPrefix}-audio-choice-required", "Verify BlackHole 2ch is selected as both the ${options.platform.displayName} microphone and speaker before starting talk-back."); } else if (micMuted !== false) { - manualActionReason = "${options.platform.manualActionReasonPrefix}-microphone-required"; - manualActionMessage = "Unmute the ${options.platform.displayName} microphone and verify the microphone control shows it is on before starting talk-back."; + manualAction = manualActionFor("${options.platform.manualActionReasonPrefix}-microphone-required", "Unmute the ${options.platform.displayName} microphone and verify the microphone control shows it is on before starting talk-back."); } } return JSON.stringify({ @@ -641,9 +639,7 @@ export function createMeetingStatusCallSource(options: MeetingStatusCallSourceOp audioOutputDeviceLabel, audioOutputRouteError, audioOutputRouteRetryable, - manualActionRequired: Boolean(manualActionReason), - manualActionReason, - manualActionMessage, + manualAction, title: document.title, url: location.href, notes, diff --git a/src/meeting-bot/status-prejoin-source.ts b/src/meeting-bot/status-prejoin-source.ts index 34635d69cc12..39954926fa16 100644 --- a/src/meeting-bot/status-prejoin-source.ts +++ b/src/meeting-bot/status-prejoin-source.ts @@ -64,6 +64,7 @@ export function createMeetingStatusPreludeSource( node?.getAttribute?.("data-tid"), text(node), ].filter(Boolean).join(" "); + const manualActionFor = (reason, message) => ({ reason, message }); const clickable = (node) => node?.matches?.("button") ? node : node?.querySelector?.("button") || node?.closest?.("button") || node; @@ -306,9 +307,7 @@ export function createMeetingStatusPreludeSource( if (requestOwnsMeeting) delete window[${meetingGlobal}]; return JSON.stringify({ inCall: false, - manualActionRequired: true, - manualActionReason: "${options.platform.manualActionReasonPrefix}-session-conflict", - manualActionMessage: "The tracked ${options.platform.displayName} tab now shows a different meeting. Return to the requested meeting link, then retry.", + manualAction: manualActionFor("${options.platform.manualActionReasonPrefix}-session-conflict", "The tracked ${options.platform.displayName} tab now shows a different meeting. Return to the requested meeting link, then retry."), title: document.title, url: location.href, notes,