From 900ba7cf33e081018f1d77dc8bc8bc56f215b698 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 24 Apr 2026 23:06:45 +0100 Subject: [PATCH] fix(google-meet): handle browser mic prompt --- CHANGELOG.md | 1 + docs/plugins/google-meet.md | 31 ++- extensions/google-meet/index.test.ts | 57 +++++ extensions/google-meet/index.ts | 1 + .../google-meet/src/transports/chrome.ts | 230 ++++++++++++++---- .../google-meet/src/transports/types.ts | 1 + 6 files changed, 272 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748085a8b54..616b7bca01f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ Docs: https://docs.openclaw.ai - Group chats/silent replies: tighten `NO_REPLY` prompt guidance so groups stay quiet without narrating silence or emitting fallback chatter when silence is the intended outcome. (#70954, #71209) Thanks @Takhoffman. - WhatsApp/groups+direct: setting `systemPrompt: ""` on a specific `groups.` or `direct.` entry now suppresses the wildcard system prompt instead of falling through to it, so users can silence the global prompt for a specific group or peer. (#70381) Thanks @Bluetegu. - Browser/tool: tell agents not to pass per-call `timeoutMs` on existing-session type, evaluate, and other Chrome MCP actions that reject timeout overrides. +- Plugins/Google Meet: use browser automation to classify and clear Meet's microphone-choice interstitial during browser meeting creation instead of reporting a false Google login failure. - Codex/GPT-5.4: harden fallback, auth-profile, tool-schema, and replay edge cases across native and embedded runtime paths. (#70743) Thanks @100yenadmin. - Models/fallback: resolve bare fallback model provider ids before model switching, so configured fallback chains keep working when a fallback is named without an explicit provider prefix. - Voice-call/Telnyx: preserve inbound/outbound callback metadata and read transcription text from Telnyx's current `transcription_data` payload. diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index 45af386aa2f..67d7e278daf 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -112,6 +112,8 @@ openclaw googlemeet join https://meet.google.com/new-abcd-xyz --transport chrome pinned Chrome node, opens `https://meet.google.com/new`, waits for Google to redirect to a real meeting-code URL, then returns that URL. This path requires the OpenClaw Chrome profile on the node to already be signed in to Google. + Browser automation handles Meet's own first-run microphone prompt; that prompt + is not treated as a Google login failure. The command output includes a `source` field (`api` or `browser`) so agents can explain which path was used. @@ -271,11 +273,17 @@ phrase, and prints session health: openclaw googlemeet test-speech https://meet.google.com/abc-defg-hij ``` -If the browser profile is not signed in, Meet is waiting for host admission, or -Chrome needs microphone/camera permission, the join/test-speech result reports +During join, OpenClaw browser automation fills the guest name, clicks Join/Ask +to join, and accepts Meet's first-run "Use microphone" choice when that prompt +appears. During browser-only meeting creation, it can also continue past the +same prompt without microphone if Meet does not expose the use-microphone button. +If the browser profile is not signed in, Meet is waiting for host +admission, Chrome needs microphone/camera permission, or Meet is stuck on a +prompt automation could not resolve, the join/test-speech result reports `manualActionRequired: true` with `manualActionReason` and -`manualActionMessage`. Agents should stop retrying the join, report that message -to the operator, and retry only after the manual browser action is complete. +`manualActionMessage`. Agents should stop retrying the join, report that exact +message plus the current `browserUrl`/`browserTitle`, and retry only after the +manual browser action is complete. If `chromeNode.node` is omitted, OpenClaw auto-selects only when exactly one connected node advertises both `googlemeet.chrome` and browser control. If @@ -784,9 +792,17 @@ Common manual actions: - Sign in to the Chrome profile. - Admit the guest from the Meet host account. -- Grant Chrome microphone/camera permissions. +- Grant Chrome microphone/camera permissions when Chrome's native permission + prompt appears. - Close or repair a stuck Meet permission dialog. +Do not report "not signed in" just because Meet shows "Do you want people to +hear you in the meeting?" That is Meet's audio-choice interstitial; OpenClaw +clicks **Use microphone** through browser automation when available and keeps +waiting for the real meeting state. For create-only browser fallback, OpenClaw +may click **Continue without microphone** because creating the URL does not need +the realtime audio path. + ### Meeting creation fails `googlemeet create` first uses the Google Meet API `spaces.create` endpoint @@ -803,6 +819,11 @@ to the pinned Chrome node browser. Confirm: `googlemeet.chrome`. - For browser fallback: the OpenClaw Chrome profile on that node is signed in to Google and can open `https://meet.google.com/new`. +- For browser fallback: if Meet shows "Do you want people to hear you in the + meeting?", leave the tab open. OpenClaw should click **Use microphone** or, for + create-only fallback, **Continue without microphone** through browser + automation and continue waiting for the generated Meet URL. If it cannot, the + error should mention `meet-audio-choice-required`, not `google-login-required`. ### Agent joins but does not talk diff --git a/extensions/google-meet/index.test.ts b/extensions/google-meet/index.test.ts index 0528b8c0a6f..6f2bb3478a4 100644 --- a/extensions/google-meet/index.test.ts +++ b/extensions/google-meet/index.test.ts @@ -23,6 +23,7 @@ import { startNodeRealtimeAudioBridge } from "./src/realtime-node.js"; import { startCommandRealtimeAudioBridge } from "./src/realtime.js"; import { normalizeMeetUrl } from "./src/runtime.js"; import type { GoogleMeetRuntime } from "./src/runtime.js"; +import { CREATE_MEET_FROM_BROWSER_SCRIPT } from "./src/transports/chrome.js"; import { buildMeetDtmfSequence, normalizeDialInNumber } from "./src/transports/twilio.js"; const voiceCallMocks = vi.hoisted(() => ({ @@ -843,6 +844,62 @@ describe("google-meet plugin", () => { ); }); + it.each([ + ["Use microphone", "Accepted Meet microphone prompt with browser automation."], + [ + "Continue without microphone", + "Continued through Meet microphone prompt with browser automation.", + ], + ])( + "uses browser automation for Meet's %s choice during browser creation", + async (buttonText, note) => { + const location = { + href: "https://meet.google.com/new", + hostname: "meet.google.com", + }; + const button = { + disabled: false, + innerText: buttonText, + textContent: buttonText, + getAttribute: (name: string) => (name === "aria-label" ? buttonText : null), + click: vi.fn(() => { + location.href = "https://meet.google.com/abc-defg-hij"; + }), + }; + const document = { + title: "Meet", + body: { + innerText: "Do you want people to hear you in the meeting?", + textContent: "Do you want people to hear you in the meeting?", + }, + querySelectorAll: (selector: string) => (selector === "button" ? [button] : []), + }; + vi.stubGlobal("document", document); + vi.stubGlobal("location", location); + vi.useFakeTimers(); + + try { + const fn = (0, eval)(`(${CREATE_MEET_FROM_BROWSER_SCRIPT})`) as () => Promise<{ + meetingUri?: string; + manualActionReason?: string; + notes?: string[]; + retryAfterMs?: number; + }>; + const result = await fn(); + + expect(result).toMatchObject({ + retryAfterMs: 1000, + notes: [note], + }); + expect(button.click).toHaveBeenCalledTimes(1); + expect(result.meetingUri).toBeUndefined(); + expect(result.manualActionReason).toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }, + ); + it("launches Chrome after the BlackHole check", async () => { const originalPlatform = process.platform; Object.defineProperty(process, "platform", { value: "darwin" }); diff --git a/extensions/google-meet/index.ts b/extensions/google-meet/index.ts index 85fb7b1514d..57b9ec8b466 100644 --- a/extensions/google-meet/index.ts +++ b/extensions/google-meet/index.ts @@ -265,6 +265,7 @@ async function createMeetFromParams(params: { targetId: browser.targetId, browserUrl: browser.browserUrl, browserTitle: browser.browserTitle, + notes: browser.notes, }, }; } diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index f46324efb95..6f9f5b72e93 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -231,12 +231,24 @@ type BrowserTab = { url?: string; }; +function formatBrowserAutomationError(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + try { + return JSON.stringify(error); + } catch { + return "unknown error"; + } +} + export type GoogleMeetBrowserCreateResult = { meetingUri: string; nodeId: string; targetId?: string; browserUrl?: string; browserTitle?: string; + notes?: string[]; source: "browser"; }; @@ -297,6 +309,9 @@ function readBrowserCreateResult(result: unknown): { browserUrl?: string; browserTitle?: string; manualAction?: string; + manualActionReason?: GoogleMeetChromeHealth["manualActionReason"]; + notes?: string[]; + retryAfterMs?: number; } { const record = result && typeof result === "object" ? (result as Record) : {}; const nested = @@ -308,39 +323,110 @@ function readBrowserCreateResult(result: unknown): { browserUrl: typeof nested.browserUrl === "string" ? nested.browserUrl : undefined, browserTitle: typeof nested.browserTitle === "string" ? nested.browserTitle : undefined, manualAction: typeof nested.manualAction === "string" ? nested.manualAction : undefined, + manualActionReason: + typeof nested.manualActionReason === "string" + ? (nested.manualActionReason as GoogleMeetChromeHealth["manualActionReason"]) + : undefined, + notes: Array.isArray(nested.notes) + ? nested.notes.filter((note): note is string => typeof note === "string") + : undefined, + retryAfterMs: + typeof nested.retryAfterMs === "number" && Number.isFinite(nested.retryAfterMs) + ? nested.retryAfterMs + : undefined, }; } -const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { +export const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { const meetUrlPattern = /^https:\\/\\/meet\\.google\\.com\\/[a-z]{3}-[a-z]{4}-[a-z]{3}(?:$|[/?#])/i; - const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const text = (node) => (node?.innerText || node?.textContent || "").trim(); const current = () => location.href; + const notes = []; + const findButton = (pattern) => + [...document.querySelectorAll("button")].find((button) => { + const label = [ + button.getAttribute("aria-label"), + button.getAttribute("data-tooltip"), + text(button), + ] + .filter(Boolean) + .join(" "); + return pattern.test(label) && !button.disabled; + }); + const clickButton = (pattern, note) => { + const button = findButton(pattern); + if (!button) { + return false; + } + button.click(); + notes.push(note); + return true; + }; if (!current().startsWith("https://meet.google.com/")) { return { + manualActionReason: "google-login-required", manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", browserUrl: current(), browserTitle: document.title, + notes, }; } - for (let i = 0; i < 80; i += 1) { - const href = current(); - if (meetUrlPattern.test(href)) { - return { meetingUri: href, browserUrl: href, browserTitle: document.title }; - } - const text = document.body?.innerText ?? ""; - if (/sign in|use your google account|couldn't create|unable to create/i.test(text)) { - return { - manualAction: "Sign in to Google in the OpenClaw browser profile or resolve the Meet page prompt, then retry meeting creation.", - browserUrl: href, - browserTitle: document.title, - }; - } - await sleep(500); + const href = current(); + if (meetUrlPattern.test(href)) { + return { meetingUri: href, browserUrl: href, browserTitle: document.title, notes }; + } + const pageText = text(document.body); + if (clickButton(/\\buse microphone\\b/i, "Accepted Meet microphone prompt with browser automation.")) { + return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; + } + if ( + clickButton( + /continue without microphone/i, + "Continued through Meet microphone prompt with browser automation.", + ) + ) { + return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; + } + if (/do you want people to hear you in the meeting/i.test(pageText)) { + return { + manualActionReason: "meet-audio-choice-required", + manualAction: "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (/allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) { + return { + manualActionReason: "meet-permission-required", + manualAction: "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (/couldn't create|unable to create/i.test(pageText)) { + return { + manualAction: "Resolve the Google Meet page prompt in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (location.hostname.toLowerCase() === "accounts.google.com" || /use your google account|to continue to google meet|choose an account|sign in to (join|continue)/i.test(pageText)) { + return { + manualActionReason: "google-login-required", + manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; } return { - manualAction: "Google Meet did not return a meeting URL from the browser create flow before timeout.", + retryAfterMs: 500, browserUrl: current(), browserTitle: document.title, + notes, }; }`; @@ -367,32 +453,62 @@ export async function createMeetWithBrowserProxyOnNode(params: { if (!targetId) { throw new Error("Browser fallback opened Google Meet but did not return a targetId."); } - const evaluated = await callBrowserProxyOnNode({ - runtime: params.runtime, - nodeId, - method: "POST", - path: "/act", - body: { - kind: "evaluate", - targetId, - fn: CREATE_MEET_FROM_BROWSER_SCRIPT, - }, - timeoutMs, - }); - const result = readBrowserCreateResult(evaluated); - if (result.meetingUri) { - return { - source: "browser", - nodeId, - targetId, - meetingUri: result.meetingUri, - browserUrl: result.browserUrl, - browserTitle: result.browserTitle, - }; + const notes = new Set(); + let lastResult: ReturnType | undefined; + let lastError: unknown; + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + try { + const evaluated = await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId, + method: "POST", + path: "/act", + body: { + kind: "evaluate", + targetId, + fn: CREATE_MEET_FROM_BROWSER_SCRIPT, + }, + timeoutMs: Math.min(timeoutMs, 10_000), + }); + const result = readBrowserCreateResult(evaluated); + lastResult = result; + for (const note of result.notes ?? []) { + notes.add(note); + } + if (result.meetingUri) { + return { + source: "browser", + nodeId, + targetId, + meetingUri: result.meetingUri, + browserUrl: result.browserUrl, + browserTitle: result.browserTitle, + notes: [...notes], + }; + } + if (result.manualAction) { + if (result.manualActionReason) { + throw new Error(`${result.manualActionReason}: ${result.manualAction}`); + } + throw new Error(result.manualAction); + } + await new Promise((resolve) => setTimeout(resolve, result.retryAfterMs ?? 500)); + } catch (error) { + lastError = error; + if (!/execution context was destroyed|navigation|target closed/i.test(String(error))) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } } throw new Error( - result.manualAction ?? - "Browser fallback could not create a Google Meet URL. Sign in to the OpenClaw browser profile, then retry.", + lastResult?.manualAction ?? + `Google Meet did not return a meeting URL from the browser create flow before timeout.${ + lastError + ? ` Last browser automation error: ${formatBrowserAutomationError(lastError)}` + : "" + }`, ); } @@ -410,6 +526,7 @@ function parseMeetBrowserStatus(result: unknown): GoogleMeetChromeHealth | undef manualActionMessage?: string; url?: string; title?: string; + notes?: string[]; }; return { inCall: parsed.inCall, @@ -420,12 +537,28 @@ function parseMeetBrowserStatus(result: unknown): GoogleMeetChromeHealth | undef browserUrl: parsed.url, browserTitle: parsed.title, status: "browser-control", + notes: Array.isArray(parsed.notes) + ? parsed.notes.filter((note): note is string => typeof note === "string") + : undefined, }; } function meetStatusScript(params: { guestName: string; autoJoin: boolean }) { return `() => { const text = (node) => (node?.innerText || node?.textContent || "").trim(); + const buttons = [...document.querySelectorAll('button')]; + const notes = []; + const findButton = (pattern) => + buttons.find((button) => { + const label = [ + button.getAttribute("aria-label"), + button.getAttribute("data-tooltip"), + text(button), + ] + .filter(Boolean) + .join(" "); + return pattern.test(label) && !button.disabled; + }); const input = [...document.querySelectorAll('input')].find((el) => /your name/i.test(el.getAttribute('aria-label') || el.placeholder || '') ); @@ -435,14 +568,18 @@ function meetStatusScript(params: { guestName: string; autoJoin: boolean }) { input.dispatchEvent(new Event('input', { bubbles: true })); input.dispatchEvent(new Event('change', { bubbles: true })); } - const buttons = [...document.querySelectorAll('button')]; const pageText = text(document.body).toLowerCase(); const host = location.hostname.toLowerCase(); const pageUrl = location.href; const join = ${JSON.stringify(params.autoJoin)} - ? buttons.find((button) => /join now|ask to join/i.test(text(button)) && !button.disabled) + ? findButton(/join now|ask to join/i) : null; if (join) join.click(); + const microphoneChoice = findButton(/\\buse microphone\\b/i); + if (microphoneChoice) { + microphoneChoice.click(); + notes.push("Accepted Meet microphone prompt with browser automation."); + } const mic = buttons.find((button) => /turn off microphone|turn on microphone|microphone/i.test(button.getAttribute('aria-label') || text(button))); const inCall = buttons.some((button) => /leave call/i.test(button.getAttribute('aria-label') || text(button))); let manualActionReason; @@ -456,16 +593,21 @@ function meetStatusScript(params: { guestName: string; autoJoin: boolean }) { } else if (!inCall && /allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) { manualActionReason = "meet-permission-required"; manualActionMessage = "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry."; + } else if (!inCall && !microphoneChoice && /do you want people to hear you in the meeting/i.test(pageText)) { + manualActionReason = "meet-audio-choice-required"; + manualActionMessage = "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry."; } return JSON.stringify({ clickedJoin: Boolean(join), + clickedMicrophoneChoice: Boolean(microphoneChoice), inCall, micMuted: mic ? /turn on microphone/i.test(mic.getAttribute('aria-label') || text(mic)) : undefined, manualActionRequired: Boolean(manualActionReason), manualActionReason, manualActionMessage, title: document.title, - url: pageUrl + url: pageUrl, + notes }); }`; } diff --git a/extensions/google-meet/src/transports/types.ts b/extensions/google-meet/src/transports/types.ts index 3bf6f1a4754..29f855a5418 100644 --- a/extensions/google-meet/src/transports/types.ts +++ b/extensions/google-meet/src/transports/types.ts @@ -16,6 +16,7 @@ export type GoogleMeetManualActionReason = | "google-login-required" | "meet-admission-required" | "meet-permission-required" + | "meet-audio-choice-required" | "browser-control-unavailable"; export type GoogleMeetChromeHealth = {