From 63dc5089b2111b2116d773a867e2af6183fea89c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Apr 2026 01:40:45 +0100 Subject: [PATCH] refactor(google-meet): split create browser flow --- extensions/google-meet/index.create.test.ts | 489 ++++++++++++++++++ extensions/google-meet/index.test.ts | 339 ------------ extensions/google-meet/index.ts | 98 +--- extensions/google-meet/src/create.ts | 103 ++++ extensions/google-meet/src/runtime.ts | 7 +- .../src/transports/chrome-browser-proxy.ts | 109 ++++ .../src/transports/chrome-create.ts | 323 ++++++++++++ .../google-meet/src/transports/chrome.ts | 428 +-------------- 8 files changed, 1038 insertions(+), 858 deletions(-) create mode 100644 extensions/google-meet/index.create.test.ts create mode 100644 extensions/google-meet/src/create.ts create mode 100644 extensions/google-meet/src/transports/chrome-browser-proxy.ts create mode 100644 extensions/google-meet/src/transports/chrome-create.ts diff --git a/extensions/google-meet/index.create.test.ts b/extensions/google-meet/index.create.test.ts new file mode 100644 index 00000000000..ec659a7af06 --- /dev/null +++ b/extensions/google-meet/index.create.test.ts @@ -0,0 +1,489 @@ +import { Command } from "commander"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createTestPluginApi } from "../../test/helpers/plugins/plugin-api.ts"; +import plugin from "./index.js"; +import { registerGoogleMeetCli } from "./src/cli.js"; +import { resolveGoogleMeetConfig } from "./src/config.js"; +import type { GoogleMeetRuntime } from "./src/runtime.js"; +import { CREATE_MEET_FROM_BROWSER_SCRIPT } from "./src/transports/chrome-create.js"; + +const voiceCallMocks = vi.hoisted(() => ({ + joinMeetViaVoiceCallGateway: vi.fn(async () => ({ callId: "call-1", dtmfSent: true })), + endMeetVoiceCallGatewayCall: vi.fn(async () => {}), +})); + +const fetchGuardMocks = vi.hoisted(() => ({ + fetchWithSsrFGuard: vi.fn( + async (params: { + url: string; + init?: RequestInit; + }): Promise<{ + response: Response; + release: () => Promise; + }> => ({ + response: await fetch(params.url, params.init), + release: vi.fn(async () => {}), + }), + ), +})); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ + fetchWithSsrFGuard: fetchGuardMocks.fetchWithSsrFGuard, +})); + +vi.mock("./src/voice-call-gateway.js", () => ({ + joinMeetViaVoiceCallGateway: voiceCallMocks.joinMeetViaVoiceCallGateway, + endMeetVoiceCallGatewayCall: voiceCallMocks.endMeetVoiceCallGatewayCall, +})); + +const noopLogger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), +}; + +function captureStdout() { + let output = ""; + const writeSpy = vi.spyOn(process.stdout, "write").mockImplementation(((chunk: unknown) => { + output += String(chunk); + return true; + }) as typeof process.stdout.write); + return { + output: () => output, + restore: () => writeSpy.mockRestore(), + }; +} + +async function runCreateMeetBrowserScript(params: { buttonText: string }) { + const location = { + href: "https://meet.google.com/new", + hostname: "meet.google.com", + }; + const button = { + disabled: false, + innerText: params.buttonText, + textContent: params.buttonText, + getAttribute: (name: string) => (name === "aria-label" ? params.buttonText : null), + click: vi.fn(() => { + location.href = "https://meet.google.com/abc-defg-hij"; + }), + }; + const document = { + title: "Meet", + body: { + innerText: "Do you want people to hear you in the meeting?", + textContent: "Do you want people to hear you in the meeting?", + }, + querySelectorAll: (selector: string) => (selector === "button" ? [button] : []), + }; + vi.stubGlobal("document", document); + vi.stubGlobal("location", location); + const fn = (0, eval)(`(${CREATE_MEET_FROM_BROWSER_SCRIPT})`) as () => Promise<{ + meetingUri?: string; + manualActionReason?: string; + notes?: string[]; + retryAfterMs?: number; + }>; + return { button, result: await fn() }; +} + +function setup( + config: Record = {}, + options: { + nodesInvokeHandler?: (params: { + nodeId: string; + command: string; + params?: unknown; + timeoutMs?: number; + }) => Promise; + } = {}, +) { + const methods = new Map(); + const tools: unknown[] = []; + const nodesList = vi.fn(async () => ({ + nodes: [ + { + nodeId: "node-1", + displayName: "parallels-macos", + connected: true, + caps: ["browser"], + commands: ["browser.proxy", "googlemeet.chrome"], + }, + ], + })); + const nodesInvoke = vi.fn(async (params) => { + if (options.nodesInvokeHandler) { + return options.nodesInvokeHandler(params); + } + if (params.command === "browser.proxy") { + const proxy = params.params as { path?: string; body?: { url?: string; targetId?: string } }; + if (proxy.path === "/tabs") { + return { payload: { result: { running: true, tabs: [] } } }; + } + if (proxy.path === "/tabs/open") { + return { + payload: { + result: { + targetId: "tab-1", + title: "Meet", + url: proxy.body?.url ?? "https://meet.google.com/abc-defg-hij", + }, + }, + }; + } + return { payload: { result: { ok: true } } }; + } + return { payload: { launched: true } }; + }); + const runCommandWithTimeout = vi.fn(async (argv: string[]) => { + if (argv[0] === "/usr/sbin/system_profiler") { + return { code: 0, stdout: "BlackHole 2ch", stderr: "" }; + } + return { code: 0, stdout: "", stderr: "" }; + }); + const api = createTestPluginApi({ + id: "google-meet", + name: "Google Meet", + description: "test", + version: "0", + source: "test", + config: {}, + pluginConfig: config, + runtime: { + system: { + runCommandWithTimeout, + formatNativeDependencyHint: vi.fn(() => "Install with brew install blackhole-2ch."), + }, + nodes: { + list: nodesList, + invoke: nodesInvoke, + }, + } as unknown as OpenClawPluginApi["runtime"], + logger: noopLogger, + registerGatewayMethod: (method: string, handler: unknown) => methods.set(method, handler), + registerTool: (tool: unknown) => tools.push(tool), + }); + plugin.register(api); + return { + methods, + tools, + nodesInvoke, + }; +} + +describe("google-meet create flow", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("CLI create prints the new meeting URL", async () => { + const fetchMock = vi.fn(async (input: RequestInfo | URL, _init?: RequestInit) => { + const url = input instanceof Request ? input.url : input.toString(); + if (url.includes("oauth2.googleapis.com")) { + return new Response( + JSON.stringify({ + access_token: "new-access-token", + expires_in: 3600, + token_type: "Bearer", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response( + JSON.stringify({ + name: "spaces/new-space", + meetingCode: "new-abcd-xyz", + meetingUri: "https://meet.google.com/new-abcd-xyz", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + const program = new Command(); + const stdout = captureStdout(); + registerGoogleMeetCli({ + program, + config: resolveGoogleMeetConfig({ + oauth: { clientId: "client-id", refreshToken: "refresh-token" }, + }), + ensureRuntime: async () => ({}) as GoogleMeetRuntime, + }); + + try { + await program.parseAsync(["googlemeet", "create", "--no-join"], { from: "user" }); + expect(stdout.output()).toContain("meeting uri: https://meet.google.com/new-abcd-xyz"); + expect(stdout.output()).toContain("space: spaces/new-space"); + } finally { + stdout.restore(); + } + }); + + it("can create a Meet through browser fallback without joining when requested", async () => { + const { methods, nodesInvoke } = setup( + { + defaultTransport: "chrome-node", + chromeNode: { node: "parallels-macos" }, + }, + { + nodesInvokeHandler: async (params) => { + const proxy = params.params as { path?: string; body?: { url?: string } }; + if (proxy.path === "/tabs") { + return { payload: { result: { tabs: [] } } }; + } + if (proxy.path === "/tabs/open") { + return { + payload: { + result: { + targetId: "tab-1", + title: "Meet", + url: proxy.body?.url, + }, + }, + }; + } + if (proxy.path === "/act") { + return { + payload: { + result: { + ok: true, + targetId: "tab-1", + result: { + meetingUri: "https://meet.google.com/browser-made-url", + browserUrl: "https://meet.google.com/browser-made-url", + browserTitle: "Meet", + }, + }, + }, + }; + } + return { payload: { result: { ok: true } } }; + }, + }, + ); + const handler = methods.get("googlemeet.create") as + | ((ctx: { + params: Record; + respond: ReturnType; + }) => Promise) + | undefined; + const respond = vi.fn(); + + await handler?.({ params: { join: false }, respond }); + + expect(respond.mock.calls[0]?.[0]).toBe(true); + expect(respond.mock.calls[0]?.[1]).toMatchObject({ + source: "browser", + meetingUri: "https://meet.google.com/browser-made-url", + joined: false, + browser: { nodeId: "node-1", targetId: "tab-1" }, + }); + expect(nodesInvoke).toHaveBeenCalledWith( + expect.objectContaining({ + command: "browser.proxy", + params: expect.objectContaining({ + path: "/tabs/open", + body: { url: "https://meet.google.com/new" }, + }), + }), + ); + }); + + it("creates and joins a Meet through the create tool action by default", async () => { + const { tools, nodesInvoke } = setup( + { + defaultTransport: "chrome-node", + defaultMode: "transcribe", + chromeNode: { node: "parallels-macos" }, + }, + { + nodesInvokeHandler: async (params) => { + if (params.command === "googlemeet.chrome") { + return { payload: { launched: true } }; + } + const proxy = params.params as { + path?: string; + body?: { url?: string; targetId?: string; fn?: string }; + }; + if (proxy.path === "/tabs") { + return { payload: { result: { tabs: [] } } }; + } + if (proxy.path === "/tabs/open") { + return { + payload: { + result: { + targetId: + proxy.body?.url === "https://meet.google.com/new" ? "create-tab" : "join-tab", + title: "Meet", + url: proxy.body?.url, + }, + }, + }; + } + if (proxy.path === "/act" && proxy.body?.fn?.includes("meetUrlPattern")) { + return { + payload: { + result: { + ok: true, + targetId: "create-tab", + result: { + meetingUri: "https://meet.google.com/new-abcd-xyz", + browserUrl: "https://meet.google.com/new-abcd-xyz", + browserTitle: "Meet", + }, + }, + }, + }; + } + if (proxy.path === "/act") { + return { + payload: { + result: { + ok: true, + targetId: "join-tab", + result: JSON.stringify({ + inCall: true, + micMuted: false, + title: "Meet call", + url: "https://meet.google.com/new-abcd-xyz", + }), + }, + }, + }; + } + return { payload: { result: { ok: true } } }; + }, + }, + ); + const tool = tools[0] as { + execute: ( + id: string, + params: unknown, + ) => Promise<{ + details: { joined?: boolean; meetingUri?: string; join?: { session: { url: string } } }; + }>; + }; + + const result = await tool.execute("id", { action: "create" }); + + expect(result.details).toMatchObject({ + source: "browser", + joined: true, + meetingUri: "https://meet.google.com/new-abcd-xyz", + join: { session: { url: "https://meet.google.com/new-abcd-xyz" } }, + }); + expect(nodesInvoke).toHaveBeenCalledWith( + expect.objectContaining({ + command: "googlemeet.chrome", + params: expect.objectContaining({ + action: "start", + url: "https://meet.google.com/new-abcd-xyz", + launch: false, + }), + }), + ); + }); + + it("reuses an existing browser create tab instead of opening duplicates", async () => { + const { methods, nodesInvoke } = setup( + { + defaultTransport: "chrome-node", + chromeNode: { node: "parallels-macos" }, + }, + { + nodesInvokeHandler: async (params) => { + const proxy = params.params as { path?: string; body?: { targetId?: string } }; + if (proxy.path === "/tabs") { + return { + payload: { + result: { + tabs: [ + { + targetId: "existing-create-tab", + title: "Meet", + url: "https://meet.google.com/new", + }, + ], + }, + }, + }; + } + if (proxy.path === "/tabs/focus") { + return { payload: { result: { ok: true } } }; + } + if (proxy.path === "/act") { + return { + payload: { + result: { + ok: true, + targetId: proxy.body?.targetId ?? "existing-create-tab", + result: { + meetingUri: "https://meet.google.com/reu-sedx-tab", + browserUrl: "https://meet.google.com/reu-sedx-tab", + browserTitle: "Meet", + }, + }, + }, + }; + } + throw new Error(`unexpected browser proxy path ${proxy.path}`); + }, + }, + ); + const handler = methods.get("googlemeet.create") as + | ((ctx: { + params: Record; + respond: ReturnType; + }) => Promise) + | undefined; + const respond = vi.fn(); + + await handler?.({ params: { join: false }, respond }); + + expect(respond.mock.calls[0]?.[0]).toBe(true); + expect(respond.mock.calls[0]?.[1]).toMatchObject({ + source: "browser", + meetingUri: "https://meet.google.com/reu-sedx-tab", + browser: { nodeId: "node-1", targetId: "existing-create-tab" }, + }); + expect(nodesInvoke).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ + path: "/tabs/focus", + body: { targetId: "existing-create-tab" }, + }), + }), + ); + expect(nodesInvoke).not.toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ path: "/tabs/open" }), + }), + ); + }); + + it.each([ + ["Use microphone", "Accepted Meet microphone prompt with browser automation."], + [ + "Continue without microphone", + "Continued through Meet microphone prompt with browser automation.", + ], + ])( + "uses browser automation for Meet's %s choice during browser creation", + async (buttonText, note) => { + const { button, result } = await runCreateMeetBrowserScript({ buttonText }); + + expect(result).toMatchObject({ + retryAfterMs: 1000, + notes: [note], + }); + expect(button.click).toHaveBeenCalledTimes(1); + expect(result.meetingUri).toBeUndefined(); + expect(result.manualActionReason).toBeUndefined(); + }, + ); +}); diff --git a/extensions/google-meet/index.test.ts b/extensions/google-meet/index.test.ts index e2338bfde8f..34de65415cb 100644 --- a/extensions/google-meet/index.test.ts +++ b/extensions/google-meet/index.test.ts @@ -23,7 +23,6 @@ import { startNodeRealtimeAudioBridge } from "./src/realtime-node.js"; import { startCommandRealtimeAudioBridge } from "./src/realtime.js"; import { normalizeMeetUrl } from "./src/runtime.js"; import type { GoogleMeetRuntime } from "./src/runtime.js"; -import { CREATE_MEET_FROM_BROWSER_SCRIPT } from "./src/transports/chrome.js"; import { buildMeetDtmfSequence, normalizeDialInNumber } from "./src/transports/twilio.js"; const voiceCallMocks = vi.hoisted(() => ({ @@ -74,39 +73,6 @@ function captureStdout() { }; } -async function runCreateMeetBrowserScript(params: { buttonText: string }) { - const location = { - href: "https://meet.google.com/new", - hostname: "meet.google.com", - }; - const button = { - disabled: false, - innerText: params.buttonText, - textContent: params.buttonText, - getAttribute: (name: string) => (name === "aria-label" ? params.buttonText : null), - click: vi.fn(() => { - location.href = "https://meet.google.com/abc-defg-hij"; - }), - }; - const document = { - title: "Meet", - body: { - innerText: "Do you want people to hear you in the meeting?", - textContent: "Do you want people to hear you in the meeting?", - }, - querySelectorAll: (selector: string) => (selector === "button" ? [button] : []), - }; - vi.stubGlobal("document", document); - vi.stubGlobal("location", location); - const fn = (0, eval)(`(${CREATE_MEET_FROM_BROWSER_SCRIPT})`) as () => Promise<{ - meetingUri?: string; - manualActionReason?: string; - notes?: string[]; - retryAfterMs?: number; - }>; - return { button, result: await fn() }; -} - type TestBridgeProcess = { stdin?: { write(chunk: unknown): unknown } | null; stdout?: { on(event: "data", listener: (chunk: unknown) => void): unknown } | null; @@ -769,311 +735,6 @@ describe("google-meet plugin", () => { } }); - it("CLI create prints the new meeting URL", async () => { - const fetchMock = vi.fn(async (input: RequestInfo | URL, _init?: RequestInit) => { - const url = input instanceof Request ? input.url : input.toString(); - if (url.includes("oauth2.googleapis.com")) { - return new Response( - JSON.stringify({ - access_token: "new-access-token", - expires_in: 3600, - token_type: "Bearer", - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ); - } - return new Response( - JSON.stringify({ - name: "spaces/new-space", - meetingCode: "new-abcd-xyz", - meetingUri: "https://meet.google.com/new-abcd-xyz", - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ); - }); - vi.stubGlobal("fetch", fetchMock); - const program = new Command(); - const stdout = captureStdout(); - registerGoogleMeetCli({ - program, - config: resolveGoogleMeetConfig({ - oauth: { clientId: "client-id", refreshToken: "refresh-token" }, - }), - ensureRuntime: async () => ({}) as GoogleMeetRuntime, - }); - - try { - await program.parseAsync(["googlemeet", "create", "--no-join"], { from: "user" }); - expect(stdout.output()).toContain("meeting uri: https://meet.google.com/new-abcd-xyz"); - expect(stdout.output()).toContain("space: spaces/new-space"); - } finally { - stdout.restore(); - } - }); - - it("can create a Meet through browser fallback without joining when requested", async () => { - const { methods, nodesInvoke } = setup( - { - defaultTransport: "chrome-node", - chromeNode: { node: "parallels-macos" }, - }, - { - nodesInvokeHandler: async (params) => { - const proxy = params.params as { path?: string; body?: { url?: string } }; - if (proxy.path === "/tabs") { - return { payload: { result: { tabs: [] } } }; - } - if (proxy.path === "/tabs/open") { - return { - payload: { - result: { - targetId: "tab-1", - title: "Meet", - url: proxy.body?.url, - }, - }, - }; - } - if (proxy.path === "/act") { - return { - payload: { - result: { - ok: true, - targetId: "tab-1", - result: { - meetingUri: "https://meet.google.com/browser-made-url", - browserUrl: "https://meet.google.com/browser-made-url", - browserTitle: "Meet", - }, - }, - }, - }; - } - return { payload: { result: { ok: true } } }; - }, - }, - ); - const handler = methods.get("googlemeet.create") as - | ((ctx: { - params: Record; - respond: ReturnType; - }) => Promise) - | undefined; - const respond = vi.fn(); - - await handler?.({ params: { join: false }, respond }); - - expect(respond.mock.calls[0]?.[0]).toBe(true); - expect(respond.mock.calls[0]?.[1]).toMatchObject({ - source: "browser", - meetingUri: "https://meet.google.com/browser-made-url", - joined: false, - browser: { nodeId: "node-1", targetId: "tab-1" }, - }); - expect(nodesInvoke).toHaveBeenCalledWith( - expect.objectContaining({ - command: "browser.proxy", - params: expect.objectContaining({ - path: "/tabs/open", - body: { url: "https://meet.google.com/new" }, - }), - }), - ); - }); - - it("creates and joins a Meet through the create tool action by default", async () => { - const { tools, nodesInvoke } = setup( - { - defaultTransport: "chrome-node", - defaultMode: "transcribe", - chromeNode: { node: "parallels-macos" }, - }, - { - nodesInvokeHandler: async (params) => { - if (params.command === "googlemeet.chrome") { - return { payload: { launched: true } }; - } - const proxy = params.params as { - path?: string; - body?: { url?: string; targetId?: string; fn?: string }; - }; - if (proxy.path === "/tabs") { - return { payload: { result: { tabs: [] } } }; - } - if (proxy.path === "/tabs/open") { - return { - payload: { - result: { - targetId: - proxy.body?.url === "https://meet.google.com/new" ? "create-tab" : "join-tab", - title: "Meet", - url: proxy.body?.url, - }, - }, - }; - } - if (proxy.path === "/act" && proxy.body?.fn?.includes("meetUrlPattern")) { - return { - payload: { - result: { - ok: true, - targetId: "create-tab", - result: { - meetingUri: "https://meet.google.com/new-abcd-xyz", - browserUrl: "https://meet.google.com/new-abcd-xyz", - browserTitle: "Meet", - }, - }, - }, - }; - } - if (proxy.path === "/act") { - return { - payload: { - result: { - ok: true, - targetId: "join-tab", - result: JSON.stringify({ - inCall: true, - micMuted: false, - title: "Meet call", - url: "https://meet.google.com/new-abcd-xyz", - }), - }, - }, - }; - } - return { payload: { result: { ok: true } } }; - }, - }, - ); - const tool = tools[0] as { - execute: ( - id: string, - params: unknown, - ) => Promise<{ - details: { joined?: boolean; meetingUri?: string; join?: { session: { url: string } } }; - }>; - }; - - const result = await tool.execute("id", { action: "create" }); - - expect(result.details).toMatchObject({ - source: "browser", - joined: true, - meetingUri: "https://meet.google.com/new-abcd-xyz", - join: { session: { url: "https://meet.google.com/new-abcd-xyz" } }, - }); - expect(nodesInvoke).toHaveBeenCalledWith( - expect.objectContaining({ - command: "googlemeet.chrome", - params: expect.objectContaining({ - action: "start", - url: "https://meet.google.com/new-abcd-xyz", - launch: false, - }), - }), - ); - }); - - it("reuses an existing browser create tab instead of opening duplicates", async () => { - const { methods, nodesInvoke } = setup( - { - defaultTransport: "chrome-node", - chromeNode: { node: "parallels-macos" }, - }, - { - nodesInvokeHandler: async (params) => { - const proxy = params.params as { path?: string; body?: { targetId?: string } }; - if (proxy.path === "/tabs") { - return { - payload: { - result: { - tabs: [ - { - targetId: "existing-create-tab", - title: "Meet", - url: "https://meet.google.com/new", - }, - ], - }, - }, - }; - } - if (proxy.path === "/tabs/focus") { - return { payload: { result: { ok: true } } }; - } - if (proxy.path === "/act") { - return { - payload: { - result: { - ok: true, - targetId: proxy.body?.targetId ?? "existing-create-tab", - result: { - meetingUri: "https://meet.google.com/reu-sedx-tab", - browserUrl: "https://meet.google.com/reu-sedx-tab", - browserTitle: "Meet", - }, - }, - }, - }; - } - throw new Error(`unexpected browser proxy path ${proxy.path}`); - }, - }, - ); - const handler = methods.get("googlemeet.create") as - | ((ctx: { - params: Record; - respond: ReturnType; - }) => Promise) - | undefined; - const respond = vi.fn(); - - await handler?.({ params: { join: false }, respond }); - - expect(respond.mock.calls[0]?.[0]).toBe(true); - expect(respond.mock.calls[0]?.[1]).toMatchObject({ - source: "browser", - meetingUri: "https://meet.google.com/reu-sedx-tab", - browser: { nodeId: "node-1", targetId: "existing-create-tab" }, - }); - expect(nodesInvoke).toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ - path: "/tabs/focus", - body: { targetId: "existing-create-tab" }, - }), - }), - ); - expect(nodesInvoke).not.toHaveBeenCalledWith( - expect.objectContaining({ - params: expect.objectContaining({ path: "/tabs/open" }), - }), - ); - }); - - it.each([ - ["Use microphone", "Accepted Meet microphone prompt with browser automation."], - [ - "Continue without microphone", - "Continued through Meet microphone prompt with browser automation.", - ], - ])( - "uses browser automation for Meet's %s choice during browser creation", - async (buttonText, note) => { - const { button, result } = await runCreateMeetBrowserScript({ buttonText }); - - expect(result).toMatchObject({ - retryAfterMs: 1000, - notes: [note], - }); - expect(button.click).toHaveBeenCalledTimes(1); - expect(result.meetingUri).toBeUndefined(); - expect(result.manualActionReason).toBeUndefined(); - }, - ); - it("launches Chrome after the BlackHole check", async () => { const originalPlatform = process.platform; Object.defineProperty(process, "platform", { value: "darwin" }); diff --git a/extensions/google-meet/index.ts b/extensions/google-meet/index.ts index fc5582f803d..7c9a19f1bf9 100644 --- a/extensions/google-meet/index.ts +++ b/extensions/google-meet/index.ts @@ -11,14 +11,14 @@ import { type GoogleMeetTransport, } from "./src/config.js"; import { - buildGoogleMeetPreflightReport, - createGoogleMeetSpace, - fetchGoogleMeetSpace, -} from "./src/meet.js"; + createAndJoinMeetFromParams, + createMeetFromParams, + shouldJoinCreatedMeet, +} from "./src/create.js"; +import { buildGoogleMeetPreflightReport, fetchGoogleMeetSpace } from "./src/meet.js"; import { handleGoogleMeetNodeHostCommand } from "./src/node-host.js"; import { resolveGoogleMeetAccessToken } from "./src/oauth.js"; import { GoogleMeetRuntime } from "./src/runtime.js"; -import { createMeetWithBrowserProxyOnNode } from "./src/transports/chrome.js"; const googleMeetConfigSchema = { parse(value: unknown) { @@ -225,94 +225,6 @@ async function resolveSpaceFromParams(config: GoogleMeetConfig, raw: Record) { - const token = await resolveGoogleMeetAccessToken({ - clientId: normalizeOptionalString(raw.clientId) ?? config.oauth.clientId, - clientSecret: normalizeOptionalString(raw.clientSecret) ?? config.oauth.clientSecret, - refreshToken: normalizeOptionalString(raw.refreshToken) ?? config.oauth.refreshToken, - accessToken: normalizeOptionalString(raw.accessToken) ?? config.oauth.accessToken, - expiresAt: typeof raw.expiresAt === "number" ? raw.expiresAt : config.oauth.expiresAt, - }); - const result = await createGoogleMeetSpace({ accessToken: token.accessToken }); - return { source: "api" as const, token, ...result }; -} - -function hasGoogleMeetOAuth(config: GoogleMeetConfig, raw: Record): boolean { - return Boolean( - normalizeOptionalString(raw.accessToken) ?? - normalizeOptionalString(raw.refreshToken) ?? - config.oauth.accessToken ?? - config.oauth.refreshToken, - ); -} - -function shouldJoinCreatedMeet(raw: Record): boolean { - return raw.join !== false && raw.join !== "false"; -} - -async function createMeetFromParams(params: { - config: GoogleMeetConfig; - runtime: OpenClawPluginApi["runtime"]; - raw: Record; -}) { - if (hasGoogleMeetOAuth(params.config, params.raw)) { - const { token: _token, ...result } = await createSpaceFromParams(params.config, params.raw); - return { - ...result, - joined: false, - nextAction: - "URL-only creation was requested. Call google_meet with action=join and url=meetingUri to enter the meeting.", - }; - } - const browser = await createMeetWithBrowserProxyOnNode({ - runtime: params.runtime, - config: params.config, - }); - return { - source: browser.source, - meetingUri: browser.meetingUri, - joined: false, - nextAction: - "URL-only creation was requested. Call google_meet with action=join and url=meetingUri to enter the meeting.", - space: { - name: `browser/${browser.meetingUri.split("/").pop()}`, - meetingUri: browser.meetingUri, - }, - browser: { - nodeId: browser.nodeId, - targetId: browser.targetId, - browserUrl: browser.browserUrl, - browserTitle: browser.browserTitle, - notes: browser.notes, - }, - }; -} - -async function createAndJoinMeetFromParams(params: { - config: GoogleMeetConfig; - runtime: OpenClawPluginApi["runtime"]; - raw: Record; - ensureRuntime: () => Promise; -}) { - const created = await createMeetFromParams(params); - const rt = await params.ensureRuntime(); - const join = await rt.join({ - url: created.meetingUri, - transport: normalizeTransport(params.raw.transport), - mode: normalizeMode(params.raw.mode), - dialInNumber: normalizeOptionalString(params.raw.dialInNumber), - pin: normalizeOptionalString(params.raw.pin), - dtmfSequence: normalizeOptionalString(params.raw.dtmfSequence), - message: normalizeOptionalString(params.raw.message), - }); - return { - ...created, - joined: true, - nextAction: "Share meetingUri with participants; the OpenClaw agent has started the join flow.", - join, - }; -} - export default definePluginEntry({ id: "google-meet", name: "Google Meet", diff --git a/extensions/google-meet/src/create.ts b/extensions/google-meet/src/create.ts new file mode 100644 index 00000000000..b5f892c6f89 --- /dev/null +++ b/extensions/google-meet/src/create.ts @@ -0,0 +1,103 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; +import type { GoogleMeetConfig, GoogleMeetMode, GoogleMeetTransport } from "./config.js"; +import { createGoogleMeetSpace } from "./meet.js"; +import { resolveGoogleMeetAccessToken } from "./oauth.js"; +import type { GoogleMeetRuntime } from "./runtime.js"; +import { createMeetWithBrowserProxyOnNode } from "./transports/chrome-create.js"; + +function normalizeTransport(value: unknown): GoogleMeetTransport | undefined { + return value === "chrome" || value === "chrome-node" || value === "twilio" ? value : undefined; +} + +function normalizeMode(value: unknown): GoogleMeetMode | undefined { + return value === "realtime" || value === "transcribe" ? value : undefined; +} + +async function createSpaceFromParams(config: GoogleMeetConfig, raw: Record) { + const token = await resolveGoogleMeetAccessToken({ + clientId: normalizeOptionalString(raw.clientId) ?? config.oauth.clientId, + clientSecret: normalizeOptionalString(raw.clientSecret) ?? config.oauth.clientSecret, + refreshToken: normalizeOptionalString(raw.refreshToken) ?? config.oauth.refreshToken, + accessToken: normalizeOptionalString(raw.accessToken) ?? config.oauth.accessToken, + expiresAt: typeof raw.expiresAt === "number" ? raw.expiresAt : config.oauth.expiresAt, + }); + const result = await createGoogleMeetSpace({ accessToken: token.accessToken }); + return { source: "api" as const, token, ...result }; +} + +function hasGoogleMeetOAuth(config: GoogleMeetConfig, raw: Record): boolean { + return Boolean( + normalizeOptionalString(raw.accessToken) ?? + normalizeOptionalString(raw.refreshToken) ?? + config.oauth.accessToken ?? + config.oauth.refreshToken, + ); +} + +export function shouldJoinCreatedMeet(raw: Record): boolean { + return raw.join !== false && raw.join !== "false"; +} + +export async function createMeetFromParams(params: { + config: GoogleMeetConfig; + runtime: OpenClawPluginApi["runtime"]; + raw: Record; +}) { + if (hasGoogleMeetOAuth(params.config, params.raw)) { + const { token: _token, ...result } = await createSpaceFromParams(params.config, params.raw); + return { + ...result, + joined: false, + nextAction: + "URL-only creation was requested. Call google_meet with action=join and url=meetingUri to enter the meeting.", + }; + } + const browser = await createMeetWithBrowserProxyOnNode({ + runtime: params.runtime, + config: params.config, + }); + return { + source: browser.source, + meetingUri: browser.meetingUri, + joined: false, + nextAction: + "URL-only creation was requested. Call google_meet with action=join and url=meetingUri to enter the meeting.", + space: { + name: `browser/${browser.meetingUri.split("/").pop()}`, + meetingUri: browser.meetingUri, + }, + browser: { + nodeId: browser.nodeId, + targetId: browser.targetId, + browserUrl: browser.browserUrl, + browserTitle: browser.browserTitle, + notes: browser.notes, + }, + }; +} + +export async function createAndJoinMeetFromParams(params: { + config: GoogleMeetConfig; + runtime: OpenClawPluginApi["runtime"]; + raw: Record; + ensureRuntime: () => Promise; +}) { + const created = await createMeetFromParams(params); + const rt = await params.ensureRuntime(); + const join = await rt.join({ + url: created.meetingUri, + transport: normalizeTransport(params.raw.transport), + mode: normalizeMode(params.raw.mode), + dialInNumber: normalizeOptionalString(params.raw.dialInNumber), + pin: normalizeOptionalString(params.raw.pin), + dtmfSequence: normalizeOptionalString(params.raw.dtmfSequence), + message: normalizeOptionalString(params.raw.message), + }); + return { + ...created, + joined: true, + nextAction: "Share meetingUri with participants; the OpenClaw agent has started the join flow.", + join, + }; +} diff --git a/extensions/google-meet/src/runtime.ts b/extensions/google-meet/src/runtime.ts index 92b122cca41..ae3d44350ce 100644 --- a/extensions/google-meet/src/runtime.ts +++ b/extensions/google-meet/src/runtime.ts @@ -5,11 +5,8 @@ import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-ru import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { GoogleMeetConfig, GoogleMeetMode, GoogleMeetTransport } from "./config.js"; import { getGoogleMeetSetupStatus } from "./setup.js"; -import { - createMeetWithBrowserProxyOnNode, - launchChromeMeet, - launchChromeMeetOnNode, -} from "./transports/chrome.js"; +import { createMeetWithBrowserProxyOnNode } from "./transports/chrome-create.js"; +import { launchChromeMeet, launchChromeMeetOnNode } from "./transports/chrome.js"; import { buildMeetDtmfSequence, normalizeDialInNumber } from "./transports/twilio.js"; import type { GoogleMeetChromeHealth, diff --git a/extensions/google-meet/src/transports/chrome-browser-proxy.ts b/extensions/google-meet/src/transports/chrome-browser-proxy.ts new file mode 100644 index 00000000000..292ccde9026 --- /dev/null +++ b/extensions/google-meet/src/transports/chrome-browser-proxy.ts @@ -0,0 +1,109 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; + +type BrowserProxyResult = { + result?: unknown; +}; + +export type BrowserTab = { + targetId?: string; + title?: string; + url?: string; +}; + +function isGoogleMeetNode(node: { + caps?: string[]; + commands?: string[]; + connected?: boolean; + nodeId?: string; + displayName?: string; + remoteIp?: string; +}) { + const commands = Array.isArray(node.commands) ? node.commands : []; + const caps = Array.isArray(node.caps) ? node.caps : []; + return ( + node.connected === true && + commands.includes("googlemeet.chrome") && + (commands.includes("browser.proxy") || caps.includes("browser")) + ); +} + +export async function resolveChromeNode(params: { + runtime: PluginRuntime; + requestedNode?: string; +}): Promise { + const list = await params.runtime.nodes.list({ connected: true }); + const nodes = list.nodes.filter(isGoogleMeetNode); + if (nodes.length === 0) { + throw new Error( + "No connected Google Meet-capable node with browser proxy. Run `openclaw node run` on the Chrome host with browser proxy enabled, approve pairing, and allow googlemeet.chrome plus browser.proxy.", + ); + } + const requested = params.requestedNode?.trim(); + if (requested) { + const matches = nodes.filter((node) => + [node.nodeId, node.displayName, node.remoteIp].some((value) => value === requested), + ); + if (matches.length === 1) { + return matches[0].nodeId; + } + throw new Error(`Google Meet node not found or ambiguous: ${requested}`); + } + if (nodes.length === 1) { + return nodes[0].nodeId; + } + throw new Error( + "Multiple Google Meet-capable nodes connected. Set plugins.entries.google-meet.config.chromeNode.node.", + ); +} + +function unwrapNodeInvokePayload(raw: unknown): unknown { + const record = raw && typeof raw === "object" ? (raw as Record) : {}; + if (typeof record.payloadJSON === "string" && record.payloadJSON.trim()) { + return JSON.parse(record.payloadJSON); + } + if ("payload" in record) { + return record.payload; + } + return raw; +} + +function parseBrowserProxyResult(raw: unknown): unknown { + const payload = unwrapNodeInvokePayload(raw); + const proxy = + payload && typeof payload === "object" ? (payload as BrowserProxyResult) : undefined; + if (!proxy || !("result" in proxy)) { + throw new Error("Google Meet browser proxy returned an invalid result."); + } + return proxy.result; +} + +export async function callBrowserProxyOnNode(params: { + runtime: PluginRuntime; + nodeId: string; + method: "GET" | "POST" | "DELETE"; + path: string; + body?: unknown; + timeoutMs: number; +}) { + const raw = await params.runtime.nodes.invoke({ + nodeId: params.nodeId, + command: "browser.proxy", + params: { + method: params.method, + path: params.path, + body: params.body, + timeoutMs: params.timeoutMs, + }, + timeoutMs: params.timeoutMs + 5_000, + }); + return parseBrowserProxyResult(raw); +} + +export function asBrowserTabs(result: unknown): BrowserTab[] { + const record = result && typeof result === "object" ? (result as Record) : {}; + return Array.isArray(record.tabs) ? (record.tabs as BrowserTab[]) : []; +} + +export function readBrowserTab(result: unknown): BrowserTab | undefined { + return result && typeof result === "object" ? (result as BrowserTab) : undefined; +} diff --git a/extensions/google-meet/src/transports/chrome-create.ts b/extensions/google-meet/src/transports/chrome-create.ts new file mode 100644 index 00000000000..6c4bbd96015 --- /dev/null +++ b/extensions/google-meet/src/transports/chrome-create.ts @@ -0,0 +1,323 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import type { GoogleMeetConfig } from "../config.js"; +import { + asBrowserTabs, + callBrowserProxyOnNode, + readBrowserTab, + resolveChromeNode, + type BrowserTab, +} from "./chrome-browser-proxy.js"; +import type { GoogleMeetChromeHealth } from "./types.js"; + +const GOOGLE_MEET_NEW_URL = "https://meet.google.com/new"; +const GOOGLE_MEET_BROWSER_CREATE_TIMEOUT_MS = 60_000; +const GOOGLE_MEET_BROWSER_STEP_TIMEOUT_MS = 10_000; +const GOOGLE_MEET_BROWSER_NAVIGATION_RETRY_MS = 1_000; +const GOOGLE_MEET_BROWSER_POLL_MS = 500; + +type BrowserCreateStepResult = { + meetingUri?: string; + browserUrl?: string; + browserTitle?: string; + manualAction?: string; + manualActionReason?: GoogleMeetChromeHealth["manualActionReason"]; + notes?: string[]; + retryAfterMs?: number; +}; + +export type GoogleMeetBrowserCreateResult = { + meetingUri: string; + nodeId: string; + targetId?: string; + browserUrl?: string; + browserTitle?: string; + notes?: string[]; + source: "browser"; +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function formatBrowserAutomationError(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + try { + return JSON.stringify(error); + } catch { + return "unknown error"; + } +} + +function isBrowserNavigationInterruption(error: unknown): boolean { + return /execution context was destroyed|navigation|target closed/i.test( + formatBrowserAutomationError(error), + ); +} + +function isGoogleMeetCreateTab(tab: BrowserTab): boolean { + const url = tab.url ?? ""; + if (/^https:\/\/meet\.google\.com\/(?:new|[a-z]{3}-[a-z]{4}-[a-z]{3})(?:$|[/?#])/i.test(url)) { + return true; + } + return ( + url.startsWith("https://accounts.google.com/") && + /sign in|google accounts|meet/i.test(tab.title ?? "") + ); +} + +async function findGoogleMeetCreateTab(params: { + runtime: PluginRuntime; + nodeId: string; + timeoutMs: number; +}): Promise { + const tabs = asBrowserTabs( + await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId: params.nodeId, + method: "GET", + path: "/tabs", + timeoutMs: params.timeoutMs, + }), + ); + return tabs.find(isGoogleMeetCreateTab); +} + +async function focusBrowserTab(params: { + runtime: PluginRuntime; + nodeId: string; + targetId: string; + timeoutMs: number; +}): Promise { + await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId: params.nodeId, + method: "POST", + path: "/tabs/focus", + body: { targetId: params.targetId }, + timeoutMs: params.timeoutMs, + }); +} + +function readStringArray(value: unknown): string[] | undefined { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === "string") + : undefined; +} + +function readBrowserCreateResult(result: unknown): BrowserCreateStepResult { + const record = result && typeof result === "object" ? (result as Record) : {}; + const nested = + record.result && typeof record.result === "object" + ? (record.result as Record) + : record; + return { + meetingUri: typeof nested.meetingUri === "string" ? nested.meetingUri : undefined, + browserUrl: typeof nested.browserUrl === "string" ? nested.browserUrl : undefined, + browserTitle: typeof nested.browserTitle === "string" ? nested.browserTitle : undefined, + manualAction: typeof nested.manualAction === "string" ? nested.manualAction : undefined, + manualActionReason: + typeof nested.manualActionReason === "string" + ? (nested.manualActionReason as GoogleMeetChromeHealth["manualActionReason"]) + : undefined, + notes: readStringArray(nested.notes), + retryAfterMs: + typeof nested.retryAfterMs === "number" && Number.isFinite(nested.retryAfterMs) + ? nested.retryAfterMs + : undefined, + }; +} + +export const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { + const meetUrlPattern = /^https:\\/\\/meet\\.google\\.com\\/[a-z]{3}-[a-z]{4}-[a-z]{3}(?:$|[/?#])/i; + const text = (node) => (node?.innerText || node?.textContent || "").trim(); + const current = () => location.href; + const notes = []; + const findButton = (pattern) => + [...document.querySelectorAll("button")].find((button) => { + const label = [ + button.getAttribute("aria-label"), + button.getAttribute("data-tooltip"), + text(button), + ] + .filter(Boolean) + .join(" "); + return pattern.test(label) && !button.disabled; + }); + const clickButton = (pattern, note) => { + const button = findButton(pattern); + if (!button) { + return false; + } + button.click(); + notes.push(note); + return true; + }; + if (!current().startsWith("https://meet.google.com/")) { + return { + manualActionReason: "google-login-required", + manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: current(), + browserTitle: document.title, + notes, + }; + } + const href = current(); + if (meetUrlPattern.test(href)) { + return { meetingUri: href, browserUrl: href, browserTitle: document.title, notes }; + } + const pageText = text(document.body); + if (clickButton(/\\buse microphone\\b/i, "Accepted Meet microphone prompt with browser automation.")) { + return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; + } + if ( + clickButton( + /continue without microphone/i, + "Continued through Meet microphone prompt with browser automation.", + ) + ) { + return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; + } + if (/do you want people to hear you in the meeting/i.test(pageText)) { + return { + manualActionReason: "meet-audio-choice-required", + manualAction: "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (/allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) { + return { + manualActionReason: "meet-permission-required", + manualAction: "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (/couldn't create|unable to create/i.test(pageText)) { + return { + manualAction: "Resolve the Google Meet page prompt in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + if (location.hostname.toLowerCase() === "accounts.google.com" || /use your google account|to continue to google meet|choose an account|sign in to (join|continue)/i.test(pageText)) { + return { + manualActionReason: "google-login-required", + manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", + browserUrl: href, + browserTitle: document.title, + notes, + }; + } + return { + retryAfterMs: 500, + browserUrl: current(), + browserTitle: document.title, + notes, + }; +}`; + +export async function createMeetWithBrowserProxyOnNode(params: { + runtime: PluginRuntime; + config: GoogleMeetConfig; +}): Promise { + const nodeId = await resolveChromeNode({ + runtime: params.runtime, + requestedNode: params.config.chromeNode.node, + }); + const timeoutMs = Math.max( + GOOGLE_MEET_BROWSER_CREATE_TIMEOUT_MS, + params.config.chrome.joinTimeoutMs, + ); + const stepTimeoutMs = Math.min(timeoutMs, GOOGLE_MEET_BROWSER_STEP_TIMEOUT_MS); + let tab = await findGoogleMeetCreateTab({ + runtime: params.runtime, + nodeId, + timeoutMs: stepTimeoutMs, + }); + if (tab?.targetId) { + await focusBrowserTab({ + runtime: params.runtime, + nodeId, + targetId: tab.targetId, + timeoutMs: stepTimeoutMs, + }); + } else { + tab = readBrowserTab( + await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId, + method: "POST", + path: "/tabs/open", + body: { url: GOOGLE_MEET_NEW_URL }, + timeoutMs: stepTimeoutMs, + }), + ); + } + const targetId = tab?.targetId; + if (!targetId) { + throw new Error("Browser fallback opened Google Meet but did not return a targetId."); + } + const notes = new Set(); + let lastResult: BrowserCreateStepResult | undefined; + let lastError: unknown; + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + try { + const evaluated = await callBrowserProxyOnNode({ + runtime: params.runtime, + nodeId, + method: "POST", + path: "/act", + body: { + kind: "evaluate", + targetId, + fn: CREATE_MEET_FROM_BROWSER_SCRIPT, + }, + timeoutMs: stepTimeoutMs, + }); + const result = readBrowserCreateResult(evaluated); + lastResult = result; + for (const note of result.notes ?? []) { + notes.add(note); + } + if (result.meetingUri) { + return { + source: "browser", + nodeId, + targetId, + meetingUri: result.meetingUri, + browserUrl: result.browserUrl, + browserTitle: result.browserTitle, + notes: [...notes], + }; + } + if (result.manualAction) { + if (result.manualActionReason) { + throw new Error(`${result.manualActionReason}: ${result.manualAction}`); + } + throw new Error(result.manualAction); + } + await sleep(result.retryAfterMs ?? GOOGLE_MEET_BROWSER_POLL_MS); + } catch (error) { + lastError = error; + if (!isBrowserNavigationInterruption(error)) { + throw error; + } + await sleep(GOOGLE_MEET_BROWSER_NAVIGATION_RETRY_MS); + } + } + throw new Error( + lastResult?.manualAction ?? + `Google Meet did not return a meeting URL from the browser create flow before timeout.${ + lastError + ? ` Last browser automation error: ${formatBrowserAutomationError(lastError)}` + : "" + }`, + ); +} diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index 0f2f8d7a39a..39e0b6d7f91 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -10,6 +10,13 @@ import { startCommandRealtimeAudioBridge, type ChromeRealtimeAudioBridgeHandle, } from "../realtime.js"; +import { + asBrowserTabs, + callBrowserProxyOnNode, + readBrowserTab, + resolveChromeNode, + type BrowserTab, +} from "./chrome-browser-proxy.js"; import type { GoogleMeetChromeHealth } from "./types.js"; export const GOOGLE_MEET_SYSTEM_PROFILER_COMMAND = "/usr/sbin/system_profiler"; @@ -154,52 +161,6 @@ export async function launchChromeMeet(params: { } } -function isGoogleMeetNode(node: { - caps?: string[]; - commands?: string[]; - connected?: boolean; - nodeId?: string; - displayName?: string; - remoteIp?: string; -}) { - const commands = Array.isArray(node.commands) ? node.commands : []; - const caps = Array.isArray(node.caps) ? node.caps : []; - return ( - node.connected === true && - commands.includes("googlemeet.chrome") && - (commands.includes("browser.proxy") || caps.includes("browser")) - ); -} - -async function resolveChromeNode(params: { - runtime: PluginRuntime; - requestedNode?: string; -}): Promise { - const list = await params.runtime.nodes.list({ connected: true }); - const nodes = list.nodes.filter(isGoogleMeetNode); - if (nodes.length === 0) { - throw new Error( - "No connected Google Meet-capable node with browser proxy. Run `openclaw node run` on the Chrome host with browser proxy enabled, approve pairing, and allow googlemeet.chrome plus browser.proxy.", - ); - } - const requested = params.requestedNode?.trim(); - if (requested) { - const matches = nodes.filter((node) => - [node.nodeId, node.displayName, node.remoteIp].some((value) => value === requested), - ); - if (matches.length === 1) { - return matches[0].nodeId; - } - throw new Error(`Google Meet node not found or ambiguous: ${requested}`); - } - if (nodes.length === 1) { - return nodes[0].nodeId; - } - throw new Error( - "Multiple Google Meet-capable nodes connected. Set plugins.entries.google-meet.config.chromeNode.node.", - ); -} - function parseNodeStartResult(raw: unknown): { launched?: boolean; bridgeId?: string; @@ -221,381 +182,6 @@ function parseNodeStartResult(raw: unknown): { }; } -type BrowserProxyResult = { - result?: unknown; -}; - -type BrowserTab = { - targetId?: string; - title?: string; - url?: string; -}; - -const GOOGLE_MEET_NEW_URL = "https://meet.google.com/new"; -const GOOGLE_MEET_BROWSER_CREATE_TIMEOUT_MS = 60_000; -const GOOGLE_MEET_BROWSER_STEP_TIMEOUT_MS = 10_000; -const GOOGLE_MEET_BROWSER_NAVIGATION_RETRY_MS = 1_000; -const GOOGLE_MEET_BROWSER_POLL_MS = 500; - -type BrowserCreateStepResult = { - meetingUri?: string; - browserUrl?: string; - browserTitle?: string; - manualAction?: string; - manualActionReason?: GoogleMeetChromeHealth["manualActionReason"]; - notes?: string[]; - retryAfterMs?: number; -}; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function formatBrowserAutomationError(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - try { - return JSON.stringify(error); - } catch { - return "unknown error"; - } -} - -function isBrowserNavigationInterruption(error: unknown): boolean { - return /execution context was destroyed|navigation|target closed/i.test( - formatBrowserAutomationError(error), - ); -} - -export type GoogleMeetBrowserCreateResult = { - meetingUri: string; - nodeId: string; - targetId?: string; - browserUrl?: string; - browserTitle?: string; - notes?: string[]; - source: "browser"; -}; - -function unwrapNodeInvokePayload(raw: unknown): unknown { - const record = raw && typeof raw === "object" ? (raw as Record) : {}; - if (typeof record.payloadJSON === "string" && record.payloadJSON.trim()) { - return JSON.parse(record.payloadJSON); - } - if ("payload" in record) { - return record.payload; - } - return raw; -} - -function parseBrowserProxyResult(raw: unknown): unknown { - const payload = unwrapNodeInvokePayload(raw); - const proxy = - payload && typeof payload === "object" ? (payload as BrowserProxyResult) : undefined; - if (!proxy || !("result" in proxy)) { - throw new Error("Google Meet browser proxy returned an invalid result."); - } - return proxy.result; -} - -async function callBrowserProxyOnNode(params: { - runtime: PluginRuntime; - nodeId: string; - method: "GET" | "POST" | "DELETE"; - path: string; - body?: unknown; - timeoutMs: number; -}) { - const raw = await params.runtime.nodes.invoke({ - nodeId: params.nodeId, - command: "browser.proxy", - params: { - method: params.method, - path: params.path, - body: params.body, - timeoutMs: params.timeoutMs, - }, - timeoutMs: params.timeoutMs + 5_000, - }); - return parseBrowserProxyResult(raw); -} - -function asBrowserTabs(result: unknown): BrowserTab[] { - const record = result && typeof result === "object" ? (result as Record) : {}; - return Array.isArray(record.tabs) ? (record.tabs as BrowserTab[]) : []; -} - -function readBrowserTab(result: unknown): BrowserTab | undefined { - return result && typeof result === "object" ? (result as BrowserTab) : undefined; -} - -function isGoogleMeetCreateTab(tab: BrowserTab): boolean { - const url = tab.url ?? ""; - if (/^https:\/\/meet\.google\.com\/(?:new|[a-z]{3}-[a-z]{4}-[a-z]{3})(?:$|[/?#])/i.test(url)) { - return true; - } - return ( - url.startsWith("https://accounts.google.com/") && - /sign in|google accounts|meet/i.test(tab.title ?? "") - ); -} - -async function findGoogleMeetCreateTab(params: { - runtime: PluginRuntime; - nodeId: string; - timeoutMs: number; -}): Promise { - const tabs = asBrowserTabs( - await callBrowserProxyOnNode({ - runtime: params.runtime, - nodeId: params.nodeId, - method: "GET", - path: "/tabs", - timeoutMs: params.timeoutMs, - }), - ); - return tabs.find(isGoogleMeetCreateTab); -} - -async function focusBrowserTab(params: { - runtime: PluginRuntime; - nodeId: string; - targetId: string; - timeoutMs: number; -}): Promise { - await callBrowserProxyOnNode({ - runtime: params.runtime, - nodeId: params.nodeId, - method: "POST", - path: "/tabs/focus", - body: { targetId: params.targetId }, - timeoutMs: params.timeoutMs, - }); -} - -function readStringArray(value: unknown): string[] | undefined { - return Array.isArray(value) - ? value.filter((entry): entry is string => typeof entry === "string") - : undefined; -} - -function readBrowserCreateResult(result: unknown): BrowserCreateStepResult { - const record = result && typeof result === "object" ? (result as Record) : {}; - const nested = - record.result && typeof record.result === "object" - ? (record.result as Record) - : record; - return { - meetingUri: typeof nested.meetingUri === "string" ? nested.meetingUri : undefined, - browserUrl: typeof nested.browserUrl === "string" ? nested.browserUrl : undefined, - browserTitle: typeof nested.browserTitle === "string" ? nested.browserTitle : undefined, - manualAction: typeof nested.manualAction === "string" ? nested.manualAction : undefined, - manualActionReason: - typeof nested.manualActionReason === "string" - ? (nested.manualActionReason as GoogleMeetChromeHealth["manualActionReason"]) - : undefined, - notes: readStringArray(nested.notes), - retryAfterMs: - typeof nested.retryAfterMs === "number" && Number.isFinite(nested.retryAfterMs) - ? nested.retryAfterMs - : undefined, - }; -} - -export const CREATE_MEET_FROM_BROWSER_SCRIPT = `async () => { - const meetUrlPattern = /^https:\\/\\/meet\\.google\\.com\\/[a-z]{3}-[a-z]{4}-[a-z]{3}(?:$|[/?#])/i; - const text = (node) => (node?.innerText || node?.textContent || "").trim(); - const current = () => location.href; - const notes = []; - const findButton = (pattern) => - [...document.querySelectorAll("button")].find((button) => { - const label = [ - button.getAttribute("aria-label"), - button.getAttribute("data-tooltip"), - text(button), - ] - .filter(Boolean) - .join(" "); - return pattern.test(label) && !button.disabled; - }); - const clickButton = (pattern, note) => { - const button = findButton(pattern); - if (!button) { - return false; - } - button.click(); - notes.push(note); - return true; - }; - if (!current().startsWith("https://meet.google.com/")) { - return { - manualActionReason: "google-login-required", - manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", - browserUrl: current(), - browserTitle: document.title, - notes, - }; - } - const href = current(); - if (meetUrlPattern.test(href)) { - return { meetingUri: href, browserUrl: href, browserTitle: document.title, notes }; - } - const pageText = text(document.body); - if (clickButton(/\\buse microphone\\b/i, "Accepted Meet microphone prompt with browser automation.")) { - return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; - } - if ( - clickButton( - /continue without microphone/i, - "Continued through Meet microphone prompt with browser automation.", - ) - ) { - return { browserUrl: href, browserTitle: document.title, notes, retryAfterMs: 1000 }; - } - if (/do you want people to hear you in the meeting/i.test(pageText)) { - return { - manualActionReason: "meet-audio-choice-required", - manualAction: "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry meeting creation.", - browserUrl: href, - browserTitle: document.title, - notes, - }; - } - if (/allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) { - return { - manualActionReason: "meet-permission-required", - manualAction: "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry meeting creation.", - browserUrl: href, - browserTitle: document.title, - notes, - }; - } - if (/couldn't create|unable to create/i.test(pageText)) { - return { - manualAction: "Resolve the Google Meet page prompt in the OpenClaw browser profile, then retry meeting creation.", - browserUrl: href, - browserTitle: document.title, - notes, - }; - } - if (location.hostname.toLowerCase() === "accounts.google.com" || /use your google account|to continue to google meet|choose an account|sign in to (join|continue)/i.test(pageText)) { - return { - manualActionReason: "google-login-required", - manualAction: "Sign in to Google in the OpenClaw browser profile, then retry meeting creation.", - browserUrl: href, - browserTitle: document.title, - notes, - }; - } - return { - retryAfterMs: 500, - browserUrl: current(), - browserTitle: document.title, - notes, - }; -}`; - -export async function createMeetWithBrowserProxyOnNode(params: { - runtime: PluginRuntime; - config: GoogleMeetConfig; -}): Promise { - const nodeId = await resolveChromeNode({ - runtime: params.runtime, - requestedNode: params.config.chromeNode.node, - }); - const timeoutMs = Math.max( - GOOGLE_MEET_BROWSER_CREATE_TIMEOUT_MS, - params.config.chrome.joinTimeoutMs, - ); - const stepTimeoutMs = Math.min(timeoutMs, GOOGLE_MEET_BROWSER_STEP_TIMEOUT_MS); - let tab = await findGoogleMeetCreateTab({ - runtime: params.runtime, - nodeId, - timeoutMs: stepTimeoutMs, - }); - if (tab?.targetId) { - await focusBrowserTab({ - runtime: params.runtime, - nodeId, - targetId: tab.targetId, - timeoutMs: stepTimeoutMs, - }); - } else { - tab = readBrowserTab( - await callBrowserProxyOnNode({ - runtime: params.runtime, - nodeId, - method: "POST", - path: "/tabs/open", - body: { url: GOOGLE_MEET_NEW_URL }, - timeoutMs: stepTimeoutMs, - }), - ); - } - const targetId = tab?.targetId; - if (!targetId) { - throw new Error("Browser fallback opened Google Meet but did not return a targetId."); - } - const notes = new Set(); - let lastResult: BrowserCreateStepResult | undefined; - let lastError: unknown; - const deadline = Date.now() + timeoutMs; - while (Date.now() <= deadline) { - try { - const evaluated = await callBrowserProxyOnNode({ - runtime: params.runtime, - nodeId, - method: "POST", - path: "/act", - body: { - kind: "evaluate", - targetId, - fn: CREATE_MEET_FROM_BROWSER_SCRIPT, - }, - timeoutMs: stepTimeoutMs, - }); - const result = readBrowserCreateResult(evaluated); - lastResult = result; - for (const note of result.notes ?? []) { - notes.add(note); - } - if (result.meetingUri) { - return { - source: "browser", - nodeId, - targetId, - meetingUri: result.meetingUri, - browserUrl: result.browserUrl, - browserTitle: result.browserTitle, - notes: [...notes], - }; - } - if (result.manualAction) { - if (result.manualActionReason) { - throw new Error(`${result.manualActionReason}: ${result.manualAction}`); - } - throw new Error(result.manualAction); - } - await sleep(result.retryAfterMs ?? GOOGLE_MEET_BROWSER_POLL_MS); - } catch (error) { - lastError = error; - if (!isBrowserNavigationInterruption(error)) { - throw error; - } - await sleep(GOOGLE_MEET_BROWSER_NAVIGATION_RETRY_MS); - } - } - throw new Error( - lastResult?.manualAction ?? - `Google Meet did not return a meeting URL from the browser create flow before timeout.${ - lastError - ? ` Last browser automation error: ${formatBrowserAutomationError(lastError)}` - : "" - }`, - ); -} - function parseMeetBrowserStatus(result: unknown): GoogleMeetChromeHealth | undefined { const record = result && typeof result === "object" ? (result as Record) : {}; const raw = record.result;