fix(google-meet): grant browser media permissions

extensions/browser/src/browser/routes/index.ts

@@ -1,11 +1,13 @@
 import type { BrowserRouteContext } from "../server-context.js";
 import { registerBrowserAgentRoutes } from "./agent.js";
 import { registerBrowserBasicRoutes } from "./basic.js";
+import { registerBrowserPermissionRoutes } from "./permissions.js";
 import { registerBrowserTabRoutes } from "./tabs.js";
 import type { BrowserRouteRegistrar } from "./types.js";
 
 export function registerBrowserRoutes(app: BrowserRouteRegistrar, ctx: BrowserRouteContext) {
   registerBrowserBasicRoutes(app, ctx);
   registerBrowserTabRoutes(app, ctx);
+  registerBrowserPermissionRoutes(app, ctx);
   registerBrowserAgentRoutes(app, ctx);
 }

extensions/browser/src/browser/routes/permissions.test.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { createBrowserRouteApp, createBrowserRouteResponse } from "./test-helpers.js";
+
+const cdpMocks = vi.hoisted(() => ({
+  getChromeWebSocketUrl: vi.fn(async () => "ws://127.0.0.1:18800/devtools/browser/test"),
+  send: vi.fn(
+    async (
+      _method: string,
+      _params?: Record<string, unknown>,
+    ): Promise<Record<string, unknown>> => ({}),
+  ),
+  withCdpSocket: vi.fn(
+    async (
+      _wsUrl: string,
+      fn: (
+        send: (method: string, params?: Record<string, unknown>) => Promise<unknown>,
+      ) => Promise<unknown>,
+    ) => await fn(cdpMocks.send),
+  ),
+}));
+
+vi.mock("../chrome.js", () => ({
+  getChromeWebSocketUrl: cdpMocks.getChromeWebSocketUrl,
+}));
+
+vi.mock("../cdp.helpers.js", () => ({
+  withCdpSocket: cdpMocks.withCdpSocket,
+}));
+
+const { registerBrowserPermissionRoutes } = await import("./permissions.js");
+
+function createProfileContext() {
+  return {
+    profile: {
+      name: "openclaw",
+      cdpUrl: "http://127.0.0.1:18800",
+    },
+    ensureBrowserAvailable: vi.fn(async () => {}),
+    ensureTabAvailable: vi.fn(),
+    isHttpReachable: vi.fn(),
+    isTransportAvailable: vi.fn(),
+    isReachable: vi.fn(),
+    listTabs: vi.fn(),
+    openTab: vi.fn(),
+    labelTab: vi.fn(),
+    focusTab: vi.fn(),
+    closeTab: vi.fn(),
+    stopRunningBrowser: vi.fn(),
+    resetProfile: vi.fn(),
+  };
+}
+
+function createRouteContext(profileCtx: ReturnType<typeof createProfileContext>) {
+  return {
+    state: () => ({ resolved: { ssrfPolicy: { allowPrivateNetwork: false } } }),
+    forProfile: () => profileCtx,
+    listProfiles: vi.fn(async () => []),
+    mapTabError: vi.fn(() => null),
+    ...profileCtx,
+  };
+}
+
+async function callGrant(body: Record<string, unknown>) {
+  const { app, postHandlers } = createBrowserRouteApp();
+  const profileCtx = createProfileContext();
+  registerBrowserPermissionRoutes(app, createRouteContext(profileCtx) as never);
+  const handler = postHandlers.get("/permissions/grant");
+  expect(handler).toBeTypeOf("function");
+
+  const response = createBrowserRouteResponse();
+  await handler?.({ params: {}, query: {}, body }, response.res);
+  return { response, profileCtx };
+}
+
+describe("browser permission routes", () => {
+  beforeEach(() => {
+    cdpMocks.getChromeWebSocketUrl.mockClear();
+    cdpMocks.send.mockReset().mockResolvedValue({});
+    cdpMocks.withCdpSocket.mockClear();
+  });
+
+  it("grants required and optional Chrome permissions for an origin", async () => {
+    const { response, profileCtx } = await callGrant({
+      origin: "https://meet.google.com/abc-defg-hij",
+      permissions: ["audioCapture", "videoCapture"],
+      optionalPermissions: ["speakerSelection"],
+      timeoutMs: 1234,
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.body).toMatchObject({
+      ok: true,
+      origin: "https://meet.google.com",
+      grantedPermissions: ["audioCapture", "videoCapture", "speakerSelection"],
+      unsupportedPermissions: [],
+    });
+    expect(profileCtx.ensureBrowserAvailable).toHaveBeenCalled();
+    expect(cdpMocks.getChromeWebSocketUrl).toHaveBeenCalledWith("http://127.0.0.1:18800", 1234, {
+      allowPrivateNetwork: false,
+    });
+    expect(cdpMocks.send).toHaveBeenCalledWith("Browser.grantPermissions", {
+      origin: "https://meet.google.com",
+      permissions: ["audioCapture", "videoCapture", "speakerSelection"],
+    });
+  });
+
+  it("keeps required permissions when an optional permission is unsupported", async () => {
+    cdpMocks.send.mockImplementation(async (_method: string, params?: Record<string, unknown>) => {
+      const permissions = Array.isArray(params?.permissions) ? params.permissions : [];
+      if (permissions.includes("speakerSelection")) {
+        throw new Error("Unknown permission type");
+      }
+      return {};
+    });
+
+    const { response } = await callGrant({
+      origin: "https://meet.google.com",
+      permissions: ["audioCapture", "videoCapture"],
+      optionalPermissions: ["speakerSelection"],
+    });
+
+    expect(response.statusCode).toBe(200);
+    expect(response.body).toMatchObject({
+      ok: true,
+      grantedPermissions: ["audioCapture", "videoCapture"],
+      unsupportedPermissions: ["speakerSelection"],
+    });
+    expect(cdpMocks.send).toHaveBeenNthCalledWith(2, "Browser.grantPermissions", {
+      origin: "https://meet.google.com",
+      permissions: ["audioCapture", "videoCapture"],
+    });
+  });
+});

extensions/browser/src/browser/routes/permissions.ts

@@ -0,0 +1,135 @@
+import { withCdpSocket } from "../cdp.helpers.js";
+import { getChromeWebSocketUrl } from "../chrome.js";
+import type { BrowserRouteContext } from "../server-context.js";
+import type { BrowserRouteRegistrar } from "./types.js";
+import {
+  asyncBrowserRoute,
+  getProfileContext,
+  jsonError,
+  toNumber,
+  toStringOrEmpty,
+} from "./utils.js";
+
+type GrantPermissionsBody = {
+  origin?: unknown;
+  permissions?: unknown;
+  optionalPermissions?: unknown;
+  timeoutMs?: unknown;
+};
+
+function readOrigin(raw: unknown): string | null {
+  const value = toStringOrEmpty(raw);
+  if (!value) {
+    return null;
+  }
+  try {
+    const parsed = new URL(value);
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+      return null;
+    }
+    return parsed.origin;
+  } catch {
+    return null;
+  }
+}
+
+function readPermissions(raw: unknown): string[] | null {
+  if (!Array.isArray(raw)) {
+    return null;
+  }
+  const permissions = raw
+    .map((value) => (typeof value === "string" ? value.trim() : ""))
+    .filter(Boolean);
+  if (permissions.length !== raw.length) {
+    return null;
+  }
+  return [...new Set(permissions)];
+}
+
+async function grantPermissions(params: {
+  wsUrl: string;
+  origin: string;
+  requiredPermissions: string[];
+  optionalPermissions: string[];
+  timeoutMs: number;
+}) {
+  const allPermissions = [
+    ...new Set([...params.requiredPermissions, ...params.optionalPermissions]),
+  ];
+  let unsupportedPermissions: string[] = [];
+  await withCdpSocket(
+    params.wsUrl,
+    async (send) => {
+      try {
+        await send("Browser.grantPermissions", {
+          origin: params.origin,
+          permissions: allPermissions,
+        });
+        return;
+      } catch (error) {
+        if (params.optionalPermissions.length === 0) {
+          throw error;
+        }
+      }
+      await send("Browser.grantPermissions", {
+        origin: params.origin,
+        permissions: params.requiredPermissions,
+      });
+      unsupportedPermissions = params.optionalPermissions;
+    },
+    { commandTimeoutMs: params.timeoutMs },
+  );
+  return {
+    grantedPermissions: allPermissions.filter((value) => !unsupportedPermissions.includes(value)),
+    unsupportedPermissions,
+  };
+}
+
+export function registerBrowserPermissionRoutes(
+  app: BrowserRouteRegistrar,
+  ctx: BrowserRouteContext,
+) {
+  app.post(
+    "/permissions/grant",
+    asyncBrowserRoute(async (req, res) => {
+      const profileCtx = getProfileContext(req, ctx);
+      if ("error" in profileCtx) {
+        return jsonError(res, profileCtx.status, profileCtx.error);
+      }
+
+      const body = (req.body ?? {}) as GrantPermissionsBody;
+      const origin = readOrigin(body.origin);
+      if (!origin) {
+        return jsonError(res, 400, "origin must be an http(s) origin");
+      }
+      const requiredPermissions = readPermissions(body.permissions);
+      if (!requiredPermissions || requiredPermissions.length === 0) {
+        return jsonError(res, 400, "permissions must be a non-empty string array");
+      }
+      const optionalPermissions = readPermissions(body.optionalPermissions ?? []) ?? [];
+      const timeoutMs = Math.max(1_000, toNumber(body.timeoutMs) ?? 5_000);
+
+      try {
+        await profileCtx.ensureBrowserAvailable();
+        const wsUrl = await getChromeWebSocketUrl(
+          profileCtx.profile.cdpUrl,
+          timeoutMs,
+          ctx.state().resolved.ssrfPolicy,
+        );
+        if (!wsUrl) {
+          return jsonError(res, 409, "browser CDP WebSocket unavailable");
+        }
+        const granted = await grantPermissions({
+          wsUrl,
+          origin,
+          requiredPermissions,
+          optionalPermissions,
+          timeoutMs,
+        });
+        return res.json({ ok: true, origin, ...granted });
+      } catch (error) {
+        return jsonError(res, 500, error instanceof Error ? error.message : String(error));
+      }
+    }),
+  );
+}

extensions/google-meet/index.test.ts

@@ -124,6 +124,14 @@ function mockLocalMeetBrowserRequest(
       if (request.path === "/tabs/focus") {
         return { ok: true };
       }
+      if (request.path === "/permissions/grant") {
+        return {
+          ok: true,
+          origin: "https://meet.google.com",
+          grantedPermissions: ["audioCapture", "videoCapture", "speakerSelection"],
+          unsupportedPermissions: [],
+        };
+      }
       if (request.path === "/act") {
         return { result: JSON.stringify(browserActResult) };
       }
@@ -299,9 +307,12 @@ describe("google-meet plugin", () => {
         waitForInCallMs: 20000,
         audioFormat: "pcm16-24khz",
         audioInputCommand: [
-          "rec",
+          "sox",
           "-q",
           "-t",
+          "coreaudio",
+          "BlackHole 2ch",
+          "-t",
           "raw",
           "-r",
           "24000",
@@ -315,7 +326,7 @@ describe("google-meet plugin", () => {
           "-",
         ],
         audioOutputCommand: [
-          "play",
+          "sox",
           "-q",
           "-t",
           "raw",
@@ -329,6 +340,9 @@ describe("google-meet plugin", () => {
           "16",
           "-L",
           "-",
+          "-t",
+          "coreaudio",
+          "BlackHole 2ch",
         ],
       },
       voiceCall: { enabled: true, requestTimeoutMs: 30000, dtmfDelayMs: 2500 },
@@ -1253,7 +1267,7 @@ describe("google-meet plugin", () => {
             if (argv[0] === "/usr/sbin/system_profiler") {
               return { code: 0, stdout: "BlackHole 2ch", stderr: "" };
             }
-            if (argv[0] === "/bin/sh" && argv.at(-1) === "play") {
+            if (argv[0] === "/bin/sh" && argv.at(-1) === "sox") {
               return { code: 1, stdout: "", stderr: "" };
             }
             return { code: 0, stdout: "", stderr: "" };
@@ -1275,7 +1289,7 @@ describe("google-meet plugin", () => {
           expect.objectContaining({
             id: "chrome-local-audio-commands",
             ok: false,
-            message: "Chrome audio command missing: play",
+            message: "Chrome audio command missing: sox",
           }),
         ]),
       );
@@ -1410,6 +1424,20 @@ describe("google-meet plugin", () => {
         }),
         { progress: false },
       );
+      expect(callGatewayFromCli).toHaveBeenCalledWith(
+        "browser.request",
+        expect.any(Object),
+        expect.objectContaining({
+          method: "POST",
+          path: "/permissions/grant",
+          body: expect.objectContaining({
+            origin: "https://meet.google.com",
+            permissions: ["audioCapture", "videoCapture"],
+            optionalPermissions: ["speakerSelection"],
+          }),
+        }),
+        { progress: false },
+      );
     } finally {
       Object.defineProperty(process, "platform", { value: originalPlatform });
     }

extensions/google-meet/openclaw.plugin.json

@@ -245,9 +245,12 @@
           "audioInputCommand": {
             "type": "array",
             "default": [
-              "rec",
+              "sox",
               "-q",
               "-t",
+              "coreaudio",
+              "BlackHole 2ch",
+              "-t",
               "raw",
               "-r",
               "24000",
@@ -267,7 +270,7 @@
           "audioOutputCommand": {
             "type": "array",
             "default": [
-              "play",
+              "sox",
               "-q",
               "-t",
               "raw",
@@ -280,7 +283,10 @@
               "-b",
               "16",
               "-L",
-              "-"
+              "-",
+              "-t",
+              "coreaudio",
+              "BlackHole 2ch"
             ],
             "items": {
               "type": "string"

extensions/google-meet/src/config.ts

@@ -79,9 +79,12 @@ export type GoogleMeetConfig = {
 };
 
 export const DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND = [
-  "rec",
+  "sox",
   "-q",
   "-t",
+  "coreaudio",
+  "BlackHole 2ch",
+  "-t",
   "raw",
   "-r",
   "24000",
@@ -96,7 +99,7 @@ export const DEFAULT_GOOGLE_MEET_AUDIO_INPUT_COMMAND = [
 ] as const;
 
 export const DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND = [
-  "play",
+  "sox",
   "-q",
   "-t",
   "raw",
@@ -110,6 +113,9 @@ export const DEFAULT_GOOGLE_MEET_AUDIO_OUTPUT_COMMAND = [
   "16",
   "-L",
   "-",
+  "-t",
+  "coreaudio",
+  "BlackHole 2ch",
 ] as const;
 
 export const LEGACY_GOOGLE_MEET_AUDIO_INPUT_COMMAND = [

extensions/google-meet/src/transports/chrome.ts

@@ -245,6 +245,57 @@ async function callLocalBrowserRequest(params: BrowserRequestParams) {
   );
 }
 
+function mergeBrowserNotes(
+  browser: GoogleMeetChromeHealth | undefined,
+  notes: string[],
+): GoogleMeetChromeHealth | undefined {
+  if (!browser || notes.length === 0) {
+    return browser;
+  }
+  return {
+    ...browser,
+    notes: [...new Set([...(browser.notes ?? []), ...notes])],
+  };
+}
+
+function parsePermissionGrantNotes(result: unknown): string[] {
+  const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
+  const unsupportedPermissions = Array.isArray(record.unsupportedPermissions)
+    ? record.unsupportedPermissions.filter((value): value is string => typeof value === "string")
+    : [];
+  const notes = ["Granted Meet microphone/camera permissions through browser control."];
+  if (unsupportedPermissions.includes("speakerSelection")) {
+    notes.push("Chrome did not accept the optional Meet speaker-selection permission.");
+  }
+  return notes;
+}
+
+async function grantMeetMediaPermissions(params: {
+  callBrowser: BrowserRequestCaller;
+  timeoutMs: number;
+}): Promise<string[]> {
+  try {
+    const result = await params.callBrowser({
+      method: "POST",
+      path: "/permissions/grant",
+      body: {
+        origin: "https://meet.google.com",
+        permissions: ["audioCapture", "videoCapture"],
+        optionalPermissions: ["speakerSelection"],
+        timeoutMs: Math.min(params.timeoutMs, 5_000),
+      },
+      timeoutMs: Math.min(params.timeoutMs, 5_000),
+    });
+    return parsePermissionGrantNotes(result);
+  } catch (error) {
+    return [
+      `Could not grant Meet media permissions automatically: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    ];
+  }
+}
+
 function meetStatusScript(params: { guestName: string; autoJoin: boolean }) {
   return `() => {
   const text = (node) => (node?.innerText || node?.textContent || "").trim();
@@ -273,6 +324,7 @@ function meetStatusScript(params: { guestName: string; autoJoin: boolean }) {
   const pageText = text(document.body).toLowerCase();
   const host = location.hostname.toLowerCase();
   const pageUrl = location.href;
+  const permissionNeeded = /permission needed|allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera|speaker)/i.test(pageText);
   const join = ${JSON.stringify(params.autoJoin)}
     ? findButton(/join now|ask to join/i)
     : null;
@@ -292,9 +344,9 @@ function meetStatusScript(params: { guestName: string; autoJoin: boolean }) {
   } else if (!inCall && /asking to be let in|you.?ll join when someone lets you in|waiting to be let in|ask to join/i.test(pageText)) {
     manualActionReason = "meet-admission-required";
     manualActionMessage = "Admit the OpenClaw browser participant in Google Meet, then retry speech.";
-  } else if (!inCall && /allow.*(microphone|camera)|blocked.*(microphone|camera)|permission.*(microphone|camera)/i.test(pageText)) {
+  } else if (permissionNeeded) {
     manualActionReason = "meet-permission-required";
-    manualActionMessage = "Allow microphone/camera permissions for Meet in the OpenClaw browser profile, then retry.";
+    manualActionMessage = "Allow microphone/camera/speaker permissions for Meet in the OpenClaw browser profile, then retry.";
   } else if (!inCall && !microphoneChoice && /do you want people to hear you in the meeting/i.test(pageText)) {
     manualActionReason = "meet-audio-choice-required";
     manualActionMessage = "Meet is showing the microphone choice. Click Use microphone in the OpenClaw browser profile, then retry.";
@@ -389,11 +441,16 @@ async function openMeetWithBrowserRequest(params: {
     };
   }
 
+  const permissionNotes = await grantMeetMediaPermissions({
+    callBrowser: params.callBrowser,
+    timeoutMs,
+  });
   const deadline = Date.now() + Math.max(0, params.config.chrome.waitForInCallMs);
   let browser: GoogleMeetChromeHealth | undefined = {
     status: "browser-control",
     browserUrl: tab?.url,
     browserTitle: tab?.title,
+    notes: permissionNotes,
   };
   do {
     try {
@@ -410,7 +467,7 @@ async function openMeetWithBrowserRequest(params: {
         },
         timeoutMs: Math.min(timeoutMs, 10_000),
       });
-      browser = parseMeetBrowserStatus(evaluated) ?? browser;
+      browser = mergeBrowserNotes(parseMeetBrowserStatus(evaluated) ?? browser, permissionNotes);
       if (browser?.inCall === true) {
         return { launched: true, browser };
       }
@@ -426,6 +483,7 @@ async function openMeetWithBrowserRequest(params: {
         manualActionMessage:
           "Open the OpenClaw browser profile, finish Google Meet login, admission, or permission prompts, then retry.",
         notes: [
+          ...permissionNotes,
           `Browser control could not inspect or auto-join Meet: ${
             error instanceof Error ? error.message : String(error)
           }`,
@@ -467,6 +525,10 @@ async function inspectRecoverableMeetTab(params: {
     body: { targetId: params.targetId },
     timeoutMs: Math.min(params.timeoutMs, 5_000),
   });
+  const permissionNotes = await grantMeetMediaPermissions({
+    callBrowser: params.callBrowser,
+    timeoutMs: params.timeoutMs,
+  });
   const evaluated = await params.callBrowser({
     method: "POST",
     path: "/act",
@@ -480,7 +542,14 @@ async function inspectRecoverableMeetTab(params: {
     },
     timeoutMs: Math.min(params.timeoutMs, 10_000),
   });
-  const browser = parseMeetBrowserStatus(evaluated);
+  const browser = mergeBrowserNotes(
+    parseMeetBrowserStatus(evaluated) ?? {
+      status: "browser-control",
+      browserUrl: params.tab.url,
+      browserTitle: params.tab.title,
+    },
+    permissionNotes,
+  );
   const manual = browser?.manualActionRequired
     ? browser.manualActionMessage || browser.manualActionReason
     : undefined;