feat(google-meet): add browser recovery diagnostics

2026-05-06 05:50:43 +00:00 · 2026-04-25 03:30:50 +01:00
parent 996e9226e5
commit e442065970
7 changed files with 459 additions and 2 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ Docs: https://docs.openclaw.ai
 - Plugins/Google Meet: add a `chrome-node` transport so a paired macOS node, such as a Parallels VM, can own Chrome, BlackHole, and SoX while the Gateway machine keeps the agent and model key. Thanks @steipete.
 - Plugins/Voice Call: expose the shared `openclaw_agent_consult` realtime tool so live phone calls can ask the full OpenClaw agent for deeper/tool-backed answers. Thanks @steipete.
 - Plugins/Voice Call: add `voicecall setup` and a dry-run-by-default `voicecall smoke` command so Twilio/provider readiness can be checked before placing a live test call. Thanks @steipete.
+- Plugins/Google Meet: add `googlemeet doctor` and a `recover_current_tab`/`recover-tab` flow so agents can inspect an already-open Meet tab and report the blocker without opening another window. Thanks @steipete.
 - Plugins/Bonjour: move LAN Gateway discovery advertising into a default-enabled bundled plugin with its own `@homebridge/ciao` dependency, so users can disable Bonjour without cutting wide-area discovery. Thanks @vincentkoc.
 - Providers/Google: add a Gemini Live realtime voice provider for backend Voice Call and Google Meet audio bridges, with bidirectional audio and function-call support. Thanks @steipete.
 - Plugins/Google Meet: let realtime Meet sessions consult the full OpenClaw agent for deeper answers while staying in the live voice loop. Thanks @steipete.
--- a/docs/plugins/google-meet.md
+++ b/docs/plugins/google-meet.md
@@ -900,7 +900,7 @@ Check the realtime path:

 ```bash
 openclaw googlemeet setup
-openclaw googlemeet status
+openclaw googlemeet doctor
 ```

 Use `mode: "realtime"` for listen/talk-back. `mode: "transcribe"` intentionally
@@ -915,6 +915,24 @@ Also verify:
 - Meet microphone and speaker are routed through the virtual audio path used by
  OpenClaw.

+`googlemeet doctor [session-id]` prints the session, node, in-call state,
+manual action reason, realtime provider connection, `realtimeReady`, audio
+input/output activity, last audio timestamps, byte counters, and browser URL.
+Use `googlemeet status [session-id]` when you need the raw JSON.
+
+If an agent timed out and you can see a Meet tab already open, inspect that tab
+without opening another one:
+
+```bash
+openclaw googlemeet recover-tab
+openclaw googlemeet recover-tab https://meet.google.com/abc-defg-hij
+```
+
+The equivalent tool action is `recover_current_tab`. It focuses and inspects an
+existing Meet tab on the configured Chrome node. It does not open a new tab or
+create a new session; it reports the current blocker, such as login, admission,
+permissions, or audio-choice state.
+
 ### Twilio setup checks fail

 `twilio-voice-call-plugin` fails when `voice-call` is not allowed or not enabled.
@@ -934,6 +952,21 @@ Then restart or reload the Gateway and run:

 ```bash
 openclaw googlemeet setup
+openclaw voicecall setup
+openclaw voicecall smoke
+```
+
+`voicecall smoke` is readiness-only by default. To dry-run a specific number:
+
+```bash
+openclaw voicecall smoke --to "+15555550123"
+```
+
+Only add `--yes` when you intentionally want to place a live outbound notify
+call:
+
+```bash
+openclaw voicecall smoke --to "+15555550123" --yes
 ```

 ### Twilio call starts but never enters the meeting
--- a/extensions/google-meet/index.test.ts
+++ b/extensions/google-meet/index.test.ts
@@ -217,6 +217,7 @@ describe("google-meet plugin", () => {
            "setup_status",
            "resolve_space",
            "preflight",
+            "recover_current_tab",
            "leave",
            "speak",
            "test_speech",
@@ -627,6 +628,95 @@ describe("google-meet plugin", () => {
    }
  });

+  it("CLI doctor prints human-readable session health", async () => {
+    const program = new Command();
+    const stdout = captureStdout();
+    registerGoogleMeetCli({
+      program,
+      config: resolveGoogleMeetConfig({}),
+      ensureRuntime: async () =>
+        ({
+          status: () => ({
+            found: true,
+            session: {
+              id: "meet_1",
+              url: "https://meet.google.com/abc-defg-hij",
+              state: "active",
+              transport: "chrome-node",
+              mode: "realtime",
+              participantIdentity: "signed-in Google Chrome profile on a paired node",
+              createdAt: "2026-04-25T00:00:00.000Z",
+              updatedAt: "2026-04-25T00:00:01.000Z",
+              realtime: { enabled: true, provider: "openai", toolPolicy: "safe-read-only" },
+              chrome: {
+                audioBackend: "blackhole-2ch",
+                launched: true,
+                nodeId: "node-1",
+                audioBridge: { type: "node-command-pair", provider: "openai" },
+                health: {
+                  inCall: true,
+                  providerConnected: true,
+                  realtimeReady: true,
+                  audioInputActive: true,
+                  audioOutputActive: false,
+                  lastInputAt: "2026-04-25T00:00:02.000Z",
+                  lastInputBytes: 160,
+                  lastOutputBytes: 0,
+                },
+              },
+              notes: [],
+            },
+          }),
+        }) as unknown as GoogleMeetRuntime,
+    });
+
+    try {
+      await program.parseAsync(["googlemeet", "doctor", "meet_1"], { from: "user" });
+      expect(stdout.output()).toContain("session: meet_1");
+      expect(stdout.output()).toContain("node: node-1");
+      expect(stdout.output()).toContain("provider connected: yes");
+      expect(stdout.output()).toContain("audio input active: yes");
+      expect(stdout.output()).toContain("audio output active: no");
+    } finally {
+      stdout.restore();
+    }
+  });
+
+  it("CLI recover-tab focuses and summarizes an existing Meet tab", async () => {
+    const program = new Command();
+    const stdout = captureStdout();
+    registerGoogleMeetCli({
+      program,
+      config: resolveGoogleMeetConfig({ defaultTransport: "chrome-node" }),
+      ensureRuntime: async () =>
+        ({
+          recoverCurrentTab: async () => ({
+            nodeId: "node-1",
+            found: true,
+            targetId: "tab-1",
+            tab: { targetId: "tab-1", url: "https://meet.google.com/abc-defg-hij" },
+            browser: {
+              inCall: false,
+              manualActionRequired: true,
+              manualActionReason: "meet-admission-required",
+              manualActionMessage: "Admit the OpenClaw browser participant in Google Meet.",
+              browserUrl: "https://meet.google.com/abc-defg-hij",
+            },
+            message: "Admit the OpenClaw browser participant in Google Meet.",
+          }),
+        }) as unknown as GoogleMeetRuntime,
+    });
+
+    try {
+      await program.parseAsync(["googlemeet", "recover-tab"], { from: "user" });
+      expect(stdout.output()).toContain("Google Meet current tab: found");
+      expect(stdout.output()).toContain("target: tab-1");
+      expect(stdout.output()).toContain("manual reason: meet-admission-required");
+    } finally {
+      stdout.restore();
+    }
+  });
+
  it("launches Chrome after the BlackHole check", async () => {
    const originalPlatform = process.platform;
    Object.defineProperty(process, "platform", { value: "darwin" });
@@ -888,6 +978,90 @@ describe("google-meet plugin", () => {
    );
  });

+  it("recovers and inspects an existing Meet tab without opening a new one", async () => {
+    const { tools, nodesInvoke } = setup(
+      {
+        defaultTransport: "chrome-node",
+      },
+      {
+        nodesInvokeHandler: async (params) => {
+          if (params.command !== "browser.proxy") {
+            throw new Error(`unexpected command ${params.command}`);
+          }
+          const proxy = params.params as { path?: string; body?: { targetId?: string } };
+          if (proxy.path === "/tabs") {
+            return {
+              payload: {
+                result: {
+                  tabs: [
+                    {
+                      targetId: "existing-meet-tab",
+                      title: "Meet",
+                      url: "https://meet.google.com/abc-defg-hij?authuser=me@example.com",
+                    },
+                  ],
+                },
+              },
+            };
+          }
+          if (proxy.path === "/tabs/focus") {
+            return { payload: { result: { ok: true } } };
+          }
+          if (proxy.path === "/act") {
+            return {
+              payload: {
+                result: {
+                  result: JSON.stringify({
+                    inCall: false,
+                    manualActionRequired: true,
+                    manualActionReason: "meet-admission-required",
+                    manualActionMessage: "Admit the OpenClaw browser participant in Google Meet.",
+                    title: "Meet",
+                    url: "https://meet.google.com/abc-defg-hij?authuser=me@example.com",
+                  }),
+                },
+              },
+            };
+          }
+          throw new Error(`unexpected browser proxy path ${proxy.path}`);
+        },
+      },
+    );
+    const tool = tools[0] as {
+      execute: (
+        id: string,
+        params: unknown,
+      ) => Promise<{ details: { found?: boolean; browser?: unknown } }>;
+    };
+
+    const result = await tool.execute("id", {
+      action: "recover_current_tab",
+      url: "https://meet.google.com/abc-defg-hij",
+    });
+
+    expect(result.details).toMatchObject({
+      found: true,
+      targetId: "existing-meet-tab",
+      browser: {
+        manualActionRequired: true,
+        manualActionReason: "meet-admission-required",
+      },
+    });
+    expect(nodesInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        params: expect.objectContaining({
+          path: "/tabs/focus",
+          body: { targetId: "existing-meet-tab" },
+        }),
+      }),
+    );
+    expect(nodesInvoke).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        params: expect.objectContaining({ path: "/tabs/open" }),
+      }),
+    );
+  });
+
  it("exposes a test-speech action that joins the requested meeting", async () => {
    const { tools, nodesInvoke } = setup(
      {
--- a/extensions/google-meet/index.ts
+++ b/extensions/google-meet/index.ts
@@ -144,6 +144,7 @@ const GoogleMeetToolSchema = Type.Object({
      "setup_status",
      "resolve_space",
      "preflight",
+      "recover_current_tab",
      "leave",
      "speak",
      "test_speech",
@@ -308,6 +309,18 @@ export default definePluginEntry({
      },
    );

+    api.registerGatewayMethod(
+      "googlemeet.recoverCurrentTab",
+      async ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const rt = await ensureRuntime();
+          respond(true, await rt.recoverCurrentTab({ url: normalizeOptionalString(params?.url) }));
+        } catch (err) {
+          sendError(respond, err);
+        }
+      },
+    );
+
    api.registerGatewayMethod(
      "googlemeet.setup",
      async ({ respond }: GatewayRequestHandlerOptions) => {
@@ -428,6 +441,10 @@ export default definePluginEntry({
              const rt = await ensureRuntime();
              return json(rt.status(normalizeOptionalString(raw.sessionId)));
            }
+            case "recover_current_tab": {
+              const rt = await ensureRuntime();
+              return json(await rt.recoverCurrentTab({ url: normalizeOptionalString(raw.url) }));
+            }
            case "setup_status": {
              const rt = await ensureRuntime();
              return json(await rt.setupStatus());
--- a/extensions/google-meet/src/cli.ts
+++ b/extensions/google-meet/src/cli.ts
@@ -48,6 +48,10 @@ type SetupOptions = {
  json?: boolean;
 };

+type JsonOptions = {
+  json?: boolean;
+};
+
 type CreateOptions = {
  accessToken?: string;
  refreshToken?: string;
@@ -102,6 +106,101 @@ function writeSetupStatus(status: Awaited<ReturnType<GoogleMeetRuntime["setupSta
  }
 }

+function formatBoolean(value: boolean | undefined): string {
+  return typeof value === "boolean" ? (value ? "yes" : "no") : "unknown";
+}
+
+function formatOptional(value: unknown): string {
+  return typeof value === "string" && value.trim() ? value : "n/a";
+}
+
+function writeDoctorStatus(status: ReturnType<GoogleMeetRuntime["status"]>): void {
+  if (!status.found) {
+    writeStdoutLine("Google Meet session: not found");
+    return;
+  }
+  const sessions = status.session ? [status.session] : (status.sessions ?? []);
+  if (sessions.length === 0) {
+    writeStdoutLine("Google Meet sessions: none");
+    return;
+  }
+  writeStdoutLine("Google Meet sessions: %d", sessions.length);
+  for (const session of sessions) {
+    const health = session.chrome?.health;
+    writeStdoutLine("");
+    writeStdoutLine("session: %s", session.id);
+    writeStdoutLine("url: %s", session.url);
+    writeStdoutLine("state: %s", session.state);
+    writeStdoutLine("transport: %s", session.transport);
+    writeStdoutLine("mode: %s", session.mode);
+    writeStdoutLine("node: %s", session.chrome?.nodeId ?? "local/none");
+    writeStdoutLine("audio bridge: %s", session.chrome?.audioBridge?.type ?? "none");
+    writeStdoutLine(
+      "provider: %s",
+      session.chrome?.audioBridge?.provider ?? session.realtime.provider ?? "n/a",
+    );
+    writeStdoutLine("in call: %s", formatBoolean(health?.inCall));
+    writeStdoutLine("manual action: %s", formatBoolean(health?.manualActionRequired));
+    if (health?.manualActionRequired) {
+      writeStdoutLine("manual reason: %s", formatOptional(health.manualActionReason));
+      writeStdoutLine("manual message: %s", formatOptional(health.manualActionMessage));
+    }
+    writeStdoutLine("provider connected: %s", formatBoolean(health?.providerConnected));
+    writeStdoutLine("realtime ready: %s", formatBoolean(health?.realtimeReady));
+    writeStdoutLine("audio input active: %s", formatBoolean(health?.audioInputActive));
+    writeStdoutLine("audio output active: %s", formatBoolean(health?.audioOutputActive));
+    writeStdoutLine(
+      "last input: %s (%s bytes)",
+      formatOptional(health?.lastInputAt),
+      health?.lastInputBytes ?? 0,
+    );
+    writeStdoutLine(
+      "last output: %s (%s bytes)",
+      formatOptional(health?.lastOutputAt),
+      health?.lastOutputBytes ?? 0,
+    );
+    writeStdoutLine("bridge closed: %s", formatBoolean(health?.bridgeClosed));
+    writeStdoutLine("browser url: %s", formatOptional(health?.browserUrl));
+  }
+}
+
+function writeRecoverCurrentTabResult(
+  result: Awaited<ReturnType<GoogleMeetRuntime["recoverCurrentTab"]>>,
+): void {
+  writeStdoutLine("Google Meet current tab: %s", result.found ? "found" : "not found");
+  writeStdoutLine("node: %s", result.nodeId);
+  if (result.targetId) {
+    writeStdoutLine("target: %s", result.targetId);
+  }
+  if (result.tab?.url) {
+    writeStdoutLine("tab url: %s", result.tab.url);
+  }
+  writeStdoutLine("message: %s", result.message);
+  if (result.browser) {
+    writeDoctorStatus({
+      found: true,
+      session: {
+        id: "current-tab",
+        url: result.browser.browserUrl ?? result.tab?.url ?? "unknown",
+        transport: "chrome-node",
+        mode: "transcribe",
+        state: "active",
+        createdAt: "",
+        updatedAt: "",
+        participantIdentity: "signed-in Google Chrome profile on a paired node",
+        realtime: { enabled: false, toolPolicy: "safe-read-only" },
+        chrome: {
+          audioBackend: "blackhole-2ch",
+          launched: true,
+          nodeId: result.nodeId,
+          health: result.browser,
+        },
+        notes: [],
+      },
+    });
+  }
+}
+
 function resolveMeetingInput(config: GoogleMeetConfig, value?: string): string {
  const meeting = value?.trim() || config.defaults.meeting;
  if (!meeting) {
@@ -479,6 +578,36 @@ export function registerGoogleMeetCli(params: {
      writeStdoutJson(rt.status(sessionId));
    });

+  root
+    .command("doctor")
+    .description("Show human-readable Meet session/browser/realtime health")
+    .argument("[session-id]", "Meet session ID")
+    .option("--json", "Print JSON output", false)
+    .action(async (sessionId: string | undefined, options: JsonOptions) => {
+      const rt = await params.ensureRuntime();
+      const status = rt.status(sessionId);
+      if (options.json) {
+        writeStdoutJson(status);
+        return;
+      }
+      writeDoctorStatus(status);
+    });
+
+  root
+    .command("recover-tab")
+    .description("Focus and inspect an existing Google Meet tab on the Chrome node")
+    .argument("[url]", "Optional Meet URL to match")
+    .option("--json", "Print JSON output", false)
+    .action(async (url: string | undefined, options: JsonOptions) => {
+      const rt = await params.ensureRuntime();
+      const result = await rt.recoverCurrentTab({ url });
+      if (options.json) {
+        writeStdoutJson(result);
+        return;
+      }
+      writeRecoverCurrentTabResult(result);
+    });
+
  root
    .command("setup")
    .description("Show Google Meet transport setup status")
--- a/extensions/google-meet/src/runtime.ts
+++ b/extensions/google-meet/src/runtime.ts
@@ -7,7 +7,11 @@ import type { GoogleMeetConfig, GoogleMeetMode, GoogleMeetTransport } from "./co
 import { addGoogleMeetSetupCheck, getGoogleMeetSetupStatus } from "./setup.js";
 import { isSameMeetUrlForReuse, resolveChromeNodeInfo } from "./transports/chrome-browser-proxy.js";
 import { createMeetWithBrowserProxyOnNode } from "./transports/chrome-create.js";
-import { launchChromeMeet, launchChromeMeetOnNode } from "./transports/chrome.js";
+import {
+  launchChromeMeet,
+  launchChromeMeetOnNode,
+  recoverCurrentMeetTabOnNode,
+} from "./transports/chrome.js";
 import { buildMeetDtmfSequence, normalizeDialInNumber } from "./transports/twilio.js";
 import type {
  GoogleMeetChromeHealth,
@@ -119,6 +123,14 @@ export class GoogleMeetRuntime {
    });
  }

+  async recoverCurrentTab(request: { url?: string } = {}) {
+    return recoverCurrentMeetTabOnNode({
+      runtime: this.params.runtime,
+      config: this.params.config,
+      url: request.url ? normalizeMeetUrl(request.url) : undefined,
+    });
+  }
+
  async join(request: GoogleMeetJoinRequest): Promise<GoogleMeetJoinResult> {
    const url = normalizeMeetUrl(request.url);
    const transport = resolveTransport(request.transport, this.params.config);
--- a/extensions/google-meet/src/transports/chrome.ts
+++ b/extensions/google-meet/src/transports/chrome.ts
@@ -14,6 +14,7 @@ import {
  asBrowserTabs,
  callBrowserProxyOnNode,
  isSameMeetUrlForReuse,
+  normalizeMeetUrlForReuse,
  readBrowserTab,
  resolveChromeNode,
  type BrowserTab,
@@ -397,6 +398,96 @@ async function openMeetWithBrowserProxy(params: {
  return { launched: true, browser };
 }

+function isRecoverableMeetTab(tab: BrowserTab, url?: string): boolean {
+  if (url) {
+    return isSameMeetUrlForReuse(tab.url, url);
+  }
+  if (normalizeMeetUrlForReuse(tab.url)) {
+    return true;
+  }
+  const tabUrl = tab.url ?? "";
+  return (
+    tabUrl.startsWith("https://accounts.google.com/") &&
+    /sign in|google accounts|meet/i.test(tab.title ?? "")
+  );
+}
+
+export async function recoverCurrentMeetTabOnNode(params: {
+  runtime: PluginRuntime;
+  config: GoogleMeetConfig;
+  url?: string;
+}): Promise<{
+  nodeId: string;
+  found: boolean;
+  targetId?: string;
+  tab?: BrowserTab;
+  browser?: GoogleMeetChromeHealth;
+  message: string;
+}> {
+  const nodeId = await resolveChromeNode({
+    runtime: params.runtime,
+    requestedNode: params.config.chromeNode.node,
+  });
+  const timeoutMs = Math.max(1_000, params.config.chrome.joinTimeoutMs);
+  const tabs = asBrowserTabs(
+    await callBrowserProxyOnNode({
+      runtime: params.runtime,
+      nodeId,
+      method: "GET",
+      path: "/tabs",
+      timeoutMs: Math.min(timeoutMs, 5_000),
+    }),
+  );
+  const tab = tabs.find((entry) => isRecoverableMeetTab(entry, params.url));
+  const targetId = tab?.targetId;
+  if (!tab || !targetId) {
+    return {
+      nodeId,
+      found: false,
+      tab,
+      message: params.url
+        ? `No existing Meet tab matched ${params.url}.`
+        : "No existing Meet tab found on the selected Chrome node.",
+    };
+  }
+  await callBrowserProxyOnNode({
+    runtime: params.runtime,
+    nodeId,
+    method: "POST",
+    path: "/tabs/focus",
+    body: { targetId },
+    timeoutMs: Math.min(timeoutMs, 5_000),
+  });
+  const evaluated = await callBrowserProxyOnNode({
+    runtime: params.runtime,
+    nodeId,
+    method: "POST",
+    path: "/act",
+    body: {
+      kind: "evaluate",
+      targetId,
+      fn: meetStatusScript({
+        guestName: params.config.chrome.guestName,
+        autoJoin: false,
+      }),
+    },
+    timeoutMs: Math.min(timeoutMs, 10_000),
+  });
+  const browser = parseMeetBrowserStatus(evaluated);
+  const manual = browser?.manualActionRequired
+    ? browser.manualActionMessage || browser.manualActionReason
+    : undefined;
+  return {
+    nodeId,
+    found: true,
+    targetId,
+    tab,
+    browser,
+    message:
+      manual ?? (browser?.inCall ? "Existing Meet tab is in-call." : "Existing Meet tab focused."),
+  };
+}
+
 export async function launchChromeMeetOnNode(params: {
  runtime: PluginRuntime;
  config: GoogleMeetConfig;