mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:20:43 +00:00
fix: use browser automation for Google Meet join
This commit is contained in:
@@ -166,7 +166,8 @@ openclaw devices list
|
||||
openclaw devices approve <requestId>
|
||||
```
|
||||
|
||||
Confirm the Gateway sees the node and that it advertises `googlemeet.chrome`:
|
||||
Confirm the Gateway sees the node and that it advertises both `googlemeet.chrome`
|
||||
and browser capability/`browser.proxy`:
|
||||
|
||||
```bash
|
||||
openclaw nodes status
|
||||
@@ -178,7 +179,7 @@ Route Meet through that node on the Gateway host:
|
||||
{
|
||||
gateway: {
|
||||
nodes: {
|
||||
allowCommands: ["googlemeet.chrome"],
|
||||
allowCommands: ["googlemeet.chrome", "browser.proxy"],
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
@@ -218,21 +219,24 @@ openclaw googlemeet test-speech https://meet.google.com/abc-defg-hij
|
||||
```
|
||||
|
||||
If `chromeNode.node` is omitted, OpenClaw auto-selects only when exactly one
|
||||
connected node advertises `googlemeet.chrome`. If several capable nodes are
|
||||
connected, set `chromeNode.node` to the node id, display name, or remote IP.
|
||||
connected node advertises both `googlemeet.chrome` and browser control. If
|
||||
several capable nodes are connected, set `chromeNode.node` to the node id,
|
||||
display name, or remote IP.
|
||||
|
||||
Common failure checks:
|
||||
|
||||
- `No connected Google Meet-capable node`: start `openclaw node run` in the VM,
|
||||
approve pairing, and make sure `openclaw plugins enable google-meet` was run
|
||||
in the VM. Also confirm the Gateway host allows the node command with
|
||||
`gateway.nodes.allowCommands: ["googlemeet.chrome"]`.
|
||||
approve pairing, and make sure `openclaw plugins enable google-meet` and
|
||||
`openclaw plugins enable browser` were run in the VM. Also confirm the
|
||||
Gateway host allows both node commands with
|
||||
`gateway.nodes.allowCommands: ["googlemeet.chrome", "browser.proxy"]`.
|
||||
- `BlackHole 2ch audio device not found on the node`: install `blackhole-2ch`
|
||||
in the VM and reboot the VM.
|
||||
- Chrome opens but cannot join: sign in to Chrome inside the VM, or keep
|
||||
`chrome.guestName` set for guest join. Guest auto-join uses Chrome Apple
|
||||
Events; if it reports an automation warning, enable Chrome > View > Developer
|
||||
> Allow JavaScript from Apple Events, then retry.
|
||||
- Chrome opens but cannot join: sign in to the browser profile inside the VM, or
|
||||
keep `chrome.guestName` set for guest join. Guest auto-join uses OpenClaw
|
||||
browser automation through the node browser proxy; make sure the node browser
|
||||
config points at the profile you want, for example
|
||||
`browser.defaultProfile: "user"` or a named existing-session profile.
|
||||
- Duplicate Meet tabs: leave `chrome.reuseExistingTab: true` enabled. OpenClaw
|
||||
activates an existing tab for the same Meet URL before opening a new one.
|
||||
- No audio: in Meet, route microphone/speaker through the virtual audio device
|
||||
@@ -372,6 +376,7 @@ Defaults:
|
||||
- `chrome.guestName: "OpenClaw Agent"`: name used on the signed-out Meet guest
|
||||
screen
|
||||
- `chrome.autoJoin: true`: best-effort guest-name fill and Join Now click
|
||||
through OpenClaw browser automation on `chrome-node`
|
||||
- `chrome.reuseExistingTab: true`: activate an existing Meet tab instead of
|
||||
opening duplicates
|
||||
- `chrome.waitForInCallMs: 20000`: wait for the Meet tab to report in-call
|
||||
|
||||
@@ -72,6 +72,7 @@ type NodeListResult = {
|
||||
displayName?: string;
|
||||
connected?: boolean;
|
||||
commands?: string[];
|
||||
caps?: string[];
|
||||
remoteIp?: string;
|
||||
}>;
|
||||
};
|
||||
@@ -101,16 +102,52 @@ function setup(
|
||||
nodeId: "node-1",
|
||||
displayName: "parallels-macos",
|
||||
connected: true,
|
||||
commands: ["googlemeet.chrome"],
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy", "googlemeet.chrome"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
const nodesInvoke = vi.fn(async (params) =>
|
||||
options.nodesInvokeHandler
|
||||
? options.nodesInvokeHandler(params)
|
||||
: (options.nodesInvokeResult ?? { launched: true }),
|
||||
);
|
||||
const nodesInvoke = vi.fn(async (params) => {
|
||||
if (options.nodesInvokeHandler) {
|
||||
return options.nodesInvokeHandler(params);
|
||||
}
|
||||
if (params.command === "browser.proxy") {
|
||||
const proxy = params.params as { path?: string; body?: { url?: string; targetId?: string } };
|
||||
if (proxy.path === "/tabs") {
|
||||
return { payload: { result: { running: true, tabs: [] } } };
|
||||
}
|
||||
if (proxy.path === "/tabs/open") {
|
||||
return {
|
||||
payload: {
|
||||
result: {
|
||||
targetId: "tab-1",
|
||||
title: "Meet",
|
||||
url: proxy.body?.url ?? "https://meet.google.com/abc-defg-hij",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (proxy.path === "/act") {
|
||||
return {
|
||||
payload: {
|
||||
result: {
|
||||
ok: true,
|
||||
targetId: proxy.body?.targetId ?? "tab-1",
|
||||
result: JSON.stringify({
|
||||
inCall: true,
|
||||
micMuted: false,
|
||||
title: "Meet call",
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
}),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return { payload: { result: { ok: true } } };
|
||||
}
|
||||
return options.nodesInvokeResult ?? { launched: true };
|
||||
});
|
||||
const runCommandWithTimeout = vi.fn(async (argv: string[]) => {
|
||||
if (argv[0] === "/usr/sbin/system_profiler") {
|
||||
return { code: 0, stdout: "BlackHole 2ch", stderr: "" };
|
||||
@@ -559,6 +596,16 @@ describe("google-meet plugin", () => {
|
||||
|
||||
expect(respond.mock.calls[0]?.[0]).toBe(true);
|
||||
expect(nodesList).toHaveBeenCalledWith({ connected: true });
|
||||
expect(nodesInvoke).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
nodeId: "node-1",
|
||||
command: "browser.proxy",
|
||||
params: expect.objectContaining({
|
||||
path: "/tabs/open",
|
||||
body: { url: "https://meet.google.com/abc-defg-hij" },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(nodesInvoke).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
nodeId: "node-1",
|
||||
@@ -567,10 +614,7 @@ describe("google-meet plugin", () => {
|
||||
action: "start",
|
||||
url: "https://meet.google.com/abc-defg-hij",
|
||||
mode: "transcribe",
|
||||
guestName: "OpenClaw Agent",
|
||||
reuseExistingTab: true,
|
||||
autoJoin: true,
|
||||
waitForInCallMs: 20000,
|
||||
launch: false,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
@@ -618,7 +662,9 @@ describe("google-meet plugin", () => {
|
||||
respond: second,
|
||||
});
|
||||
|
||||
expect(nodesInvoke).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
nodesInvoke.mock.calls.filter(([call]) => call.command === "googlemeet.chrome"),
|
||||
).toHaveLength(1);
|
||||
expect(second.mock.calls[0]?.[1]).toMatchObject({
|
||||
session: {
|
||||
chrome: { health: { inCall: true, micMuted: false } },
|
||||
@@ -696,13 +742,15 @@ describe("google-meet plugin", () => {
|
||||
nodeId: "node-1",
|
||||
displayName: "parallels-macos",
|
||||
connected: true,
|
||||
commands: ["googlemeet.chrome"],
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy", "googlemeet.chrome"],
|
||||
},
|
||||
{
|
||||
nodeId: "node-2",
|
||||
displayName: "mac-studio-vm",
|
||||
connected: true,
|
||||
commands: ["googlemeet.chrome"],
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy", "googlemeet.chrome"],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
@@ -53,7 +53,7 @@ const googleMeetConfigSchema = {
|
||||
},
|
||||
"chrome.autoJoin": {
|
||||
label: "Auto Join Guest Screen",
|
||||
help: "Best-effort guest-name fill and Join Now click when Chrome allows JavaScript from Apple Events.",
|
||||
help: "Best-effort guest-name fill and Join Now click through OpenClaw browser automation.",
|
||||
},
|
||||
"chrome.waitForInCallMs": {
|
||||
label: "Wait For In-Call (ms)",
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
},
|
||||
"chrome.autoJoin": {
|
||||
"label": "Auto Join Guest Screen",
|
||||
"help": "Best-effort guest-name fill and Join Now click when Chrome allows JavaScript from Apple Events."
|
||||
"help": "Best-effort guest-name fill and Join Now click through OpenClaw browser automation."
|
||||
},
|
||||
"chrome.waitForInCallMs": {
|
||||
"label": "Wait For In-Call (ms)",
|
||||
|
||||
@@ -25,15 +25,6 @@ type NodeBridgeSession = {
|
||||
lastOutputBytes: number;
|
||||
};
|
||||
|
||||
type BrowserStatus = {
|
||||
inCall?: boolean;
|
||||
micMuted?: boolean;
|
||||
browserUrl?: string;
|
||||
browserTitle?: string;
|
||||
status?: string;
|
||||
notes?: string[];
|
||||
};
|
||||
|
||||
const sessions = new Map<string, NodeBridgeSession>();
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
@@ -60,10 +51,6 @@ function readNumber(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readBoolean(value: unknown, fallback: boolean): boolean {
|
||||
return typeof value === "boolean" ? value : fallback;
|
||||
}
|
||||
|
||||
function runCommandWithTimeout(argv: string[], timeoutMs: number) {
|
||||
const [command, ...args] = argv;
|
||||
if (!command) {
|
||||
@@ -80,164 +67,6 @@ function runCommandWithTimeout(argv: string[], timeoutMs: number) {
|
||||
};
|
||||
}
|
||||
|
||||
function runAppleScript(script: string, timeoutMs: number) {
|
||||
return runCommandWithTimeout(["/usr/bin/osascript", "-e", script], timeoutMs);
|
||||
}
|
||||
|
||||
function normalizeAppleScriptString(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function activeMeetTabStatus(timeoutMs: number): BrowserStatus {
|
||||
const script = `
|
||||
tell application "Google Chrome"
|
||||
repeat with w in windows
|
||||
repeat with t in tabs of w
|
||||
set tabUrl to URL of t
|
||||
if tabUrl starts with "https://meet.google.com/" then
|
||||
set active tab index of w to index of t
|
||||
set index of w to 1
|
||||
set tabTitle to title of t
|
||||
return tabUrl & linefeed & tabTitle
|
||||
end if
|
||||
end repeat
|
||||
end repeat
|
||||
end tell`;
|
||||
const result = runAppleScript(script, timeoutMs);
|
||||
if (result.code !== 0) {
|
||||
return {
|
||||
inCall: false,
|
||||
status: "browser-unavailable",
|
||||
notes: [result.stderr || result.stdout || "Google Chrome tab status unavailable"],
|
||||
};
|
||||
}
|
||||
const [browserUrl = "", browserTitle = ""] = result.stdout.split(/\r?\n/u);
|
||||
const trimmedBrowserTitle = browserTitle.trim();
|
||||
return {
|
||||
inCall: Boolean(browserUrl.trim()) && !trimmedBrowserTitle.endsWith("Meet"),
|
||||
browserUrl: browserUrl.trim() || undefined,
|
||||
browserTitle: trimmedBrowserTitle || undefined,
|
||||
status: "ok",
|
||||
};
|
||||
}
|
||||
|
||||
function activateExistingMeetTab(url: string, timeoutMs: number): boolean {
|
||||
const script = `
|
||||
set targetUrl to ${normalizeAppleScriptString(url)}
|
||||
tell application "Google Chrome"
|
||||
repeat with w in windows
|
||||
repeat with t in tabs of w
|
||||
if URL of t is targetUrl then
|
||||
set active tab index of w to index of t
|
||||
set index of w to 1
|
||||
activate
|
||||
return "found"
|
||||
end if
|
||||
end repeat
|
||||
end repeat
|
||||
end tell
|
||||
return "missing"`;
|
||||
const result = runAppleScript(script, timeoutMs);
|
||||
return result.code === 0 && result.stdout.trim() === "found";
|
||||
}
|
||||
|
||||
function executeMeetTabScript(url: string, javascript: string, timeoutMs: number) {
|
||||
const script = `
|
||||
set targetUrl to ${normalizeAppleScriptString(url)}
|
||||
set source to ${normalizeAppleScriptString(javascript)}
|
||||
tell application "Google Chrome"
|
||||
repeat with w in windows
|
||||
repeat with t in tabs of w
|
||||
if URL of t starts with targetUrl then
|
||||
set active tab index of w to index of t
|
||||
set index of w to 1
|
||||
return execute t javascript source
|
||||
end if
|
||||
end repeat
|
||||
end repeat
|
||||
end tell
|
||||
return ""`;
|
||||
return runAppleScript(script, timeoutMs);
|
||||
}
|
||||
|
||||
function tryAutoJoinMeet(params: {
|
||||
url: string;
|
||||
guestName: string;
|
||||
timeoutMs: number;
|
||||
}): BrowserStatus {
|
||||
const js = `
|
||||
(() => {
|
||||
const text = (node) => (node?.innerText || node?.textContent || "").trim();
|
||||
const input = [...document.querySelectorAll('input')].find((el) =>
|
||||
/your name/i.test(el.getAttribute('aria-label') || el.placeholder || '')
|
||||
);
|
||||
if (input && !input.value) {
|
||||
input.focus();
|
||||
input.value = ${JSON.stringify(params.guestName)};
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
const buttons = [...document.querySelectorAll('button')];
|
||||
const join = buttons.find((button) => /join now|ask to join/i.test(text(button)) && !button.disabled);
|
||||
if (join) join.click();
|
||||
const mic = buttons.find((button) => /turn off microphone|turn on microphone|microphone/i.test(button.getAttribute('aria-label') || text(button)));
|
||||
return JSON.stringify({
|
||||
clickedJoin: Boolean(join),
|
||||
inCall: buttons.some((button) => /leave call/i.test(button.getAttribute('aria-label') || text(button))),
|
||||
micMuted: mic ? /turn on microphone/i.test(mic.getAttribute('aria-label') || text(mic)) : undefined,
|
||||
title: document.title,
|
||||
url: location.href
|
||||
});
|
||||
})();`;
|
||||
const result = executeMeetTabScript(params.url, js, Math.min(params.timeoutMs, 5_000));
|
||||
if (result.code !== 0) {
|
||||
return {
|
||||
...activeMeetTabStatus(Math.min(params.timeoutMs, 2_000)),
|
||||
notes: [
|
||||
"Chrome JavaScript automation is unavailable; enable Chrome > View > Developer > Allow JavaScript from Apple Events for guest auto-join.",
|
||||
result.stderr || result.stdout || "unknown Apple Events failure",
|
||||
],
|
||||
};
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(result.stdout.trim()) as {
|
||||
inCall?: boolean;
|
||||
micMuted?: boolean;
|
||||
url?: string;
|
||||
title?: string;
|
||||
};
|
||||
return {
|
||||
inCall: parsed.inCall,
|
||||
micMuted: parsed.micMuted,
|
||||
browserUrl: parsed.url,
|
||||
browserTitle: parsed.title,
|
||||
status: "ok",
|
||||
};
|
||||
} catch {
|
||||
return activeMeetTabStatus(Math.min(params.timeoutMs, 2_000));
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForInCall(params: {
|
||||
url: string;
|
||||
guestName: string;
|
||||
autoJoin: boolean;
|
||||
timeoutMs: number;
|
||||
}): Promise<BrowserStatus> {
|
||||
const deadline = Date.now() + Math.max(0, params.timeoutMs);
|
||||
let status: BrowserStatus = activeMeetTabStatus(2_000);
|
||||
while (Date.now() <= deadline) {
|
||||
status = params.autoJoin
|
||||
? tryAutoJoinMeet({ url: params.url, guestName: params.guestName, timeoutMs: 5_000 })
|
||||
: activeMeetTabStatus(2_000);
|
||||
if (status.inCall === true) {
|
||||
return status;
|
||||
}
|
||||
await sleep(750);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
function assertBlackHoleAvailable(timeoutMs: number) {
|
||||
if (process.platform !== "darwin") {
|
||||
throw new Error("Chrome Meet transport with blackhole-2ch audio is currently macOS-only");
|
||||
@@ -409,44 +238,42 @@ function startChrome(params: Record<string, unknown>) {
|
||||
if (browserProfile) {
|
||||
argv.push("--args", `--profile-directory=${browserProfile}`);
|
||||
}
|
||||
const reused = readBoolean(params.reuseExistingTab, true)
|
||||
? activateExistingMeetTab(url, Math.min(timeoutMs, 5_000))
|
||||
: false;
|
||||
if (!reused) {
|
||||
argv.push(url);
|
||||
const result = runCommandWithTimeout(argv, timeoutMs);
|
||||
if (result.code !== 0) {
|
||||
if (bridgeId) {
|
||||
const session = sessions.get(bridgeId);
|
||||
if (session) {
|
||||
stopSession(session);
|
||||
}
|
||||
argv.push(url);
|
||||
const result = runCommandWithTimeout(argv, timeoutMs);
|
||||
if (result.code !== 0) {
|
||||
if (bridgeId) {
|
||||
const session = sessions.get(bridgeId);
|
||||
if (session) {
|
||||
stopSession(session);
|
||||
}
|
||||
throw new Error(
|
||||
`failed to launch Chrome for Meet: ${result.stderr || result.stdout || result.code}`,
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`failed to launch Chrome for Meet: ${result.stderr || result.stdout || result.code}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const waitForInCallMs = readNumber(params.waitForInCallMs, 20_000);
|
||||
return Promise.resolve(
|
||||
params.launch !== false && waitForInCallMs > 0
|
||||
? waitForInCall({
|
||||
url,
|
||||
guestName: readString(params.guestName) ?? "OpenClaw Agent",
|
||||
autoJoin: readBoolean(params.autoJoin, true),
|
||||
timeoutMs: waitForInCallMs,
|
||||
})
|
||||
: activeMeetTabStatus(2_000),
|
||||
).then((browser) => ({ launched: params.launch !== false, bridgeId, audioBridge, browser }));
|
||||
return {
|
||||
launched: params.launch !== false,
|
||||
bridgeId,
|
||||
audioBridge,
|
||||
browser:
|
||||
params.launch !== false
|
||||
? {
|
||||
status: "chrome-opened",
|
||||
browserUrl: url,
|
||||
notes: [
|
||||
"Browser page control is handled by OpenClaw browser automation when using chrome-node.",
|
||||
],
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function bridgeStatus(params: Record<string, unknown>) {
|
||||
const bridgeId = readString(params.bridgeId);
|
||||
const session = bridgeId ? sessions.get(bridgeId) : undefined;
|
||||
return {
|
||||
browser: activeMeetTabStatus(2_000),
|
||||
bridge: session
|
||||
? {
|
||||
bridgeId,
|
||||
|
||||
@@ -155,16 +155,19 @@ export async function launchChromeMeet(params: {
|
||||
}
|
||||
|
||||
function isGoogleMeetNode(node: {
|
||||
caps?: string[];
|
||||
commands?: string[];
|
||||
connected?: boolean;
|
||||
nodeId?: string;
|
||||
displayName?: string;
|
||||
remoteIp?: string;
|
||||
}) {
|
||||
const commands = Array.isArray(node.commands) ? node.commands : [];
|
||||
const caps = Array.isArray(node.caps) ? node.caps : [];
|
||||
return (
|
||||
node.connected === true &&
|
||||
Array.isArray(node.commands) &&
|
||||
node.commands.includes("googlemeet.chrome")
|
||||
commands.includes("googlemeet.chrome") &&
|
||||
(commands.includes("browser.proxy") || caps.includes("browser"))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -176,7 +179,7 @@ async function resolveChromeNode(params: {
|
||||
const nodes = list.nodes.filter(isGoogleMeetNode);
|
||||
if (nodes.length === 0) {
|
||||
throw new Error(
|
||||
"No connected Google Meet-capable node. Run `openclaw node run` on the Chrome host and approve pairing.",
|
||||
"No connected Google Meet-capable node with browser proxy. Run `openclaw node run` on the Chrome host with browser proxy enabled, approve pairing, and allow googlemeet.chrome plus browser.proxy.",
|
||||
);
|
||||
}
|
||||
const requested = params.requestedNode?.trim();
|
||||
@@ -218,6 +221,224 @@ function parseNodeStartResult(raw: unknown): {
|
||||
};
|
||||
}
|
||||
|
||||
type BrowserProxyResult = {
|
||||
result?: unknown;
|
||||
};
|
||||
|
||||
type BrowserTab = {
|
||||
targetId?: string;
|
||||
title?: string;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
function unwrapNodeInvokePayload(raw: unknown): unknown {
|
||||
const record = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
|
||||
if (typeof record.payloadJSON === "string" && record.payloadJSON.trim()) {
|
||||
return JSON.parse(record.payloadJSON);
|
||||
}
|
||||
if ("payload" in record) {
|
||||
return record.payload;
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function parseBrowserProxyResult(raw: unknown): unknown {
|
||||
const payload = unwrapNodeInvokePayload(raw);
|
||||
const proxy =
|
||||
payload && typeof payload === "object" ? (payload as BrowserProxyResult) : undefined;
|
||||
if (!proxy || !("result" in proxy)) {
|
||||
throw new Error("Google Meet browser proxy returned an invalid result.");
|
||||
}
|
||||
return proxy.result;
|
||||
}
|
||||
|
||||
async function callBrowserProxyOnNode(params: {
|
||||
runtime: PluginRuntime;
|
||||
nodeId: string;
|
||||
method: "GET" | "POST" | "DELETE";
|
||||
path: string;
|
||||
body?: unknown;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const raw = await params.runtime.nodes.invoke({
|
||||
nodeId: params.nodeId,
|
||||
command: "browser.proxy",
|
||||
params: {
|
||||
method: params.method,
|
||||
path: params.path,
|
||||
body: params.body,
|
||||
timeoutMs: params.timeoutMs,
|
||||
},
|
||||
timeoutMs: params.timeoutMs + 5_000,
|
||||
});
|
||||
return parseBrowserProxyResult(raw);
|
||||
}
|
||||
|
||||
function asBrowserTabs(result: unknown): BrowserTab[] {
|
||||
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
|
||||
return Array.isArray(record.tabs) ? (record.tabs as BrowserTab[]) : [];
|
||||
}
|
||||
|
||||
function readBrowserTab(result: unknown): BrowserTab | undefined {
|
||||
return result && typeof result === "object" ? (result as BrowserTab) : undefined;
|
||||
}
|
||||
|
||||
function parseMeetBrowserStatus(result: unknown): GoogleMeetChromeHealth | undefined {
|
||||
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
|
||||
const raw = record.result;
|
||||
if (typeof raw !== "string" || !raw.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = JSON.parse(raw) as {
|
||||
inCall?: boolean;
|
||||
micMuted?: boolean;
|
||||
url?: string;
|
||||
title?: string;
|
||||
};
|
||||
return {
|
||||
inCall: parsed.inCall,
|
||||
micMuted: parsed.micMuted,
|
||||
browserUrl: parsed.url,
|
||||
browserTitle: parsed.title,
|
||||
status: "browser-control",
|
||||
};
|
||||
}
|
||||
|
||||
function meetStatusScript(params: { guestName: string; autoJoin: boolean }) {
|
||||
return `() => {
|
||||
const text = (node) => (node?.innerText || node?.textContent || "").trim();
|
||||
const input = [...document.querySelectorAll('input')].find((el) =>
|
||||
/your name/i.test(el.getAttribute('aria-label') || el.placeholder || '')
|
||||
);
|
||||
if (${JSON.stringify(params.autoJoin)} && input && !input.value) {
|
||||
input.focus();
|
||||
input.value = ${JSON.stringify(params.guestName)};
|
||||
input.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
input.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
const buttons = [...document.querySelectorAll('button')];
|
||||
const join = ${JSON.stringify(params.autoJoin)}
|
||||
? buttons.find((button) => /join now|ask to join/i.test(text(button)) && !button.disabled)
|
||||
: null;
|
||||
if (join) join.click();
|
||||
const mic = buttons.find((button) => /turn off microphone|turn on microphone|microphone/i.test(button.getAttribute('aria-label') || text(button)));
|
||||
return JSON.stringify({
|
||||
clickedJoin: Boolean(join),
|
||||
inCall: buttons.some((button) => /leave call/i.test(button.getAttribute('aria-label') || text(button))),
|
||||
micMuted: mic ? /turn on microphone/i.test(mic.getAttribute('aria-label') || text(mic)) : undefined,
|
||||
title: document.title,
|
||||
url: location.href
|
||||
});
|
||||
}`;
|
||||
}
|
||||
|
||||
async function openMeetWithBrowserProxy(params: {
|
||||
runtime: PluginRuntime;
|
||||
nodeId: string;
|
||||
config: GoogleMeetConfig;
|
||||
url: string;
|
||||
}): Promise<{ launched: boolean; browser?: GoogleMeetChromeHealth }> {
|
||||
if (!params.config.chrome.launch) {
|
||||
return { launched: false };
|
||||
}
|
||||
|
||||
const timeoutMs = Math.max(1_000, params.config.chrome.joinTimeoutMs);
|
||||
let targetId: string | undefined;
|
||||
let tab: BrowserTab | undefined;
|
||||
if (params.config.chrome.reuseExistingTab) {
|
||||
const tabs = asBrowserTabs(
|
||||
await callBrowserProxyOnNode({
|
||||
runtime: params.runtime,
|
||||
nodeId: params.nodeId,
|
||||
method: "GET",
|
||||
path: "/tabs",
|
||||
timeoutMs: Math.min(timeoutMs, 5_000),
|
||||
}),
|
||||
);
|
||||
tab = tabs.find((entry) => entry.url === params.url);
|
||||
targetId = tab?.targetId;
|
||||
if (targetId) {
|
||||
await callBrowserProxyOnNode({
|
||||
runtime: params.runtime,
|
||||
nodeId: params.nodeId,
|
||||
method: "POST",
|
||||
path: "/tabs/focus",
|
||||
body: { targetId },
|
||||
timeoutMs: Math.min(timeoutMs, 5_000),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!targetId) {
|
||||
tab = readBrowserTab(
|
||||
await callBrowserProxyOnNode({
|
||||
runtime: params.runtime,
|
||||
nodeId: params.nodeId,
|
||||
method: "POST",
|
||||
path: "/tabs/open",
|
||||
body: { url: params.url },
|
||||
timeoutMs,
|
||||
}),
|
||||
);
|
||||
targetId = tab?.targetId;
|
||||
}
|
||||
if (!targetId) {
|
||||
return {
|
||||
launched: true,
|
||||
browser: {
|
||||
status: "browser-control",
|
||||
notes: ["Browser proxy opened Meet but did not return a targetId."],
|
||||
browserUrl: tab?.url,
|
||||
browserTitle: tab?.title,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const deadline = Date.now() + Math.max(0, params.config.chrome.waitForInCallMs);
|
||||
let browser: GoogleMeetChromeHealth | undefined = {
|
||||
status: "browser-control",
|
||||
browserUrl: tab?.url,
|
||||
browserTitle: tab?.title,
|
||||
};
|
||||
do {
|
||||
try {
|
||||
const evaluated = await callBrowserProxyOnNode({
|
||||
runtime: params.runtime,
|
||||
nodeId: params.nodeId,
|
||||
method: "POST",
|
||||
path: "/act",
|
||||
body: {
|
||||
kind: "evaluate",
|
||||
targetId,
|
||||
fn: meetStatusScript({
|
||||
guestName: params.config.chrome.guestName,
|
||||
autoJoin: params.config.chrome.autoJoin,
|
||||
}),
|
||||
},
|
||||
timeoutMs: Math.min(timeoutMs, 10_000),
|
||||
});
|
||||
browser = parseMeetBrowserStatus(evaluated) ?? browser;
|
||||
if (browser?.inCall === true) {
|
||||
return { launched: true, browser };
|
||||
}
|
||||
} catch (error) {
|
||||
browser = {
|
||||
...browser,
|
||||
inCall: false,
|
||||
notes: [
|
||||
`Browser control could not inspect or auto-join Meet: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
],
|
||||
};
|
||||
break;
|
||||
}
|
||||
if (Date.now() <= deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 750));
|
||||
}
|
||||
} while (Date.now() <= deadline);
|
||||
return { launched: true, browser };
|
||||
}
|
||||
|
||||
export async function launchChromeMeetOnNode(params: {
|
||||
runtime: PluginRuntime;
|
||||
config: GoogleMeetConfig;
|
||||
@@ -238,6 +459,12 @@ export async function launchChromeMeetOnNode(params: {
|
||||
runtime: params.runtime,
|
||||
requestedNode: params.config.chromeNode.node,
|
||||
});
|
||||
const browserControl = await openMeetWithBrowserProxy({
|
||||
runtime: params.runtime,
|
||||
nodeId,
|
||||
config: params.config,
|
||||
url: params.url,
|
||||
});
|
||||
const raw = await params.runtime.nodes.invoke({
|
||||
nodeId,
|
||||
command: "googlemeet.chrome",
|
||||
@@ -245,17 +472,13 @@ export async function launchChromeMeetOnNode(params: {
|
||||
action: "start",
|
||||
url: params.url,
|
||||
mode: params.mode,
|
||||
launch: params.config.chrome.launch,
|
||||
launch: false,
|
||||
browserProfile: params.config.chrome.browserProfile,
|
||||
joinTimeoutMs: params.config.chrome.joinTimeoutMs,
|
||||
audioInputCommand: params.config.chrome.audioInputCommand,
|
||||
audioOutputCommand: params.config.chrome.audioOutputCommand,
|
||||
audioBridgeCommand: params.config.chrome.audioBridgeCommand,
|
||||
audioBridgeHealthCommand: params.config.chrome.audioBridgeHealthCommand,
|
||||
guestName: params.config.chrome.guestName,
|
||||
reuseExistingTab: params.config.chrome.reuseExistingTab,
|
||||
autoJoin: params.config.chrome.autoJoin,
|
||||
waitForInCallMs: params.config.chrome.waitForInCallMs,
|
||||
},
|
||||
timeoutMs: params.config.chrome.joinTimeoutMs + 5_000,
|
||||
});
|
||||
@@ -275,18 +498,22 @@ export async function launchChromeMeetOnNode(params: {
|
||||
});
|
||||
return {
|
||||
nodeId,
|
||||
launched: result.launched === true,
|
||||
launched: browserControl.launched || result.launched === true,
|
||||
audioBridge: bridge,
|
||||
browser: result.browser,
|
||||
browser: browserControl.browser ?? result.browser,
|
||||
};
|
||||
}
|
||||
if (result.audioBridge?.type === "external-command") {
|
||||
return {
|
||||
nodeId,
|
||||
launched: result.launched === true,
|
||||
launched: browserControl.launched || result.launched === true,
|
||||
audioBridge: { type: "external-command" },
|
||||
browser: result.browser,
|
||||
browser: browserControl.browser ?? result.browser,
|
||||
};
|
||||
}
|
||||
return { nodeId, launched: result.launched === true, browser: result.browser };
|
||||
return {
|
||||
nodeId,
|
||||
launched: browserControl.launched || result.launched === true,
|
||||
browser: browserControl.browser ?? result.browser,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user