From 593964560ba7abf458c1069dc03e7ad35afa5dc2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Mar 2026 20:08:42 +0000 Subject: [PATCH] feat(browser): add chrome MCP existing-session support --- CHANGELOG.md | 2 + docs/tools/browser.md | 72 +++ docs/tools/chrome-extension.md | 7 + package.json | 1 + pnpm-lock.yaml | 124 ++++- src/browser/chrome-mcp.snapshot.test.ts | 68 +++ src/browser/chrome-mcp.snapshot.ts | 246 +++++++++ src/browser/chrome-mcp.test.ts | 100 ++++ src/browser/chrome-mcp.ts | 488 ++++++++++++++++++ src/browser/client.ts | 4 +- src/browser/config.ts | 11 +- src/browser/profile-capabilities.ts | 23 +- src/browser/profiles-service.test.ts | 92 +++- src/browser/profiles-service.ts | 17 +- src/browser/routes/agent.act.download.ts | 39 +- src/browser/routes/agent.act.hooks.ts | 108 +++- src/browser/routes/agent.act.ts | 370 ++++++++++++- src/browser/routes/agent.snapshot.ts | 242 ++++++++- src/browser/routes/basic.ts | 15 +- src/browser/server-context.availability.ts | 24 + .../server-context.existing-session.test.ts | 102 ++++ src/browser/server-context.selection.ts | 13 + src/browser/server-context.tab-ops.ts | 14 + src/browser/server-context.ts | 20 +- src/browser/server-context.types.ts | 1 + src/cli/browser-cli-manage.ts | 21 +- src/config/types.browser.ts | 2 +- src/config/zod-schema.ts | 7 +- 28 files changed, 2165 insertions(+), 68 deletions(-) create mode 100644 src/browser/chrome-mcp.snapshot.test.ts create mode 100644 src/browser/chrome-mcp.snapshot.ts create mode 100644 src/browser/chrome-mcp.test.ts create mode 100644 src/browser/chrome-mcp.ts create mode 100644 src/browser/server-context.existing-session.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bec1b32283a..057a4997e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,11 @@ Docs: https://docs.openclaw.ai - Android/chat settings: redesign the chat settings sheet with grouped device and media sections, refresh the Connect and Voice tabs, and tighten the chat composer/session header for a denser mobile layout. (#44894) Thanks @obviyus. - Docker/timezone override: add `OPENCLAW_TZ` so `docker-setup.sh` can pin gateway and CLI containers to a chosen IANA timezone instead of inheriting the daemon default. (#34119) Thanks @Lanfei. - iOS/onboarding: add a first-run welcome pager before gateway setup, stop auto-opening the QR scanner, and show `/pair qr` instructions on the connect step. (#45054) Thanks @ngutman. +- Browser/existing-session: add an official Chrome DevTools MCP attach mode for signed-in live Chrome sessions, with docs for `chrome://inspect/#remote-debugging` enablement and direct backlinks to Chrome’s own setup guides. ### Fixes +- Browser/existing-session: accept text-only `list_pages` and `new_page` responses from Chrome DevTools MCP so live-session tab discovery and new-tab open flows keep working when the server omits structured page metadata. - Ollama/reasoning visibility: stop promoting native `thinking` and `reasoning` fields into final assistant text so local reasoning models no longer leak internal thoughts in normal replies. (#45330) Thanks @xi7ang. - Windows/gateway install: bound `schtasks` calls and fall back to the Startup-folder login item when task creation hangs, so native `openclaw gateway install` fails fast instead of wedging forever on broken Scheduled Task setups. - Windows/gateway auth: stop attaching device identity on local loopback shared-token and password gateway calls, so native Windows agent replies no longer log stale `device signature expired` fallback noise before succeeding. diff --git a/docs/tools/browser.md b/docs/tools/browser.md index d632e713068..8a7abe93209 100644 --- a/docs/tools/browser.md +++ b/docs/tools/browser.md @@ -48,6 +48,8 @@ Gateway. - `openclaw`: managed, isolated browser (no extension required). - `chrome`: extension relay to your **system browser** (requires the OpenClaw extension to be attached to a tab). +- `existing-session`: official Chrome MCP attach flow for a running Chrome + profile. Set `browser.defaultProfile: "openclaw"` if you want managed mode by default. @@ -77,6 +79,12 @@ Browser settings live in `~/.openclaw/openclaw.json`. profiles: { openclaw: { cdpPort: 18800, color: "#FF4500" }, work: { cdpPort: 18801, color: "#0066CC" }, + chromeLive: { + cdpPort: 18802, + driver: "existing-session", + attachOnly: true, + color: "#00AA00", + }, remote: { cdpUrl: "http://10.0.0.42:9222", color: "#00AA00" }, }, }, @@ -100,6 +108,8 @@ Notes: - Default profile is `openclaw` (OpenClaw-managed standalone browser). Use `defaultProfile: "chrome"` to opt into the Chrome extension relay. - Auto-detect order: system default browser if Chromium-based; otherwise Chrome → Brave → Edge → Chromium → Chrome Canary. - Local `openclaw` profiles auto-assign `cdpPort`/`cdpUrl` — set those only for remote CDP. +- `driver: "existing-session"` uses Chrome DevTools MCP instead of raw CDP. Do + not set `cdpUrl` for that driver. ## Use Brave (or another Chromium-based browser) @@ -264,11 +274,13 @@ OpenClaw supports multiple named profiles (routing configs). Profiles can be: - **openclaw-managed**: a dedicated Chromium-based browser instance with its own user data directory + CDP port - **remote**: an explicit CDP URL (Chromium-based browser running elsewhere) - **extension relay**: your existing Chrome tab(s) via the local relay + Chrome extension +- **existing session**: your existing Chrome profile via Chrome DevTools MCP auto-connect Defaults: - The `openclaw` profile is auto-created if missing. - The `chrome` profile is built-in for the Chrome extension relay (points at `http://127.0.0.1:18792` by default). +- Existing-session profiles are opt-in; create them with `--driver existing-session`. - Local CDP ports allocate from **18800–18899** by default. - Deleting a profile moves its local data directory to Trash. @@ -328,6 +340,66 @@ Notes: - This mode relies on Playwright-on-CDP for most operations (screenshots/snapshots/actions). - Detach by clicking the extension icon again. + +## Chrome existing-session via MCP + +OpenClaw can also attach to a running Chrome profile through the official +Chrome DevTools MCP server. This reuses the tabs and login state already open in +that Chrome profile. + +Official background and setup references: + +- [Chrome for Developers: Use Chrome DevTools MCP with your browser session](https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session) +- [Chrome DevTools MCP README](https://github.com/ChromeDevTools/chrome-devtools-mcp) + +Create a profile: + +```bash +openclaw browser create-profile \ + --name chrome-live \ + --driver existing-session \ + --color "#00AA00" +``` + +Then in Chrome: + +1. Open `chrome://inspect/#remote-debugging` +2. Enable remote debugging +3. Keep Chrome running and approve the connection prompt when OpenClaw attaches + +Live attach smoke test: + +```bash +openclaw browser --browser-profile chrome-live start +openclaw browser --browser-profile chrome-live status +openclaw browser --browser-profile chrome-live tabs +openclaw browser --browser-profile chrome-live snapshot --format ai +``` + +What success looks like: + +- `status` shows `driver: existing-session` +- `status` shows `running: true` +- `tabs` lists your already-open Chrome tabs +- `snapshot` returns refs from the selected live tab + +What to check if attach does not work: + +- Chrome is version `144+` +- remote debugging is enabled at `chrome://inspect/#remote-debugging` +- Chrome showed and you accepted the attach consent prompt +- the Gateway or node host can spawn `npx chrome-devtools-mcp@latest --autoConnect` + +Notes: + +- This path is higher-risk than the isolated `openclaw` profile because it can + act inside your signed-in browser session. +- OpenClaw does not launch Chrome for this driver; it attaches to an existing + session only. +- OpenClaw uses the official Chrome DevTools MCP `--autoConnect` flow here, not + the legacy default-profile remote debugging port workflow. +- Some features still require the extension relay or managed browser path, such + as PDF export and download interception. - Leave the relay loopback-only by default. If the relay must be reachable from a different network namespace (for example Gateway in WSL2, Chrome on Windows), set `browser.relayBindHost` to an explicit bind address such as `0.0.0.0` while keeping the surrounding network private and authenticated. WSL2 / cross-namespace example: diff --git a/docs/tools/chrome-extension.md b/docs/tools/chrome-extension.md index ce4b271ae9c..dcf2150409b 100644 --- a/docs/tools/chrome-extension.md +++ b/docs/tools/chrome-extension.md @@ -13,6 +13,13 @@ The OpenClaw Chrome extension lets the agent control your **existing Chrome tabs Attach/detach happens via a **single Chrome toolbar button**. +If you want Chrome’s official DevTools MCP attach flow instead of the OpenClaw +extension relay, use an `existing-session` browser profile instead. See +[Browser](/tools/browser#chrome-existing-session-via-mcp). For Chrome’s own +setup docs, see [Chrome for Developers: Use Chrome DevTools MCP with your +browser session](https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session) +and the [Chrome DevTools MCP README](https://github.com/ChromeDevTools/chrome-devtools-mcp). + ## What it is (concept) There are three parts: diff --git a/package.json b/package.json index c63e72f66fa..54d897eb66f 100644 --- a/package.json +++ b/package.json @@ -353,6 +353,7 @@ "@mariozechner/pi-ai": "0.57.1", "@mariozechner/pi-coding-agent": "0.57.1", "@mariozechner/pi-tui": "0.57.1", + "@modelcontextprotocol/sdk": "1.27.1", "@mozilla/readability": "^0.6.0", "@sinclair/typebox": "0.34.48", "@slack/bolt": "^4.6.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac32d145c57..ecc73c421c7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,16 +60,19 @@ importers: version: 1.2.0-beta.3 '@mariozechner/pi-agent-core': specifier: 0.57.1 - version: 0.57.1(ws@8.19.0)(zod@4.3.6) + version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@mariozechner/pi-ai': specifier: 0.57.1 - version: 0.57.1(ws@8.19.0)(zod@4.3.6) + version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@mariozechner/pi-coding-agent': specifier: 0.57.1 - version: 0.57.1(ws@8.19.0)(zod@4.3.6) + version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@mariozechner/pi-tui': specifier: 0.57.1 version: 0.57.1 + '@modelcontextprotocol/sdk': + specifier: 1.27.1 + version: 1.27.1(zod@4.3.6) '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 @@ -346,7 +349,7 @@ importers: version: 10.6.1 openclaw: specifier: '>=2026.3.11' - version: 2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)) + version: 2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)) extensions/imessage: {} @@ -377,7 +380,7 @@ importers: dependencies: '@mariozechner/pi-agent-core': specifier: 0.57.1 - version: 0.57.1(ws@8.19.0)(zod@4.3.6) + version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@matrix-org/matrix-sdk-crypto-nodejs': specifier: ^0.4.0 version: 0.4.0 @@ -407,7 +410,7 @@ importers: dependencies: openclaw: specifier: '>=2026.3.11' - version: 2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)) + version: 2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)) extensions/memory-lancedb: dependencies: @@ -1828,6 +1831,16 @@ packages: '@mistralai/mistralai@1.14.1': resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==} + '@modelcontextprotocol/sdk@1.27.1': + resolution: {integrity: sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==} + engines: {node: '>=18'} + peerDependencies: + '@cfworker/json-schema': ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@cfworker/json-schema': + optional: true + '@mozilla/readability@0.6.0': resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} engines: {node: '>=14.0.0'} @@ -4271,6 +4284,10 @@ packages: core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + croner@10.0.1: resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==} engines: {node: '>=18.0'} @@ -4550,6 +4567,14 @@ packages: events-universal@1.0.1: resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + eventsource-parser@3.0.6: + resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + execa@4.1.0: resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==} engines: {node: '>=10'} @@ -4561,6 +4586,12 @@ packages: exponential-backoff@3.1.3: resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} + express-rate-limit@8.3.1: + resolution: {integrity: sha512-D1dKN+cmyPWuvB+G2SREQDzPY1agpBIcTa9sJxOPMCNeH3gwzhqJRDWCXW3gg0y//+LQ/8j52JbMROWyrKdMdw==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + express@4.22.1: resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} engines: {node: '>= 0.10.0'} @@ -5058,6 +5089,9 @@ packages: jose@4.15.9: resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} + jose@6.2.1: + resolution: {integrity: sha512-jUaKr1yrbfaImV7R2TN/b3IcZzsw38/chqMpo2XJ7i2F8AfM/lA4G1goC3JVEwg0H7UldTmSt3P68nt31W7/mw==} + js-stringify@1.0.2: resolution: {integrity: sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g==} @@ -5102,6 +5136,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-schema-typed@8.0.2: + resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-schema@0.4.0: resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} @@ -5870,6 +5907,10 @@ packages: resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} hasBin: true + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + playwright-core@1.58.2: resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} engines: {node: '>=18'} @@ -8645,12 +8686,14 @@ snapshots: optionalDependencies: '@noble/hashes': 2.0.1 - '@google/genai@1.44.0': + '@google/genai@1.44.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))': dependencies: google-auth-library: 10.6.1 p-retry: 4.6.2 protobufjs: 7.5.4 ws: 8.19.0 + optionalDependencies: + '@modelcontextprotocol/sdk': 1.27.1(zod@4.3.6) transitivePeerDependencies: - bufferutil - supports-color @@ -8698,7 +8741,6 @@ snapshots: '@hono/node-server@1.19.10(hono@4.12.7)': dependencies: hono: 4.12.7 - optional: true '@huggingface/jinja@0.5.5': {} @@ -9025,9 +9067,9 @@ snapshots: std-env: 3.10.0 yoctocolors: 2.1.2 - '@mariozechner/pi-agent-core@0.57.1(ws@8.19.0)(zod@4.3.6)': + '@mariozechner/pi-agent-core@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)': dependencies: - '@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) transitivePeerDependencies: - '@modelcontextprotocol/sdk' - aws-crt @@ -9037,11 +9079,11 @@ snapshots: - ws - zod - '@mariozechner/pi-ai@0.57.1(ws@8.19.0)(zod@4.3.6)': + '@mariozechner/pi-ai@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)': dependencies: '@anthropic-ai/sdk': 0.73.0(zod@4.3.6) '@aws-sdk/client-bedrock-runtime': 3.1004.0 - '@google/genai': 1.44.0 + '@google/genai': 1.44.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6)) '@mistralai/mistralai': 1.14.1 '@sinclair/typebox': 0.34.48 ajv: 8.18.0 @@ -9061,11 +9103,11 @@ snapshots: - ws - zod - '@mariozechner/pi-coding-agent@0.57.1(ws@8.19.0)(zod@4.3.6)': + '@mariozechner/pi-coding-agent@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)': dependencies: '@mariozechner/jiti': 2.6.5 - '@mariozechner/pi-agent-core': 0.57.1(ws@8.19.0)(zod@4.3.6) - '@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-agent-core': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@mariozechner/pi-tui': 0.57.1 '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 @@ -9141,6 +9183,28 @@ snapshots: - bufferutil - utf-8-validate + '@modelcontextprotocol/sdk@1.27.1(zod@4.3.6)': + dependencies: + '@hono/node-server': 1.19.10(hono@4.12.7) + ajv: 8.18.0 + ajv-formats: 3.0.1(ajv@8.18.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.6 + express: 5.2.1 + express-rate-limit: 8.3.1(express@5.2.1) + hono: 4.12.7 + jose: 6.2.1 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.3.6 + zod-to-json-schema: 3.25.1(zod@4.3.6) + transitivePeerDependencies: + - supports-color + '@mozilla/readability@0.6.0': {} '@napi-rs/canvas-android-arm64@0.1.95': @@ -11916,6 +11980,11 @@ snapshots: core-util-is@1.0.3: {} + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + croner@10.0.1: {} cross-spawn@7.0.6: @@ -12167,6 +12236,12 @@ snapshots: transitivePeerDependencies: - bare-abort-controller + eventsource-parser@3.0.6: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.0.6 + execa@4.1.0: dependencies: cross-spawn: 7.0.6 @@ -12183,6 +12258,11 @@ snapshots: exponential-backoff@3.1.3: {} + express-rate-limit@8.3.1(express@5.2.1): + dependencies: + express: 5.2.1 + ip-address: 10.1.0 + express@4.22.1: dependencies: accepts: 1.3.8 @@ -12826,6 +12906,8 @@ snapshots: jose@4.15.9: {} + jose@6.2.1: {} + js-stringify@1.0.2: {} js-tokens@10.0.0: {} @@ -12893,6 +12975,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-schema-typed@8.0.2: {} + json-schema@0.4.0: {} json-stringify-safe@5.0.1: {} @@ -13497,7 +13581,7 @@ snapshots: ws: 8.19.0 zod: 4.3.6 - openclaw@2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)): + openclaw@2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)): dependencies: '@agentclientprotocol/sdk': 0.16.1(zod@4.3.6) '@aws-sdk/client-bedrock': 3.1007.0 @@ -13510,9 +13594,9 @@ snapshots: '@larksuiteoapi/node-sdk': 1.59.0 '@line/bot-sdk': 10.6.0 '@lydell/node-pty': 1.2.0-beta.3 - '@mariozechner/pi-agent-core': 0.57.1(ws@8.19.0)(zod@4.3.6) - '@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6) - '@mariozechner/pi-coding-agent': 0.57.1(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-agent-core': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) + '@mariozechner/pi-coding-agent': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6) '@mariozechner/pi-tui': 0.57.1 '@mozilla/readability': 0.6.0 '@napi-rs/canvas': 0.1.95 @@ -13784,6 +13868,8 @@ snapshots: sonic-boom: 4.2.1 thread-stream: 3.1.0 + pkce-challenge@5.0.1: {} + playwright-core@1.58.2: {} playwright@1.58.2: diff --git a/src/browser/chrome-mcp.snapshot.test.ts b/src/browser/chrome-mcp.snapshot.test.ts new file mode 100644 index 00000000000..3fe3288848f --- /dev/null +++ b/src/browser/chrome-mcp.snapshot.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + buildAiSnapshotFromChromeMcpSnapshot, + flattenChromeMcpSnapshotToAriaNodes, +} from "./chrome-mcp.snapshot.js"; + +const snapshot = { + id: "root", + role: "document", + name: "Example", + children: [ + { + id: "btn-1", + role: "button", + name: "Continue", + }, + { + id: "txt-1", + role: "textbox", + name: "Email", + value: "peter@example.com", + }, + ], +}; + +describe("chrome MCP snapshot conversion", () => { + it("flattens structured snapshots into aria-style nodes", () => { + const nodes = flattenChromeMcpSnapshotToAriaNodes(snapshot, 10); + expect(nodes).toEqual([ + { + ref: "root", + role: "document", + name: "Example", + value: undefined, + description: undefined, + depth: 0, + }, + { + ref: "btn-1", + role: "button", + name: "Continue", + value: undefined, + description: undefined, + depth: 1, + }, + { + ref: "txt-1", + role: "textbox", + name: "Email", + value: "peter@example.com", + description: undefined, + depth: 1, + }, + ]); + }); + + it("builds AI snapshots that preserve Chrome MCP uids as refs", () => { + const result = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot }); + + expect(result.snapshot).toContain('- button "Continue" [ref=btn-1]'); + expect(result.snapshot).toContain('- textbox "Email" [ref=txt-1] value="peter@example.com"'); + expect(result.refs).toEqual({ + "btn-1": { role: "button", name: "Continue" }, + "txt-1": { role: "textbox", name: "Email" }, + }); + expect(result.stats.refs).toBe(2); + }); +}); diff --git a/src/browser/chrome-mcp.snapshot.ts b/src/browser/chrome-mcp.snapshot.ts new file mode 100644 index 00000000000..e92709df6f2 --- /dev/null +++ b/src/browser/chrome-mcp.snapshot.ts @@ -0,0 +1,246 @@ +import type { SnapshotAriaNode } from "./client.js"; +import { + getRoleSnapshotStats, + type RoleRefMap, + type RoleSnapshotOptions, +} from "./pw-role-snapshot.js"; + +export type ChromeMcpSnapshotNode = { + id?: string; + role?: string; + name?: string; + value?: string | number | boolean; + description?: string; + children?: ChromeMcpSnapshotNode[]; +}; + +const INTERACTIVE_ROLES = new Set([ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); + +const CONTENT_ROLES = new Set([ + "article", + "cell", + "columnheader", + "gridcell", + "heading", + "listitem", + "main", + "navigation", + "region", + "rowheader", +]); + +const STRUCTURAL_ROLES = new Set([ + "application", + "directory", + "document", + "generic", + "group", + "ignored", + "list", + "menu", + "menubar", + "none", + "presentation", + "row", + "rowgroup", + "tablist", + "table", + "toolbar", + "tree", + "treegrid", +]); + +function normalizeRole(node: ChromeMcpSnapshotNode): string { + const role = typeof node.role === "string" ? node.role.trim().toLowerCase() : ""; + return role || "generic"; +} + +function normalizeString(value: unknown): string | undefined { + if (typeof value === "string") { + const trimmed = value.trim(); + return trimmed || undefined; + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return undefined; +} + +function escapeQuoted(value: string): string { + return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"'); +} + +function shouldIncludeNode(params: { + role: string; + name?: string; + options?: RoleSnapshotOptions; +}): boolean { + if (params.options?.interactive && !INTERACTIVE_ROLES.has(params.role)) { + return false; + } + if (params.options?.compact && STRUCTURAL_ROLES.has(params.role) && !params.name) { + return false; + } + return true; +} + +function shouldCreateRef(role: string, name?: string): boolean { + return INTERACTIVE_ROLES.has(role) || (CONTENT_ROLES.has(role) && Boolean(name)); +} + +type DuplicateTracker = { + counts: Map; + keysByRef: Map; + duplicates: Set; +}; + +function createDuplicateTracker(): DuplicateTracker { + return { + counts: new Map(), + keysByRef: new Map(), + duplicates: new Set(), + }; +} + +function registerRef( + tracker: DuplicateTracker, + ref: string, + role: string, + name?: string, +): number | undefined { + const key = `${role}:${name ?? ""}`; + const count = tracker.counts.get(key) ?? 0; + tracker.counts.set(key, count + 1); + tracker.keysByRef.set(ref, key); + if (count > 0) { + tracker.duplicates.add(key); + return count; + } + return undefined; +} + +export function flattenChromeMcpSnapshotToAriaNodes( + root: ChromeMcpSnapshotNode, + limit = 500, +): SnapshotAriaNode[] { + const boundedLimit = Math.max(1, Math.min(2000, Math.floor(limit))); + const out: SnapshotAriaNode[] = []; + + const visit = (node: ChromeMcpSnapshotNode, depth: number) => { + if (out.length >= boundedLimit) { + return; + } + const ref = normalizeString(node.id); + if (ref) { + out.push({ + ref, + role: normalizeRole(node), + name: normalizeString(node.name) ?? "", + value: normalizeString(node.value), + description: normalizeString(node.description), + depth, + }); + } + for (const child of node.children ?? []) { + visit(child, depth + 1); + if (out.length >= boundedLimit) { + return; + } + } + }; + + visit(root, 0); + return out; +} + +export function buildAiSnapshotFromChromeMcpSnapshot(params: { + root: ChromeMcpSnapshotNode; + options?: RoleSnapshotOptions; + maxChars?: number; +}): { + snapshot: string; + truncated?: boolean; + refs: RoleRefMap; + stats: { lines: number; chars: number; refs: number; interactive: number }; +} { + const refs: RoleRefMap = {}; + const tracker = createDuplicateTracker(); + const lines: string[] = []; + + const visit = (node: ChromeMcpSnapshotNode, depth: number) => { + const role = normalizeRole(node); + const name = normalizeString(node.name); + const value = normalizeString(node.value); + const description = normalizeString(node.description); + const maxDepth = params.options?.maxDepth; + if (maxDepth !== undefined && depth > maxDepth) { + return; + } + + const includeNode = shouldIncludeNode({ role, name, options: params.options }); + if (includeNode) { + let line = `${" ".repeat(depth)}- ${role}`; + if (name) { + line += ` "${escapeQuoted(name)}"`; + } + const ref = normalizeString(node.id); + if (ref && shouldCreateRef(role, name)) { + const nth = registerRef(tracker, ref, role, name); + refs[ref] = nth === undefined ? { role, name } : { role, name, nth }; + line += ` [ref=${ref}]`; + } + if (value) { + line += ` value="${escapeQuoted(value)}"`; + } + if (description) { + line += ` description="${escapeQuoted(description)}"`; + } + lines.push(line); + } + + for (const child of node.children ?? []) { + visit(child, depth + 1); + } + }; + + visit(params.root, 0); + + for (const [ref, data] of Object.entries(refs)) { + const key = tracker.keysByRef.get(ref); + if (key && !tracker.duplicates.has(key)) { + delete data.nth; + } + } + + let snapshot = lines.join("\n"); + let truncated = false; + const maxChars = + typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0 + ? Math.floor(params.maxChars) + : undefined; + if (maxChars && snapshot.length > maxChars) { + snapshot = `${snapshot.slice(0, maxChars)}\n\n[...TRUNCATED - page too large]`; + truncated = true; + } + + const stats = getRoleSnapshotStats(snapshot, refs); + return truncated ? { snapshot, truncated, refs, stats } : { snapshot, refs, stats }; +} diff --git a/src/browser/chrome-mcp.test.ts b/src/browser/chrome-mcp.test.ts new file mode 100644 index 00000000000..82149b67792 --- /dev/null +++ b/src/browser/chrome-mcp.test.ts @@ -0,0 +1,100 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + listChromeMcpTabs, + openChromeMcpTab, + resetChromeMcpSessionsForTest, + setChromeMcpSessionFactoryForTest, +} from "./chrome-mcp.js"; + +type ToolCall = { + name: string; + arguments?: Record; +}; + +function createFakeSession() { + const callTool = vi.fn(async ({ name }: ToolCall) => { + if (name === "list_pages") { + return { + content: [ + { + type: "text", + text: [ + "## Pages", + "1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session [selected]", + "2: https://github.com/openclaw/openclaw/pull/45318", + ].join("\n"), + }, + ], + }; + } + if (name === "new_page") { + return { + content: [ + { + type: "text", + text: [ + "## Pages", + "1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session", + "2: https://github.com/openclaw/openclaw/pull/45318", + "3: https://example.com/ [selected]", + ].join("\n"), + }, + ], + }; + } + throw new Error(`unexpected tool ${name}`); + }); + + return { + client: { + callTool, + listTools: vi.fn().mockResolvedValue({ tools: [{ name: "list_pages" }] }), + close: vi.fn().mockResolvedValue(undefined), + connect: vi.fn().mockResolvedValue(undefined), + }, + transport: { + pid: 123, + }, + ready: Promise.resolve(), + }; +} + +describe("chrome MCP page parsing", () => { + beforeEach(async () => { + await resetChromeMcpSessionsForTest(); + }); + + it("parses list_pages text responses when structuredContent is missing", async () => { + setChromeMcpSessionFactoryForTest(async () => createFakeSession()); + + const tabs = await listChromeMcpTabs("chrome-live"); + + expect(tabs).toEqual([ + { + targetId: "1", + title: "", + url: "https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session", + type: "page", + }, + { + targetId: "2", + title: "", + url: "https://github.com/openclaw/openclaw/pull/45318", + type: "page", + }, + ]); + }); + + it("parses new_page text responses and returns the created tab", async () => { + setChromeMcpSessionFactoryForTest(async () => createFakeSession()); + + const tab = await openChromeMcpTab("chrome-live", "https://example.com/"); + + expect(tab).toEqual({ + targetId: "3", + title: "", + url: "https://example.com/", + type: "page", + }); + }); +}); diff --git a/src/browser/chrome-mcp.ts b/src/browser/chrome-mcp.ts new file mode 100644 index 00000000000..7719a2338e3 --- /dev/null +++ b/src/browser/chrome-mcp.ts @@ -0,0 +1,488 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import type { ChromeMcpSnapshotNode } from "./chrome-mcp.snapshot.js"; +import type { BrowserTab } from "./client.js"; +import { BrowserProfileUnavailableError, BrowserTabNotFoundError } from "./errors.js"; + +type ChromeMcpStructuredPage = { + id: number; + url?: string; + selected?: boolean; +}; + +type ChromeMcpToolResult = { + structuredContent?: Record; + content?: Array>; + isError?: boolean; +}; + +type ChromeMcpSession = { + client: Client; + transport: StdioClientTransport; + ready: Promise; +}; + +type ChromeMcpSessionFactory = (profileName: string) => Promise; + +const DEFAULT_CHROME_MCP_COMMAND = "npx"; +const DEFAULT_CHROME_MCP_ARGS = [ + "-y", + "chrome-devtools-mcp@latest", + "--autoConnect", + "--experimental-page-id-routing", +]; + +const sessions = new Map(); +let sessionFactory: ChromeMcpSessionFactory | null = null; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function asPages(value: unknown): ChromeMcpStructuredPage[] { + if (!Array.isArray(value)) { + return []; + } + const out: ChromeMcpStructuredPage[] = []; + for (const entry of value) { + const record = asRecord(entry); + if (!record || typeof record.id !== "number") { + continue; + } + out.push({ + id: record.id, + url: typeof record.url === "string" ? record.url : undefined, + selected: record.selected === true, + }); + } + return out; +} + +function parsePageId(targetId: string): number { + const parsed = Number.parseInt(targetId.trim(), 10); + if (!Number.isFinite(parsed)) { + throw new BrowserTabNotFoundError(); + } + return parsed; +} + +function toBrowserTabs(pages: ChromeMcpStructuredPage[]): BrowserTab[] { + return pages.map((page) => ({ + targetId: String(page.id), + title: "", + url: page.url ?? "", + type: "page", + })); +} + +function extractStructuredContent(result: ChromeMcpToolResult): Record { + return asRecord(result.structuredContent) ?? {}; +} + +function extractTextContent(result: ChromeMcpToolResult): string[] { + const content = Array.isArray(result.content) ? result.content : []; + return content + .map((entry) => { + const record = asRecord(entry); + return record && typeof record.text === "string" ? record.text : ""; + }) + .filter(Boolean); +} + +function extractTextPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] { + const pages: ChromeMcpStructuredPage[] = []; + for (const block of extractTextContent(result)) { + for (const line of block.split(/\r?\n/)) { + const match = line.match(/^\s*(\d+):\s+(.+?)(?:\s+\[(selected)\])?\s*$/i); + if (!match) { + continue; + } + pages.push({ + id: Number.parseInt(match[1] ?? "", 10), + url: match[2]?.trim() || undefined, + selected: Boolean(match[3]), + }); + } + } + return pages; +} + +function extractStructuredPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] { + const structured = asPages(extractStructuredContent(result).pages); + return structured.length > 0 ? structured : extractTextPages(result); +} + +function extractSnapshot(result: ChromeMcpToolResult): ChromeMcpSnapshotNode { + const structured = extractStructuredContent(result); + const snapshot = asRecord(structured.snapshot); + if (!snapshot) { + throw new Error("Chrome MCP snapshot response was missing structured snapshot data."); + } + return snapshot as unknown as ChromeMcpSnapshotNode; +} + +function extractJsonBlock(text: string): unknown { + const match = text.match(/```json\s*([\s\S]*?)\s*```/i); + const raw = match?.[1]?.trim() || text.trim(); + return raw ? JSON.parse(raw) : null; +} + +async function createRealSession(profileName: string): Promise { + const transport = new StdioClientTransport({ + command: DEFAULT_CHROME_MCP_COMMAND, + args: DEFAULT_CHROME_MCP_ARGS, + stderr: "pipe", + }); + const client = new Client( + { + name: "openclaw-browser", + version: "0.0.0", + }, + {}, + ); + + const ready = (async () => { + try { + await client.connect(transport); + const tools = await client.listTools(); + if (!tools.tools.some((tool) => tool.name === "list_pages")) { + throw new Error("Chrome MCP server did not expose the expected navigation tools."); + } + } catch (err) { + await client.close().catch(() => {}); + throw new BrowserProfileUnavailableError( + `Chrome MCP existing-session attach failed for profile "${profileName}". ` + + `Make sure Chrome is running, enable chrome://inspect/#remote-debugging, and approve the connection. ` + + `Details: ${String(err)}`, + ); + } + })(); + + return { + client, + transport, + ready, + }; +} + +async function getSession(profileName: string): Promise { + let session = sessions.get(profileName); + if (session && session.transport.pid === null) { + sessions.delete(profileName); + session = undefined; + } + if (!session) { + session = await (sessionFactory ?? createRealSession)(profileName); + sessions.set(profileName, session); + } + try { + await session.ready; + return session; + } catch (err) { + const current = sessions.get(profileName); + if (current?.transport === session.transport) { + sessions.delete(profileName); + } + throw err; + } +} + +async function callTool( + profileName: string, + name: string, + args: Record = {}, +): Promise { + const session = await getSession(profileName); + try { + return (await session.client.callTool({ + name, + arguments: args, + })) as ChromeMcpToolResult; + } catch (err) { + sessions.delete(profileName); + await session.client.close().catch(() => {}); + throw err; + } +} + +async function withTempFile(fn: (filePath: string) => Promise): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-chrome-mcp-")); + const filePath = path.join(dir, randomUUID()); + try { + return await fn(filePath); + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); + } +} + +async function findPageById(profileName: string, pageId: number): Promise { + const pages = await listChromeMcpPages(profileName); + const page = pages.find((entry) => entry.id === pageId); + if (!page) { + throw new BrowserTabNotFoundError(); + } + return page; +} + +export async function ensureChromeMcpAvailable(profileName: string): Promise { + await getSession(profileName); +} + +export function getChromeMcpPid(profileName: string): number | null { + return sessions.get(profileName)?.transport.pid ?? null; +} + +export async function closeChromeMcpSession(profileName: string): Promise { + const session = sessions.get(profileName); + if (!session) { + return false; + } + sessions.delete(profileName); + await session.client.close().catch(() => {}); + return true; +} + +export async function stopAllChromeMcpSessions(): Promise { + const names = [...sessions.keys()]; + for (const name of names) { + await closeChromeMcpSession(name).catch(() => {}); + } +} + +export async function listChromeMcpPages(profileName: string): Promise { + const result = await callTool(profileName, "list_pages"); + return extractStructuredPages(result); +} + +export async function listChromeMcpTabs(profileName: string): Promise { + return toBrowserTabs(await listChromeMcpPages(profileName)); +} + +export async function openChromeMcpTab(profileName: string, url: string): Promise { + const result = await callTool(profileName, "new_page", { url }); + const pages = extractStructuredPages(result); + const chosen = pages.find((page) => page.selected) ?? pages.at(-1); + if (!chosen) { + throw new Error("Chrome MCP did not return the created page."); + } + return { + targetId: String(chosen.id), + title: "", + url: chosen.url ?? url, + type: "page", + }; +} + +export async function focusChromeMcpTab(profileName: string, targetId: string): Promise { + await callTool(profileName, "select_page", { + pageId: parsePageId(targetId), + bringToFront: true, + }); +} + +export async function closeChromeMcpTab(profileName: string, targetId: string): Promise { + await callTool(profileName, "close_page", { pageId: parsePageId(targetId) }); +} + +export async function navigateChromeMcpPage(params: { + profileName: string; + targetId: string; + url: string; + timeoutMs?: number; +}): Promise<{ url: string }> { + await callTool(params.profileName, "navigate_page", { + pageId: parsePageId(params.targetId), + type: "url", + url: params.url, + ...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}), + }); + const page = await findPageById(params.profileName, parsePageId(params.targetId)); + return { url: page.url ?? params.url }; +} + +export async function takeChromeMcpSnapshot(params: { + profileName: string; + targetId: string; +}): Promise { + const result = await callTool(params.profileName, "take_snapshot", { + pageId: parsePageId(params.targetId), + }); + return extractSnapshot(result); +} + +export async function takeChromeMcpScreenshot(params: { + profileName: string; + targetId: string; + uid?: string; + fullPage?: boolean; + format?: "png" | "jpeg"; +}): Promise { + return await withTempFile(async (filePath) => { + await callTool(params.profileName, "take_screenshot", { + pageId: parsePageId(params.targetId), + filePath, + format: params.format ?? "png", + ...(params.uid ? { uid: params.uid } : {}), + ...(params.fullPage ? { fullPage: true } : {}), + }); + return await fs.readFile(filePath); + }); +} + +export async function clickChromeMcpElement(params: { + profileName: string; + targetId: string; + uid: string; + doubleClick?: boolean; +}): Promise { + await callTool(params.profileName, "click", { + pageId: parsePageId(params.targetId), + uid: params.uid, + ...(params.doubleClick ? { dblClick: true } : {}), + }); +} + +export async function fillChromeMcpElement(params: { + profileName: string; + targetId: string; + uid: string; + value: string; +}): Promise { + await callTool(params.profileName, "fill", { + pageId: parsePageId(params.targetId), + uid: params.uid, + value: params.value, + }); +} + +export async function fillChromeMcpForm(params: { + profileName: string; + targetId: string; + elements: Array<{ uid: string; value: string }>; +}): Promise { + await callTool(params.profileName, "fill_form", { + pageId: parsePageId(params.targetId), + elements: params.elements, + }); +} + +export async function hoverChromeMcpElement(params: { + profileName: string; + targetId: string; + uid: string; +}): Promise { + await callTool(params.profileName, "hover", { + pageId: parsePageId(params.targetId), + uid: params.uid, + }); +} + +export async function dragChromeMcpElement(params: { + profileName: string; + targetId: string; + fromUid: string; + toUid: string; +}): Promise { + await callTool(params.profileName, "drag", { + pageId: parsePageId(params.targetId), + from_uid: params.fromUid, + to_uid: params.toUid, + }); +} + +export async function uploadChromeMcpFile(params: { + profileName: string; + targetId: string; + uid: string; + filePath: string; +}): Promise { + await callTool(params.profileName, "upload_file", { + pageId: parsePageId(params.targetId), + uid: params.uid, + filePath: params.filePath, + }); +} + +export async function pressChromeMcpKey(params: { + profileName: string; + targetId: string; + key: string; +}): Promise { + await callTool(params.profileName, "press_key", { + pageId: parsePageId(params.targetId), + key: params.key, + }); +} + +export async function resizeChromeMcpPage(params: { + profileName: string; + targetId: string; + width: number; + height: number; +}): Promise { + await callTool(params.profileName, "resize_page", { + pageId: parsePageId(params.targetId), + width: params.width, + height: params.height, + }); +} + +export async function handleChromeMcpDialog(params: { + profileName: string; + targetId: string; + action: "accept" | "dismiss"; + promptText?: string; +}): Promise { + await callTool(params.profileName, "handle_dialog", { + pageId: parsePageId(params.targetId), + action: params.action, + ...(params.promptText ? { promptText: params.promptText } : {}), + }); +} + +export async function evaluateChromeMcpScript(params: { + profileName: string; + targetId: string; + fn: string; + args?: string[]; +}): Promise { + const result = await callTool(params.profileName, "evaluate_script", { + pageId: parsePageId(params.targetId), + function: params.fn, + ...(params.args?.length ? { args: params.args } : {}), + }); + const message = extractStructuredContent(result).message; + const text = typeof message === "string" ? message : ""; + if (!text.trim()) { + return null; + } + return extractJsonBlock(text); +} + +export async function waitForChromeMcpText(params: { + profileName: string; + targetId: string; + text: string[]; + timeoutMs?: number; +}): Promise { + await callTool(params.profileName, "wait_for", { + pageId: parsePageId(params.targetId), + text: params.text, + ...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}), + }); +} + +export function setChromeMcpSessionFactoryForTest(factory: ChromeMcpSessionFactory | null): void { + sessionFactory = factory; +} + +export async function resetChromeMcpSessionsForTest(): Promise { + sessionFactory = null; + await stopAllChromeMcpSessions(); +} diff --git a/src/browser/client.ts b/src/browser/client.ts index 953c9efcd11..dc418cf3b4a 100644 --- a/src/browser/client.ts +++ b/src/browser/client.ts @@ -3,6 +3,7 @@ import { fetchBrowserJson } from "./client-fetch.js"; export type BrowserStatus = { enabled: boolean; profile?: string; + driver?: "openclaw" | "extension" | "existing-session"; running: boolean; cdpReady?: boolean; cdpHttp?: boolean; @@ -26,6 +27,7 @@ export type ProfileStatus = { cdpPort: number; cdpUrl: string; color: string; + driver: "openclaw" | "extension" | "existing-session"; running: boolean; tabCount: number; isDefault: boolean; @@ -165,7 +167,7 @@ export async function browserCreateProfile( name: string; color?: string; cdpUrl?: string; - driver?: "openclaw" | "extension"; + driver?: "openclaw" | "extension" | "existing-session"; }, ): Promise { return await fetchBrowserJson( diff --git a/src/browser/config.ts b/src/browser/config.ts index 6d24a07a287..529ee791c40 100644 --- a/src/browser/config.ts +++ b/src/browser/config.ts @@ -46,7 +46,7 @@ export type ResolvedBrowserProfile = { cdpHost: string; cdpIsLoopback: boolean; color: string; - driver: "openclaw" | "extension"; + driver: "openclaw" | "extension" | "existing-session"; attachOnly: boolean; }; @@ -335,7 +335,12 @@ export function resolveProfile( let cdpHost = resolved.cdpHost; let cdpPort = profile.cdpPort ?? 0; let cdpUrl = ""; - const driver = profile.driver === "extension" ? "extension" : "openclaw"; + const driver = + profile.driver === "extension" + ? "extension" + : profile.driver === "existing-session" + ? "existing-session" + : "openclaw"; if (rawProfileUrl) { const parsed = parseHttpUrl(rawProfileUrl, `browser.profiles.${profileName}.cdpUrl`); @@ -356,7 +361,7 @@ export function resolveProfile( cdpIsLoopback: isLoopbackHost(cdpHost), color: profile.color, driver, - attachOnly: profile.attachOnly ?? resolved.attachOnly, + attachOnly: driver === "existing-session" ? true : (profile.attachOnly ?? resolved.attachOnly), }; } diff --git a/src/browser/profile-capabilities.ts b/src/browser/profile-capabilities.ts index 07a70ba00c4..2bcf4f8fe9e 100644 --- a/src/browser/profile-capabilities.ts +++ b/src/browser/profile-capabilities.ts @@ -1,6 +1,10 @@ import type { ResolvedBrowserProfile } from "./config.js"; -export type BrowserProfileMode = "local-managed" | "local-extension-relay" | "remote-cdp"; +export type BrowserProfileMode = + | "local-managed" + | "local-extension-relay" + | "local-existing-session" + | "remote-cdp"; export type BrowserProfileCapabilities = { mode: BrowserProfileMode; @@ -31,6 +35,20 @@ export function getBrowserProfileCapabilities( }; } + if (profile.driver === "existing-session") { + return { + mode: "local-existing-session", + isRemote: false, + requiresRelay: false, + requiresAttachedTab: false, + usesPersistentPlaywright: false, + supportsPerTabWs: false, + supportsJsonTabEndpoints: false, + supportsReset: false, + supportsManagedTabLimit: false, + }; + } + if (!profile.cdpIsLoopback) { return { mode: "remote-cdp", @@ -75,6 +93,9 @@ export function resolveDefaultSnapshotFormat(params: { if (capabilities.mode === "local-extension-relay") { return "aria"; } + if (capabilities.mode === "local-existing-session") { + return "ai"; + } return params.hasPlaywright ? "ai" : "aria"; } diff --git a/src/browser/profiles-service.test.ts b/src/browser/profiles-service.test.ts index 3dc714d33f3..f70e23ddb67 100644 --- a/src/browser/profiles-service.test.ts +++ b/src/browser/profiles-service.test.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { resolveBrowserConfig } from "./config.js"; import { createBrowserProfilesService } from "./profiles-service.js"; import type { BrowserRouteContext, BrowserServerState } from "./server-context.js"; @@ -57,6 +57,10 @@ async function createWorkProfileWithConfig(params: { } describe("BrowserProfilesService", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + it("allocates next local port for new profiles", async () => { const { result, state } = await createWorkProfileWithConfig({ resolved: resolveBrowserConfig({}), @@ -163,6 +167,56 @@ describe("BrowserProfilesService", () => { ).rejects.toThrow(/requires an explicit loopback cdpUrl/i); }); + it("creates existing-session profiles as attach-only local entries", async () => { + const resolved = resolveBrowserConfig({}); + const { ctx, state } = createCtx(resolved); + vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } }); + + const service = createBrowserProfilesService(ctx); + const result = await service.createProfile({ + name: "chrome-live", + driver: "existing-session", + }); + + expect(result.cdpPort).toBe(18801); + expect(result.isRemote).toBe(false); + expect(state.resolved.profiles["chrome-live"]).toEqual({ + cdpPort: 18801, + driver: "existing-session", + attachOnly: true, + color: expect.any(String), + }); + expect(writeConfigFile).toHaveBeenCalledWith( + expect.objectContaining({ + browser: expect.objectContaining({ + profiles: expect.objectContaining({ + "chrome-live": expect.objectContaining({ + cdpPort: 18801, + driver: "existing-session", + attachOnly: true, + }), + }), + }), + }), + ); + }); + + it("rejects driver=existing-session when cdpUrl is provided", async () => { + const resolved = resolveBrowserConfig({}); + const { ctx } = createCtx(resolved); + vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } }); + + const service = createBrowserProfilesService(ctx); + + await expect( + service.createProfile({ + name: "chrome-live", + driver: "existing-session", + cdpUrl: "http://127.0.0.1:9222", + }), + ).rejects.toThrow(/does not accept cdpUrl/i); + }); + it("deletes remote profiles without stopping or removing local data", async () => { const resolved = resolveBrowserConfig({ profiles: { @@ -218,4 +272,40 @@ describe("BrowserProfilesService", () => { expect(result.deleted).toBe(true); expect(movePathToTrash).toHaveBeenCalledWith(path.dirname(userDataDir)); }); + + it("deletes existing-session profiles without touching local browser data", async () => { + const resolved = resolveBrowserConfig({ + profiles: { + "chrome-live": { + cdpPort: 18801, + color: "#0066CC", + driver: "existing-session", + attachOnly: true, + }, + }, + }); + const { ctx } = createCtx(resolved); + + vi.mocked(loadConfig).mockReturnValue({ + browser: { + defaultProfile: "openclaw", + profiles: { + openclaw: { cdpPort: 18800, color: "#FF4500" }, + "chrome-live": { + cdpPort: 18801, + color: "#0066CC", + driver: "existing-session", + attachOnly: true, + }, + }, + }, + }); + + const service = createBrowserProfilesService(ctx); + const result = await service.deleteProfile("chrome-live"); + + expect(result.deleted).toBe(false); + expect(ctx.forProfile).not.toHaveBeenCalled(); + expect(movePathToTrash).not.toHaveBeenCalled(); + }); }); diff --git a/src/browser/profiles-service.ts b/src/browser/profiles-service.ts index 962c6408522..936a55c1ffa 100644 --- a/src/browser/profiles-service.ts +++ b/src/browser/profiles-service.ts @@ -27,7 +27,7 @@ export type CreateProfileParams = { name: string; color?: string; cdpUrl?: string; - driver?: "openclaw" | "extension"; + driver?: "openclaw" | "extension" | "existing-session"; }; export type CreateProfileResult = { @@ -79,7 +79,12 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) { const createProfile = async (params: CreateProfileParams): Promise => { const name = params.name.trim(); const rawCdpUrl = params.cdpUrl?.trim() || undefined; - const driver = params.driver === "extension" ? "extension" : undefined; + const driver = + params.driver === "extension" + ? "extension" + : params.driver === "existing-session" + ? "existing-session" + : undefined; if (!isValidProfileName(name)) { throw new BrowserValidationError( @@ -118,6 +123,11 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) { ); } } + if (driver === "existing-session") { + throw new BrowserValidationError( + "driver=existing-session does not accept cdpUrl; it attaches via the Chrome MCP auto-connect flow", + ); + } profileConfig = { cdpUrl: parsed.normalized, ...(driver ? { driver } : {}), @@ -136,6 +146,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) { profileConfig = { cdpPort, ...(driver ? { driver } : {}), + ...(driver === "existing-session" ? { attachOnly: true } : {}), color: profileColor, }; } @@ -195,7 +206,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) { const state = ctx.state(); const resolved = resolveProfile(state.resolved, name); - if (resolved?.cdpIsLoopback) { + if (resolved?.cdpIsLoopback && resolved.driver === "openclaw") { try { await ctx.forProfile(name).stopRunningBrowser(); } catch { diff --git a/src/browser/routes/agent.act.download.ts b/src/browser/routes/agent.act.download.ts index d08287fea59..9ed04469c26 100644 --- a/src/browser/routes/agent.act.download.ts +++ b/src/browser/routes/agent.act.download.ts @@ -1,5 +1,10 @@ import type { BrowserRouteContext } from "../server-context.js"; -import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js"; +import { + readBody, + requirePwAi, + resolveTargetIdFromBody, + withRouteTabContext, +} from "./agent.shared.js"; import { ensureOutputRootDir, resolveWritableOutputPathOrRespond } from "./output-paths.js"; import { DEFAULT_DOWNLOAD_DIR } from "./path-output.js"; import type { BrowserRouteRegistrar } from "./types.js"; @@ -23,13 +28,23 @@ export function registerBrowserAgentActDownloadRoutes( const out = toStringOrEmpty(body.path) || ""; const timeoutMs = toNumber(body.timeoutMs); - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "wait for download", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { + if (profileCtx.profile.driver === "existing-session") { + return jsonError( + res, + 501, + "download waiting is not supported for existing-session profiles yet.", + ); + } + const pw = await requirePwAi(res, "wait for download"); + if (!pw) { + return; + } await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR); let downloadPath: string | undefined; if (out.trim()) { @@ -67,13 +82,23 @@ export function registerBrowserAgentActDownloadRoutes( return jsonError(res, 400, "path is required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "download", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { + if (profileCtx.profile.driver === "existing-session") { + return jsonError( + res, + 501, + "downloads are not supported for existing-session profiles yet.", + ); + } + const pw = await requirePwAi(res, "download"); + if (!pw) { + return; + } await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR); const downloadPath = await resolveWritableOutputPathOrRespond({ res, diff --git a/src/browser/routes/agent.act.hooks.ts b/src/browser/routes/agent.act.hooks.ts index 56d97bb03d3..bb1f03b7a7c 100644 --- a/src/browser/routes/agent.act.hooks.ts +++ b/src/browser/routes/agent.act.hooks.ts @@ -1,5 +1,11 @@ +import { evaluateChromeMcpScript, uploadChromeMcpFile } from "../chrome-mcp.js"; import type { BrowserRouteContext } from "../server-context.js"; -import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js"; +import { + readBody, + requirePwAi, + resolveTargetIdFromBody, + withRouteTabContext, +} from "./agent.shared.js"; import { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./path-output.js"; import type { BrowserRouteRegistrar } from "./types.js"; import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js"; @@ -20,13 +26,12 @@ export function registerBrowserAgentActHookRoutes( return jsonError(res, 400, "paths are required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "file chooser hook", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { const uploadPathsResult = await resolveExistingPathsWithinRoot({ rootDir: DEFAULT_UPLOAD_DIR, requestedPaths: paths, @@ -38,6 +43,39 @@ export function registerBrowserAgentActHookRoutes( } const resolvedPaths = uploadPathsResult.paths; + if (profileCtx.profile.driver === "existing-session") { + if (element) { + return jsonError( + res, + 501, + "existing-session file uploads do not support element selectors; use ref/inputRef.", + ); + } + if (resolvedPaths.length !== 1) { + return jsonError( + res, + 501, + "existing-session file uploads currently support one file at a time.", + ); + } + const uid = inputRef || ref; + if (!uid) { + return jsonError(res, 501, "existing-session file uploads require ref or inputRef."); + } + await uploadChromeMcpFile({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + uid, + filePath: resolvedPaths[0] ?? "", + }); + return res.json({ ok: true }); + } + + const pw = await requirePwAi(res, "file chooser hook"); + if (!pw) { + return; + } + if (inputRef || element) { if (ref) { return jsonError(res, 400, "ref cannot be combined with inputRef/element"); @@ -79,13 +117,69 @@ export function registerBrowserAgentActHookRoutes( return jsonError(res, 400, "accept is required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "dialog hook", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { + if (profileCtx.profile.driver === "existing-session") { + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session dialog handling does not support timeoutMs.", + ); + } + await evaluateChromeMcpScript({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + fn: `() => { + const state = (window.__openclawDialogHook ??= {}); + if (!state.originals) { + state.originals = { + alert: window.alert.bind(window), + confirm: window.confirm.bind(window), + prompt: window.prompt.bind(window), + }; + } + const originals = state.originals; + const restore = () => { + window.alert = originals.alert; + window.confirm = originals.confirm; + window.prompt = originals.prompt; + delete window.__openclawDialogHook; + }; + window.alert = (...args) => { + try { + return undefined; + } finally { + restore(); + } + }; + window.confirm = (...args) => { + try { + return ${accept ? "true" : "false"}; + } finally { + restore(); + } + }; + window.prompt = (...args) => { + try { + return ${accept ? JSON.stringify(promptText ?? "") : "null"}; + } finally { + restore(); + } + }; + return true; + }`, + }); + return res.json({ ok: true }); + } + const pw = await requirePwAi(res, "dialog hook"); + if (!pw) { + return; + } await pw.armDialogViaPlaywright({ cdpUrl, targetId: tab.targetId, diff --git a/src/browser/routes/agent.act.ts b/src/browser/routes/agent.act.ts index 2ae6073c7cf..8928a8a7d06 100644 --- a/src/browser/routes/agent.act.ts +++ b/src/browser/routes/agent.act.ts @@ -1,3 +1,14 @@ +import { + clickChromeMcpElement, + closeChromeMcpTab, + dragChromeMcpElement, + evaluateChromeMcpScript, + fillChromeMcpElement, + fillChromeMcpForm, + hoverChromeMcpElement, + pressChromeMcpKey, + resizeChromeMcpPage, +} from "../chrome-mcp.js"; import type { BrowserFormField } from "../client-actions-core.js"; import { normalizeBrowserFormField } from "../form-fields.js"; import type { BrowserRouteContext } from "../server-context.js"; @@ -11,13 +22,88 @@ import { } from "./agent.act.shared.js"; import { readBody, + requirePwAi, resolveTargetIdFromBody, - withPlaywrightRouteContext, + withRouteTabContext, SELECTOR_UNSUPPORTED_MESSAGE, } from "./agent.shared.js"; import type { BrowserRouteRegistrar } from "./types.js"; import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js"; +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function buildExistingSessionWaitPredicate(params: { + text?: string; + textGone?: string; + selector?: string; + url?: string; + loadState?: "load" | "domcontentloaded" | "networkidle"; + fn?: string; +}): string | null { + const checks: string[] = []; + if (params.text) { + checks.push(`Boolean(document.body?.innerText?.includes(${JSON.stringify(params.text)}))`); + } + if (params.textGone) { + checks.push(`!document.body?.innerText?.includes(${JSON.stringify(params.textGone)})`); + } + if (params.selector) { + checks.push(`Boolean(document.querySelector(${JSON.stringify(params.selector)}))`); + } + if (params.url) { + checks.push(`window.location.href === ${JSON.stringify(params.url)}`); + } + if (params.loadState === "domcontentloaded") { + checks.push(`document.readyState === "interactive" || document.readyState === "complete"`); + } else if (params.loadState === "load" || params.loadState === "networkidle") { + checks.push(`document.readyState === "complete"`); + } + if (params.fn) { + checks.push(`Boolean(await (${params.fn})())`); + } + if (checks.length === 0) { + return null; + } + return checks.length === 1 ? checks[0] : checks.map((check) => `(${check})`).join(" && "); +} + +async function waitForExistingSessionCondition(params: { + profileName: string; + targetId: string; + timeMs?: number; + text?: string; + textGone?: string; + selector?: string; + url?: string; + loadState?: "load" | "domcontentloaded" | "networkidle"; + fn?: string; + timeoutMs?: number; +}): Promise { + if (params.timeMs && params.timeMs > 0) { + await sleep(params.timeMs); + } + const predicate = buildExistingSessionWaitPredicate(params); + if (!predicate) { + return; + } + const timeoutMs = Math.max(250, params.timeoutMs ?? 10_000); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const ready = await evaluateChromeMcpScript({ + profileName: params.profileName, + targetId: params.targetId, + fn: `async () => ${predicate}`, + }); + if (ready) { + return; + } + await sleep(250); + } + throw new Error("Timed out waiting for condition"); +} + export function registerBrowserAgentActRoutes( app: BrowserRouteRegistrar, ctx: BrowserRouteContext, @@ -34,14 +120,15 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: `act:${kind}`, - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { const evaluateEnabled = ctx.state().resolved.evaluateEnabled; + const isExistingSession = profileCtx.profile.driver === "existing-session"; + const profileName = profileCtx.profile.name; switch (kind) { case "click": { @@ -63,6 +150,26 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, parsedModifiers.error); } const modifiers = parsedModifiers.modifiers; + if (isExistingSession) { + if ((button && button !== "left") || (modifiers && modifiers.length > 0)) { + return jsonError( + res, + 501, + "existing-session click currently supports left-click only (no button overrides/modifiers).", + ); + } + await clickChromeMcpElement({ + profileName, + targetId: tab.targetId, + uid: ref, + doubleClick, + }); + return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } const clickRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, @@ -93,6 +200,33 @@ export function registerBrowserAgentActRoutes( const submit = toBoolean(body.submit) ?? false; const slowly = toBoolean(body.slowly) ?? false; const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (slowly) { + return jsonError( + res, + 501, + "existing-session type does not support slowly=true; use fill/press instead.", + ); + } + await fillChromeMcpElement({ + profileName, + targetId: tab.targetId, + uid: ref, + value: text, + }); + if (submit) { + await pressChromeMcpKey({ + profileName, + targetId: tab.targetId, + key: "Enter", + }); + } + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } const typeRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, @@ -113,6 +247,17 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "key is required"); } const delayMs = toNumber(body.delayMs); + if (isExistingSession) { + if (delayMs) { + return jsonError(res, 501, "existing-session press does not support delayMs."); + } + await pressChromeMcpKey({ profileName, targetId: tab.targetId, key }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.pressKeyViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -127,6 +272,21 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "ref is required"); } const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session hover does not support timeoutMs overrides.", + ); + } + await hoverChromeMcpElement({ profileName, targetId: tab.targetId, uid: ref }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.hoverViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -141,6 +301,26 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "ref is required"); } const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session scrollIntoView does not support timeoutMs overrides.", + ); + } + await evaluateChromeMcpScript({ + profileName, + targetId: tab.targetId, + fn: `(el) => { el.scrollIntoView({ block: "center", inline: "center" }); return true; }`, + args: [ref], + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } const scrollRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, @@ -159,6 +339,26 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "startRef and endRef are required"); } const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session drag does not support timeoutMs overrides.", + ); + } + await dragChromeMcpElement({ + profileName, + targetId: tab.targetId, + fromUid: startRef, + toUid: endRef, + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.dragViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -175,6 +375,33 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "ref and values are required"); } const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (values.length !== 1) { + return jsonError( + res, + 501, + "existing-session select currently supports a single value only.", + ); + } + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session select does not support timeoutMs overrides.", + ); + } + await fillChromeMcpElement({ + profileName, + targetId: tab.targetId, + uid: ref, + value: values[0] ?? "", + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.selectOptionViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -198,6 +425,28 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "fields are required"); } const timeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (timeoutMs) { + return jsonError( + res, + 501, + "existing-session fill does not support timeoutMs overrides.", + ); + } + await fillChromeMcpForm({ + profileName, + targetId: tab.targetId, + elements: fields.map((field) => ({ + uid: field.ref, + value: String(field.value ?? ""), + })), + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.fillFormViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -212,6 +461,19 @@ export function registerBrowserAgentActRoutes( if (!width || !height) { return jsonError(res, 400, "width and height are required"); } + if (isExistingSession) { + await resizeChromeMcpPage({ + profileName, + targetId: tab.targetId, + width, + height, + }); + return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.resizeViewportViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -260,6 +522,25 @@ export function registerBrowserAgentActRoutes( "wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn", ); } + if (isExistingSession) { + await waitForExistingSessionCondition({ + profileName, + targetId: tab.targetId, + timeMs, + text, + textGone, + selector, + url, + loadState, + fn, + timeoutMs, + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.waitForViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -291,6 +572,31 @@ export function registerBrowserAgentActRoutes( } const ref = toStringOrEmpty(body.ref) || undefined; const evalTimeoutMs = toNumber(body.timeoutMs); + if (isExistingSession) { + if (evalTimeoutMs !== undefined) { + return jsonError( + res, + 501, + "existing-session evaluate does not support timeoutMs overrides.", + ); + } + const result = await evaluateChromeMcpScript({ + profileName, + targetId: tab.targetId, + fn, + args: ref ? [ref] : undefined, + }); + return res.json({ + ok: true, + targetId: tab.targetId, + url: tab.url, + result, + }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } const evalRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, @@ -310,6 +616,14 @@ export function registerBrowserAgentActRoutes( }); } case "close": { + if (isExistingSession) { + await closeChromeMcpTab(profileName, tab.targetId); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -334,13 +648,23 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "url is required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "response body", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { + if (profileCtx.profile.driver === "existing-session") { + return jsonError( + res, + 501, + "response body is not supported for existing-session profiles yet.", + ); + } + const pw = await requirePwAi(res, "response body"); + if (!pw) { + return; + } const result = await pw.responseBodyViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -361,13 +685,39 @@ export function registerBrowserAgentActRoutes( return jsonError(res, 400, "ref is required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "highlight", - run: async ({ cdpUrl, tab, pw }) => { + run: async ({ profileCtx, cdpUrl, tab }) => { + if (profileCtx.profile.driver === "existing-session") { + await evaluateChromeMcpScript({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + args: [ref], + fn: `(el) => { + if (!(el instanceof Element)) { + return false; + } + el.scrollIntoView({ block: "center", inline: "center" }); + const previousOutline = el.style.outline; + const previousOffset = el.style.outlineOffset; + el.style.outline = "3px solid #FF4500"; + el.style.outlineOffset = "2px"; + setTimeout(() => { + el.style.outline = previousOutline; + el.style.outlineOffset = previousOffset; + }, 2000); + return true; + }`, + }); + return res.json({ ok: true, targetId: tab.targetId }); + } + const pw = await requirePwAi(res, "highlight"); + if (!pw) { + return; + } await pw.highlightViaPlaywright({ cdpUrl, targetId: tab.targetId, diff --git a/src/browser/routes/agent.snapshot.ts b/src/browser/routes/agent.snapshot.ts index c750cafe723..1b8626141b5 100644 --- a/src/browser/routes/agent.snapshot.ts +++ b/src/browser/routes/agent.snapshot.ts @@ -1,6 +1,20 @@ import path from "node:path"; import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js"; import { captureScreenshot, snapshotAria } from "../cdp.js"; +import { + evaluateChromeMcpScript, + navigateChromeMcpPage, + takeChromeMcpScreenshot, + takeChromeMcpSnapshot, +} from "../chrome-mcp.js"; +import { + buildAiSnapshotFromChromeMcpSnapshot, + flattenChromeMcpSnapshotToAriaNodes, +} from "../chrome-mcp.snapshot.js"; +import { + assertBrowserNavigationAllowed, + assertBrowserNavigationResultAllowed, +} from "../navigation-guard.js"; import { withBrowserNavigationPolicy } from "../navigation-guard.js"; import { DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, @@ -25,6 +39,89 @@ import { import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js"; import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js"; +const CHROME_MCP_OVERLAY_ATTR = "data-openclaw-mcp-overlay"; + +async function clearChromeMcpOverlay(params: { + profileName: string; + targetId: string; +}): Promise { + await evaluateChromeMcpScript({ + profileName: params.profileName, + targetId: params.targetId, + fn: `() => { + document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove()); + return true; + }`, + }).catch(() => {}); +} + +async function renderChromeMcpLabels(params: { + profileName: string; + targetId: string; + refs: string[]; +}): Promise<{ labels: number; skipped: number }> { + const refList = JSON.stringify(params.refs); + const result = await evaluateChromeMcpScript({ + profileName: params.profileName, + targetId: params.targetId, + args: params.refs, + fn: `(...elements) => { + const refs = ${refList}; + document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove()); + const root = document.createElement("div"); + root.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "labels"); + root.style.position = "fixed"; + root.style.inset = "0"; + root.style.pointerEvents = "none"; + root.style.zIndex = "2147483647"; + let labels = 0; + let skipped = 0; + elements.forEach((el, index) => { + if (!(el instanceof Element)) { + skipped += 1; + return; + } + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 && rect.height <= 0) { + skipped += 1; + return; + } + labels += 1; + const badge = document.createElement("div"); + badge.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "label"); + badge.textContent = refs[index] || String(labels); + badge.style.position = "fixed"; + badge.style.left = \`\${Math.max(0, rect.left)}px\`; + badge.style.top = \`\${Math.max(0, rect.top)}px\`; + badge.style.transform = "translateY(-100%)"; + badge.style.padding = "2px 6px"; + badge.style.borderRadius = "999px"; + badge.style.background = "#FF4500"; + badge.style.color = "#fff"; + badge.style.font = "600 12px ui-monospace, SFMono-Regular, Menlo, monospace"; + badge.style.boxShadow = "0 2px 6px rgba(0,0,0,0.35)"; + badge.style.whiteSpace = "nowrap"; + root.appendChild(badge); + }); + document.documentElement.appendChild(root); + return { labels, skipped }; + }`, + }); + const labels = + result && + typeof result === "object" && + typeof (result as { labels?: unknown }).labels === "number" + ? (result as { labels: number }).labels + : 0; + const skipped = + result && + typeof result === "object" && + typeof (result as { skipped?: unknown }).skipped === "number" + ? (result as { skipped: number }).skipped + : 0; + return { labels, skipped }; +} + async function saveBrowserMediaResponse(params: { res: BrowserResponse; buffer: Buffer; @@ -96,13 +193,27 @@ export function registerBrowserAgentSnapshotRoutes( if (!url) { return jsonError(res, 400, "url is required"); } - await withPlaywrightRouteContext({ + await withRouteTabContext({ req, res, ctx, targetId, - feature: "navigate", - run: async ({ cdpUrl, tab, pw, profileCtx }) => { + run: async ({ profileCtx, tab, cdpUrl }) => { + if (profileCtx.profile.driver === "existing-session") { + const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy); + await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts }); + const result = await navigateChromeMcpPage({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + url, + }); + await assertBrowserNavigationResultAllowed({ url: result.url, ...ssrfPolicyOpts }); + return res.json({ ok: true, targetId: tab.targetId, ...result }); + } + const pw = await requirePwAi(res, "navigate"); + if (!pw) { + return; + } const result = await pw.navigateViaPlaywright({ cdpUrl, targetId: tab.targetId, @@ -122,6 +233,17 @@ export function registerBrowserAgentSnapshotRoutes( app.post("/pdf", async (req, res) => { const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) { + return; + } + if (profileCtx.profile.driver === "existing-session") { + return jsonError( + res, + 501, + "pdf is not supported for existing-session profiles yet; use screenshot/snapshot instead.", + ); + } await withPlaywrightRouteContext({ req, res, @@ -163,6 +285,36 @@ export function registerBrowserAgentSnapshotRoutes( ctx, targetId, run: async ({ profileCtx, tab, cdpUrl }) => { + if (profileCtx.profile.driver === "existing-session") { + if (element) { + return jsonError( + res, + 400, + "element screenshots are not supported for existing-session profiles; use ref from snapshot.", + ); + } + const buffer = await takeChromeMcpScreenshot({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + uid: ref, + fullPage, + format: type, + }); + const normalized = await normalizeBrowserScreenshot(buffer, { + maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, + maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, + }); + await saveBrowserMediaResponse({ + res, + buffer: normalized.buffer, + contentType: normalized.contentType ?? `image/${type}`, + maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, + targetId: tab.targetId, + url: tab.url, + }); + return; + } + let buffer: Buffer; const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({ profile: profileCtx.profile, @@ -227,6 +379,90 @@ export function registerBrowserAgentSnapshotRoutes( if ((plan.labels || plan.mode === "efficient") && plan.format === "aria") { return jsonError(res, 400, "labels/mode=efficient require format=ai"); } + if (profileCtx.profile.driver === "existing-session") { + if (plan.labels) { + return jsonError(res, 501, "labels are not supported for existing-session profiles yet."); + } + if (plan.selectorValue || plan.frameSelectorValue) { + return jsonError( + res, + 400, + "selector/frame snapshots are not supported for existing-session profiles; snapshot the whole page and use refs.", + ); + } + const snapshot = await takeChromeMcpSnapshot({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + }); + if (plan.format === "aria") { + return res.json({ + ok: true, + format: "aria", + targetId: tab.targetId, + url: tab.url, + nodes: flattenChromeMcpSnapshotToAriaNodes(snapshot, plan.limit), + }); + } + const built = buildAiSnapshotFromChromeMcpSnapshot({ + root: snapshot, + options: { + interactive: plan.interactive ?? undefined, + compact: plan.compact ?? undefined, + maxDepth: plan.depth ?? undefined, + }, + maxChars: plan.resolvedMaxChars, + }); + if (plan.labels) { + const refs = Object.keys(built.refs); + const labelResult = await renderChromeMcpLabels({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + refs, + }); + try { + const labeled = await takeChromeMcpScreenshot({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + format: "png", + }); + const normalized = await normalizeBrowserScreenshot(labeled, { + maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, + maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, + }); + await ensureMediaDir(); + const saved = await saveMediaBuffer( + normalized.buffer, + normalized.contentType ?? "image/png", + "browser", + DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, + ); + return res.json({ + ok: true, + format: "ai", + targetId: tab.targetId, + url: tab.url, + labels: true, + labelsCount: labelResult.labels, + labelsSkipped: labelResult.skipped, + imagePath: path.resolve(saved.path), + imageType: normalized.contentType?.includes("jpeg") ? "jpeg" : "png", + ...built, + }); + } finally { + await clearChromeMcpOverlay({ + profileName: profileCtx.profile.name, + targetId: tab.targetId, + }); + } + } + return res.json({ + ok: true, + format: "ai", + targetId: tab.targetId, + url: tab.url, + ...built, + }); + } if (plan.format === "ai") { const pw = await requirePwAi(res, "ai snapshot"); if (!pw) { diff --git a/src/browser/routes/basic.ts b/src/browser/routes/basic.ts index 5f32c86729b..9991744107d 100644 --- a/src/browser/routes/basic.ts +++ b/src/browser/routes/basic.ts @@ -1,3 +1,4 @@ +import { getChromeMcpPid } from "../chrome-mcp.js"; import { resolveBrowserExecutableForPlatform } from "../chrome.executables.js"; import { toBrowserErrorResponse } from "../errors.js"; import { createBrowserProfilesService } from "../profiles-service.js"; @@ -76,10 +77,14 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow res.json({ enabled: current.resolved.enabled, profile: profileCtx.profile.name, + driver: profileCtx.profile.driver, running: cdpReady, cdpReady, cdpHttp, - pid: profileState?.running?.pid ?? null, + pid: + profileCtx.profile.driver === "existing-session" + ? getChromeMcpPid(profileCtx.profile.name) + : (profileState?.running?.pid ?? null), cdpPort: profileCtx.profile.cdpPort, cdpUrl: profileCtx.profile.cdpUrl, chosenBrowser: profileState?.running?.exe.kind ?? null, @@ -146,6 +151,7 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow const driver = toStringOrEmpty((req.body as { driver?: unknown })?.driver) as | "openclaw" | "extension" + | "existing-session" | ""; if (!name) { @@ -158,7 +164,12 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow name, color: color || undefined, cdpUrl: cdpUrl || undefined, - driver: driver === "extension" ? "extension" : undefined, + driver: + driver === "extension" + ? "extension" + : driver === "existing-session" + ? "existing-session" + : undefined, }); res.json(result); } catch (err) { diff --git a/src/browser/server-context.availability.ts b/src/browser/server-context.availability.ts index 3b00ff99dff..d2d9944d964 100644 --- a/src/browser/server-context.availability.ts +++ b/src/browser/server-context.availability.ts @@ -3,6 +3,11 @@ import { PROFILE_POST_RESTART_WS_TIMEOUT_MS, resolveCdpReachabilityTimeouts, } from "./cdp-timeouts.js"; +import { + closeChromeMcpSession, + ensureChromeMcpAvailable, + listChromeMcpTabs, +} from "./chrome-mcp.js"; import { isChromeCdpReady, isChromeReachable, @@ -60,11 +65,19 @@ export function createProfileAvailability({ }); const isReachable = async (timeoutMs?: number) => { + if (profile.driver === "existing-session") { + await ensureChromeMcpAvailable(profile.name); + await listChromeMcpTabs(profile.name); + return true; + } const { httpTimeoutMs, wsTimeoutMs } = resolveTimeouts(timeoutMs); return await isChromeCdpReady(profile.cdpUrl, httpTimeoutMs, wsTimeoutMs); }; const isHttpReachable = async (timeoutMs?: number) => { + if (profile.driver === "existing-session") { + return await isReachable(timeoutMs); + } const { httpTimeoutMs } = resolveTimeouts(timeoutMs); return await isChromeReachable(profile.cdpUrl, httpTimeoutMs); }; @@ -109,6 +122,9 @@ export function createProfileAvailability({ if (previousProfile.driver === "extension") { await stopChromeExtensionRelayServer({ cdpUrl: previousProfile.cdpUrl }).catch(() => false); } + if (previousProfile.driver === "existing-session") { + await closeChromeMcpSession(previousProfile.name).catch(() => false); + } await closePlaywrightBrowserConnectionForProfile(previousProfile.cdpUrl); if (previousProfile.cdpUrl !== profile.cdpUrl) { await closePlaywrightBrowserConnectionForProfile(profile.cdpUrl); @@ -138,6 +154,10 @@ export function createProfileAvailability({ const ensureBrowserAvailable = async (): Promise => { await reconcileProfileRuntime(); + if (profile.driver === "existing-session") { + await ensureChromeMcpAvailable(profile.name); + return; + } const current = state(); const remoteCdp = capabilities.isRemote; const attachOnly = profile.attachOnly; @@ -238,6 +258,10 @@ export function createProfileAvailability({ const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => { await reconcileProfileRuntime(); + if (profile.driver === "existing-session") { + const stopped = await closeChromeMcpSession(profile.name); + return { stopped }; + } if (capabilities.requiresRelay) { const stopped = await stopChromeExtensionRelayServer({ cdpUrl: profile.cdpUrl, diff --git a/src/browser/server-context.existing-session.test.ts b/src/browser/server-context.existing-session.test.ts new file mode 100644 index 00000000000..abbd222342e --- /dev/null +++ b/src/browser/server-context.existing-session.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createBrowserRouteContext } from "./server-context.js"; +import type { BrowserServerState } from "./server-context.js"; + +vi.mock("./chrome-mcp.js", () => ({ + closeChromeMcpSession: vi.fn(async () => true), + ensureChromeMcpAvailable: vi.fn(async () => {}), + focusChromeMcpTab: vi.fn(async () => {}), + listChromeMcpTabs: vi.fn(async () => [ + { targetId: "7", title: "", url: "https://example.com", type: "page" }, + ]), + openChromeMcpTab: vi.fn(async () => ({ + targetId: "8", + title: "", + url: "https://openclaw.ai", + type: "page", + })), + closeChromeMcpTab: vi.fn(async () => {}), + getChromeMcpPid: vi.fn(() => 4321), +})); + +import * as chromeMcp from "./chrome-mcp.js"; + +function makeState(): BrowserServerState { + return { + server: null, + port: 0, + resolved: { + enabled: true, + evaluateEnabled: true, + controlPort: 18791, + cdpPortRangeStart: 18800, + cdpPortRangeEnd: 18899, + cdpProtocol: "http", + cdpHost: "127.0.0.1", + cdpIsLoopback: true, + remoteCdpTimeoutMs: 1500, + remoteCdpHandshakeTimeoutMs: 3000, + color: "#FF4500", + headless: false, + noSandbox: false, + attachOnly: false, + defaultProfile: "chrome-live", + profiles: { + "chrome-live": { + cdpPort: 18801, + color: "#0066CC", + driver: "existing-session", + attachOnly: true, + }, + }, + extraArgs: [], + ssrfPolicy: { dangerouslyAllowPrivateNetwork: true }, + }, + profiles: new Map(), + }; +} + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe("browser server-context existing-session profile", () => { + it("routes tab operations through the Chrome MCP backend", async () => { + const state = makeState(); + const ctx = createBrowserRouteContext({ getState: () => state }); + const live = ctx.forProfile("chrome-live"); + + vi.mocked(chromeMcp.listChromeMcpTabs) + .mockResolvedValueOnce([ + { targetId: "7", title: "", url: "https://example.com", type: "page" }, + ]) + .mockResolvedValueOnce([ + { targetId: "8", title: "", url: "https://openclaw.ai", type: "page" }, + ]) + .mockResolvedValueOnce([ + { targetId: "8", title: "", url: "https://openclaw.ai", type: "page" }, + ]) + .mockResolvedValueOnce([ + { targetId: "7", title: "", url: "https://example.com", type: "page" }, + ]); + + await live.ensureBrowserAvailable(); + const tabs = await live.listTabs(); + expect(tabs.map((tab) => tab.targetId)).toEqual(["7"]); + + const opened = await live.openTab("https://openclaw.ai"); + expect(opened.targetId).toBe("8"); + + const selected = await live.ensureTabAvailable(); + expect(selected.targetId).toBe("8"); + + await live.focusTab("7"); + await live.stopRunningBrowser(); + + expect(chromeMcp.ensureChromeMcpAvailable).toHaveBeenCalledWith("chrome-live"); + expect(chromeMcp.listChromeMcpTabs).toHaveBeenCalledWith("chrome-live"); + expect(chromeMcp.openChromeMcpTab).toHaveBeenCalledWith("chrome-live", "https://openclaw.ai"); + expect(chromeMcp.focusChromeMcpTab).toHaveBeenCalledWith("chrome-live", "7"); + expect(chromeMcp.closeChromeMcpSession).toHaveBeenCalledWith("chrome-live"); + }); +}); diff --git a/src/browser/server-context.selection.ts b/src/browser/server-context.selection.ts index 8a9cfa19c42..9e1fb728b2a 100644 --- a/src/browser/server-context.selection.ts +++ b/src/browser/server-context.selection.ts @@ -1,5 +1,6 @@ import { fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js"; import { appendCdpPath } from "./cdp.js"; +import { closeChromeMcpTab, focusChromeMcpTab } from "./chrome-mcp.js"; import type { ResolvedBrowserProfile } from "./config.js"; import { BrowserTabNotFoundError, BrowserTargetAmbiguousError } from "./errors.js"; import { getBrowserProfileCapabilities } from "./profile-capabilities.js"; @@ -111,6 +112,13 @@ export function createProfileSelectionOps({ const focusTab = async (targetId: string): Promise => { const resolvedTargetId = await resolveTargetIdOrThrow(targetId); + if (profile.driver === "existing-session") { + await focusChromeMcpTab(profile.name, resolvedTargetId); + const profileState = getProfileState(); + profileState.lastTargetId = resolvedTargetId; + return; + } + if (capabilities.usesPersistentPlaywright) { const mod = await getPwAiModule({ mode: "strict" }); const focusPageByTargetIdViaPlaywright = (mod as Partial | null) @@ -134,6 +142,11 @@ export function createProfileSelectionOps({ const closeTab = async (targetId: string): Promise => { const resolvedTargetId = await resolveTargetIdOrThrow(targetId); + if (profile.driver === "existing-session") { + await closeChromeMcpTab(profile.name, resolvedTargetId); + return; + } + // For remote profiles, use Playwright's persistent connection to close tabs if (capabilities.usesPersistentPlaywright) { const mod = await getPwAiModule({ mode: "strict" }); diff --git a/src/browser/server-context.tab-ops.ts b/src/browser/server-context.tab-ops.ts index 24985430bdc..067536fd017 100644 --- a/src/browser/server-context.tab-ops.ts +++ b/src/browser/server-context.tab-ops.ts @@ -1,6 +1,7 @@ import { CDP_JSON_NEW_TIMEOUT_MS } from "./cdp-timeouts.js"; import { fetchJson, fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js"; import { appendCdpPath, createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js"; +import { listChromeMcpTabs, openChromeMcpTab } from "./chrome-mcp.js"; import type { ResolvedBrowserProfile } from "./config.js"; import { assertBrowserNavigationAllowed, @@ -65,6 +66,10 @@ export function createProfileTabOps({ const capabilities = getBrowserProfileCapabilities(profile); const listTabs = async (): Promise => { + if (profile.driver === "existing-session") { + return await listChromeMcpTabs(profile.name); + } + if (capabilities.usesPersistentPlaywright) { const mod = await getPwAiModule({ mode: "strict" }); const listPagesViaPlaywright = (mod as Partial | null)?.listPagesViaPlaywright; @@ -134,6 +139,15 @@ export function createProfileTabOps({ const openTab = async (url: string): Promise => { const ssrfPolicyOpts = withBrowserNavigationPolicy(state().resolved.ssrfPolicy); + if (profile.driver === "existing-session") { + await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts }); + const page = await openChromeMcpTab(profile.name, url); + const profileState = getProfileState(); + profileState.lastTargetId = page.targetId; + await assertBrowserNavigationResultAllowed({ url: page.url, ...ssrfPolicyOpts }); + return page; + } + if (capabilities.usesPersistentPlaywright) { const mod = await getPwAiModule({ mode: "strict" }); const createPageViaPlaywright = (mod as Partial | null)?.createPageViaPlaywright; diff --git a/src/browser/server-context.ts b/src/browser/server-context.ts index d75b14c2471..37e182f1e69 100644 --- a/src/browser/server-context.ts +++ b/src/browser/server-context.ts @@ -162,12 +162,22 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon let tabCount = 0; let running = false; + const profileCtx = createProfileContext(opts, profile); - if (profileState?.running) { + if (profile.driver === "existing-session") { + try { + running = await profileCtx.isReachable(300); + if (running) { + const tabs = await profileCtx.listTabs(); + tabCount = tabs.filter((t) => t.type === "page").length; + } + } catch { + // Chrome MCP not available + } + } else if (profileState?.running) { running = true; try { - const ctx = createProfileContext(opts, profile); - const tabs = await ctx.listTabs(); + const tabs = await profileCtx.listTabs(); tabCount = tabs.filter((t) => t.type === "page").length; } catch { // Browser might not be responsive @@ -178,8 +188,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon const reachable = await isChromeReachable(profile.cdpUrl, 200); if (reachable) { running = true; - const ctx = createProfileContext(opts, profile); - const tabs = await ctx.listTabs().catch(() => []); + const tabs = await profileCtx.listTabs().catch(() => []); tabCount = tabs.filter((t) => t.type === "page").length; } } catch { @@ -192,6 +201,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon cdpPort: profile.cdpPort, cdpUrl: profile.cdpUrl, color: profile.color, + driver: profile.driver, running, tabCount, isDefault: name === current.resolved.defaultProfile, diff --git a/src/browser/server-context.types.ts b/src/browser/server-context.types.ts index f05e90e9e77..8f949b96da6 100644 --- a/src/browser/server-context.types.ts +++ b/src/browser/server-context.types.ts @@ -56,6 +56,7 @@ export type ProfileStatus = { cdpPort: number; cdpUrl: string; color: string; + driver: ResolvedBrowserProfile["driver"]; running: boolean; tabCount: number; isDefault: boolean; diff --git a/src/cli/browser-cli-manage.ts b/src/cli/browser-cli-manage.ts index 53b83ca3f97..31d4b02c2aa 100644 --- a/src/cli/browser-cli-manage.ts +++ b/src/cli/browser-cli-manage.ts @@ -407,7 +407,8 @@ export function registerBrowserManageCommands( const def = p.isDefault ? " [default]" : ""; const loc = p.isRemote ? `cdpUrl: ${p.cdpUrl}` : `port: ${p.cdpPort}`; const remote = p.isRemote ? " [remote]" : ""; - return `${p.name}: ${status}${tabs}${def}${remote}\n ${loc}, color: ${p.color}`; + const driver = p.driver !== "openclaw" ? ` [${p.driver}]` : ""; + return `${p.name}: ${status}${tabs}${def}${remote}${driver}\n ${loc}, color: ${p.color}`; }) .join("\n"), ); @@ -420,7 +421,10 @@ export function registerBrowserManageCommands( .requiredOption("--name ", "Profile name (lowercase, numbers, hyphens)") .option("--color ", "Profile color (hex format, e.g. #0066CC)") .option("--cdp-url ", "CDP URL for remote Chrome (http/https)") - .option("--driver ", "Profile driver (openclaw|extension). Default: openclaw") + .option( + "--driver ", + "Profile driver (openclaw|extension|existing-session). Default: openclaw", + ) .action( async (opts: { name: string; color?: string; cdpUrl?: string; driver?: string }, cmd) => { const parent = parentOpts(cmd); @@ -434,7 +438,12 @@ export function registerBrowserManageCommands( name: opts.name, color: opts.color, cdpUrl: opts.cdpUrl, - driver: opts.driver === "extension" ? "extension" : undefined, + driver: + opts.driver === "extension" + ? "extension" + : opts.driver === "existing-session" + ? "existing-session" + : undefined, }, }, { timeoutMs: 10_000 }, @@ -446,7 +455,11 @@ export function registerBrowserManageCommands( defaultRuntime.log( info( `🦞 Created profile "${result.profile}"\n${loc}\n color: ${result.color}${ - opts.driver === "extension" ? "\n driver: extension" : "" + opts.driver === "extension" + ? "\n driver: extension" + : opts.driver === "existing-session" + ? "\n driver: existing-session" + : "" }`, ), ); diff --git a/src/config/types.browser.ts b/src/config/types.browser.ts index 57d036bd88c..5f8e28a0ebe 100644 --- a/src/config/types.browser.ts +++ b/src/config/types.browser.ts @@ -4,7 +4,7 @@ export type BrowserProfileConfig = { /** CDP URL for this profile (use for remote Chrome). */ cdpUrl?: string; /** Profile driver (default: openclaw). */ - driver?: "openclaw" | "clawd" | "extension"; + driver?: "openclaw" | "clawd" | "extension" | "existing-session"; /** If true, never launch a browser for this profile; only attach. Falls back to browser.attachOnly. */ attachOnly?: boolean; /** Profile color (hex). Auto-assigned at creation. */ diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 0064afddd20..741b4bcc0c9 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -360,7 +360,12 @@ export const OpenClawSchema = z cdpPort: z.number().int().min(1).max(65535).optional(), cdpUrl: z.string().optional(), driver: z - .union([z.literal("openclaw"), z.literal("clawd"), z.literal("extension")]) + .union([ + z.literal("openclaw"), + z.literal("clawd"), + z.literal("extension"), + z.literal("existing-session"), + ]) .optional(), attachOnly: z.boolean().optional(), color: HexColorSchema,