From 1cf68b9243aa2e8683cd2c920623e2b624250dfb Mon Sep 17 00:00:00 2001 From: Val Alexander <68980965+BunsDev@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:35:34 -0500 Subject: [PATCH] fix(control-ui): keep google talk off webrtc Keep Google Live Talk browser sessions on the supported WebSocket/gateway-relay paths instead of falling back to browser WebRTC, remove stale browser-native voice controls that bypass Talk/TTS provider settings, and harden the Google Live URL plus realtime relay resource controls. Verification: - pnpm test ui/src/ui/realtime-talk.test.ts ui/src/ui/realtime-talk-google-live.test.ts src/gateway/talk-realtime-relay.test.ts src/gateway/server-methods/talk.test.ts - pnpm check:changed --- CHANGELOG.md | 1 + src/gateway/server-methods/talk.test.ts | 92 +++++++++++++++++++++ src/gateway/server-methods/talk.ts | 17 +++- src/gateway/talk-realtime-relay.test.ts | 34 ++++++++ src/gateway/talk-realtime-relay.ts | 31 +++++++ ui/src/styles/chat/grouped.css | 8 +- ui/src/ui/chat/grouped-render.test.ts | 19 +++-- ui/src/ui/chat/grouped-render.ts | 61 +------------- ui/src/ui/chat/realtime-talk-google-live.ts | 29 ++++++- ui/src/ui/chat/realtime-talk.ts | 29 ++++++- ui/src/ui/realtime-talk-google-live.test.ts | 52 ++++++++++++ ui/src/ui/realtime-talk.test.ts | 51 ++++++++++++ ui/src/ui/views/chat.test.ts | 9 ++ ui/src/ui/views/chat.ts | 71 +--------------- 14 files changed, 354 insertions(+), 150 deletions(-) create mode 100644 ui/src/ui/realtime-talk-google-live.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aeeae584be..40a173efe42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Gateway/device tokens: stop echoing rotated bearer tokens from shared/admin `device.token.rotate` responses while preserving the same-device token handoff needed by token-only clients before reconnect. (#66773) Thanks @MoerAI. +- Control UI/Talk: keep Google Live browser sessions on the WebSocket transport instead of falling back to WebRTC, validate browser Google Live WebSocket endpoints, cap Gateway relay sessions per browser connection, and remove stale browser-native voice buttons that did not use the configured Talk/TTS provider. Thanks @BunsDev. - Agents/subagents: enforce `subagents.allowAgents` for explicit same-agent `sessions_spawn(agentId=...)` calls instead of auto-allowing requester self-targets. Fixes #72827. Thanks @oiGaDio. - ACP/sessions_spawn: let explicit `sessions_spawn(runtime="acp")` bootstrap turns run while `acp.dispatch.enabled=false` still blocks automatic ACP thread dispatch. Fixes #63591. Thanks @moeedahmed. - CLI/update: install npm global updates into a verified temporary prefix before swapping the package tree into place, preventing mixed old/new installs and stale packaged files from breaking `openclaw update` verification. Thanks @shakkernerd. diff --git a/src/gateway/server-methods/talk.test.ts b/src/gateway/server-methods/talk.test.ts index d30f5a017d3..89a34299a2d 100644 --- a/src/gateway/server-methods/talk.test.ts +++ b/src/gateway/server-methods/talk.test.ts @@ -8,6 +8,9 @@ const mocks = vi.hoisted(() => ({ canonicalizeSpeechProviderId: vi.fn((providerId: string | undefined) => providerId), getSpeechProvider: vi.fn(), synthesizeSpeech: vi.fn(), + getRealtimeVoiceProvider: vi.fn(), + resolveConfiguredRealtimeVoiceProvider: vi.fn(), + createTalkRealtimeRelaySession: vi.fn(), })); vi.mock("../../config/config.js", () => ({ @@ -23,6 +26,22 @@ vi.mock("../../tts/tts.js", () => ({ synthesizeSpeech: mocks.synthesizeSpeech, })); +vi.mock("../../realtime-voice/provider-registry.js", () => ({ + getRealtimeVoiceProvider: mocks.getRealtimeVoiceProvider, +})); + +vi.mock("../../realtime-voice/provider-resolver.js", () => ({ + resolveConfiguredRealtimeVoiceProvider: mocks.resolveConfiguredRealtimeVoiceProvider, +})); + +vi.mock("../talk-realtime-relay.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createTalkRealtimeRelaySession: mocks.createTalkRealtimeRelaySession, + }; +}); + function createTalkConfig(apiKey: unknown): OpenClawConfig { return { talk: { @@ -112,3 +131,76 @@ describe("talk.speak handler", () => { ); }); }); + +describe("talk.realtime.session handler", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("falls back to the gateway relay when Google returns a WebRTC-shaped browser session", async () => { + const createBrowserSession = vi.fn(async () => ({ + provider: "google", + clientSecret: "legacy-google-secret", + })); + const createBridge = vi.fn(); + const provider = { + id: "google", + label: "Google Live Voice", + isConfigured: () => true, + createBrowserSession, + createBridge, + }; + mocks.getRealtimeVoiceProvider.mockReturnValue(provider); + mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({ + provider, + providerConfig: { apiKey: "gemini-key" }, + }); + mocks.createTalkRealtimeRelaySession.mockReturnValue({ + provider: "google", + transport: "gateway-relay", + relaySessionId: "relay-1", + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 24000, + outputEncoding: "pcm16", + outputSampleRateHz: 24000, + }, + }); + + const respond = vi.fn(); + await talkHandlers["talk.realtime.session"]({ + req: { type: "req", id: "1", method: "talk.realtime.session" }, + params: { sessionKey: "main", provider: "google" }, + client: { connId: "conn-1" } as never, + isWebchatConnect: () => false, + respond: respond as never, + context: { + getRuntimeConfig: () => + ({ + talk: { + provider: "google", + providers: { google: { apiKey: "gemini-key" } }, + }, + }) as OpenClawConfig, + } as never, + }); + + expect(createBrowserSession).toHaveBeenCalledTimes(1); + expect(mocks.createTalkRealtimeRelaySession).toHaveBeenCalledWith( + expect.objectContaining({ + connId: "conn-1", + provider, + providerConfig: { apiKey: "gemini-key" }, + }), + ); + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ + provider: "google", + transport: "gateway-relay", + relaySessionId: "relay-1", + }), + undefined, + ); + }); +}); diff --git a/src/gateway/server-methods/talk.ts b/src/gateway/server-methods/talk.ts index 87f745ebb40..4d77b08ca16 100644 --- a/src/gateway/server-methods/talk.ts +++ b/src/gateway/server-methods/talk.ts @@ -13,7 +13,10 @@ import { } from "../../realtime-voice/agent-consult-tool.js"; import { getRealtimeVoiceProvider } from "../../realtime-voice/provider-registry.js"; import { resolveConfiguredRealtimeVoiceProvider } from "../../realtime-voice/provider-resolver.js"; -import type { RealtimeVoiceProviderConfig } from "../../realtime-voice/provider-types.js"; +import type { + RealtimeVoiceBrowserSession, + RealtimeVoiceProviderConfig, +} from "../../realtime-voice/provider-types.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, @@ -226,6 +229,12 @@ function withRealtimeBrowserOverrides( return Object.keys(overrides).length > 0 ? { ...providerConfig, ...overrides } : providerConfig; } +function isUnsupportedBrowserWebRtcSession(session: RealtimeVoiceBrowserSession): boolean { + const provider = normalizeLowercaseStringOrEmpty(session.provider); + const transport = (session as { transport?: string }).transport ?? "webrtc-sdp"; + return provider === "google" && transport === "webrtc-sdp"; +} + function isFallbackEligibleTalkReason(reason: TalkSpeakReason): boolean { return ( reason === "talk_unconfigured" || @@ -459,8 +468,10 @@ export const talkHandlers: GatewayRequestHandlers = { model: normalizeOptionalString(typedParams.model), voice: normalizeOptionalString(typedParams.voice), }); - respond(true, session, undefined); - return; + if (!isUnsupportedBrowserWebRtcSession(session)) { + respond(true, session, undefined); + return; + } } const connId = client?.connId; diff --git a/src/gateway/talk-realtime-relay.test.ts b/src/gateway/talk-realtime-relay.test.ts index 1a7125b2ae3..0c76eb15cab 100644 --- a/src/gateway/talk-realtime-relay.test.ts +++ b/src/gateway/talk-realtime-relay.test.ts @@ -178,4 +178,38 @@ describe("talk realtime gateway relay", () => { }), ).toThrow("Unknown realtime relay session"); }); + + it("caps active relay sessions per browser connection", () => { + const provider: RealtimeVoiceProviderPlugin = { + id: "relay-test", + label: "Relay Test", + isConfigured: () => true, + createBridge: () => ({ + connect: vi.fn(async () => undefined), + sendAudio: vi.fn(), + setMediaTimestamp: vi.fn(), + submitToolResult: vi.fn(), + acknowledgeMark: vi.fn(), + close: vi.fn(), + isConnected: vi.fn(() => true), + }), + }; + const createSession = (connId: string) => + createTalkRealtimeRelaySession({ + context: { broadcastToConnIds: vi.fn() } as never, + connId, + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + }); + + createSession("conn-1"); + createSession("conn-1"); + + expect(() => createSession("conn-1")).toThrow( + "Too many active realtime relay sessions for this connection", + ); + expect(() => createSession("conn-2")).not.toThrow(); + }); }); diff --git a/src/gateway/talk-realtime-relay.ts b/src/gateway/talk-realtime-relay.ts index 9af95d1d25e..65244f93844 100644 --- a/src/gateway/talk-realtime-relay.ts +++ b/src/gateway/talk-realtime-relay.ts @@ -14,6 +14,8 @@ import type { GatewayRequestContext } from "./server-methods/shared-types.js"; const RELAY_SESSION_TTL_MS = 30 * 60 * 1000; const MAX_AUDIO_BASE64_BYTES = 512 * 1024; +const MAX_RELAY_SESSIONS_PER_CONN = 2; +const MAX_RELAY_SESSIONS_GLOBAL = 64; const RELAY_EVENT = "talk.realtime.relay"; export type TalkRealtimeRelayEvent = @@ -94,9 +96,38 @@ function closeRelaySession(session: RelaySession, reason: "completed" | "error") }); } +function pruneExpiredRelaySessions(nowMs = Date.now()): void { + for (const session of relaySessions.values()) { + if (nowMs > session.expiresAtMs) { + closeRelaySession(session, "completed"); + } + } +} + +function countRelaySessionsForConn(connId: string): number { + let count = 0; + for (const session of relaySessions.values()) { + if (session.connId === connId) { + count += 1; + } + } + return count; +} + +function enforceRelaySessionLimits(connId: string): void { + pruneExpiredRelaySessions(); + if (relaySessions.size >= MAX_RELAY_SESSIONS_GLOBAL) { + throw new Error("Too many active realtime relay sessions"); + } + if (countRelaySessionsForConn(connId) >= MAX_RELAY_SESSIONS_PER_CONN) { + throw new Error("Too many active realtime relay sessions for this connection"); + } +} + export function createTalkRealtimeRelaySession( params: CreateTalkRealtimeRelaySessionParams, ): TalkRealtimeRelaySessionResult { + enforceRelaySessionLimits(params.connId); const relaySessionId = randomUUID(); const expiresAtMs = Date.now() + RELAY_SESSION_TTL_MS; let relay: RelaySession | undefined; diff --git a/ui/src/styles/chat/grouped.css b/ui/src/styles/chat/grouped.css index c19deb16fee..b5329e6b5be 100644 --- a/ui/src/styles/chat/grouped.css +++ b/ui/src/styles/chat/grouped.css @@ -65,7 +65,7 @@ line-height: 1.2; } -/* ── Group footer action buttons (TTS, delete) ── */ +/* ── Group footer action buttons ── */ .chat-group-footer button { background: none; border: none; @@ -106,12 +106,6 @@ stroke-linejoin: round; } -.chat-tts-btn--active { - opacity: 1 !important; - pointer-events: auto !important; - color: var(--accent); -} - .chat-group-delete:hover { color: var(--danger) !important; } diff --git a/ui/src/ui/chat/grouped-render.test.ts b/ui/src/ui/chat/grouped-render.test.ts index 67a116707ba..2af5f1dd279 100644 --- a/ui/src/ui/chat/grouped-render.test.ts +++ b/ui/src/ui/chat/grouped-render.test.ts @@ -89,13 +89,6 @@ vi.mock("../tool-display.ts", () => ({ }), })); -vi.mock("./speech.ts", () => ({ - isTtsSpeaking: () => false, - isTtsSupported: () => false, - speakText: () => false, - stopTts: () => undefined, -})); - type RenderMessageGroupOptions = Parameters[1]; function renderAssistantMessage( @@ -262,6 +255,18 @@ afterEach(() => { }); describe("grouped chat rendering", () => { + it("does not render the stale assistant read-aloud footer action", () => { + const container = document.createElement("div"); + renderAssistantMessage(container, { + role: "assistant", + content: "hello from assistant", + timestamp: 1000, + }); + + expect(container.querySelector(".chat-tts-btn")).toBeNull(); + expect(container.querySelector('[aria-label="Read aloud"]')).toBeNull(); + }); + it("positions delete confirm by message side", () => { const container = document.createElement("div"); clearDeleteConfirmSkip(); diff --git a/ui/src/ui/chat/grouped-render.ts b/ui/src/ui/chat/grouped-render.ts index 0630558b68b..80423d2be68 100644 --- a/ui/src/ui/chat/grouped-render.ts +++ b/ui/src/ui/chat/grouped-render.ts @@ -19,14 +19,9 @@ import { resolveLocalUserName } from "../user-identity.ts"; export { resolveAssistantTextAvatar } from "../views/agents-utils.ts"; import { renderChatAvatar } from "./chat-avatar.ts"; import { renderCopyAsMarkdownButton } from "./copy-as-markdown.ts"; -import { - extractTextCached, - extractThinkingCached, - formatReasoningMarkdown, -} from "./message-extract.ts"; +import { extractThinkingCached, formatReasoningMarkdown } from "./message-extract.ts"; import { isToolResultMessage, normalizeMessage } from "./message-normalizer.ts"; import { normalizeRoleForGrouping } from "./role-normalizer.ts"; -import { isTtsSupported, speakText, stopTts, isTtsSpeaking } from "./speech.ts"; import { extractToolCards, renderExpandedToolCardContent, @@ -465,7 +460,6 @@ export function renderMessageGroup(
${who} ${renderChatTimestamp(group.timestamp)} ${renderMessageMeta(meta)} - ${normalizedRole === "assistant" && isTtsSupported() ? renderTtsButton(group) : nothing} ${opts.onDelete ? renderDeleteButton(opts.onDelete, normalizedRole === "user" ? "left" : "right") : nothing} @@ -609,17 +603,6 @@ function renderMessageMeta(meta: GroupMeta | null) { `; } -function extractGroupText(group: MessageGroup): string { - const parts: string[] = []; - for (const { message } of group.messages) { - const text = extractTextCached(message); - if (text?.trim()) { - parts.push(text.trim()); - } - } - return parts.join("\n\n"); -} - const SKIP_DELETE_CONFIRM_KEY = "openclaw:skipDeleteConfirm"; type DeleteConfirmSide = "left" | "right"; @@ -697,48 +680,6 @@ function renderDeleteButton(onDelete: () => void, side: DeleteConfirmSide) { `; } -function renderTtsButton(group: MessageGroup) { - return html` - - `; -} - function resolveRenderableMessageImages( images: ImageBlock[], opts?: ImageRenderOptions, diff --git a/ui/src/ui/chat/realtime-talk-google-live.ts b/ui/src/ui/chat/realtime-talk-google-live.ts index 5cd77ace1ae..7108af42280 100644 --- a/ui/src/ui/chat/realtime-talk-google-live.ts +++ b/ui/src/ui/chat/realtime-talk-google-live.ts @@ -36,8 +36,30 @@ type PendingFunctionCall = { args: unknown; }; -function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult): string { - const url = new URL(session.websocketUrl); +const GOOGLE_LIVE_WEBSOCKET_HOST = "generativelanguage.googleapis.com"; +const GOOGLE_LIVE_WEBSOCKET_PATH = + /^\/ws\/google\.ai\.generativelanguage\.v[0-9a-z]+\.GenerativeService\.BidiGenerateContent(?:Constrained)?$/; + +export function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult): string { + let url: URL; + try { + url = new URL(session.websocketUrl); + } catch { + throw new Error("Invalid Google Live WebSocket URL"); + } + if (url.protocol !== "wss:") { + throw new Error("Google Live WebSocket URL must use wss://"); + } + if (url.hostname.toLowerCase() !== GOOGLE_LIVE_WEBSOCKET_HOST) { + throw new Error("Untrusted Google Live WebSocket host"); + } + if (url.username || url.password) { + throw new Error("Google Live WebSocket URL must not include credentials"); + } + if (!GOOGLE_LIVE_WEBSOCKET_PATH.test(url.pathname)) { + throw new Error("Untrusted Google Live WebSocket path"); + } + url.search = ""; url.searchParams.set("access_token", session.clientSecret); return url.toString(); } @@ -65,11 +87,12 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport { if (this.session.protocol !== "google-live-bidi") { throw new Error(`Unsupported realtime WebSocket protocol: ${this.session.protocol}`); } + const wsUrl = buildGoogleLiveUrl(this.session); this.closed = false; this.media = await navigator.mediaDevices.getUserMedia({ audio: true }); this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz }); this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz }); - this.ws = new WebSocket(buildGoogleLiveUrl(this.session)); + this.ws = new WebSocket(wsUrl); this.ws.addEventListener("open", () => { this.send(this.session.initialMessage ?? { setup: {} }); this.startMicrophonePump(); diff --git a/ui/src/ui/chat/realtime-talk.ts b/ui/src/ui/chat/realtime-talk.ts index 67b1abaffe4..e348490fadf 100644 --- a/ui/src/ui/chat/realtime-talk.ts +++ b/ui/src/ui/chat/realtime-talk.ts @@ -19,7 +19,7 @@ function createTransport( session: RealtimeTalkSessionResult, ctx: RealtimeTalkTransportContext, ): RealtimeTalkTransport { - const transport = session.transport ?? "webrtc-sdp"; + const transport = resolveTransport(session); if (transport === "webrtc-sdp") { return new WebRtcSdpRealtimeTalkTransport(session as RealtimeTalkWebRtcSdpSessionResult, ctx); } @@ -42,6 +42,33 @@ function createTransport( throw new Error(`Unsupported realtime Talk transport: ${unknownTransport}`); } +function resolveTransport(session: RealtimeTalkSessionResult): string { + if (session.transport) { + return session.transport; + } + const raw = session as { + provider?: string; + protocol?: string; + websocketUrl?: string; + }; + const provider = raw.provider?.trim().toLowerCase(); + if (provider === "google" && (raw.protocol === "google-live-bidi" || raw.websocketUrl)) { + return "json-pcm-websocket"; + } + if (provider === "google") { + throw new Error(buildGoogleWebRtcUnsupportedMessage()); + } + return "webrtc-sdp"; +} + +function buildGoogleWebRtcUnsupportedMessage(): string { + return [ + 'Realtime voice provider "google" does not support browser WebRTC sessions.', + "Control UI Talk can use Google through the gateway relay or a Google Live WebSocket session instead.", + 'Restart the gateway so it returns "gateway-relay" or "json-pcm-websocket", or switch Talk realtime to a WebRTC-capable provider such as OpenAI.', + ].join(" "); +} + export class RealtimeTalkSession { private transport: RealtimeTalkTransport | null = null; private closed = false; diff --git a/ui/src/ui/realtime-talk-google-live.test.ts b/ui/src/ui/realtime-talk-google-live.test.ts new file mode 100644 index 00000000000..005ed7be482 --- /dev/null +++ b/ui/src/ui/realtime-talk-google-live.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { buildGoogleLiveUrl } from "./chat/realtime-talk-google-live.ts"; +import type { RealtimeTalkJsonPcmWebSocketSessionResult } from "./chat/realtime-talk-shared.ts"; + +function createSession( + websocketUrl: string, + clientSecret = "auth_tokens/browser-session", +): RealtimeTalkJsonPcmWebSocketSessionResult { + return { + provider: "google", + transport: "json-pcm-websocket", + protocol: "google-live-bidi", + clientSecret, + websocketUrl, + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 16000, + outputEncoding: "pcm16", + outputSampleRateHz: 24000, + }, + }; +} + +describe("Google Live realtime Talk URL", () => { + it("only preserves the allowlisted Google Live endpoint and appends the ephemeral token", () => { + const url = buildGoogleLiveUrl( + createSession( + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?ignored=1", + ), + ); + + expect(url).toBe( + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained?access_token=auth_tokens%2Fbrowser-session", + ); + }); + + it("rejects attacker-controlled Google Live WebSocket URLs", () => { + expect(() => + buildGoogleLiveUrl(createSession("ws://generativelanguage.googleapis.com/ws/google.ai")), + ).toThrow("wss://"); + expect(() => + buildGoogleLiveUrl( + createSession( + "wss://attacker.test/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained", + ), + ), + ).toThrow("Untrusted Google Live WebSocket host"); + expect(() => + buildGoogleLiveUrl(createSession("wss://generativelanguage.googleapis.com/evil")), + ).toThrow("Untrusted Google Live WebSocket path"); + }); +}); diff --git a/ui/src/ui/realtime-talk.test.ts b/ui/src/ui/realtime-talk.test.ts index 8ededad9c8f..b2a15f12a58 100644 --- a/ui/src/ui/realtime-talk.test.ts +++ b/ui/src/ui/realtime-talk.test.ts @@ -83,6 +83,57 @@ describe("RealtimeTalkSession", () => { expect(onStatus).toHaveBeenCalledWith("connecting"); }); + it("keeps Google Live WebSocket sessions off the WebRTC fallback when transport is omitted", async () => { + const request = vi.fn(async () => ({ + provider: "google", + protocol: "google-live-bidi", + clientSecret: "auth_tokens/session", + websocketUrl: "wss://example.test/live", + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 16000, + outputEncoding: "pcm16", + outputSampleRateHz: 24000, + }, + })); + const session = new RealtimeTalkSession({ request } as never, "main"); + + await session.start(); + + expect(googleCtor).toHaveBeenCalledTimes(1); + expect(googleStart).toHaveBeenCalledTimes(1); + expect(webRtcCtor).not.toHaveBeenCalled(); + }); + + it("does not treat ambiguous Google sessions as browser WebRTC sessions", async () => { + const request = vi.fn(async () => ({ + provider: "google", + clientSecret: "secret", + })); + const session = new RealtimeTalkSession({ request } as never, "main"); + + await expect(session.start()).rejects.toThrow( + 'Realtime voice provider "google" does not support browser WebRTC sessions. Control UI Talk can use Google through the gateway relay or a Google Live WebSocket session instead. Restart the gateway so it returns "gateway-relay" or "json-pcm-websocket", or switch Talk realtime to a WebRTC-capable provider such as OpenAI.', + ); + + expect(webRtcCtor).not.toHaveBeenCalled(); + expect(googleCtor).not.toHaveBeenCalled(); + }); + + it("does not infer Google Live transport from websocketUrl on non-Google sessions", async () => { + const request = vi.fn(async () => ({ + provider: "example", + clientSecret: "secret", + websocketUrl: "wss://example.test/live", + })); + const session = new RealtimeTalkSession({ request } as never, "main"); + + await session.start(); + + expect(webRtcCtor).toHaveBeenCalledTimes(1); + expect(googleCtor).not.toHaveBeenCalled(); + }); + it("starts the Gateway relay transport for backend-only realtime providers", async () => { const request = vi.fn(async () => ({ provider: "example", diff --git a/ui/src/ui/views/chat.test.ts b/ui/src/ui/views/chat.test.ts index 5c82c0aa85e..e9939a4d099 100644 --- a/ui/src/ui/views/chat.test.ts +++ b/ui/src/ui/views/chat.test.ts @@ -438,6 +438,15 @@ describe("chat loading skeleton", () => { }); }); +describe("chat voice controls", () => { + it("keeps Talk visible without the stale browser dictation button", () => { + const container = renderChatView(); + + expect(container.querySelector('[aria-label="Start Talk"]')).not.toBeNull(); + expect(container.querySelector('[aria-label="Voice input"]')).toBeNull(); + }); +}); + describe("chat attachment picker", () => { it("accepts and previews non-video file attachments", async () => { const onAttachmentsChange = vi.fn(); diff --git a/ui/src/ui/views/chat.ts b/ui/src/ui/views/chat.ts index 2c39b1021be..b98e61e1903 100644 --- a/ui/src/ui/views/chat.ts +++ b/ui/src/ui/views/chat.ts @@ -34,7 +34,6 @@ import { type SlashCommandCategory, type SlashCommandDef, } from "../chat/slash-commands.ts"; -import { isSttSupported, startStt, stopStt } from "../chat/speech.ts"; import { renderCompactionIndicator, renderFallbackIndicator } from "../chat/status-indicators.ts"; import { getExpandedToolCards, syncToolCardExpansionState } from "../chat/tool-expansion-state.ts"; import type { EmbedSandboxMode } from "../embed-sandbox.ts"; @@ -145,8 +144,6 @@ function getDeletedMessages(sessionKey: string): DeletedMessages { } interface ChatEphemeralState { - sttRecording: boolean; - sttInterimText: string; slashMenuOpen: boolean; slashMenuItems: SlashCommandDef[]; slashMenuIndex: number; @@ -161,8 +158,6 @@ interface ChatEphemeralState { function createChatEphemeralState(): ChatEphemeralState { return { - sttRecording: false, - sttInterimText: "", slashMenuOpen: false, slashMenuItems: [], slashMenuIndex: 0, @@ -180,12 +175,9 @@ const vs = createChatEphemeralState(); /** * Reset chat view ephemeral state when navigating away. - * Stops STT recording and clears search/slash UI that should not survive navigation. + * Clears search/slash UI that should not survive navigation. */ export function resetChatViewState() { - if (vs.sttRecording) { - stopStt(); - } Object.assign(vs, createChatEphemeralState()); } @@ -743,8 +735,6 @@ export function renderChat(props: ChatProps) { : "Connect to the gateway to start chatting..."; const requestUpdate = props.onRequestUpdate ?? (() => {}); - const getDraft = props.getDraft ?? (() => props.draft); - const splitRatio = props.splitRatio ?? 0.6; const sidebarOpen = Boolean(props.sidebarOpen && props.onCloseSidebar); @@ -1129,9 +1119,6 @@ export function renderChat(props: ChatProps) { @change=${(e: Event) => handleFileSelect(e, props)} /> - ${vs.sttRecording && vs.sttInterimText - ? html`
${vs.sttInterimText}
` - : nothing} ${props.realtimeTalkActive || props.realtimeTalkDetail || props.realtimeTalkTranscript ? html`
@@ -1154,7 +1141,7 @@ export function renderChat(props: ChatProps) { @keydown=${handleKeyDown} @input=${handleInput} @paste=${(e: ClipboardEvent) => handlePaste(e, props)} - placeholder=${vs.sttRecording ? "Listening..." : placeholder} + placeholder=${placeholder} rows="1" > @@ -1172,60 +1159,6 @@ export function renderChat(props: ChatProps) { ${icons.paperclip} - ${isSttSupported() - ? html` - - ` - : nothing} ${props.onToggleRealtimeTalk ? html`