diff --git a/extensions/google/realtime-voice-provider.test.ts b/extensions/google/realtime-voice-provider.test.ts index d880f9535ed6..3f2fd0786a81 100644 --- a/extensions/google/realtime-voice-provider.test.ts +++ b/extensions/google/realtime-voice-provider.test.ts @@ -1,5 +1,6 @@ // Google tests cover realtime voice provider plugin behavior. import { REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ } from "openclaw/plugin-sdk/realtime-voice"; +import type { RealtimeVoiceTool } from "openclaw/plugin-sdk/realtime-voice"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildGoogleRealtimeVoiceProvider } from "./realtime-voice-provider.js"; @@ -90,6 +91,35 @@ function requireFirstAudio(mock: ReturnType): unknown { return requireFirstMockArg(mock, "Google Live audio"); } +function createRealtimeTool(name: string): RealtimeVoiceTool { + return { + type: "function", + name, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + }; +} + +function createUnreadableToolName(): RealtimeVoiceTool { + return { + type: "function", + get name(): string { + throw new Error("unreadable tool name"); + }, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + }; +} + +function createMalformedToolName(name: unknown): RealtimeVoiceTool { + return { + type: "function", + name, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + } as unknown as RealtimeVoiceTool; +} + describe("buildGoogleRealtimeVoiceProvider", () => { beforeEach(() => { envSnapshot = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]])); @@ -302,6 +332,36 @@ describe("buildGoogleRealtimeVoiceProvider", () => { expect(declarations[1]?.behavior).toBe("NON_BLOCKING"); }); + it("omits tool names that Google Live cannot accept", async () => { + const provider = buildGoogleRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "gemini-key" }, + tools: [ + createRealtimeTool("_lookup"), + createRealtimeTool("calendar.lookup:next"), + createRealtimeTool("1_lookup"), + createRealtimeTool("bad/name"), + createRealtimeTool(`x${"a".repeat(128)}`), + createMalformedToolName(undefined), + createMalformedToolName(null), + createMalformedToolName(42), + createUnreadableToolName(), + ], + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + + await bridge.connect(); + + const config = lastConnectParams().config as { + tools?: Array<{ functionDeclarations?: Array<{ name?: string }> }>; + }; + expect(config.tools?.[0]?.functionDeclarations?.map((declaration) => declaration.name)).toEqual([ + "_lookup", + "calendar.lookup:next", + ]); + }); + it("omits zero temperature for native audio responses", async () => { const provider = buildGoogleRealtimeVoiceProvider(); const bridge = provider.createBridge({ diff --git a/extensions/google/realtime-voice-provider.ts b/extensions/google/realtime-voice-provider.ts index eaf264ba0e18..e77be5924796 100644 --- a/extensions/google/realtime-voice-provider.ts +++ b/extensions/google/realtime-voice-provider.ts @@ -22,6 +22,7 @@ import { timestampMsToIsoString, } from "openclaw/plugin-sdk/number-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard"; +import { warn } from "openclaw/plugin-sdk/runtime-env"; import type { RealtimeVoiceAudioFormat, RealtimeVoiceBridge, @@ -63,6 +64,8 @@ const GOOGLE_REALTIME_BROWSER_NEW_SESSION_TTL_MS = 60 * 1000; const GOOGLE_REALTIME_RECONNECT_MAX_ATTEMPTS = 3; const GOOGLE_REALTIME_RECONNECT_BASE_DELAY_MS = 250; const GOOGLE_REALTIME_RECONNECT_MAX_DELAY_MS = 2_000; +// Google Live requires a leading letter/underscore and caps function names at 128 characters. +const GOOGLE_REALTIME_TOOL_NAME_RE = /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/; const MULAW_LINEAR_SAMPLES = new Int16Array(256); for (let i = 0; i < MULAW_LINEAR_SAMPLES.length; i += 1) { @@ -339,19 +342,34 @@ function buildRealtimeInputConfig( } function buildFunctionDeclarations(tools: RealtimeVoiceTool[] | undefined): FunctionDeclaration[] { - return (tools ?? []).map((tool) => { - // Live preview models honor the OpenAPI `parameters` field; the SDK normalizes - // our lowercase JSON Schema types before sending the mutually exclusive field. - const declaration: FunctionDeclaration = { - name: tool.name, - description: tool.description, - parameters: tool.parameters as unknown as FunctionDeclaration["parameters"], - }; - if (tool.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { - declaration.behavior = "NON_BLOCKING" as Behavior; + const declarations: FunctionDeclaration[] = []; + let omitted = 0; + for (const tool of tools ?? []) { + try { + const name = tool.name; + if (typeof name !== "string" || !GOOGLE_REALTIME_TOOL_NAME_RE.test(name)) { + omitted += 1; + continue; + } + // Live preview models honor the OpenAPI `parameters` field; the SDK normalizes + // our lowercase JSON Schema types before sending the mutually exclusive field. + const declaration: FunctionDeclaration = { + name, + description: tool.description, + parameters: tool.parameters as unknown as FunctionDeclaration["parameters"], + }; + if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) { + declaration.behavior = "NON_BLOCKING" as Behavior; + } + declarations.push(declaration); + } catch { + omitted += 1; } - return declaration; - }); + } + if (omitted > 0) { + warn(`google realtime: omitted ${omitted} tool definition(s) with unsupported names`); + } + return declarations; } function buildGoogleLiveConnectConfig(config: GoogleRealtimeLiveConfig): LiveConnectConfig { diff --git a/extensions/openai/realtime-voice-provider.test.ts b/extensions/openai/realtime-voice-provider.test.ts index a2f83046228f..71cf8066b955 100644 --- a/extensions/openai/realtime-voice-provider.test.ts +++ b/extensions/openai/realtime-voice-provider.test.ts @@ -1,6 +1,9 @@ // Openai tests cover realtime voice provider plugin behavior. import { REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ } from "openclaw/plugin-sdk/realtime-voice"; -import type { RealtimeVoiceBridge } from "openclaw/plugin-sdk/realtime-voice"; +import type { + RealtimeVoiceBridge, + RealtimeVoiceTool, +} from "openclaw/plugin-sdk/realtime-voice"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; @@ -123,6 +126,7 @@ type SentRealtimeEvent = { create_response?: boolean; }; output_modalities?: string[]; + tools?: Array<{ name?: string }>; audio?: { input?: { format?: Record; @@ -239,6 +243,35 @@ function hasSentEventType(socket: FakeWebSocketInstance, type: string): boolean return parseSent(socket).some((event) => event.type === type); } +function createRealtimeTool(name: string): RealtimeVoiceTool { + return { + type: "function", + name, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + }; +} + +function createUnreadableToolName(): RealtimeVoiceTool { + return { + type: "function", + get name(): string { + throw new Error("unreadable tool name"); + }, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + }; +} + +function createMalformedToolName(name: unknown): RealtimeVoiceTool { + return { + type: "function", + name, + description: "Contract test tool", + parameters: { type: "object", properties: {} }, + } as unknown as RealtimeVoiceTool; +} + describe("buildOpenAIRealtimeVoiceProvider", () => { beforeEach(() => { FakeWebSocket.instances = []; @@ -472,6 +505,31 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { expect((session as { offerHeaders?: Record }).offerHeaders).toBeUndefined(); }); + it("omits unsupported OpenAI tool names from browser sessions", async () => { + fetchWithSsrFGuardMock.mockResolvedValueOnce({ + response: createJsonResponse({ client_secret: { value: "client-secret-123" } }), + release: vi.fn(async () => undefined), + }); + const provider = buildOpenAIRealtimeVoiceProvider(); + if (!provider.createBrowserSession) { + throw new Error("expected OpenAI realtime provider to support browser sessions"); + } + + await provider.createBrowserSession({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + tools: [ + createRealtimeTool("1_lookup"), + createRealtimeTool("calendar.lookup:next"), + createMalformedToolName(undefined), + createUnreadableToolName(), + ], + }); + + const bodySession = requireRecord(requireFetchJsonBody().session, "fetch session"); + const tools = bodySession.tools as Array<{ name?: string }>; + expect(tools.map((tool) => tool.name)).toEqual(["1_lookup"]); + }); + it("resolves keychain OPENAI_API_KEY refs before creating browser sessions", async () => { vi.stubEnv("OPENAI_API_KEY", "keychain:openclaw:OPENAI_REALTIME_BROWSER_TEST"); execFileSyncMock.mockReturnValueOnce("sk-browser-env\n"); // pragma: allowlist secret @@ -866,6 +924,40 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { ); }); + it("omits unsupported OpenAI tool names from GA session updates", async () => { + const provider = buildOpenAIRealtimeVoiceProvider(); + const bridge = provider.createBridge({ + providerConfig: { apiKey: "sk-test" }, // pragma: allowlist secret + tools: [ + createRealtimeTool("1_lookup"), + createRealtimeTool("calendar.lookup:next"), + createRealtimeTool("bad/name"), + createRealtimeTool("x".repeat(65)), + createMalformedToolName(null), + createMalformedToolName(42), + createUnreadableToolName(), + ], + onAudio: vi.fn(), + onClearAudio: vi.fn(), + }); + const connecting = bridge.connect(); + const socket = FakeWebSocket.instances[0]; + if (!socket) { + throw new Error("expected bridge to create a websocket"); + } + + socket.readyState = FakeWebSocket.OPEN; + socket.emit("open"); + + const tools = requireSession(socket).tools as Array<{ name?: string }>; + expect(tools.map((tool) => tool.name)).toEqual([ + "1_lookup", + "x".repeat(65), + ]); + socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); + await connecting; + }); + it("rotates realtime bridges on provider max-duration events without reporting an error", async () => { vi.useFakeTimers(); const provider = buildOpenAIRealtimeVoiceProvider(); @@ -953,6 +1045,11 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { }, audioFormat: REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, instructions: "Be helpful.", + tools: [ + createRealtimeTool("1_lookup"), + createRealtimeTool("calendar.lookup:next"), + createRealtimeTool("x".repeat(65)), + ], onAudio: vi.fn(), onClearAudio: vi.fn(), }); @@ -989,6 +1086,8 @@ describe("buildOpenAIRealtimeVoiceProvider", () => { ); expect(session).not.toHaveProperty("type"); expect(session).not.toHaveProperty("audio"); + const tools = session.tools as Array<{ name?: string }>; + expect(tools.map((tool) => tool.name)).toEqual(["1_lookup"]); socket.emit("message", Buffer.from(JSON.stringify({ type: "session.updated" }))); await connecting; diff --git a/extensions/openai/realtime-voice-provider.ts b/extensions/openai/realtime-voice-provider.ts index 9409b1821ac1..7b3ba472d275 100644 --- a/extensions/openai/realtime-voice-provider.ts +++ b/extensions/openai/realtime-voice-provider.ts @@ -23,6 +23,7 @@ import type { RealtimeVoiceTool, RealtimeVoiceToolResultOptions, } from "openclaw/plugin-sdk/realtime-voice"; +import { warn } from "openclaw/plugin-sdk/runtime-env"; import { REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ, @@ -93,6 +94,10 @@ const OPENAI_REALTIME_NO_ACTIVE_RESPONSE_CANCEL_ERROR = "Cancellation failed: no active response found"; const OPENAI_REALTIME_MAX_SESSION_DURATION_FRAGMENT = "maximum duration"; const OPENAI_REALTIME_DEFAULT_MIN_BARGE_IN_AUDIO_END_MS = 250; +// Realtime validates this character set but accepts names beyond the 64-character +// cap used by other OpenAI tool surfaces. +const OPENAI_REALTIME_TOOL_NAME_RE = /^[A-Za-z0-9_-]+$/; +const AZURE_OPENAI_REALTIME_TOOL_NAME_MAX_LENGTH = 64; const OPENAI_REALTIME_VOICES = [ "alloy", "ash", @@ -336,6 +341,40 @@ function hasOpenAIRealtimeApiKeyInput(configuredApiKey: string | undefined): boo ); } +function normalizeOpenAIRealtimeTools( + tools: RealtimeVoiceTool[] | undefined, + maxNameLength?: number, +): RealtimeVoiceTool[] | undefined { + const normalized: RealtimeVoiceTool[] = []; + let omitted = 0; + for (const tool of tools ?? []) { + try { + const name = tool.name; + if (typeof name !== "string") { + omitted += 1; + continue; + } + const exceedsLengthLimit = maxNameLength !== undefined && name.length > maxNameLength; + if (exceedsLengthLimit || !OPENAI_REALTIME_TOOL_NAME_RE.test(name)) { + omitted += 1; + continue; + } + normalized.push({ + type: "function", + name, + description: tool.description, + parameters: tool.parameters, + }); + } catch { + omitted += 1; + } + } + if (omitted > 0) { + warn(`openai realtime: omitted ${omitted} tool definition(s) with unsupported names`); + } + return normalized.length > 0 ? normalized : undefined; +} + async function resolveOpenAIRealtimePlatformApiKey(params: { configuredApiKey: string | undefined; cfg: RealtimeVoiceBrowserSessionCreateRequest["cfg"] | undefined; @@ -846,6 +885,7 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private buildGaSessionUpdate(): RealtimeGaSessionUpdate { const cfg = this.config; + const tools = normalizeOpenAIRealtimeTools(cfg.tools); return { type: "session.update", session: { @@ -866,9 +906,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { }, }, ...(cfg.reasoningEffort ? { reasoning: { effort: cfg.reasoningEffort } } : {}), - ...(cfg.tools && cfg.tools.length > 0 + ...(tools ? { - tools: cfg.tools, + tools, tool_choice: "auto", } : {}), @@ -883,6 +923,10 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { private buildAzureDeploymentSessionUpdate(): RealtimeAzureDeploymentSessionUpdate { const cfg = this.config; const format = this.resolveLegacyRealtimeAudioFormat(); + const tools = normalizeOpenAIRealtimeTools( + cfg.tools, + AZURE_OPENAI_REALTIME_TOOL_NAME_MAX_LENGTH, + ); return { type: "session.update", session: { @@ -894,9 +938,9 @@ class OpenAIRealtimeVoiceBridge implements RealtimeVoiceBridge { input_audio_transcription: { model: "whisper-1" }, turn_detection: this.buildTurnDetectionConfig(), temperature: cfg.temperature ?? 0.8, - ...(cfg.tools && cfg.tools.length > 0 + ...(tools ? { - tools: cfg.tools, + tools, tool_choice: "auto", } : {}), @@ -1399,6 +1443,7 @@ async function createOpenAIRealtimeBrowserSession( cfg: req.cfg, }); const voice = normalizeOpenAIRealtimeVoice(req.voice) ?? config.voice ?? "alloy"; + const tools = normalizeOpenAIRealtimeTools(req.tools); const session: Record = { type: "realtime", model, @@ -1425,8 +1470,8 @@ async function createOpenAIRealtimeBrowserSession( output: { voice }, }, }; - if (req.tools && req.tools.length > 0) { - session.tools = req.tools; + if (tools) { + session.tools = tools; session.tool_choice = "auto"; } const reasoningEffort = trimToUndefined(req.reasoningEffort) ?? config.reasoningEffort;