From 95e3b9243be8ee331f93e7571c47c4f7a5bbdfa5 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 6 Jul 2026 16:25:33 -0700 Subject: [PATCH] test: execute media and Talk runtime boundaries (#101091) * test: execute media and Talk runtime boundaries * test(qa): centralize gateway fixture startup * test(qa): check in voice call plugin fixture --- extensions/qa-lab/src/gateway-child.test.ts | 59 ++ extensions/qa-lab/src/gateway-child.ts | 97 ++- extensions/voice-call/src/config.test.ts | 14 +- extensions/voice-call/src/config.ts | 5 +- .../voice-call-cli-rpc-agent-tool.e2e.test.ts | 204 ------ packages/gateway-client/src/client.ts | 2 + .../src/client.watchdog.test.ts | 17 + qa/scenarios/media/webchat-auto-tts.yaml | 15 +- .../voice-call-cli-rpc-agent-tool.yaml | 25 +- .../runtime/active-talk-agent-run-status.yaml | 25 +- src/gateway/client.ts | 1 + src/gateway/server-methods/talk.test.ts | 32 + src/gateway/talk-agent-consult.ts | 87 ++- .../qa-lab/media/webchat-auto-tts.e2e.test.ts | 193 ------ .../voice-call-runtime-plugin/index.js | 83 +++ .../openclaw.plugin.json | 11 + .../gateway-mcp-real-transports.test.ts | 30 - .../runtime/gateway-mcp-real-transports.ts | 59 +- test/e2e/qa-lab/runtime/media-talk-gateway.ts | 615 ++++++++++++++++++ test/e2e/qa-lab/runtime/voice-call-gateway.ts | 364 +++++++++++ .../active-talk-agent-run-status.e2e.test.ts | 173 ----- 21 files changed, 1390 insertions(+), 721 deletions(-) delete mode 100644 extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts delete mode 100644 test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts create mode 100644 test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/index.js create mode 100644 test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/openclaw.plugin.json create mode 100644 test/e2e/qa-lab/runtime/media-talk-gateway.ts create mode 100644 test/e2e/qa-lab/runtime/voice-call-gateway.ts delete mode 100644 test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index 80a91d7b810c..ae68ec53b0e4 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -115,6 +115,65 @@ async function expectPathMissing(filePath: string): Promise { throw new Error(`expected ${filePath} to be missing`); } +describe("runQaGatewayCliCommand", () => { + it("runs CLI commands with the Gateway fixture environment", async () => { + const output = await testing.runQaGatewayCliCommand({ + executablePath: process.execPath, + argsPrefix: [ + "--eval", + 'process.stdout.write(`${process.env.OPENCLAW_CLI}:${process.env.QA_VALUE}:${process.argv.slice(1).join(",")}`)', + ], + args: ["voicecall", "start"], + cwd: process.cwd(), + env: { ...process.env, QA_VALUE: "fixture" }, + }); + + expect(output).toBe("1:fixture:voicecall,start"); + }); + + it("reports CLI stderr when a fixture command fails", async () => { + await expect( + testing.runQaGatewayCliCommand({ + executablePath: process.execPath, + argsPrefix: ["--eval", 'process.stderr.write("fixture failure"); process.exit(7)'], + args: [], + cwd: process.cwd(), + env: process.env, + }), + ).rejects.toThrow("OpenClaw CLI exited 7: fixture failure"); + }); +}); + +describe("Gateway child fixture helpers", () => { + it("creates an empty transport config seam", () => { + expect(testing.createQaGatewayEmptyTransport()).toEqual({ + requiredPluginIds: [], + createGatewayConfig: expect.any(Function), + }); + }); + + it("resolves source and built Gateway CLI commands", async () => { + const repoRoot = await tempDirs.makeTempDir("qa-gateway-command-"); + await mkdir(path.join(repoRoot, "src"), { recursive: true }); + await writeFile(path.join(repoRoot, "src", "entry.ts"), "export {};\n", "utf8"); + + expect(testing.resolveQaGatewayChildCommand(repoRoot)).toEqual({ + executablePath: process.execPath, + argsPrefix: ["--import", "tsx", path.join(repoRoot, "src", "entry.ts")], + cwd: repoRoot, + }); + + await mkdir(path.join(repoRoot, "dist"), { recursive: true }); + await writeFile(path.join(repoRoot, "dist", "index.js"), "export {};\n", "utf8"); + expect(testing.resolveQaGatewayChildCommand(repoRoot)).toEqual({ + executablePath: process.execPath, + argsPrefix: [path.join(repoRoot, "dist", "index.js")], + cwd: repoRoot, + usePackagedPlugins: true, + }); + }); +}); + describe("buildQaRuntimeEnv", () => { it("cleans up temp QA gateway roots when node path resolution fails before startup", async () => { const tempParent = await mkdtemp(path.join(os.tmpdir(), "qa-gateway-node-exec-fail-")); diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index ba88e6c03fd3..2fb18be8d855 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -24,6 +24,14 @@ import { resolveQaOwnerPluginIdsForProviderIds, resolveQaRuntimeHostVersion, } from "./bundled-plugin-staging.js"; +import { + appendQaChildOutput, + appendQaChildOutputTail, + createQaChildOutputCapture, + createQaChildOutputTail, + formatQaChildOutputTail, + readQaChildOutput, +} from "./child-output.js"; import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js"; import { QaSuiteInfraError, toQaErrorObject } from "./errors.js"; import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js"; @@ -86,6 +94,66 @@ export type QaGatewayChildListeningContext = { runtimeEnv: NodeJS.ProcessEnv; }; +function createQaGatewayEmptyTransport() { + return { + requiredPluginIds: [] as const, + createGatewayConfig: () => ({}), + } satisfies Pick; +} + +function resolveQaGatewayChildCommand(repoRoot: string): QaGatewayChildCommand { + for (const relativePath of ["dist/index.mjs", "dist/index.js"]) { + const entryPath = path.join(repoRoot, relativePath); + if (existsSync(entryPath)) { + return { + executablePath: process.execPath, + argsPrefix: [entryPath], + cwd: repoRoot, + usePackagedPlugins: true, + }; + } + } + + const sourceEntryPath = path.join(repoRoot, "src/entry.ts"); + if (existsSync(sourceEntryPath)) { + return { + executablePath: process.execPath, + argsPrefix: ["--import", "tsx", sourceEntryPath], + cwd: repoRoot, + }; + } + + throw new Error("OpenClaw CLI entry not found: expected dist/index.(m)js or src/entry.ts"); +} + +async function runQaGatewayCliCommand(params: { + executablePath: string; + argsPrefix: readonly string[]; + args: readonly string[]; + cwd: string; + env: NodeJS.ProcessEnv; +}): Promise { + const stdout = createQaChildOutputCapture(); + const stderr = createQaChildOutputTail(); + const child = spawn(params.executablePath, [...params.argsPrefix, ...params.args], { + cwd: params.cwd, + env: { ...params.env, OPENCLAW_CLI: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.on("data", (chunk) => appendQaChildOutput(stdout, chunk)); + child.stderr.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk)); + const exitCode = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => resolve(code ?? 1)); + }); + const stdoutText = readQaChildOutput(stdout); + if (exitCode !== 0) { + const stderrText = formatQaChildOutputTail(stderr, "stderr"); + throw new Error(`OpenClaw CLI exited ${exitCode}: ${stderrText || stdoutText}`); + } + return stdoutText; +} + async function getFreePort() { return await new Promise((resolve, reject) => { const server = net.createServer(); @@ -378,6 +446,8 @@ export const testing = { redactQaGatewayDebugText, readQaLiveProviderConfigOverrides, resolveQaGatewayChildProviderMode, + resolveQaGatewayChildCommand, + createQaGatewayEmptyTransport, assertQaLiveCodexAuthAvailable, stageQaLiveApiKeyProfiles, stageQaLiveAnthropicSetupToken, @@ -387,6 +457,7 @@ export const testing = { resolveQaOwnerPluginIdsForProviderIds, resolveQaBundledPluginSourceDir, resolveQaRuntimeHostVersion, + runQaGatewayCliCommand, createQaGatewayChildLogCollector, createQaBundledPluginsDir, signalQaGatewayChildProcessTree, @@ -671,8 +742,9 @@ export function resolveQaControlUiRoot(params: { repoRoot: string; controlUiEnab export async function startQaGatewayChild(params: { repoRoot: string; command?: QaGatewayChildCommand; + useRepoCli?: boolean; providerBaseUrl?: string; - transport: Pick; + transport?: Pick; transportBaseUrl: string; controlUiAllowedOrigins?: string[]; providerMode?: QaProviderMode; @@ -694,7 +766,9 @@ export async function startQaGatewayChild(params: { ); const runtimeCwd = tempRoot; const distEntryPath = path.join(params.repoRoot, "dist", "index.js"); - const gatewayCommand = params.command; + const gatewayCommand = + params.command ?? + (params.useRepoCli ? resolveQaGatewayChildCommand(params.repoRoot) : undefined); const gatewayExecutablePath = gatewayCommand?.executablePath; const gatewayArgsPrefix = gatewayCommand?.argsPrefix ?? []; const gatewayArgsSuffix = gatewayCommand?.argsSuffix ?? []; @@ -707,6 +781,7 @@ export async function startQaGatewayChild(params: { const xdgCacheHome = path.join(tempRoot, "xdg-cache"); const configPath = path.join(tempRoot, "openclaw.json"); const gatewayToken = `qa-suite-${randomUUID()}`; + const transport = params.transport ?? createQaGatewayEmptyTransport(); await seedQaAgentWorkspace({ workspaceDir, repoRoot: params.repoRoot, @@ -757,8 +832,8 @@ export async function startQaGatewayChild(params: { primaryModel: params.primaryModel, alternateModel: params.alternateModel, enabledPluginIds, - transportPluginIds: params.transport.requiredPluginIds, - transportConfig: params.transport.createGatewayConfig({ + transportPluginIds: transport.requiredPluginIds, + transportConfig: transport.createGatewayConfig({ baseUrl: params.transportBaseUrl, }), liveProviderConfigs, @@ -809,8 +884,11 @@ export async function startQaGatewayChild(params: { try { const nodeExecPath = gatewayExecutablePath ?? (await resolveQaNodeExecPath()); + const cliArgsPrefix = gatewayExecutablePath + ? gatewayArgsPrefix + : [distEntryPath, ...gatewayArgsPrefix]; const buildGatewayArgs = () => [ - ...(gatewayExecutablePath ? gatewayArgsPrefix : [distEntryPath, ...gatewayArgsPrefix]), + ...cliArgsPrefix, "gateway", "run", "--port", @@ -1104,6 +1182,15 @@ export async function startQaGatewayChild(params: { configPath, runtimeEnv: runningEnv, logs, + runCli(args: readonly string[]) { + return runQaGatewayCliCommand({ + executablePath: nodeExecPath, + argsPrefix: cliArgsPrefix, + args, + cwd: gatewayCwd, + env: runningEnv, + }); + }, signalProcess(signal: NodeJS.Signals) { if (!activeChild.pid) { throw new Error("qa gateway child has no pid"); diff --git a/extensions/voice-call/src/config.test.ts b/extensions/voice-call/src/config.test.ts index a58eff844350..baecfe1cf0ac 100644 --- a/extensions/voice-call/src/config.test.ts +++ b/extensions/voice-call/src/config.test.ts @@ -262,10 +262,20 @@ describe("validateProviderConfig", () => { const result = validateProviderConfig(config); expect(result.errors).not.toContain( - 'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true', + 'plugins.entries.voice-call.config.provider must be "twilio", "telnyx", or "mock" when realtime.enabled is true', ); }); + it("accepts realtime.enabled with provider=mock", () => { + const config = createBaseConfig("mock"); + config.realtime.enabled = true; + config.inboundPolicy = "allowlist"; + + const result = validateProviderConfig(config); + + expect(result).toEqual({ valid: true, errors: [] }); + }); + it("rejects realtime.enabled with providers that do not support it yet", () => { const config = createBaseConfig("plivo"); config.realtime.enabled = true; @@ -275,7 +285,7 @@ describe("validateProviderConfig", () => { expect(result.valid).toBe(false); expect(result.errors).toContain( - 'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true', + 'plugins.entries.voice-call.config.provider must be "twilio", "telnyx", or "mock" when realtime.enabled is true', ); }); }); diff --git a/extensions/voice-call/src/config.ts b/extensions/voice-call/src/config.ts index 14f0cceda6a7..e9ee315a67fd 100644 --- a/extensions/voice-call/src/config.ts +++ b/extensions/voice-call/src/config.ts @@ -924,10 +924,11 @@ export function validateProviderConfig(config: VoiceCallConfig): { config.realtime.enabled && config.provider && config.provider !== "twilio" && - config.provider !== "telnyx" + config.provider !== "telnyx" && + config.provider !== "mock" ) { errors.push( - 'plugins.entries.voice-call.config.provider must be "twilio" or "telnyx" when realtime.enabled is true', + 'plugins.entries.voice-call.config.provider must be "twilio", "telnyx", or "mock" when realtime.enabled is true', ); } diff --git a/extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts b/extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts deleted file mode 100644 index 71a3aecfbb10..000000000000 --- a/extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts +++ /dev/null @@ -1,204 +0,0 @@ -// Voice Call E2E tests cover CLI, Gateway RPC, and agent tool through the mock provider. -import fs from "node:fs"; -import { createServer } from "node:net"; -import path from "node:path"; -import { Command } from "commander"; -import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; -import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; -import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import plugin from "../index.js"; -import { testing as voiceCallCliTesting } from "./cli.js"; -import { - createVoiceCallStateRuntimeForTests, - createTestStorePath, - installVoiceCallStateRuntimeForTests, -} from "./manager.test-harness.js"; -import { clearVoiceCallStateRuntime } from "./runtime-state.js"; -import { createVoiceCallBaseConfig } from "./test-fixtures.js"; - -type GatewayHandler = (request: { - params?: Record; - respond: (ok: boolean, payload?: unknown, error?: { message?: string }) => void; -}) => Promise; - -function asOptionalRecord(value: unknown): Record | undefined { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : undefined; -} - -async function reserveLocalPort(): Promise { - return await new Promise((resolve, reject) => { - const server = createServer(); - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - if (!address || typeof address === "string") { - server.close(); - reject(new Error("failed to reserve local voice-call port")); - return; - } - server.close((error) => (error ? reject(error) : resolve(address.port))); - }); - }); -} - -function isAddressInUseError(error: unknown): boolean { - return error instanceof Error && `${error.message}\n${error.stack ?? ""}`.includes("EADDRINUSE"); -} - -const tempDirs = new Set(); - -async function runVoiceCallEntryPointFixture(): Promise { - const stateDir = createTestStorePath(); - tempDirs.add(stateDir); - vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); - resetPluginStateStoreForTests(); - installVoiceCallStateRuntimeForTests(); - const stateRuntime = createVoiceCallStateRuntimeForTests(); - const config = createVoiceCallBaseConfig({ provider: "mock" }); - config.maxConcurrentCalls = 3; - config.store = path.join(stateDir, "voice-calls"); - config.serve.port = await reserveLocalPort(); - - const methods = new Map(); - const tools: Array<{ - name: string; - execute: (toolCallId: string, params: unknown) => Promise<{ details?: unknown }>; - }> = []; - let cliRegistrar: ((context: { program: Command }) => void) | undefined; - let service: - | { - stop?: (context: { config: unknown; stateDir: string; logger: unknown }) => Promise; - } - | undefined; - const logger = { info() {}, warn() {}, error() {}, debug() {} }; - const api = createTestPluginApi({ - id: "voice-call", - name: "Voice Call", - source: "qa", - config: {}, - pluginConfig: config, - logger, - runtime: { - agent: {}, - state: stateRuntime, - tts: { textToSpeechTelephony: vi.fn() }, - } as unknown as OpenClawPluginApi["runtime"], - registerGatewayMethod: (method, handler) => { - methods.set(method, handler as GatewayHandler); - }, - registerTool: (tool) => { - tools.push(tool as (typeof tools)[number]); - }, - registerCli: (registrar) => { - cliRegistrar = registrar as typeof cliRegistrar; - }, - registerService: (registeredService) => { - service = registeredService as typeof service; - }, - }); - plugin.register(api); - - const invokeGateway = async ( - method: string, - params?: Record, - ): Promise> => { - const handler = methods.get(method); - if (!handler) { - throw new Error(`missing Gateway handler: ${method}`); - } - return await new Promise>((resolve, reject) => { - void handler({ - params, - respond(ok, payload, error) { - if (ok) { - const record = asOptionalRecord(payload); - if (!record) { - reject(new Error(`${method} returned a non-object payload`)); - return; - } - resolve(record); - return; - } - reject(new Error(error?.message ?? `${method} failed`)); - }, - }).catch(reject); - }); - }; - - voiceCallCliTesting.setCallGatewayFromCliForTests(async (method, _options, params) => - invokeGateway(method, asOptionalRecord(params)), - ); - const program = new Command(); - cliRegistrar?.({ program }); - const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - - try { - await program.parseAsync( - ["voicecall", "start", "--to", "+15550001111", "--message", "CLI fixture"], - { from: "user" }, - ); - - const rpcResult = await invokeGateway("voicecall.initiate", { - to: "+15550002222", - message: "RPC fixture", - mode: "notify", - }); - expect(rpcResult).toMatchObject({ initiated: true, callId: expect.any(String) }); - - const voiceCallTool = tools.find((tool) => tool.name === "voice_call"); - if (!voiceCallTool) { - throw new Error("voice_call tool was not registered"); - } - const toolResult = await voiceCallTool.execute("tool-call", { - action: "initiate_call", - to: "+15550003333", - message: "Agent tool fixture", - mode: "conversation", - }); - expect(toolResult.details).toMatchObject({ initiated: true, callId: expect.any(String) }); - - const status = (await invokeGateway("voicecall.status")) as { calls?: unknown[] }; - expect(status.calls).toHaveLength(3); - expect(stdout).toHaveBeenCalled(); - } finally { - await service?.stop?.({ config: {}, stateDir, logger }).catch(() => undefined); - voiceCallCliTesting.setCallGatewayFromCliForTests(); - clearVoiceCallStateRuntime(); - resetPluginStateStoreForTests(); - vi.unstubAllEnvs(); - vi.restoreAllMocks(); - } -} - -describe("QA Voice Call CLI, RPC, and agent tool", () => { - afterEach(() => { - voiceCallCliTesting.setCallGatewayFromCliForTests(); - clearVoiceCallStateRuntime(); - resetPluginStateStoreForTests(); - vi.unstubAllEnvs(); - vi.restoreAllMocks(); - for (const dir of tempDirs) { - fs.rmSync(dir, { force: true, recursive: true }); - } - tempDirs.clear(); - }); - - it("routes all three entry points through one mock-provider runtime", async () => { - let lastError: unknown; - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await runVoiceCallEntryPointFixture(); - return; - } catch (error) { - lastError = error; - if (!isAddressInUseError(error)) { - throw error; - } - } - } - throw lastError; - }); -}); diff --git a/packages/gateway-client/src/client.ts b/packages/gateway-client/src/client.ts index 113083d8b62b..177777be7463 100644 --- a/packages/gateway-client/src/client.ts +++ b/packages/gateway-client/src/client.ts @@ -416,6 +416,7 @@ export function isGatewayConnectAssemblyError(value: unknown): value is Error { export type GatewayClientOptions = { url?: string; // ws://127.0.0.1:18789 + origin?: string; connectChallengeTimeoutMs?: number; /** @deprecated Use connectChallengeTimeoutMs. */ connectDelayMs?: number; @@ -637,6 +638,7 @@ export class GatewayClient { this.deps.beforeConnect(); const wsOptions: FingerprintCheckingClientOptions = { maxPayload: 25 * 1024 * 1024, + ...(this.opts.origin ? { origin: this.opts.origin } : {}), }; if (url.startsWith("wss://") && this.opts.tlsFingerprint) { wsOptions.rejectUnauthorized = false; diff --git a/packages/gateway-client/src/client.watchdog.test.ts b/packages/gateway-client/src/client.watchdog.test.ts index 2e565b2b7029..b20de4436329 100644 --- a/packages/gateway-client/src/client.watchdog.test.ts +++ b/packages/gateway-client/src/client.watchdog.test.ts @@ -98,6 +98,23 @@ describe("GatewayClient", () => { } }); + test("sends the configured websocket origin", async () => { + const port = await getFreePort(); + wss = new WebSocketServer({ port, host: "127.0.0.1" }); + const receivedOrigin = new Promise((resolve) => { + wss?.once("connection", (_socket, request) => resolve(request.headers.origin)); + }); + const client = new GatewayClient({ + url: `ws://127.0.0.1:${port}`, + origin: `http://127.0.0.1:${port}`, + connectChallengeTimeoutMs: 0, + }); + client.start(); + + await expect(receivedOrigin).resolves.toBe(`http://127.0.0.1:${port}`); + client.stop(); + }); + test("prefers connectChallengeTimeoutMs and still honors the legacy alias", () => { expect(resolveGatewayClientConnectChallengeTimeoutMs({})).toBe( DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS, diff --git a/qa/scenarios/media/webchat-auto-tts.yaml b/qa/scenarios/media/webchat-auto-tts.yaml index 9736683d8dc5..4f1c93a65a55 100644 --- a/qa/scenarios/media/webchat-auto-tts.yaml +++ b/qa/scenarios/media/webchat-auto-tts.yaml @@ -5,7 +5,7 @@ scenario: surface: media-understanding-and-media-generation category: media-understanding-and-media-generation.text-to-speech-delivery coverage: - secondary: + primary: - media.tts - media.outbound-voice-audio-delivery objective: Verify WebChat auto-TTS synthesizes only the final reply tail and serves trusted local audio through scoped browser media tickets. @@ -22,8 +22,13 @@ scenario: - packages/speech-core/src/tts.ts - src/gateway/server-methods/chat-webchat-media.ts - src/gateway/control-ui.ts - - test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts + - test/e2e/qa-lab/runtime/media-talk-gateway.ts execution: - kind: vitest - path: test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts - summary: Vitest QA Lab coverage for mock WebChat TTS synthesis and real scoped media-ticket delivery. + kind: script + path: test/e2e/qa-lab/runtime/media-talk-gateway.ts + summary: Starts a real Gateway and mock model/TTS providers, sends a WebChat turn, and verifies the history attachment plus scoped media-ticket route. + args: + - --scenario + - webchat-auto-tts + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml b/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml index ac28ce596312..902de1369e81 100644 --- a/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml +++ b/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml @@ -1,4 +1,4 @@ -title: Voice Call CLI, RPC, and agent tool mock-provider flow +title: Voice Call CLI, RPC, agent tool, and realtime consult flow scenario: id: voice-call-cli-rpc-agent-tool @@ -7,12 +7,13 @@ scenario: coverage: primary: - voice-call.cli-rpc-agent-tool - objective: Verify the Voice Call CLI, Gateway RPC, and agent tool share one executable mock-provider runtime. + objective: Verify the real Voice Call CLI, Gateway RPC, and Gateway-invoked agent tool share one mock-provider runtime whose realtime media stream reaches embedded-agent consult context. successCriteria: - The CLI starts an outbound call through its Gateway RPC path. - - The registered Gateway RPC starts an outbound call through the mock provider. - - The registered agent tool starts an outbound call through the same runtime. - - Runtime status reports all calls and cleanup stops the local webhook fixture. + - The registered Gateway RPC starts an outbound call through the same mock provider runtime. + - Gateway tools.invoke starts an outbound call through the registered voice_call agent tool. + - The runtime issues a real webhook media-stream token for the tool-started call. + - The fake realtime provider invokes openclaw_agent_consult and the embedded agent receives caller transcript and provider context. docsRefs: - docs/cli/voicecall.md - docs/plugins/voice-call.md @@ -20,9 +21,13 @@ scenario: codeRefs: - extensions/voice-call/index.ts - extensions/voice-call/src/cli.ts - - extensions/voice-call/src/manager.test-harness.ts - - extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts + - extensions/voice-call/src/runtime.ts + - extensions/voice-call/src/webhook/realtime-handler.ts + - test/e2e/qa-lab/runtime/voice-call-gateway.ts execution: - kind: vitest - path: extensions/voice-call/src/voice-call-cli-rpc-agent-tool.e2e.test.ts - summary: Vitest QA Lab coverage for Voice Call CLI, RPC, and agent tool entry points using the mock provider. + kind: script + path: test/e2e/qa-lab/runtime/voice-call-gateway.ts + summary: Starts a real Gateway and Voice Call plugin, runs the actual CLI plus RPC and tools.invoke entry points, then drives the runtime-issued realtime media stream through embedded-agent consult. + args: + - --artifact-base + - ${outputDir} diff --git a/qa/scenarios/runtime/active-talk-agent-run-status.yaml b/qa/scenarios/runtime/active-talk-agent-run-status.yaml index ec847e308574..bfc2d345a095 100644 --- a/qa/scenarios/runtime/active-talk-agent-run-status.yaml +++ b/qa/scenarios/runtime/active-talk-agent-run-status.yaml @@ -5,22 +5,27 @@ scenario: surface: voice-and-realtime-talk category: voice-and-realtime-talk.realtime-talk-sessions coverage: - secondary: + primary: - voice.active-talk-agent-run-status - objective: Verify a mock realtime Talk session wires status, steering, follow-up, and cancellation through the active-run control contract. + objective: Verify a real Gateway Talk session drives status, steering, follow-up, and cancellation against one active agent run. successCriteria: - - A registered mock realtime provider creates a browser-owned Talk session with consult and control tools. - - Status formatting reports the latest supplied non-control tool progress. - - Steering and follow-up invoke the injected queue boundary with the expected modes. - - Cancellation invokes the injected abort boundary for the resolved active session. + - A registered mock realtime provider creates a browser-owned Talk session with consult and control tools through Gateway RPC. + - A consult tool call starts a real active agent run through the Gateway chat path. + - Status, steering, and follow-up RPCs observe and queue work to that active run. + - Cancellation RPC aborts that same active run from the owning browser connection. docsRefs: - docs/nodes/talk.md - docs/web/control-ui.md codeRefs: - src/gateway/server-methods/talk-client.ts - src/talk/agent-run-control.ts - - test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts + - test/e2e/qa-lab/runtime/media-talk-gateway.ts execution: - kind: vitest - path: test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts - summary: Vitest QA Lab boundary coverage for mock realtime session creation and active Talk run-control dependencies. + kind: script + path: test/e2e/qa-lab/runtime/media-talk-gateway.ts + summary: Starts a real Gateway and persistent WebChat client, creates a Talk session, starts an agent consult, and exercises status, steer, follow-up, and cancel RPCs. + args: + - --scenario + - active-talk-agent-run-status + - --artifact-base + - ${outputDir} diff --git a/src/gateway/client.ts b/src/gateway/client.ts index 555311fcc2f0..3810a0584b00 100644 --- a/src/gateway/client.ts +++ b/src/gateway/client.ts @@ -119,6 +119,7 @@ export function isGatewayConnectAssemblyError(value: unknown): value is Error { export type GatewayClientOptions = { url?: string; + origin?: string; connectChallengeTimeoutMs?: number; /** @deprecated Use connectChallengeTimeoutMs. */ connectDelayMs?: number; diff --git a/src/gateway/server-methods/talk.test.ts b/src/gateway/server-methods/talk.test.ts index 0292391413e5..136ae14e7f58 100644 --- a/src/gateway/server-methods/talk.test.ts +++ b/src/gateway/server-methods/talk.test.ts @@ -2249,6 +2249,38 @@ describe("talk.client.toolCall handler", () => { expect(response.idempotencyKey).toMatch(/^talk-call-1-/); }); + it("returns the tool-call acknowledgement while the agent run continues", async () => { + let finishRun: (() => void) | undefined; + mocks.chatSend.mockImplementationOnce( + ({ respond }: { respond: (ok: boolean, result?: unknown, error?: unknown) => void }) => + new Promise((resolve) => { + finishRun = resolve; + respond(true, { runId: "run-active" }, undefined); + }), + ); + const respond = vi.fn(); + + await talkHandlers["talk.client.toolCall"]({ + req: { type: "req", id: "1", method: "talk.client.toolCall" }, + params: { + sessionKey: "main", + callId: "call-active", + name: "openclaw_agent_consult", + args: { question: "What is running?" }, + }, + client: { connId: "conn-1" } as never, + isWebchatConnect: () => false, + respond: respond as never, + context: { + getRuntimeConfig: () => ({}) as OpenClawConfig, + logGateway: { warn: vi.fn() }, + } as never, + }); + + expectRespondOk(respond, { runId: "run-active" }); + finishRun?.(); + }); + it("passes configured consult thinking and fast-mode overrides to chat.send", async () => { const respond = vi.fn(); diff --git a/src/gateway/talk-agent-consult.ts b/src/gateway/talk-agent-consult.ts index 70d092d28941..2dfe1da63e66 100644 --- a/src/gateway/talk-agent-consult.ts +++ b/src/gateway/talk-agent-consult.ts @@ -76,36 +76,63 @@ export async function startTalkRealtimeAgentConsult(params: { } const idempotencyKey = `talk-${params.callId}-${randomUUID()}`; const normalizedTalk = normalizeTalkSection(params.context.getRuntimeConfig().talk); - let chatResponse: { ok: true; result: unknown } | { ok: false; error: ErrorShape } | undefined; - await chatHandlers["chat.send"]({ - req: { - type: "req", - id: `${params.requestId}:talk-tool-call`, - method: "chat.send", - }, - client: params.client, - isWebchatConnect: params.isWebchatConnect, - context: params.context, - params: { - sessionKey: params.sessionKey, - message, - idempotencyKey, - ...(normalizedTalk?.consultThinkingLevel - ? { thinking: normalizedTalk.consultThinkingLevel } - : {}), - ...(typeof normalizedTalk?.consultFastMode === "boolean" - ? { fastMode: normalizedTalk.consultFastMode } - : {}), - }, - respond: (ok: boolean, result?: unknown, error?: ErrorShape) => { - chatResponse = ok - ? { ok: true, result } - : { - ok: false, - error: error ?? errorShape(ErrorCodes.UNAVAILABLE, "chat.send failed without error"), - }; - }, - } as Parameters[0]); + const chatResponse = await new Promise< + { ok: true; result: unknown } | { ok: false; error: ErrorShape } | undefined + >((resolve) => { + let acknowledged = false; + const chatSendResult = chatHandlers["chat.send"]({ + req: { + type: "req", + id: `${params.requestId}:talk-tool-call`, + method: "chat.send", + }, + client: params.client, + isWebchatConnect: params.isWebchatConnect, + context: params.context, + params: { + sessionKey: params.sessionKey, + message, + idempotencyKey, + ...(normalizedTalk?.consultThinkingLevel + ? { thinking: normalizedTalk.consultThinkingLevel } + : {}), + ...(typeof normalizedTalk?.consultFastMode === "boolean" + ? { fastMode: normalizedTalk.consultFastMode } + : {}), + }, + respond: (ok: boolean, result?: unknown, error?: ErrorShape) => { + acknowledged = true; + resolve( + ok + ? { ok: true, result } + : { + ok: false, + error: + error ?? errorShape(ErrorCodes.UNAVAILABLE, "chat.send failed without error"), + }, + ); + }, + } as Parameters[0]); + void Promise.resolve(chatSendResult).then( + () => { + if (!acknowledged) { + resolve(undefined); + } + }, + (error: unknown) => { + if (acknowledged) { + params.context.logGateway.warn( + `realtime Talk agent consult failed after acknowledgement: ${formatForLog(error)}`, + ); + return; + } + resolve({ + ok: false, + error: errorShape(ErrorCodes.UNAVAILABLE, formatForLog(error)), + }); + }, + ); + }); if (!chatResponse) { return { diff --git a/test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts b/test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts deleted file mode 100644 index 45dd2fa3d46c..000000000000 --- a/test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { afterEach, describe, expect, it } from "vitest"; -import { maybeApplyTtsToPayload } from "../../../../packages/speech-core/src/tts.ts"; -import { setRuntimeConfigSnapshot } from "../../../../src/config/config.ts"; -import { buildWebchatAudioContentBlocksFromReplyPayloads } from "../../../../src/gateway/server-methods/chat-webchat-media.ts"; -import { - installGatewayTestHooks, - setTestPluginRegistry, - testState, - withGatewayServer, -} from "../../../../src/gateway/test-helpers.ts"; -import { createPluginRecord } from "../../../../src/plugins/loader-records.ts"; -import { createPluginRegistry } from "../../../../src/plugins/registry.ts"; -import { getActivePluginRegistry } from "../../../../src/plugins/runtime.ts"; -import { resetPluginRuntimeStateForTest } from "../../../../src/plugins/runtime.ts"; -import { getSpeechProvider } from "../../../../src/tts/provider-registry.ts"; - -installGatewayTestHooks({ scope: "suite" }); - -const CONTROL_UI_E2E_TOKEN = "test-gateway-token-1234567890"; - -const noopLogger = { - info() {}, - warn() {}, - error() {}, - debug() {}, -}; - -function installMockTtsProvider() { - const registry = createPluginRegistry({ - logger: noopLogger, - runtime: {}, - activateGlobalSideEffects: false, - }); - const record = createPluginRecord({ - id: "qa-webchat-auto-tts", - name: "QA WebChat Auto TTS", - source: "test/e2e/qa-lab/media/webchat-auto-tts.e2e.test.ts", - origin: "global", - enabled: true, - configSchema: false, - }); - const synthesizeCalls: string[] = []; - - registry.registerSpeechProvider(record, { - id: "mock", - label: "Mock", - autoSelectOrder: 1, - isConfigured: () => true, - synthesize: async (request) => { - synthesizeCalls.push(request.text); - return { - audioBuffer: Buffer.from("voice"), - fileExtension: ".ogg", - outputFormat: "ogg", - voiceCompatible: request.target === "voice-note", - }; - }, - }); - setTestPluginRegistry(registry.registry); - return synthesizeCalls; -} - -describe("QA WebChat auto TTS", () => { - afterEach(() => { - resetPluginRuntimeStateForTest(); - }); - - it("synthesizes only the final WebChat tail and exposes trusted local audio", async () => { - resetPluginRuntimeStateForTest(); - const synthesizeCalls = installMockTtsProvider(); - const prefsPath = path.join(os.tmpdir(), `openclaw-webchat-tts-${process.pid}.json`); - let mediaPath: string | undefined; - - try { - const text = "WebChat streams block text; dispatch synthesizes one TTS tail with kind final."; - const cfg = { - messages: { - tts: { - enabled: true, - provider: "mock", - prefsPath, - }, - }, - } satisfies OpenClawConfig; - setRuntimeConfigSnapshot(cfg, cfg); - - expect(getActivePluginRegistry()?.speechProviders.map((entry) => entry.provider.id)).toEqual([ - "mock", - ]); - expect(getSpeechProvider("mock", cfg)?.id).toBe("mock"); - - const blockResult = await maybeApplyTtsToPayload({ - payload: { text }, - cfg, - channel: "webchat", - kind: "block", - }); - expect(blockResult.mediaUrl).toBeUndefined(); - expect(blockResult.text).toBe(text); - expect(synthesizeCalls).toEqual([]); - - const tailResult = await maybeApplyTtsToPayload({ - payload: { text }, - cfg, - channel: "webchat", - kind: "final", - }); - mediaPath = tailResult.mediaUrl; - - expect(synthesizeCalls).toEqual([text]); - expect(mediaPath).toMatch(/voice-\d+\.ogg$/); - if (!mediaPath || !fs.existsSync(mediaPath)) { - throw new Error("expected final WebChat TTS to write local audio"); - } - expect(tailResult).toMatchObject({ - spokenText: text, - trustedLocalMedia: true, - }); - - const trustedBlocks = await buildWebchatAudioContentBlocksFromReplyPayloads( - [ - { - mediaUrl: mediaPath, - audioAsVoice: tailResult.audioAsVoice, - spokenText: text, - trustedLocalMedia: true, - }, - ], - { localRoots: [path.dirname(mediaPath)] }, - ); - const untrustedBlocks = await buildWebchatAudioContentBlocksFromReplyPayloads( - [{ mediaUrl: mediaPath }], - { localRoots: [path.dirname(mediaPath)] }, - ); - - expect(trustedBlocks).toHaveLength(1); - expect(trustedBlocks[0]).toMatchObject({ - type: "attachment", - attachment: { - kind: "audio", - label: path.basename(mediaPath), - mimeType: "audio/ogg", - url: fs.realpathSync(mediaPath), - }, - }); - expect(untrustedBlocks).toHaveLength(0); - - const source = trustedBlocks[0]?.type === "attachment" ? trustedBlocks[0].attachment.url : ""; - testState.gatewayAuth = { mode: "token", token: CONTROL_UI_E2E_TOKEN }; - await withGatewayServer( - async ({ port }) => { - const route = `http://127.0.0.1:${port}/__openclaw__/assistant-media`; - const sourceParam = encodeURIComponent(source); - const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, { - headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` }, - }); - expect(metadata.status).toBe(200); - const ticket = (await metadata.json()) as { - available?: boolean; - mediaTicket?: string; - }; - expect(ticket.available).toBe(true); - expect(ticket.mediaTicket).toMatch(/^v1\./); - - const withoutTicket = await fetch(`${route}?source=${sourceParam}`); - expect(withoutTicket.status).toBe(401); - - const ticketed = await fetch( - `${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(ticket.mediaTicket ?? "")}`, - ); - expect(ticketed.status).toBe(200); - expect(ticketed.headers.get("content-type")).toContain("audio/ogg"); - expect(Buffer.from(await ticketed.arrayBuffer())).toEqual(Buffer.from("voice")); - }, - { - serverOptions: { - auth: { mode: "token", token: CONTROL_UI_E2E_TOKEN }, - controlUiEnabled: true, - }, - }, - ); - } finally { - if (mediaPath) { - fs.rmSync(path.dirname(mediaPath), { recursive: true, force: true }); - } - fs.rmSync(prefsPath, { force: true }); - } - }); -}); diff --git a/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/index.js b/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/index.js new file mode 100644 index 000000000000..c00688dee4a5 --- /dev/null +++ b/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/index.js @@ -0,0 +1,83 @@ +import fs from "node:fs"; + +const append = (filePath, value) => { + fs.appendFileSync(filePath, `${JSON.stringify(value)}\n`); +}; + +export default { + id: "qa-voice-call-runtime", + register(api) { + api.registerRealtimeVoiceProvider({ + id: "qa-voice-call-realtime", + label: "QA Voice Call Realtime", + isConfigured: () => true, + createBrowserSession() { + throw new Error("Voice Call fixture is bridge-only"); + }, + createBridge(request) { + append(process.env.OPENCLAW_QA_VOICE_BRIDGE_CALLS_PATH, { + instructions: request.instructions, + tools: request.tools?.map((tool) => tool.name) ?? [], + }); + let connected = false; + return { + supportsToolResultContinuation: true, + async connect() { + connected = true; + request.onReady?.(); + request.onTranscript?.("user", "Caller partial transcript context", false); + setTimeout(() => { + request.onToolCall?.({ + itemId: "qa-consult-item", + callId: "qa-consult-call", + name: "openclaw_agent_consult", + args: { + question: "Use the embedded agent context to identify the caller request.", + context: "Realtime provider context marker: VOICE-CONSULT-42", + }, + }); + }, 10); + }, + sendAudio() {}, + setMediaTimestamp() {}, + submitToolResult(callId, result, options) { + append(process.env.OPENCLAW_QA_VOICE_TOOL_RESULTS_PATH, { callId, result, options }); + }, + acknowledgeMark() {}, + close() { + connected = false; + }, + isConnected() { + return connected; + }, + triggerGreeting() {}, + }; + }, + }); + api.registerGatewayMethod("qa.voiceCall.streamSession", async ({ params, respond }) => { + const runtime = globalThis[Symbol.for("openclaw.voice-call.runtime")]; + const callId = typeof params?.callId === "string" ? params.callId : ""; + const call = runtime?.manager?.getCall?.(callId); + const issue = runtime?.manager?.streamSessionIssuer; + if (!runtime || !call || typeof issue !== "function" || !call.providerCallId) { + respond(false, undefined, { + code: "UNAVAILABLE", + message: "Voice Call runtime stream issuer unavailable", + }); + return; + } + const session = issue({ + providerName: "twilio", + callId, + from: call.from, + to: call.to, + direction: call.direction, + }); + respond(true, { + ...session, + providerCallId: call.providerCallId, + webhookUrl: runtime.webhookUrl, + }); + }); + }, +}; diff --git a/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/openclaw.plugin.json b/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/openclaw.plugin.json new file mode 100644 index 000000000000..24378b076a60 --- /dev/null +++ b/test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin/openclaw.plugin.json @@ -0,0 +1,11 @@ +{ + "id": "qa-voice-call-runtime", + "activation": { + "onStartup": true + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts index e4241ef26057..6d586ed551ae 100644 --- a/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts +++ b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.test.ts @@ -18,36 +18,6 @@ async function writeEntry(root: string, relativePath: string) { } describe("gateway MCP real transport producer", () => { - it("uses the source CLI entry when build output is absent", async () => { - const root = createRepoRoot(); - const entryPath = await writeEntry(root, "src/entry.ts"); - - const cli = testing.resolveOpenClawCliInvocation(root); - - expect(cli.command).toBe(process.execPath); - expect(cli.argsPrefix).toStrictEqual(["--import", "tsx", entryPath]); - expect(cli.cwd).toBe(root); - expect(cli.gatewayCommand).toMatchObject({ - executablePath: process.execPath, - argsPrefix: ["--import", "tsx", entryPath], - cwd: root, - }); - }); - - it("prefers built package output when it exists", async () => { - const root = createRepoRoot(); - const distEntry = await writeEntry(root, "dist/index.mjs"); - await writeEntry(root, "src/entry.ts"); - - const cli = testing.resolveOpenClawCliInvocation(root); - - expect(cli.argsPrefix).toStrictEqual([distEntry]); - expect(cli.gatewayCommand).toMatchObject({ - argsPrefix: [distEntry], - usePackagedPlugins: true, - }); - }); - it("uses the source channel MCP module when build output is absent", async () => { const root = createRepoRoot(); const channelServerPath = await writeEntry(root, "src/mcp/channel-server.ts"); diff --git a/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts index 9e707f844c1f..9b619cfb7712 100644 --- a/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts +++ b/test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts @@ -11,7 +11,6 @@ import { WebSocket, WebSocketServer, type RawData } from "ws"; import { QA_EVIDENCE_FILENAME, startQaGatewayChild, - type QaGatewayChildCommand, type QaEvidenceSummaryJson, type QaGatewayChildListeningContext, } from "../../../../extensions/qa-lab/api.js"; @@ -58,13 +57,6 @@ type GatewayProxy = { url: string; }; -type OpenClawCliInvocation = { - argsPrefix: string[]; - command: string; - cwd: string; - gatewayCommand: QaGatewayChildCommand; -}; - type ChannelMcpInvocation = { args: string[]; command: string; @@ -220,50 +212,6 @@ function withFixturePlugin(config: OpenClawConfig, pluginDir: string): OpenClawC }; } -function emptyTransport() { - return { - requiredPluginIds: [] as string[], - createGatewayConfig: () => ({}), - }; -} - -function resolveOpenClawCliInvocation(repoRoot: string): OpenClawCliInvocation { - for (const relativePath of ["dist/index.mjs", "dist/index.js"]) { - const entryPath = path.join(repoRoot, relativePath); - if (existsSync(entryPath)) { - const argsPrefix = [entryPath]; - return { - argsPrefix, - command: process.execPath, - cwd: repoRoot, - gatewayCommand: { - executablePath: process.execPath, - argsPrefix, - cwd: repoRoot, - usePackagedPlugins: true, - }, - }; - } - } - - const sourceEntryPath = path.join(repoRoot, "src/entry.ts"); - if (existsSync(sourceEntryPath)) { - const argsPrefix = ["--import", "tsx", sourceEntryPath]; - return { - argsPrefix, - command: process.execPath, - cwd: repoRoot, - gatewayCommand: { - executablePath: process.execPath, - argsPrefix, - cwd: repoRoot, - }, - }; - } - - throw new Error("OpenClaw CLI entry not found: expected dist/index.(m)js or src/entry.ts"); -} - function resolveChannelMcpInvocation(params: { gatewayToken: string; gatewayUrl: string; @@ -525,8 +473,7 @@ async function approvePendingMcpPairing(gateway: Awaited { const gateway = await startQaGatewayChild({ repoRoot: options.repoRoot, - command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand, - transport: emptyTransport(), + useRepoCli: true, transportBaseUrl: "http://127.0.0.1", controlUiEnabled: false, }); @@ -585,8 +532,7 @@ async function runMcpGatewayStartupRetryProof(options: ProducerOptions): Promise }; gateway = await startQaGatewayChild({ repoRoot: options.repoRoot, - command: resolveOpenClawCliInvocation(options.repoRoot).gatewayCommand, - transport: emptyTransport(), + useRepoCli: true, transportBaseUrl: "http://127.0.0.1", controlUiEnabled: false, onListening, @@ -813,5 +759,4 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { export const testing = { resolveChannelMcpInvocation, - resolveOpenClawCliInvocation, }; diff --git a/test/e2e/qa-lab/runtime/media-talk-gateway.ts b/test/e2e/qa-lab/runtime/media-talk-gateway.ts new file mode 100644 index 000000000000..b1bb8e1fb408 --- /dev/null +++ b/test/e2e/qa-lab/runtime/media-talk-gateway.ts @@ -0,0 +1,615 @@ +import { randomUUID } from "node:crypto"; +// QA Lab producer exercises WebChat media delivery and Talk run control through a real Gateway. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/src/evidence-summary.js"; +import { startQaGatewayChild } from "../../../../extensions/qa-lab/src/gateway-child.js"; +import { startQaMockOpenAiServer } from "../../../../extensions/qa-lab/src/providers/mock-openai/server.js"; +import { GatewayClient, type GatewayClientOptions } from "../../../../src/gateway/client.js"; +import { + GATEWAY_CLIENT_MODES, + GATEWAY_CLIENT_NAMES, + type GatewayClientMode, + type GatewayClientName, +} from "../../../../src/utils/message-channel.js"; +import { createQaScriptEvidenceWriter, type QaScriptEvidenceStatus } from "./script-evidence.js"; + +const FIXTURE_PLUGIN_ID = "qa-media-talk-runtime"; +const FIXTURE_SPEECH_PROVIDER_ID = "qa-speech"; +const FIXTURE_REALTIME_PROVIDER_ID = "qa-realtime"; +const FIXTURE_WAV_BASE64 = + "UklGRsQAAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const SOURCE_PATH = "test/e2e/qa-lab/runtime/media-talk-gateway.ts"; + +type ScenarioId = "webchat-auto-tts" | "active-talk-agent-run-status"; + +type ProducerOptions = { + artifactBase: string; + repoRoot: string; + scenarioId: ScenarioId; +}; + +type ProofResult = { + details?: string; + durationMs: number; + status: QaScriptEvidenceStatus; +}; + +const SCENARIOS = { + "webchat-auto-tts": { + title: "WebChat auto TTS delivery", + sourcePath: "qa/scenarios/media/webchat-auto-tts.yaml", + primaryCoverageIds: ["media.tts", "media.outbound-voice-audio-delivery"], + docsRefs: ["docs/tools/tts.md", "docs/tools/media-overview.md"], + codeRefs: [ + SOURCE_PATH, + "packages/speech-core/src/tts.ts", + "src/gateway/server-methods/chat.ts", + "src/gateway/control-ui.ts", + ], + }, + "active-talk-agent-run-status": { + title: "Active Talk agent-run control boundaries", + sourcePath: "qa/scenarios/runtime/active-talk-agent-run-status.yaml", + primaryCoverageIds: ["voice.active-talk-agent-run-status"], + docsRefs: ["docs/nodes/talk.md", "docs/web/control-ui.md"], + codeRefs: [ + SOURCE_PATH, + "src/gateway/server-methods/talk-client.ts", + "src/talk/agent-run-control.ts", + "src/agents/embedded-agent-runner/runs.ts", + ], + }, +} as const; + +function parseOptions(argv: readonly string[]): ProducerOptions { + const readValue = (name: string) => { + const index = argv.indexOf(name); + return index >= 0 ? argv[index + 1] : undefined; + }; + const scenarioId = readValue("--scenario"); + if (!scenarioId || !(scenarioId in SCENARIOS)) { + throw new Error(`--scenario must be one of: ${Object.keys(SCENARIOS).join(", ")}`); + } + const artifactBase = readValue("--artifact-base"); + if (!artifactBase) { + throw new Error("--artifact-base is required"); + } + return { + artifactBase: path.resolve(artifactBase), + repoRoot: path.resolve(readValue("--repo-root") ?? process.cwd()), + scenarioId: scenarioId as ScenarioId, + }; +} + +async function createFixturePlugin(root: string) { + const pluginDir = path.join(root, FIXTURE_PLUGIN_ID); + const speechCallsPath = path.join(root, "speech-calls.jsonl"); + const realtimeCallsPath = path.join(root, "realtime-calls.jsonl"); + await fs.mkdir(pluginDir, { recursive: true }); + await fs.writeFile( + path.join(pluginDir, "openclaw.plugin.json"), + `${JSON.stringify( + { + id: FIXTURE_PLUGIN_ID, + activation: { onStartup: true }, + configSchema: { type: "object", additionalProperties: false, properties: {} }, + }, + null, + 2, + )}\n`, + "utf8", + ); + await fs.writeFile( + path.join(pluginDir, "index.js"), + `const fs = require("node:fs"); + +module.exports = { + id: ${JSON.stringify(FIXTURE_PLUGIN_ID)}, + register(api) { + api.registerSpeechProvider({ + id: ${JSON.stringify(FIXTURE_SPEECH_PROVIDER_ID)}, + label: "QA Speech", + autoSelectOrder: 1, + isConfigured: () => true, + async synthesize(request) { + fs.appendFileSync(process.env.OPENCLAW_QA_SPEECH_CALLS_PATH, JSON.stringify({ text: request.text, target: request.target }) + "\\n"); + return { + audioBuffer: Buffer.from(${JSON.stringify(FIXTURE_WAV_BASE64)}, "base64"), + fileExtension: ".wav", + outputFormat: "wav", + voiceCompatible: request.target === "voice-note", + }; + }, + }); + api.registerRealtimeVoiceProvider({ + id: ${JSON.stringify(FIXTURE_REALTIME_PROVIDER_ID)}, + label: "QA Realtime", + isConfigured: () => true, + async createBrowserSession(request) { + fs.appendFileSync(process.env.OPENCLAW_QA_REALTIME_CALLS_PATH, JSON.stringify({ tools: request.tools?.map((tool) => tool.name) ?? [] }) + "\\n"); + return { + provider: ${JSON.stringify(FIXTURE_REALTIME_PROVIDER_ID)}, + transport: "provider-websocket", + protocol: "google-live-bidi", + clientSecret: "qa-browser-token", + websocketUrl: "wss://qa.invalid/realtime", + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 16000, + outputEncoding: "pcm16", + outputSampleRateHz: 24000, + }, + }; + }, + createBridge() { + throw new Error("QA browser Talk provider does not create server bridges"); + }, + }); + }, +}; +`, + "utf8", + ); + return { pluginDir, realtimeCallsPath, speechCallsPath }; +} + +function withFixturePlugin(config: OpenClawConfig, pluginDir: string): OpenClawConfig { + return { + ...config, + plugins: { + ...config.plugins, + enabled: true, + allow: [...new Set([...(config.plugins?.allow ?? []), FIXTURE_PLUGIN_ID])], + load: { + ...config.plugins?.load, + paths: [...new Set([...(config.plugins?.load?.paths ?? []), pluginDir])], + }, + entries: { + ...config.plugins?.entries, + [FIXTURE_PLUGIN_ID]: { enabled: true }, + }, + }, + }; +} + +async function connectGatewayClient(params: { + clientName: GatewayClientName; + mode: GatewayClientMode; + onEvent?: GatewayClientOptions["onEvent"]; + token: string; + url: string; +}) { + const gatewayUrl = new URL(params.url); + gatewayUrl.protocol = gatewayUrl.protocol === "wss:" ? "https:" : "http:"; + let resolveHello: (() => void) | undefined; + let rejectHello: ((error: Error) => void) | undefined; + const hello = new Promise((resolve, reject) => { + resolveHello = resolve; + rejectHello = reject; + }); + const client = new GatewayClient({ + url: params.url, + origin: gatewayUrl.origin, + token: params.token, + clientName: params.clientName, + mode: params.mode, + role: "operator", + scopes: [ + "operator.read", + "operator.write", + "operator.admin", + "operator.approvals", + "operator.talk.secrets", + ], + platform: "qa", + requestTimeoutMs: 30_000, + onEvent: params.onEvent, + onHelloOk: () => resolveHello?.(), + onConnectError: (error) => rejectHello?.(error), + onClose: (code, reason) => rejectHello?.(new Error(`Gateway closed ${code}: ${reason}`)), + }); + client.start(); + const timer = setTimeout(() => rejectHello?.(new Error("Gateway connect timeout")), 20_000); + try { + await hello; + } catch (error) { + client.stop(); + throw error; + } finally { + clearTimeout(timer); + } + return client; +} + +function collectRecords(value: unknown, records: Record[] = []) { + if (Array.isArray(value)) { + for (const entry of value) { + collectRecords(entry, records); + } + return records; + } + if (!value || typeof value !== "object") { + return records; + } + const record = value as Record; + records.push(record); + for (const entry of Object.values(record)) { + collectRecords(entry, records); + } + return records; +} + +function findAudioAttachmentSource(value: unknown): string | undefined { + return collectRecords(value) + .filter((record) => record.kind === "audio") + .map((record) => record.url) + .find((url): url is string => typeof url === "string" && url.length > 0); +} + +async function readJsonLines(filePath: string): Promise[]> { + const raw = await fs.readFile(filePath, "utf8").catch(() => ""); + return raw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +async function waitForChatFinal( + events: Array<{ event: string; payload?: unknown }>, + runId: string, +) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const finalEvent = events.find((event) => { + if (event.event !== "chat" || !event.payload || typeof event.payload !== "object") { + return false; + } + const payload = event.payload as Record; + return payload.runId === runId && payload.state === "final"; + }); + if (finalEvent) { + return finalEvent.payload; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`timed out waiting for WebChat final event for run ${runId}`); +} + +async function waitForWebchatAudio(params: { + client: GatewayClient; + events: Array<{ event: string; payload?: unknown }>; + sessionKey: string; +}) { + const deadline = Date.now() + 15_000; + let history: unknown; + while (Date.now() < deadline) { + history = await params.client.request("chat.history", { + sessionKey: params.sessionKey, + limit: 20, + }); + const source = findAudioAttachmentSource(params.events) ?? findAudioAttachmentSource(history); + if (source) { + return { history, source }; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { history, source: undefined }; +} + +async function runWebchatAutoTtsProof(options: ProducerOptions): Promise { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-webchat-tts-")); + const fixture = await createFixturePlugin(fixtureRoot); + const mock = await startQaMockOpenAiServer(); + let gateway: Awaited> | undefined; + let client: GatewayClient | undefined; + const events: Array<{ event: string; payload?: unknown }> = []; + try { + gateway = await startQaGatewayChild({ + repoRoot: options.repoRoot, + useRepoCli: true, + providerBaseUrl: `${mock.baseUrl}/v1`, + providerMode: "mock-openai", + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: true, + runtimeEnvPatch: { + OPENCLAW_QA_SPEECH_CALLS_PATH: fixture.speechCallsPath, + OPENCLAW_QA_REALTIME_CALLS_PATH: fixture.realtimeCallsPath, + }, + mutateConfig: (config) => { + const withPlugin = withFixturePlugin(config, fixture.pluginDir); + return { + ...withPlugin, + messages: { + ...withPlugin.messages, + tts: { + auto: "always", + mode: "final", + provider: FIXTURE_SPEECH_PROVIDER_ID, + prefsPath: path.join(fixtureRoot, "tts-prefs.json"), + }, + }, + }; + }, + }); + client = await connectGatewayClient({ + clientName: GATEWAY_CLIENT_NAMES.WEBCHAT_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + onEvent: (event) => events.push(event), + token: gateway.token, + url: gateway.wsUrl, + }); + const sessionKey = "agent:qa:main"; + const runId = randomUUID(); + await client.request("chat.send", { + sessionKey, + message: "block streaming qa check; answer with one short sentence", + deliver: false, + idempotencyKey: runId, + }); + await waitForChatFinal(events, runId); + const { history, source } = await waitForWebchatAudio({ client, events, sessionKey }); + if (!source) { + const speechCalls = await readJsonLines(fixture.speechCallsPath); + throw new Error( + `WebChat history did not contain an audio attachment; speech=${JSON.stringify(speechCalls)}; gateway=${gateway.logs()}; history=${JSON.stringify(history)}`, + ); + } + const speechCalls = await readJsonLines(fixture.speechCallsPath); + if (speechCalls.length !== 1) { + throw new Error(`expected one final-tail TTS synthesis, received ${speechCalls.length}`); + } + const route = `${gateway.baseUrl}/__openclaw__/assistant-media`; + const sourceParam = encodeURIComponent(source); + const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, { + headers: { Authorization: `Bearer ${gateway.token}` }, + }); + if (!metadata.ok) { + throw new Error(`media metadata failed with ${metadata.status}: ${await metadata.text()}`); + } + const ticket = (await metadata.json()) as { available?: boolean; mediaTicket?: string }; + if (ticket.available !== true || !ticket.mediaTicket?.startsWith("v1.")) { + throw new Error(`media metadata did not mint a scoped ticket: ${JSON.stringify(ticket)}`); + } + const withoutTicket = await fetch(`${route}?source=${sourceParam}`); + if (withoutTicket.status !== 401) { + throw new Error(`media route without ticket returned ${withoutTicket.status}, expected 401`); + } + const ticketed = await fetch( + `${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(ticket.mediaTicket)}`, + ); + const body = Buffer.from(await ticketed.arrayBuffer()); + if (!ticketed.ok || !ticketed.headers.get("content-type")?.includes("audio/wav")) { + throw new Error(`ticketed media failed with ${ticketed.status}`); + } + if (!body.equals(Buffer.from(FIXTURE_WAV_BASE64, "base64"))) { + throw new Error(`ticketed media returned unexpected bytes: ${body.toString("hex")}`); + } + return `real Gateway pid=${gateway.pid ?? "unknown"}; WebChat history contained trusted audio; syntheses=1; scoped ticket served ${body.length} bytes`; + } finally { + client?.stop(); + await gateway?.stop().catch(() => undefined); + await mock.stop(); + await fs.rm(fixtureRoot, { force: true, recursive: true }); + } +} + +function assertControlResult( + value: unknown, + expected: { mode: string; active?: boolean; queued?: boolean; aborted?: boolean }, +) { + if (!value || typeof value !== "object") { + throw new Error(`Talk control returned non-object: ${JSON.stringify(value)}`); + } + const result = value as Record; + for (const [key, expectedValue] of Object.entries(expected)) { + if (result[key] !== expectedValue) { + throw new Error(`Talk ${expected.mode} returned ${JSON.stringify(result)}`); + } + } + if (result.ok !== true) { + throw new Error(`Talk ${expected.mode} failed: ${JSON.stringify(result)}`); + } +} + +async function waitForActiveTalkStatus(client: GatewayClient, sessionKey: string) { + const deadline = Date.now() + 20_000; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const status = await client.request("talk.client.steer", { + sessionKey, + text: "status", + mode: "status", + }); + assertControlResult(status, { mode: "status", active: true }); + return status; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + throw lastError instanceof Error ? lastError : new Error("timed out waiting for active Talk run"); +} + +async function waitForQueuedTalkSteer(client: GatewayClient, sessionKey: string) { + const deadline = Date.now() + 20_000; + let lastResult: unknown; + while (Date.now() < deadline) { + lastResult = await client.request("talk.client.steer", { + sessionKey, + text: "use the safer path", + mode: "steer", + }); + if ( + lastResult && + typeof lastResult === "object" && + (lastResult as Record).queued === true + ) { + return lastResult; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`timed out waiting for steerable Talk run: ${JSON.stringify(lastResult)}`); +} + +async function runActiveTalkAgentRunProof(options: ProducerOptions): Promise { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-active-talk-")); + const fixture = await createFixturePlugin(fixtureRoot); + const mock = await startQaMockOpenAiServer({ finalOnlyMarkerPauseMs: 60_000 }); + let gateway: Awaited> | undefined; + let client: GatewayClient | undefined; + try { + gateway = await startQaGatewayChild({ + repoRoot: options.repoRoot, + useRepoCli: true, + providerBaseUrl: `${mock.baseUrl}/v1`, + providerMode: "mock-openai", + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: true, + runtimeEnvPatch: { + OPENCLAW_QA_SPEECH_CALLS_PATH: fixture.speechCallsPath, + OPENCLAW_QA_REALTIME_CALLS_PATH: fixture.realtimeCallsPath, + }, + mutateConfig: (config) => { + const withPlugin = withFixturePlugin(config, fixture.pluginDir); + return { + ...withPlugin, + talk: { + ...withPlugin.talk, + realtime: { + ...withPlugin.talk?.realtime, + provider: FIXTURE_REALTIME_PROVIDER_ID, + providers: { + ...withPlugin.talk?.realtime?.providers, + [FIXTURE_REALTIME_PROVIDER_ID]: {}, + }, + }, + }, + }; + }, + }); + client = await connectGatewayClient({ + clientName: GATEWAY_CLIENT_NAMES.WEBCHAT_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + token: gateway.token, + url: gateway.wsUrl, + }); + const sessionKey = "agent:qa:main"; + const created = await client.request>("talk.client.create", { + sessionKey, + provider: FIXTURE_REALTIME_PROVIDER_ID, + }); + if (created.provider !== FIXTURE_REALTIME_PROVIDER_ID) { + throw new Error(`Talk client used unexpected provider: ${JSON.stringify(created)}`); + } + const providerCalls = await readJsonLines(fixture.realtimeCallsPath); + const tools = providerCalls[0]?.tools; + if ( + !Array.isArray(tools) || + !tools.includes("openclaw_agent_consult") || + !tools.includes("openclaw_agent_control") + ) { + throw new Error( + `Talk provider did not receive consult/control tools: ${JSON.stringify(tools)}`, + ); + } + const consultRequest = client.request("talk.client.toolCall", { + sessionKey, + callId: `qa-talk-${randomUUID()}`, + name: "openclaw_agent_consult", + args: { question: "final-only marker streaming qa check: inspect the active run" }, + }); + const steer = await waitForQueuedTalkSteer(client, sessionKey); + assertControlResult(steer, { mode: "steer", active: true, queued: true }); + await waitForActiveTalkStatus(client, sessionKey); + const followup = await client.request("talk.client.steer", { + sessionKey, + text: "also verify migration cleanup", + mode: "followup", + }); + assertControlResult(followup, { mode: "followup", active: true, queued: true }); + const cancel = await client.request("talk.client.steer", { + sessionKey, + text: "cancel", + mode: "cancel", + }); + assertControlResult(cancel, { mode: "cancel", active: true, aborted: true }); + await consultRequest; + return `real Gateway pid=${gateway.pid ?? "unknown"}; persistent WebChat connection created Talk session and completed status, steer, follow-up, cancel RPCs`; + } finally { + client?.stop(); + await gateway?.stop().catch(() => undefined); + await mock.stop(); + await fs.rm(fixtureRoot, { force: true, recursive: true }); + } +} + +async function produceProof(options: ProducerOptions): Promise { + const startedAt = Date.now(); + try { + const details = + options.scenarioId === "webchat-auto-tts" + ? await runWebchatAutoTtsProof(options) + : await runActiveTalkAgentRunProof(options); + return { details, durationMs: Math.max(1, Date.now() - startedAt), status: "pass" }; + } catch (error) { + return { + details: formatErrorMessage(error), + durationMs: Math.max(1, Date.now() - startedAt), + status: "fail", + }; + } +} + +export async function runMediaTalkGatewayProducer( + options: ProducerOptions, +): Promise { + const scenario = SCENARIOS[options.scenarioId]; + const writer = createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: `${options.scenarioId}.log`, + primaryModel: "mock-openai/gpt-5.5", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target: { + id: options.scenarioId, + title: scenario.title, + sourcePath: scenario.sourcePath, + primaryCoverageIds: scenario.primaryCoverageIds, + docsRefs: scenario.docsRefs, + codeRefs: scenario.codeRefs, + }, + }); + const result = await produceProof(options); + writer.appendLog(`${result.status}: ${result.details ?? "no details"}\n`); + return await writer.write(result); +} + +async function main(argv: readonly string[]) { + const options = parseOptions(argv); + const evidence = await runMediaTalkGatewayProducer(options); + const status = evidence.entries[0]?.result.status; + console.log(`Media/Talk Gateway evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Media/Talk Gateway status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((exitCode) => { + process.exitCode = exitCode; + }) + .catch((error) => { + console.error(formatErrorMessage(error)); + process.exitCode = 1; + }); +} diff --git a/test/e2e/qa-lab/runtime/voice-call-gateway.ts b/test/e2e/qa-lab/runtime/voice-call-gateway.ts new file mode 100644 index 000000000000..e9eab1e73689 --- /dev/null +++ b/test/e2e/qa-lab/runtime/voice-call-gateway.ts @@ -0,0 +1,364 @@ +// QA Lab producer exercises Voice Call CLI, Gateway RPC/tool, webhook, and realtime consult. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { WebSocket } from "ws"; +import { + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, +} from "../../../../extensions/qa-lab/src/evidence-summary.js"; +import { startQaGatewayChild } from "../../../../extensions/qa-lab/src/gateway-child.js"; +import { startQaMockOpenAiServer } from "../../../../extensions/qa-lab/src/providers/mock-openai/server.js"; +import { getFreePort } from "../../../../src/test-utils/ports.js"; +import { createQaScriptEvidenceWriter, type QaScriptEvidenceStatus } from "./script-evidence.js"; + +const FIXTURE_PLUGIN_ID = "qa-voice-call-runtime"; +const FIXTURE_REALTIME_PROVIDER_ID = "qa-voice-call-realtime"; +const SOURCE_PATH = "test/e2e/qa-lab/runtime/voice-call-gateway.ts"; + +type ProducerOptions = { + artifactBase: string; + repoRoot: string; +}; + +type ProofResult = { + details?: string; + durationMs: number; + status: QaScriptEvidenceStatus; +}; + +function parseOptions(argv: readonly string[]): ProducerOptions { + const readValue = (name: string) => { + const index = argv.indexOf(name); + return index >= 0 ? argv[index + 1] : undefined; + }; + const artifactBase = readValue("--artifact-base"); + if (!artifactBase) { + throw new Error("--artifact-base is required"); + } + return { + artifactBase: path.resolve(artifactBase), + repoRoot: path.resolve(readValue("--repo-root") ?? process.cwd()), + }; +} + +function createFixturePlugin(repoRoot: string, outputRoot: string) { + return { + pluginDir: path.join(repoRoot, "test/e2e/qa-lab/runtime/fixtures/voice-call-runtime-plugin"), + bridgeCallsPath: path.join(outputRoot, "bridge-calls.jsonl"), + toolResultsPath: path.join(outputRoot, "tool-results.jsonl"), + }; +} + +function withVoiceCallConfig(params: { + config: OpenClawConfig; + pluginDir: string; + servePort: number; +}): OpenClawConfig { + const config = params.config; + return { + ...config, + plugins: { + ...config.plugins, + enabled: true, + allow: [...new Set([...(config.plugins?.allow ?? []), "voice-call", FIXTURE_PLUGIN_ID])], + load: { + ...config.plugins?.load, + paths: [...new Set([...(config.plugins?.load?.paths ?? []), params.pluginDir])], + }, + entries: { + ...config.plugins?.entries, + "voice-call": { + enabled: true, + config: { + enabled: true, + provider: "mock", + inboundPolicy: "open", + maxConcurrentCalls: 4, + serve: { port: params.servePort, bind: "127.0.0.1", path: "/voice/webhook" }, + realtime: { + enabled: true, + provider: FIXTURE_REALTIME_PROVIDER_ID, + streamPath: "/voice/stream/realtime", + toolPolicy: "safe-read-only", + consultPolicy: "auto", + providers: { [FIXTURE_REALTIME_PROVIDER_ID]: {} }, + }, + }, + }, + [FIXTURE_PLUGIN_ID]: { enabled: true }, + }, + }, + }; +} + +function findStringByKey(value: unknown, key: string): string | undefined { + if (Array.isArray(value)) { + for (const entry of value) { + const found = findStringByKey(entry, key); + if (found) { + return found; + } + } + return undefined; + } + if (!value || typeof value !== "object") { + return undefined; + } + const record = value as Record; + if (typeof record[key] === "string") { + return record[key]; + } + for (const entry of Object.values(record)) { + const found = findStringByKey(entry, key); + if (found) { + return found; + } + } + return undefined; +} + +async function waitForFinalToolResult(filePath: string) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const raw = await fs.readFile(filePath, "utf8").catch(() => ""); + const entries = raw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + const final = entries.find( + (entry) => + entry.callId === "qa-consult-call" && + entry.options === undefined && + entry.result && + typeof entry.result === "object" && + !Object.hasOwn(entry.result, "status"), + ); + if (final) { + return { entries, final }; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("timed out waiting for final Voice Call consult tool result"); +} + +async function openRealtimeMediaStream(params: { + providerCallId: string; + servePort: number; + streamUrl: string; +}) { + const streamPath = new URL(params.streamUrl).pathname; + const ws = new WebSocket(`ws://127.0.0.1:${params.servePort}${streamPath}`); + await new Promise((resolve, reject) => { + ws.once("open", resolve); + ws.once("error", reject); + }); + ws.send( + JSON.stringify({ + event: "start", + start: { streamSid: "MZ-qa-voice-call", callSid: params.providerCallId }, + }), + ); + return ws; +} + +async function runVoiceCallProof(options: ProducerOptions): Promise { + const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-voice-call-gateway-")); + const fixture = createFixturePlugin(options.repoRoot, fixtureRoot); + const mock = await startQaMockOpenAiServer(); + const servePort = await getFreePort(); + let gateway: Awaited> | undefined; + let mediaStream: WebSocket | undefined; + try { + gateway = await startQaGatewayChild({ + repoRoot: options.repoRoot, + useRepoCli: true, + providerBaseUrl: `${mock.baseUrl}/v1`, + providerMode: "mock-openai", + transportBaseUrl: "http://127.0.0.1", + controlUiEnabled: false, + enabledPluginIds: ["voice-call"], + runtimeEnvPatch: { + OPENCLAW_QA_VOICE_BRIDGE_CALLS_PATH: fixture.bridgeCallsPath, + OPENCLAW_QA_VOICE_TOOL_RESULTS_PATH: fixture.toolResultsPath, + }, + mutateConfig: (config) => + withVoiceCallConfig({ config, pluginDir: fixture.pluginDir, servePort }), + }); + const cliOutput = await gateway.runCli([ + "voicecall", + "start", + "--to", + "+15550001111", + "--message", + "CLI fixture", + "--mode", + "conversation", + ]); + const cliCallId = findStringByKey(JSON.parse(cliOutput), "callId"); + if (!cliCallId) { + throw new Error(`Voice Call CLI did not return a callId: ${cliOutput}`); + } + const rpc = await gateway.call("voicecall.initiate", { + to: "+15550002222", + message: "Gateway RPC fixture", + mode: "conversation", + sessionKey: "agent:main:voice-rpc", + }); + const rpcCallId = findStringByKey(rpc, "callId"); + if (!rpcCallId) { + throw new Error(`voicecall.initiate did not return a callId: ${JSON.stringify(rpc)}`); + } + const tool = await gateway.call("tools.invoke", { + name: "voice_call", + sessionKey: "agent:main:requester", + args: { + action: "initiate_call", + to: "+15550003333", + message: "Agent tool fixture", + mode: "conversation", + sessionKey: "agent:main:voice-consult", + requesterSessionKey: "agent:main:requester", + }, + }); + const toolCallId = findStringByKey(tool, "callId"); + if (!toolCallId) { + throw new Error(`tools.invoke voice_call did not return a callId: ${JSON.stringify(tool)}`); + } + const status = (await gateway.call("voicecall.status", {})) as { + calls?: Array<{ providerCallId?: string }>; + }; + if (!Array.isArray(status.calls) || status.calls.length !== 3) { + throw new Error( + `Voice Call status did not report all three calls: ${JSON.stringify(status)}`, + ); + } + const providerCallIds = status.calls.map((call) => call.providerCallId); + if (providerCallIds.some((callId) => !callId?.startsWith("mock-"))) { + throw new Error( + `Voice Call entries did not use the mock provider: ${JSON.stringify(status)}`, + ); + } + const stream = (await gateway.call("qa.voiceCall.streamSession", { + callId: toolCallId, + })) as { providerCallId?: string; streamUrl?: string }; + if (!stream.providerCallId || !stream.streamUrl) { + throw new Error(`Voice Call stream issuer returned invalid data: ${JSON.stringify(stream)}`); + } + mediaStream = await openRealtimeMediaStream({ + providerCallId: stream.providerCallId, + servePort, + streamUrl: stream.streamUrl, + }); + const toolResults = await waitForFinalToolResult(fixture.toolResultsPath); + const bridgeCalls = (await fs.readFile(fixture.bridgeCallsPath, "utf8")) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as { tools?: string[] }); + if (!bridgeCalls.some((call) => call.tools?.includes("openclaw_agent_consult"))) { + throw new Error( + `realtime bridge did not expose embedded consult: ${JSON.stringify(bridgeCalls)}`, + ); + } + const requests = (await fetch(`${mock.baseUrl}/debug/requests`).then((response) => + response.json(), + )) as Array<{ allInputText?: string; instructions?: string }>; + const consultRequest = requests.find((request) => + request.allInputText?.includes("VOICE-CONSULT-42"), + ); + if (!consultRequest) { + throw new Error( + `embedded consult request did not include provider context: ${JSON.stringify(requests)}`, + ); + } + const promptText = `${consultRequest.instructions ?? ""}\n${consultRequest.allInputText ?? ""}`; + for (const marker of [ + "live phone call", + "Caller partial transcript context", + "VOICE-CONSULT-42", + "Use the embedded agent context", + ]) { + if (!promptText.includes(marker)) { + throw new Error(`embedded consult prompt missed ${marker}: ${promptText}`); + } + } + return `real CLI, voicecall.initiate, and tools.invoke created ${status.calls.length} mock-provider calls; runtime-issued media stream invoked embedded consult with transcript/provider context; tool results=${toolResults.entries.length}`; + } finally { + if (mediaStream && mediaStream.readyState < WebSocket.CLOSING) { + mediaStream.close(); + } + await gateway?.stop().catch(() => undefined); + await mock.stop(); + await fs.rm(fixtureRoot, { force: true, recursive: true }); + } +} + +async function produceProof(options: ProducerOptions): Promise { + const startedAt = Date.now(); + try { + return { + details: await runVoiceCallProof(options), + durationMs: Math.max(1, Date.now() - startedAt), + status: "pass", + }; + } catch (error) { + return { + details: formatErrorMessage(error), + durationMs: Math.max(1, Date.now() - startedAt), + status: "fail", + }; + } +} + +export async function runVoiceCallGatewayProducer( + options: ProducerOptions, +): Promise { + const writer = createQaScriptEvidenceWriter({ + artifactBase: options.artifactBase, + logFileName: "voice-call-cli-rpc-agent-tool.log", + primaryModel: "mock-openai/gpt-5.5", + providerMode: "mock-openai", + repoRoot: options.repoRoot, + target: { + id: "voice-call-cli-rpc-agent-tool", + title: "Voice Call CLI, RPC, and agent tool flow", + sourcePath: "qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml", + primaryCoverageIds: ["voice-call.cli-rpc-agent-tool"], + docsRefs: ["docs/cli/voicecall.md", "docs/plugins/voice-call.md"], + codeRefs: [ + SOURCE_PATH, + "extensions/voice-call/index.ts", + "extensions/voice-call/src/runtime.ts", + "extensions/voice-call/src/webhook/realtime-handler.ts", + ], + }, + }); + const result = await produceProof(options); + writer.appendLog(`${result.status}: ${result.details ?? "no details"}\n`); + return await writer.write(result); +} + +async function main(argv: readonly string[]) { + const options = parseOptions(argv); + const evidence = await runVoiceCallGatewayProducer(options); + const status = evidence.entries[0]?.result.status; + console.log(`Voice Call Gateway evidence: ${QA_EVIDENCE_FILENAME}`); + console.log(`Voice Call Gateway status: ${status}`); + return status === "pass" ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)) + .then((exitCode) => { + process.exitCode = exitCode; + }) + .catch((error) => { + console.error(formatErrorMessage(error)); + process.exitCode = 1; + }); +} diff --git a/test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts b/test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts deleted file mode 100644 index f05b6ec8b2e5..000000000000 --- a/test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts +++ /dev/null @@ -1,173 +0,0 @@ -// QA Talk E2E tests cover provider session creation and active-run voice controls. -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { talkHandlers } from "../../../../src/gateway/server-methods/talk.ts"; -import { createPluginRecord } from "../../../../src/plugins/loader-records.ts"; -import { createPluginRegistry } from "../../../../src/plugins/registry.ts"; -import { - resetPluginRuntimeStateForTest, - setActivePluginRegistry, -} from "../../../../src/plugins/runtime.ts"; -import { controlRealtimeVoiceAgentRun } from "../../../../src/talk/agent-run-control.ts"; -import type { TalkEvent } from "../../../../src/talk/talk-events.ts"; - -const noopLogger = { - info() {}, - warn() {}, - error() {}, - debug() {}, -}; - -function installMockRealtimeProvider() { - const registry = createPluginRegistry({ - logger: noopLogger, - runtime: {}, - activateGlobalSideEffects: false, - }); - const record = createPluginRecord({ - id: "qa-talk-realtime", - name: "QA Talk Realtime", - source: "test/e2e/qa-lab/voice/active-talk-agent-run-status.e2e.test.ts", - origin: "global", - enabled: true, - configSchema: false, - }); - const createBrowserSession = vi.fn(async () => ({ - provider: "qa-realtime", - transport: "provider-websocket" as const, - protocol: "google-live-bidi" as const, - clientSecret: "auth_tokens/qa-talk", - websocketUrl: - "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained", - audio: { - inputEncoding: "pcm16" as const, - inputSampleRateHz: 16_000, - outputEncoding: "pcm16" as const, - outputSampleRateHz: 24_000, - }, - })); - registry.registerRealtimeVoiceProvider(record, { - id: "qa-realtime", - label: "QA Realtime", - isConfigured: () => true, - createBrowserSession, - createBridge: vi.fn(), - }); - setActivePluginRegistry(registry.registry); - return createBrowserSession; -} - -function createControlDeps() { - return { - abortEmbeddedAgentRun: vi.fn(() => true), - queueEmbeddedAgentMessageWithOutcomeAsync: vi.fn(async (sessionId: string) => ({ - queued: true as const, - sessionId, - target: "embedded_run" as const, - gatewayHealth: "live" as const, - enqueuedAtMs: 123, - })), - getDiagnosticSessionActivitySnapshot: vi.fn(() => ({ - activeWorkKind: "embedded_run" as const, - hasActiveEmbeddedRun: true, - })), - resolveActiveEmbeddedRunSessionId: vi.fn(() => "session-active"), - }; -} - -describe("QA active Talk agent-run status", () => { - afterEach(() => { - resetPluginRuntimeStateForTest(); - }); - - it("creates a mock realtime session and controls one active agent run", async () => { - const createBrowserSession = installMockRealtimeProvider(); - const respond = vi.fn(); - await talkHandlers["talk.client.create"]({ - req: { type: "req", id: "create", method: "talk.client.create" }, - params: { sessionKey: "agent:main:main", provider: "qa-realtime" }, - client: { connId: "qa-talk" }, - isWebchatConnect: () => true, - respond, - context: { - getRuntimeConfig: () => - ({ - talk: { - realtime: { - provider: "qa-realtime", - providers: { "qa-realtime": {} }, - }, - }, - }) satisfies OpenClawConfig, - }, - } as never); - - expect(createBrowserSession).toHaveBeenCalledTimes(1); - expect(createBrowserSession.mock.calls[0]?.[0]).toMatchObject({ - tools: [ - expect.objectContaining({ name: "openclaw_agent_consult" }), - expect.objectContaining({ name: "openclaw_agent_control" }), - ], - }); - expect(respond).toHaveBeenCalledWith( - true, - expect.objectContaining({ provider: "qa-realtime", transport: "provider-websocket" }), - undefined, - ); - - const deps = createControlDeps(); - const recentEvents = [ - { - id: "event-1", - type: "tool.progress", - sessionId: "talk-1", - seq: 1, - timestamp: new Date(0).toISOString(), - mode: "realtime", - transport: "provider-websocket", - brain: "agent-consult", - payload: { name: "exec_command", phase: "running" }, - } satisfies TalkEvent, - ]; - - await expect( - controlRealtimeVoiceAgentRun( - { - sessionKey: "agent:main:main", - text: "status", - mode: "status", - recentEvents, - }, - deps, - ), - ).resolves.toMatchObject({ - ok: true, - active: true, - message: "OpenClaw is working in exec_command (running).", - }); - - await expect( - controlRealtimeVoiceAgentRun( - { sessionKey: "agent:main:main", text: "use the safer path", mode: "steer" }, - deps, - ), - ).resolves.toMatchObject({ ok: true, mode: "steer", queued: true }); - await expect( - controlRealtimeVoiceAgentRun( - { sessionKey: "agent:main:main", text: "also check migration", mode: "followup" }, - deps, - ), - ).resolves.toMatchObject({ ok: true, mode: "followup", queued: true }); - expect(deps.queueEmbeddedAgentMessageWithOutcomeAsync.mock.calls[1]?.[1]).toContain( - "Spoken follow-up for the current voice call.", - ); - - await expect( - controlRealtimeVoiceAgentRun( - { sessionKey: "agent:main:main", text: "cancel", mode: "cancel" }, - deps, - ), - ).resolves.toMatchObject({ ok: true, mode: "cancel", aborted: true }); - expect(deps.abortEmbeddedAgentRun).toHaveBeenCalledWith("session-active"); - }); -});