From 5da5551a5ce5fb9b68266554e4c32fea7ec301cd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 02:18:16 -0400 Subject: [PATCH] fix(qa): verify paired Ollama nodes over the gateway (#115644) --- .../node-inference.paired-node.e2e.test.ts | 589 ++++++++++++++++++ ...t-file-scenario-runner.e2e-routing.test.ts | 62 ++ .../qa-lab/src/test-file-scenario-runner.ts | 10 +- .../models/ollama-paired-node-inference.yaml | 13 +- 4 files changed, 669 insertions(+), 5 deletions(-) create mode 100644 extensions/ollama/src/node-inference.paired-node.e2e.test.ts create mode 100644 extensions/qa-lab/src/test-file-scenario-runner.e2e-routing.test.ts diff --git a/extensions/ollama/src/node-inference.paired-node.e2e.test.ts b/extensions/ollama/src/node-inference.paired-node.e2e.test.ts new file mode 100644 index 000000000000..60e35d7d4ffb --- /dev/null +++ b/extensions/ollama/src/node-inference.paired-node.e2e.test.ts @@ -0,0 +1,589 @@ +// Proves local Ollama inference crosses a real Gateway and paired node socket. +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { GatewayClient } from "openclaw/plugin-sdk/gateway-runtime"; +import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry"; +import { createOpenClawTestState } from "openclaw/plugin-sdk/test-state"; +import { describe, expect, it, vi } from "vitest"; +import { createOllamaNodeHostCommands } from "./node-inference.js"; + +const E2E_TIMEOUT_MS = 90_000; +const STARTUP_TIMEOUT_MS = 25_000; +const NODE_DISPLAY_NAME = "paired-ollama-e2e"; +const NODE_MODEL = "node-local:latest"; +const LOADED_NODE_MODEL = "loaded-node-local:latest"; + +type FakeOllama = { + baseUrl: string; + chatRequests: Array>; + paths: string[]; + close: () => Promise; +}; + +type NodeInvokeFrame = { + id?: string; + nodeId?: string; + command?: string; + params?: unknown; + paramsJSON?: string | null; +}; + +describe("Ollama paired-node Gateway inference", () => { + it( + "routes authenticated discovery and chat over the actual paired node socket", + { timeout: E2E_TIMEOUT_MS }, + async () => { + const nodeOllama = await startFakeOllama("node"); + const gatewayOllama = await startFakeOllama("gateway"); + const state = await createOpenClawTestState({ + label: "ollama-paired-node-e2e", + layout: "home", + }); + const gatewayPort = await reserveLoopbackPort(); + const gatewayToken = `ollama-e2e-${randomUUID()}`; + const gatewayLogs: string[] = []; + const handlerErrors: Error[] = []; + let gateway: ChildProcessWithoutNullStreams | undefined; + let operator: GatewayClient | undefined; + let node: GatewayClient | undefined; + + try { + await state.writeConfig({ + gateway: { + mode: "local", + port: gatewayPort, + bind: "loopback", + auth: { mode: "token", token: gatewayToken }, + controlUi: { enabled: false }, + nodes: { commands: { allow: ["ollama.models", "ollama.chat"] } }, + }, + plugins: { + allow: ["ollama"], + }, + agents: { + defaults: { heartbeat: { every: "0m" }, skipBootstrap: true }, + entries: { main: { default: true, tools: { allow: ["node_inference"] } } }, + }, + models: { + providers: { + ollama: { api: "ollama", baseUrl: gatewayOllama.baseUrl, models: [] }, + }, + }, + }); + + gateway = spawn( + process.execPath, + [ + "--import", + "tsx", + "src/entry.ts", + "gateway", + "--port", + String(gatewayPort), + "--bind", + "loopback", + "--allow-unconfigured", + ], + { + cwd: process.cwd(), + env: { + ...state.env, + OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(process.cwd(), "extensions"), + OPENCLAW_CLI: "1", + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "0", + OPENCLAW_GATEWAY_TOKEN: gatewayToken, + OPENCLAW_GATEWAY_STARTUP_TRACE: "1", + OPENCLAW_NO_RESPAWN: "1", + OPENCLAW_TEST_FAST: "1", + OPENCLAW_TEST_MINIMAL_GATEWAY: "1", + OPENCLAW_SKIP_CHANNELS: "1", + OPENCLAW_SKIP_PROVIDERS: "0", + OPENCLAW_SKIP_GMAIL_WATCHER: "1", + OPENCLAW_SKIP_CRON: "1", + OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1", + OPENCLAW_SKIP_CANVAS_HOST: "1", + OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1", + OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", + }, + stdio: "pipe", + }, + ); + gateway.stdout.setEncoding("utf8"); + gateway.stderr.setEncoding("utf8"); + gateway.stdout.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk)); + gateway.stderr.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk)); + gateway.once("error", (error) => appendGatewayLog(gatewayLogs, error.message)); + + await waitForGatewayHealth(gateway, gatewayPort, gatewayLogs); + + try { + operator = await vi.waitFor( + () => + connectClient({ + gatewayPort, + gatewayToken, + role: "operator", + clientName: "test", + mode: "test", + scopes: ["operator.admin", "operator.pairing", "operator.read", "operator.write"], + }), + { timeout: 15_000, interval: 250 }, + ); + } catch (error) { + throw new Error( + `Gateway did not accept its authenticated operator:\n${gatewayLogs.join("")}`, + { + cause: error, + }, + ); + } + + const commands = createOllamaNodeHostCommands({ baseUrl: nodeOllama.baseUrl }); + const commandsByName = new Map(commands.map((command) => [command.command, command])); + node = await connectPairedNode(operator, { + gatewayPort, + gatewayToken, + role: "node", + clientName: "node-host", + mode: "node", + platform: "macos", + displayName: NODE_DISPLAY_NAME, + caps: ["local-inference"], + commands: [...commandsByName.keys()], + onEvent: (event) => { + if (event.event !== "node.invoke.request") { + return; + } + void respondToNodeInvocation(node, event.payload, commandsByName).catch( + (error: unknown) => { + handlerErrors.push(error instanceof Error ? error : new Error(String(error))); + }, + ); + }, + }); + + const pairedNode = await waitForPairedInferenceNode(operator, gatewayLogs); + expect(pairedNode.commands).toEqual( + expect.arrayContaining(["ollama.models", "ollama.chat"]), + ); + + const rejected = await invokeGatewayTool({ + port: gatewayPort, + token: "not-the-gateway-token", + args: { action: "discover" }, + }); + expect(rejected.status).toBe(401); + + const discovered = await operator.request("node.invoke", { + nodeId: pairedNode.nodeId, + command: "ollama.models", + params: {}, + timeoutMs: 15_000, + idempotencyKey: randomUUID(), + }); + expect(discovered).toMatchObject({ + ok: true, + nodeId: pairedNode.nodeId, + command: "ollama.models", + payload: { + provider: "ollama", + models: [ + expect.objectContaining({ name: LOADED_NODE_MODEL, loaded: true }), + expect.objectContaining({ name: NODE_MODEL, loaded: false }), + ], + }, + }); + + const chat = await operator.request("node.invoke", { + nodeId: pairedNode.nodeId, + command: "ollama.chat", + params: { + model: NODE_MODEL, + prompt: "Reply exactly with PAIRED_NODE_OK", + maxTokens: 16, + timeoutMs: 10_000, + }, + timeoutMs: 15_000, + idempotencyKey: randomUUID(), + }); + expect(chat).toMatchObject({ + ok: true, + nodeId: pairedNode.nodeId, + command: "ollama.chat", + payload: { + provider: "ollama", + model: NODE_MODEL, + response: "PAIRED_NODE_OK", + usage: { promptTokens: 8, completionTokens: 3 }, + }, + }); + + expect(handlerErrors).toEqual([]); + expect(nodeOllama.chatRequests).toEqual([ + expect.objectContaining({ + model: NODE_MODEL, + stream: false, + think: false, + options: expect.objectContaining({ num_predict: 16 }), + }), + ]); + expect(gatewayOllama.chatRequests).toEqual([]); + expect(nodeOllama.paths).toEqual( + expect.arrayContaining(["/api/tags", "/api/ps", "/api/show", "/api/chat"]), + ); + } finally { + await Promise.allSettled([ + ...(node ? [node.stopAndWait({ timeoutMs: 1000 })] : []), + ...(operator ? [operator.stopAndWait({ timeoutMs: 1000 })] : []), + ]); + if (gateway) { + await stopGatewayProcess(gateway); + } + await Promise.allSettled([nodeOllama.close(), gatewayOllama.close()]); + await state.cleanup(); + } + }, + ); +}); + +async function reserveLoopbackPort(): Promise { + const server = createServer(); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const { port } = server.address() as AddressInfo; + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + return port; +} + +function appendGatewayLog(logs: string[], chunk: string): void { + logs.push(chunk); + if (logs.length > 32) { + logs.shift(); + } +} + +async function waitForGatewayHealth( + gateway: ChildProcessWithoutNullStreams, + port: number, + logs: string[], +): Promise { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + let lastError: unknown; + while (Date.now() < deadline) { + if (gateway.exitCode !== null || gateway.signalCode !== null) { + throw new Error( + `Gateway exited before listening (code=${String(gateway.exitCode)}, signal=${String(gateway.signalCode)}):\n${logs.join("")}`, + ); + } + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`, { + signal: AbortSignal.timeout(1_000), + }); + if (response.status === 200) { + return; + } + lastError = new Error(`Gateway health returned HTTP ${response.status}`); + } catch (error) { + lastError = error; + } + await delay(100); + } + throw new Error(`Gateway did not become healthy:\n${logs.join("")}`, { cause: lastError }); +} + +async function connectClient(params: { + gatewayPort: number; + gatewayToken: string; + role: "operator" | "node"; + clientName: "test" | "node-host"; + mode: "test" | "node"; + scopes?: string[]; + platform?: string; + displayName?: string; + caps?: string[]; + commands?: string[]; + onEvent?: (event: { event: string; payload?: unknown }) => void; +}): Promise { + return await new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (error) { + client.stop(); + reject(error); + } else { + resolve(client); + } + }; + const client = new GatewayClient({ + url: `ws://127.0.0.1:${params.gatewayPort}`, + token: params.gatewayToken, + role: params.role, + clientName: params.clientName, + clientDisplayName: params.displayName ?? "ollama-paired-node-e2e", + clientVersion: "1.0.0", + platform: params.platform ?? process.platform, + mode: params.mode, + scopes: params.scopes ?? [], + caps: params.caps, + commands: params.commands, + requestTimeoutMs: 15_000, + onEvent: params.onEvent, + onHelloOk: () => finish(), + onConnectError: (error) => finish(error), + onClose: (code, reason) => finish(new Error(`Gateway closed (${code}): ${reason}`)), + }); + const timeout = setTimeout( + () => finish(new Error("Gateway client connection timed out")), + 5_000, + ); + timeout.unref(); + client.start(); + }); +} + +async function approvePendingNodePairings(operator: GatewayClient): Promise { + const devices = await operator.request<{ + pending?: Array<{ requestId?: string; role?: string }>; + }>("device.pair.list", {}); + for (const request of devices.pending ?? []) { + if (request.requestId) { + await operator.request("device.pair.approve", { requestId: request.requestId }); + } + } + + const nodes = await operator.request<{ + pending?: Array<{ requestId?: string; displayName?: string }>; + }>("node.pair.list", {}); + for (const request of nodes.pending ?? []) { + if (request.requestId) { + await operator.request("node.pair.approve", { requestId: request.requestId }); + } + } +} + +async function connectPairedNode( + operator: GatewayClient, + params: Parameters[0], +): Promise { + try { + return await connectClient(params); + } catch (error) { + const details = (error as { details?: { code?: string } }).details; + if (details?.code !== "PAIRING_REQUIRED") { + throw error; + } + // The operator and node share this isolated device identity. Approving the + // Gateway's requested role upgrade preserves the real node pairing boundary. + await approvePendingNodePairings(operator); + return await connectClient(params); + } +} + +async function waitForPairedInferenceNode(operator: GatewayClient, logs: string[]) { + let paired: + | { nodeId: string; displayName?: string; connected?: boolean; commands?: string[] } + | undefined; + await vi.waitFor( + async () => { + await approvePendingNodePairings(operator); + const result = await operator.request<{ + nodes?: Array<{ + nodeId: string; + displayName?: string; + connected?: boolean; + commands?: string[]; + }>; + }>("node.list", {}); + paired = result.nodes?.find( + (entry) => entry.displayName === NODE_DISPLAY_NAME && entry.connected, + ); + expect(paired, logs.join("")).toBeDefined(); + expect(paired?.commands).toEqual(expect.arrayContaining(["ollama.models", "ollama.chat"])); + }, + { timeout: 25_000, interval: 100 }, + ); + if (!paired) { + throw new Error("Ollama-capable paired node never connected"); + } + return paired; +} + +async function respondToNodeInvocation( + node: GatewayClient | undefined, + payload: unknown, + commands: ReadonlyMap, +): Promise { + const frame = payload as NodeInvokeFrame; + const command = typeof frame.command === "string" ? commands.get(frame.command) : undefined; + if (!node || !frame.id || !frame.nodeId || !command) { + throw new Error("Gateway sent an invalid or unauthorized Ollama node invocation"); + } + const paramsJSON = + frame.paramsJSON ?? (frame.params === undefined ? null : JSON.stringify(frame.params)); + try { + const payloadJSON = await command.handle(paramsJSON); + await node.request("node.invoke.result", { + id: frame.id, + nodeId: frame.nodeId, + ok: true, + payloadJSON, + }); + } catch (error) { + await node.request("node.invoke.result", { + id: frame.id, + nodeId: frame.nodeId, + ok: false, + error: { + code: "UNAVAILABLE", + message: error instanceof Error ? error.message : String(error), + }, + }); + } +} + +async function invokeGatewayTool(params: { + port: number; + token: string; + args: Record; +}): Promise { + return await fetch(`http://127.0.0.1:${params.port}/tools/invoke`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${params.token}`, + }, + body: JSON.stringify({ tool: "node_inference", args: params.args }), + signal: AbortSignal.timeout(30_000), + }); +} + +async function readRequestJson(request: IncomingMessage): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; +} + +async function startFakeOllama(owner: "node" | "gateway"): Promise { + const paths: string[] = []; + const chatRequests: Array> = []; + const server = createServer((request, response) => { + void handleFakeOllamaRequest(request, response, owner, paths, chatRequests).catch( + (error: unknown) => { + response.statusCode = 500; + response.end(JSON.stringify({ error: String(error) })); + }, + ); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const { port } = server.address() as AddressInfo; + return { + baseUrl: `http://127.0.0.1:${port}`, + paths, + chatRequests, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +async function handleFakeOllamaRequest( + request: IncomingMessage, + response: ServerResponse, + owner: "node" | "gateway", + paths: string[], + chatRequests: Array>, +): Promise { + const requestPath = request.url ?? "/"; + paths.push(requestPath); + response.setHeader("content-type", "application/json"); + + if (requestPath === "/api/tags") { + const models = + owner === "node" + ? [ + { name: "remote:cloud", remote_host: "https://ollama.com" }, + ...Array.from({ length: 200 }, (_, index) => ({ name: `embedding-${index}:latest` })), + { name: NODE_MODEL, size: 600 }, + { name: LOADED_NODE_MODEL, size: 1200 }, + ] + : [{ name: "gateway-remote:latest" }]; + response.end(JSON.stringify({ models })); + return; + } + if (requestPath === "/api/ps") { + response.end(JSON.stringify({ models: [{ name: LOADED_NODE_MODEL }] })); + return; + } + if (requestPath === "/api/show") { + const body = await readRequestJson(request); + // Ollama documents `model`; the current provider sends its supported `name` alias. + const modelName = + typeof body.model === "string" + ? body.model + : typeof body.name === "string" + ? body.name + : undefined; + if (!modelName) { + response.statusCode = 400; + response.end(JSON.stringify({ error: "model is required" })); + return; + } + const isEmbedding = modelName.startsWith("embedding-"); + response.end( + JSON.stringify({ + capabilities: isEmbedding ? ["embedding"] : ["completion", "tools"], + model_info: isEmbedding ? {} : { "test.context_length": 32_768 }, + }), + ); + return; + } + if (requestPath === "/api/chat") { + const body = await readRequestJson(request); + chatRequests.push(body); + response.end( + JSON.stringify({ + model: body.model, + message: { content: owner === "node" ? "PAIRED_NODE_OK" : "WRONG_GATEWAY_ENDPOINT" }, + done_reason: "stop", + prompt_eval_count: 8, + eval_count: 3, + }), + ); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); +} + +async function stopGatewayProcess(child: ChildProcessWithoutNullStreams): Promise { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + const exited = once(child, "exit").then(() => true); + child.kill("SIGTERM"); + if (await Promise.race([exited, delay(2_000).then(() => false)])) { + return; + } + child.kill("SIGKILL"); + await Promise.race([exited, delay(2_000)]); +} diff --git a/extensions/qa-lab/src/test-file-scenario-runner.e2e-routing.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.e2e-routing.test.ts new file mode 100644 index 000000000000..c804a019792c --- /dev/null +++ b/extensions/qa-lab/src/test-file-scenario-runner.e2e-routing.test.ts @@ -0,0 +1,62 @@ +// QA native Vitest scenarios must select the configuration that owns E2E files. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { + runQaTestFileScenarios, + type QaScenarioCommandExecution, +} from "./test-file-scenario-runner.js"; + +describe("QA native Vitest scenario routing", () => { + it("runs E2E test scenarios under the existing Gateway E2E configuration", async () => { + const repoRoot = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qa-vitest-e2e-routing-")), + ); + const commands: QaScenarioCommandExecution[] = []; + const testPath = "extensions/ollama/src/node-inference.paired-node.e2e.test.ts"; + const scenario: QaSeedScenarioWithSource = { + id: "ollama-paired-node-inference", + title: "Ollama paired-node inference", + surface: "models", + category: "agent-runtime.local-and-self-hosted-providers", + coverage: { primary: [], secondary: ["gateway.remote-host-commands"] }, + objective: "Run local inference through an authenticated paired Gateway node.", + successCriteria: ["The real Gateway routes inference to its paired node."], + docsRefs: ["docs/providers/ollama.md"], + codeRefs: [testPath], + sourcePath: "qa/scenarios/models/ollama-paired-node-inference.yaml", + execution: { kind: "vitest", path: testPath }, + }; + + try { + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", scenario.id), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarios: [scenario], + runCommand: async (command) => { + commands.push(command); + return { exitCode: 0, stdout: "1 passed\n", stderr: "" }; + }, + }); + + expect(result.executionKind).toBe("vitest"); + expect(result.results).toMatchObject([{ status: "pass" }]); + expect(commands.map((command) => command.args)).toEqual([ + [ + "scripts/run-vitest.mjs", + "run", + "--config", + "test/vitest/vitest.e2e.config.ts", + testPath, + "--reporter=verbose", + ], + ]); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index 31037028d6d6..95433473aab4 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -98,10 +98,18 @@ export function isQaTestFileScenario( } function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] { + const e2eConfigArgs = scenario.execution.path.endsWith(".e2e.test.ts") + ? ["run", "--config", "test/vitest/vitest.e2e.config.ts"] + : []; return [ { command: process.execPath, - args: ["scripts/run-vitest.mjs", scenario.execution.path, "--reporter=verbose"], + args: [ + "scripts/run-vitest.mjs", + ...e2eConfigArgs, + scenario.execution.path, + "--reporter=verbose", + ], }, ]; } diff --git a/qa/scenarios/models/ollama-paired-node-inference.yaml b/qa/scenarios/models/ollama-paired-node-inference.yaml index 41545d3baa62..1a8e3d619588 100644 --- a/qa/scenarios/models/ollama-paired-node-inference.yaml +++ b/qa/scenarios/models/ollama-paired-node-inference.yaml @@ -11,8 +11,12 @@ scenario: - agent-runtime.tool-capability-flags - agent-runtime.local-smoke-checks - agent-runtime.local-failure-handling - objective: Verify paired-node Ollama discovery and inference with the provider-owned deterministic local test server. + objective: Verify Ollama discovery and inference across a real authenticated Gateway, paired node connection, and provider-owned deterministic node-local server. successCriteria: + - A real Gateway authenticates and pairs the Ollama-capable node over WebSocket. + - Authenticated Gateway node.invoke requests cross the paired node request and response boundary. + - Unauthenticated Gateway tool requests are rejected. + - The Gateway provider endpoint does not replace the node's own Ollama endpoint. - Embedding and cloud models are excluded from local chat discovery. - A chat model after 200 embedding-only models remains discoverable. - Loaded local chat models sort first. @@ -23,8 +27,9 @@ scenario: - docs/nodes/index.md codeRefs: - extensions/ollama/src/node-inference.ts - - extensions/ollama/src/node-inference.test.ts + - extensions/ollama/src/node-inference.paired-node.e2e.test.ts + - extensions/qa-lab/src/test-file-scenario-runner.ts execution: kind: vitest - path: extensions/ollama/src/node-inference.test.ts - summary: Run deterministic fake-Ollama paired-node discovery, authorization, model-selection, and bounded-chat regression tests. + path: extensions/ollama/src/node-inference.paired-node.e2e.test.ts + summary: Run actual Gateway authentication, WebSocket pairing, node command policy, provider-local discovery, and bounded inference against an isolated fake Ollama.