fix(qa): verify paired Ollama nodes over the gateway (#115644)

This commit is contained in:
Peter Steinberger
2026-07-29 02:18:16 -04:00
committed by GitHub
parent 1b9481012b
commit 5da5551a5c
4 changed files with 669 additions and 5 deletions

View File

@@ -0,0 +1,589 @@
// Proves local Ollama inference crosses a real Gateway and paired node socket.
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import { once } from "node:events";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { GatewayClient } from "openclaw/plugin-sdk/gateway-runtime";
import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry";
import { createOpenClawTestState } from "openclaw/plugin-sdk/test-state";
import { describe, expect, it, vi } from "vitest";
import { createOllamaNodeHostCommands } from "./node-inference.js";
const E2E_TIMEOUT_MS = 90_000;
const STARTUP_TIMEOUT_MS = 25_000;
const NODE_DISPLAY_NAME = "paired-ollama-e2e";
const NODE_MODEL = "node-local:latest";
const LOADED_NODE_MODEL = "loaded-node-local:latest";
type FakeOllama = {
baseUrl: string;
chatRequests: Array<Record<string, unknown>>;
paths: string[];
close: () => Promise<void>;
};
type NodeInvokeFrame = {
id?: string;
nodeId?: string;
command?: string;
params?: unknown;
paramsJSON?: string | null;
};
describe("Ollama paired-node Gateway inference", () => {
it(
"routes authenticated discovery and chat over the actual paired node socket",
{ timeout: E2E_TIMEOUT_MS },
async () => {
const nodeOllama = await startFakeOllama("node");
const gatewayOllama = await startFakeOllama("gateway");
const state = await createOpenClawTestState({
label: "ollama-paired-node-e2e",
layout: "home",
});
const gatewayPort = await reserveLoopbackPort();
const gatewayToken = `ollama-e2e-${randomUUID()}`;
const gatewayLogs: string[] = [];
const handlerErrors: Error[] = [];
let gateway: ChildProcessWithoutNullStreams | undefined;
let operator: GatewayClient | undefined;
let node: GatewayClient | undefined;
try {
await state.writeConfig({
gateway: {
mode: "local",
port: gatewayPort,
bind: "loopback",
auth: { mode: "token", token: gatewayToken },
controlUi: { enabled: false },
nodes: { commands: { allow: ["ollama.models", "ollama.chat"] } },
},
plugins: {
allow: ["ollama"],
},
agents: {
defaults: { heartbeat: { every: "0m" }, skipBootstrap: true },
entries: { main: { default: true, tools: { allow: ["node_inference"] } } },
},
models: {
providers: {
ollama: { api: "ollama", baseUrl: gatewayOllama.baseUrl, models: [] },
},
},
});
gateway = spawn(
process.execPath,
[
"--import",
"tsx",
"src/entry.ts",
"gateway",
"--port",
String(gatewayPort),
"--bind",
"loopback",
"--allow-unconfigured",
],
{
cwd: process.cwd(),
env: {
...state.env,
OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(process.cwd(), "extensions"),
OPENCLAW_CLI: "1",
OPENCLAW_DISABLE_BUNDLED_PLUGINS: "0",
OPENCLAW_GATEWAY_TOKEN: gatewayToken,
OPENCLAW_GATEWAY_STARTUP_TRACE: "1",
OPENCLAW_NO_RESPAWN: "1",
OPENCLAW_TEST_FAST: "1",
OPENCLAW_TEST_MINIMAL_GATEWAY: "1",
OPENCLAW_SKIP_CHANNELS: "1",
OPENCLAW_SKIP_PROVIDERS: "0",
OPENCLAW_SKIP_GMAIL_WATCHER: "1",
OPENCLAW_SKIP_CRON: "1",
OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
OPENCLAW_SKIP_CANVAS_HOST: "1",
OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1",
OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1",
},
stdio: "pipe",
},
);
gateway.stdout.setEncoding("utf8");
gateway.stderr.setEncoding("utf8");
gateway.stdout.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk));
gateway.stderr.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk));
gateway.once("error", (error) => appendGatewayLog(gatewayLogs, error.message));
await waitForGatewayHealth(gateway, gatewayPort, gatewayLogs);
try {
operator = await vi.waitFor(
() =>
connectClient({
gatewayPort,
gatewayToken,
role: "operator",
clientName: "test",
mode: "test",
scopes: ["operator.admin", "operator.pairing", "operator.read", "operator.write"],
}),
{ timeout: 15_000, interval: 250 },
);
} catch (error) {
throw new Error(
`Gateway did not accept its authenticated operator:\n${gatewayLogs.join("")}`,
{
cause: error,
},
);
}
const commands = createOllamaNodeHostCommands({ baseUrl: nodeOllama.baseUrl });
const commandsByName = new Map(commands.map((command) => [command.command, command]));
node = await connectPairedNode(operator, {
gatewayPort,
gatewayToken,
role: "node",
clientName: "node-host",
mode: "node",
platform: "macos",
displayName: NODE_DISPLAY_NAME,
caps: ["local-inference"],
commands: [...commandsByName.keys()],
onEvent: (event) => {
if (event.event !== "node.invoke.request") {
return;
}
void respondToNodeInvocation(node, event.payload, commandsByName).catch(
(error: unknown) => {
handlerErrors.push(error instanceof Error ? error : new Error(String(error)));
},
);
},
});
const pairedNode = await waitForPairedInferenceNode(operator, gatewayLogs);
expect(pairedNode.commands).toEqual(
expect.arrayContaining(["ollama.models", "ollama.chat"]),
);
const rejected = await invokeGatewayTool({
port: gatewayPort,
token: "not-the-gateway-token",
args: { action: "discover" },
});
expect(rejected.status).toBe(401);
const discovered = await operator.request("node.invoke", {
nodeId: pairedNode.nodeId,
command: "ollama.models",
params: {},
timeoutMs: 15_000,
idempotencyKey: randomUUID(),
});
expect(discovered).toMatchObject({
ok: true,
nodeId: pairedNode.nodeId,
command: "ollama.models",
payload: {
provider: "ollama",
models: [
expect.objectContaining({ name: LOADED_NODE_MODEL, loaded: true }),
expect.objectContaining({ name: NODE_MODEL, loaded: false }),
],
},
});
const chat = await operator.request("node.invoke", {
nodeId: pairedNode.nodeId,
command: "ollama.chat",
params: {
model: NODE_MODEL,
prompt: "Reply exactly with PAIRED_NODE_OK",
maxTokens: 16,
timeoutMs: 10_000,
},
timeoutMs: 15_000,
idempotencyKey: randomUUID(),
});
expect(chat).toMatchObject({
ok: true,
nodeId: pairedNode.nodeId,
command: "ollama.chat",
payload: {
provider: "ollama",
model: NODE_MODEL,
response: "PAIRED_NODE_OK",
usage: { promptTokens: 8, completionTokens: 3 },
},
});
expect(handlerErrors).toEqual([]);
expect(nodeOllama.chatRequests).toEqual([
expect.objectContaining({
model: NODE_MODEL,
stream: false,
think: false,
options: expect.objectContaining({ num_predict: 16 }),
}),
]);
expect(gatewayOllama.chatRequests).toEqual([]);
expect(nodeOllama.paths).toEqual(
expect.arrayContaining(["/api/tags", "/api/ps", "/api/show", "/api/chat"]),
);
} finally {
await Promise.allSettled([
...(node ? [node.stopAndWait({ timeoutMs: 1000 })] : []),
...(operator ? [operator.stopAndWait({ timeoutMs: 1000 })] : []),
]);
if (gateway) {
await stopGatewayProcess(gateway);
}
await Promise.allSettled([nodeOllama.close(), gatewayOllama.close()]);
await state.cleanup();
}
},
);
});
async function reserveLoopbackPort(): Promise<number> {
const server = createServer();
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});
const { port } = server.address() as AddressInfo;
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
return port;
}
function appendGatewayLog(logs: string[], chunk: string): void {
logs.push(chunk);
if (logs.length > 32) {
logs.shift();
}
}
async function waitForGatewayHealth(
gateway: ChildProcessWithoutNullStreams,
port: number,
logs: string[],
): Promise<void> {
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
let lastError: unknown;
while (Date.now() < deadline) {
if (gateway.exitCode !== null || gateway.signalCode !== null) {
throw new Error(
`Gateway exited before listening (code=${String(gateway.exitCode)}, signal=${String(gateway.signalCode)}):\n${logs.join("")}`,
);
}
try {
const response = await fetch(`http://127.0.0.1:${port}/healthz`, {
signal: AbortSignal.timeout(1_000),
});
if (response.status === 200) {
return;
}
lastError = new Error(`Gateway health returned HTTP ${response.status}`);
} catch (error) {
lastError = error;
}
await delay(100);
}
throw new Error(`Gateway did not become healthy:\n${logs.join("")}`, { cause: lastError });
}
async function connectClient(params: {
gatewayPort: number;
gatewayToken: string;
role: "operator" | "node";
clientName: "test" | "node-host";
mode: "test" | "node";
scopes?: string[];
platform?: string;
displayName?: string;
caps?: string[];
commands?: string[];
onEvent?: (event: { event: string; payload?: unknown }) => void;
}): Promise<GatewayClient> {
return await new Promise<GatewayClient>((resolve, reject) => {
let settled = false;
const finish = (error?: Error) => {
if (settled) {
return;
}
settled = true;
clearTimeout(timeout);
if (error) {
client.stop();
reject(error);
} else {
resolve(client);
}
};
const client = new GatewayClient({
url: `ws://127.0.0.1:${params.gatewayPort}`,
token: params.gatewayToken,
role: params.role,
clientName: params.clientName,
clientDisplayName: params.displayName ?? "ollama-paired-node-e2e",
clientVersion: "1.0.0",
platform: params.platform ?? process.platform,
mode: params.mode,
scopes: params.scopes ?? [],
caps: params.caps,
commands: params.commands,
requestTimeoutMs: 15_000,
onEvent: params.onEvent,
onHelloOk: () => finish(),
onConnectError: (error) => finish(error),
onClose: (code, reason) => finish(new Error(`Gateway closed (${code}): ${reason}`)),
});
const timeout = setTimeout(
() => finish(new Error("Gateway client connection timed out")),
5_000,
);
timeout.unref();
client.start();
});
}
async function approvePendingNodePairings(operator: GatewayClient): Promise<void> {
const devices = await operator.request<{
pending?: Array<{ requestId?: string; role?: string }>;
}>("device.pair.list", {});
for (const request of devices.pending ?? []) {
if (request.requestId) {
await operator.request("device.pair.approve", { requestId: request.requestId });
}
}
const nodes = await operator.request<{
pending?: Array<{ requestId?: string; displayName?: string }>;
}>("node.pair.list", {});
for (const request of nodes.pending ?? []) {
if (request.requestId) {
await operator.request("node.pair.approve", { requestId: request.requestId });
}
}
}
async function connectPairedNode(
operator: GatewayClient,
params: Parameters<typeof connectClient>[0],
): Promise<GatewayClient> {
try {
return await connectClient(params);
} catch (error) {
const details = (error as { details?: { code?: string } }).details;
if (details?.code !== "PAIRING_REQUIRED") {
throw error;
}
// The operator and node share this isolated device identity. Approving the
// Gateway's requested role upgrade preserves the real node pairing boundary.
await approvePendingNodePairings(operator);
return await connectClient(params);
}
}
async function waitForPairedInferenceNode(operator: GatewayClient, logs: string[]) {
let paired:
| { nodeId: string; displayName?: string; connected?: boolean; commands?: string[] }
| undefined;
await vi.waitFor(
async () => {
await approvePendingNodePairings(operator);
const result = await operator.request<{
nodes?: Array<{
nodeId: string;
displayName?: string;
connected?: boolean;
commands?: string[];
}>;
}>("node.list", {});
paired = result.nodes?.find(
(entry) => entry.displayName === NODE_DISPLAY_NAME && entry.connected,
);
expect(paired, logs.join("")).toBeDefined();
expect(paired?.commands).toEqual(expect.arrayContaining(["ollama.models", "ollama.chat"]));
},
{ timeout: 25_000, interval: 100 },
);
if (!paired) {
throw new Error("Ollama-capable paired node never connected");
}
return paired;
}
async function respondToNodeInvocation(
node: GatewayClient | undefined,
payload: unknown,
commands: ReadonlyMap<string, OpenClawPluginNodeHostCommand>,
): Promise<void> {
const frame = payload as NodeInvokeFrame;
const command = typeof frame.command === "string" ? commands.get(frame.command) : undefined;
if (!node || !frame.id || !frame.nodeId || !command) {
throw new Error("Gateway sent an invalid or unauthorized Ollama node invocation");
}
const paramsJSON =
frame.paramsJSON ?? (frame.params === undefined ? null : JSON.stringify(frame.params));
try {
const payloadJSON = await command.handle(paramsJSON);
await node.request("node.invoke.result", {
id: frame.id,
nodeId: frame.nodeId,
ok: true,
payloadJSON,
});
} catch (error) {
await node.request("node.invoke.result", {
id: frame.id,
nodeId: frame.nodeId,
ok: false,
error: {
code: "UNAVAILABLE",
message: error instanceof Error ? error.message : String(error),
},
});
}
}
async function invokeGatewayTool(params: {
port: number;
token: string;
args: Record<string, unknown>;
}): Promise<Response> {
return await fetch(`http://127.0.0.1:${params.port}/tools/invoke`, {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${params.token}`,
},
body: JSON.stringify({ tool: "node_inference", args: params.args }),
signal: AbortSignal.timeout(30_000),
});
}
async function readRequestJson(request: IncomingMessage): Promise<Record<string, unknown>> {
const chunks: Buffer[] = [];
for await (const chunk of request) {
chunks.push(Buffer.from(chunk));
}
return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
}
async function startFakeOllama(owner: "node" | "gateway"): Promise<FakeOllama> {
const paths: string[] = [];
const chatRequests: Array<Record<string, unknown>> = [];
const server = createServer((request, response) => {
void handleFakeOllamaRequest(request, response, owner, paths, chatRequests).catch(
(error: unknown) => {
response.statusCode = 500;
response.end(JSON.stringify({ error: String(error) }));
},
);
});
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});
const { port } = server.address() as AddressInfo;
return {
baseUrl: `http://127.0.0.1:${port}`,
paths,
chatRequests,
close: async () => {
server.closeAllConnections();
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
},
};
}
async function handleFakeOllamaRequest(
request: IncomingMessage,
response: ServerResponse,
owner: "node" | "gateway",
paths: string[],
chatRequests: Array<Record<string, unknown>>,
): Promise<void> {
const requestPath = request.url ?? "/";
paths.push(requestPath);
response.setHeader("content-type", "application/json");
if (requestPath === "/api/tags") {
const models =
owner === "node"
? [
{ name: "remote:cloud", remote_host: "https://ollama.com" },
...Array.from({ length: 200 }, (_, index) => ({ name: `embedding-${index}:latest` })),
{ name: NODE_MODEL, size: 600 },
{ name: LOADED_NODE_MODEL, size: 1200 },
]
: [{ name: "gateway-remote:latest" }];
response.end(JSON.stringify({ models }));
return;
}
if (requestPath === "/api/ps") {
response.end(JSON.stringify({ models: [{ name: LOADED_NODE_MODEL }] }));
return;
}
if (requestPath === "/api/show") {
const body = await readRequestJson(request);
// Ollama documents `model`; the current provider sends its supported `name` alias.
const modelName =
typeof body.model === "string"
? body.model
: typeof body.name === "string"
? body.name
: undefined;
if (!modelName) {
response.statusCode = 400;
response.end(JSON.stringify({ error: "model is required" }));
return;
}
const isEmbedding = modelName.startsWith("embedding-");
response.end(
JSON.stringify({
capabilities: isEmbedding ? ["embedding"] : ["completion", "tools"],
model_info: isEmbedding ? {} : { "test.context_length": 32_768 },
}),
);
return;
}
if (requestPath === "/api/chat") {
const body = await readRequestJson(request);
chatRequests.push(body);
response.end(
JSON.stringify({
model: body.model,
message: { content: owner === "node" ? "PAIRED_NODE_OK" : "WRONG_GATEWAY_ENDPOINT" },
done_reason: "stop",
prompt_eval_count: 8,
eval_count: 3,
}),
);
return;
}
response.statusCode = 404;
response.end(JSON.stringify({ error: "not found" }));
}
async function stopGatewayProcess(child: ChildProcessWithoutNullStreams): Promise<void> {
if (child.exitCode !== null || child.signalCode !== null) {
return;
}
const exited = once(child, "exit").then(() => true);
child.kill("SIGTERM");
if (await Promise.race([exited, delay(2_000).then(() => false)])) {
return;
}
child.kill("SIGKILL");
await Promise.race([exited, delay(2_000)]);
}

View File

@@ -0,0 +1,62 @@
// QA native Vitest scenarios must select the configuration that owns E2E files.
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
import {
runQaTestFileScenarios,
type QaScenarioCommandExecution,
} from "./test-file-scenario-runner.js";
describe("QA native Vitest scenario routing", () => {
it("runs E2E test scenarios under the existing Gateway E2E configuration", async () => {
const repoRoot = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qa-vitest-e2e-routing-")),
);
const commands: QaScenarioCommandExecution[] = [];
const testPath = "extensions/ollama/src/node-inference.paired-node.e2e.test.ts";
const scenario: QaSeedScenarioWithSource = {
id: "ollama-paired-node-inference",
title: "Ollama paired-node inference",
surface: "models",
category: "agent-runtime.local-and-self-hosted-providers",
coverage: { primary: [], secondary: ["gateway.remote-host-commands"] },
objective: "Run local inference through an authenticated paired Gateway node.",
successCriteria: ["The real Gateway routes inference to its paired node."],
docsRefs: ["docs/providers/ollama.md"],
codeRefs: [testPath],
sourcePath: "qa/scenarios/models/ollama-paired-node-inference.yaml",
execution: { kind: "vitest", path: testPath },
};
try {
const result = await runQaTestFileScenarios({
repoRoot,
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", scenario.id),
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarios: [scenario],
runCommand: async (command) => {
commands.push(command);
return { exitCode: 0, stdout: "1 passed\n", stderr: "" };
},
});
expect(result.executionKind).toBe("vitest");
expect(result.results).toMatchObject([{ status: "pass" }]);
expect(commands.map((command) => command.args)).toEqual([
[
"scripts/run-vitest.mjs",
"run",
"--config",
"test/vitest/vitest.e2e.config.ts",
testPath,
"--reporter=verbose",
],
]);
} finally {
await fs.rm(repoRoot, { recursive: true, force: true });
}
});
});

View File

@@ -98,10 +98,18 @@ export function isQaTestFileScenario(
}
function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] {
const e2eConfigArgs = scenario.execution.path.endsWith(".e2e.test.ts")
? ["run", "--config", "test/vitest/vitest.e2e.config.ts"]
: [];
return [
{
command: process.execPath,
args: ["scripts/run-vitest.mjs", scenario.execution.path, "--reporter=verbose"],
args: [
"scripts/run-vitest.mjs",
...e2eConfigArgs,
scenario.execution.path,
"--reporter=verbose",
],
},
];
}

View File

@@ -11,8 +11,12 @@ scenario:
- agent-runtime.tool-capability-flags
- agent-runtime.local-smoke-checks
- agent-runtime.local-failure-handling
objective: Verify paired-node Ollama discovery and inference with the provider-owned deterministic local test server.
objective: Verify Ollama discovery and inference across a real authenticated Gateway, paired node connection, and provider-owned deterministic node-local server.
successCriteria:
- A real Gateway authenticates and pairs the Ollama-capable node over WebSocket.
- Authenticated Gateway node.invoke requests cross the paired node request and response boundary.
- Unauthenticated Gateway tool requests are rejected.
- The Gateway provider endpoint does not replace the node's own Ollama endpoint.
- Embedding and cloud models are excluded from local chat discovery.
- A chat model after 200 embedding-only models remains discoverable.
- Loaded local chat models sort first.
@@ -23,8 +27,9 @@ scenario:
- docs/nodes/index.md
codeRefs:
- extensions/ollama/src/node-inference.ts
- extensions/ollama/src/node-inference.test.ts
- extensions/ollama/src/node-inference.paired-node.e2e.test.ts
- extensions/qa-lab/src/test-file-scenario-runner.ts
execution:
kind: vitest
path: extensions/ollama/src/node-inference.test.ts
summary: Run deterministic fake-Ollama paired-node discovery, authorization, model-selection, and bounded-chat regression tests.
path: extensions/ollama/src/node-inference.paired-node.e2e.test.ts
summary: Run actual Gateway authentication, WebSocket pairing, node command policy, provider-local discovery, and bounded inference against an isolated fake Ollama.