mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 09:51:33 +00:00
fix(qa): verify paired Ollama nodes over the gateway (#115644)
This commit is contained in:
committed by
GitHub
parent
1b9481012b
commit
5da5551a5c
589
extensions/ollama/src/node-inference.paired-node.e2e.test.ts
Normal file
589
extensions/ollama/src/node-inference.paired-node.e2e.test.ts
Normal file
@@ -0,0 +1,589 @@
|
||||
// Proves local Ollama inference crosses a real Gateway and paired node socket.
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { GatewayClient } from "openclaw/plugin-sdk/gateway-runtime";
|
||||
import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { createOpenClawTestState } from "openclaw/plugin-sdk/test-state";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createOllamaNodeHostCommands } from "./node-inference.js";
|
||||
|
||||
const E2E_TIMEOUT_MS = 90_000;
|
||||
const STARTUP_TIMEOUT_MS = 25_000;
|
||||
const NODE_DISPLAY_NAME = "paired-ollama-e2e";
|
||||
const NODE_MODEL = "node-local:latest";
|
||||
const LOADED_NODE_MODEL = "loaded-node-local:latest";
|
||||
|
||||
type FakeOllama = {
|
||||
baseUrl: string;
|
||||
chatRequests: Array<Record<string, unknown>>;
|
||||
paths: string[];
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
type NodeInvokeFrame = {
|
||||
id?: string;
|
||||
nodeId?: string;
|
||||
command?: string;
|
||||
params?: unknown;
|
||||
paramsJSON?: string | null;
|
||||
};
|
||||
|
||||
describe("Ollama paired-node Gateway inference", () => {
|
||||
it(
|
||||
"routes authenticated discovery and chat over the actual paired node socket",
|
||||
{ timeout: E2E_TIMEOUT_MS },
|
||||
async () => {
|
||||
const nodeOllama = await startFakeOllama("node");
|
||||
const gatewayOllama = await startFakeOllama("gateway");
|
||||
const state = await createOpenClawTestState({
|
||||
label: "ollama-paired-node-e2e",
|
||||
layout: "home",
|
||||
});
|
||||
const gatewayPort = await reserveLoopbackPort();
|
||||
const gatewayToken = `ollama-e2e-${randomUUID()}`;
|
||||
const gatewayLogs: string[] = [];
|
||||
const handlerErrors: Error[] = [];
|
||||
let gateway: ChildProcessWithoutNullStreams | undefined;
|
||||
let operator: GatewayClient | undefined;
|
||||
let node: GatewayClient | undefined;
|
||||
|
||||
try {
|
||||
await state.writeConfig({
|
||||
gateway: {
|
||||
mode: "local",
|
||||
port: gatewayPort,
|
||||
bind: "loopback",
|
||||
auth: { mode: "token", token: gatewayToken },
|
||||
controlUi: { enabled: false },
|
||||
nodes: { commands: { allow: ["ollama.models", "ollama.chat"] } },
|
||||
},
|
||||
plugins: {
|
||||
allow: ["ollama"],
|
||||
},
|
||||
agents: {
|
||||
defaults: { heartbeat: { every: "0m" }, skipBootstrap: true },
|
||||
entries: { main: { default: true, tools: { allow: ["node_inference"] } } },
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
ollama: { api: "ollama", baseUrl: gatewayOllama.baseUrl, models: [] },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
gateway = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"src/entry.ts",
|
||||
"gateway",
|
||||
"--port",
|
||||
String(gatewayPort),
|
||||
"--bind",
|
||||
"loopback",
|
||||
"--allow-unconfigured",
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...state.env,
|
||||
OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(process.cwd(), "extensions"),
|
||||
OPENCLAW_CLI: "1",
|
||||
OPENCLAW_DISABLE_BUNDLED_PLUGINS: "0",
|
||||
OPENCLAW_GATEWAY_TOKEN: gatewayToken,
|
||||
OPENCLAW_GATEWAY_STARTUP_TRACE: "1",
|
||||
OPENCLAW_NO_RESPAWN: "1",
|
||||
OPENCLAW_TEST_FAST: "1",
|
||||
OPENCLAW_TEST_MINIMAL_GATEWAY: "1",
|
||||
OPENCLAW_SKIP_CHANNELS: "1",
|
||||
OPENCLAW_SKIP_PROVIDERS: "0",
|
||||
OPENCLAW_SKIP_GMAIL_WATCHER: "1",
|
||||
OPENCLAW_SKIP_CRON: "1",
|
||||
OPENCLAW_SKIP_BROWSER_CONTROL_SERVER: "1",
|
||||
OPENCLAW_SKIP_CANVAS_HOST: "1",
|
||||
OPENCLAW_SKIP_STARTUP_MODEL_PREWARM: "1",
|
||||
OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1",
|
||||
},
|
||||
stdio: "pipe",
|
||||
},
|
||||
);
|
||||
gateway.stdout.setEncoding("utf8");
|
||||
gateway.stderr.setEncoding("utf8");
|
||||
gateway.stdout.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk));
|
||||
gateway.stderr.on("data", (chunk: string) => appendGatewayLog(gatewayLogs, chunk));
|
||||
gateway.once("error", (error) => appendGatewayLog(gatewayLogs, error.message));
|
||||
|
||||
await waitForGatewayHealth(gateway, gatewayPort, gatewayLogs);
|
||||
|
||||
try {
|
||||
operator = await vi.waitFor(
|
||||
() =>
|
||||
connectClient({
|
||||
gatewayPort,
|
||||
gatewayToken,
|
||||
role: "operator",
|
||||
clientName: "test",
|
||||
mode: "test",
|
||||
scopes: ["operator.admin", "operator.pairing", "operator.read", "operator.write"],
|
||||
}),
|
||||
{ timeout: 15_000, interval: 250 },
|
||||
);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Gateway did not accept its authenticated operator:\n${gatewayLogs.join("")}`,
|
||||
{
|
||||
cause: error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const commands = createOllamaNodeHostCommands({ baseUrl: nodeOllama.baseUrl });
|
||||
const commandsByName = new Map(commands.map((command) => [command.command, command]));
|
||||
node = await connectPairedNode(operator, {
|
||||
gatewayPort,
|
||||
gatewayToken,
|
||||
role: "node",
|
||||
clientName: "node-host",
|
||||
mode: "node",
|
||||
platform: "macos",
|
||||
displayName: NODE_DISPLAY_NAME,
|
||||
caps: ["local-inference"],
|
||||
commands: [...commandsByName.keys()],
|
||||
onEvent: (event) => {
|
||||
if (event.event !== "node.invoke.request") {
|
||||
return;
|
||||
}
|
||||
void respondToNodeInvocation(node, event.payload, commandsByName).catch(
|
||||
(error: unknown) => {
|
||||
handlerErrors.push(error instanceof Error ? error : new Error(String(error)));
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const pairedNode = await waitForPairedInferenceNode(operator, gatewayLogs);
|
||||
expect(pairedNode.commands).toEqual(
|
||||
expect.arrayContaining(["ollama.models", "ollama.chat"]),
|
||||
);
|
||||
|
||||
const rejected = await invokeGatewayTool({
|
||||
port: gatewayPort,
|
||||
token: "not-the-gateway-token",
|
||||
args: { action: "discover" },
|
||||
});
|
||||
expect(rejected.status).toBe(401);
|
||||
|
||||
const discovered = await operator.request("node.invoke", {
|
||||
nodeId: pairedNode.nodeId,
|
||||
command: "ollama.models",
|
||||
params: {},
|
||||
timeoutMs: 15_000,
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
expect(discovered).toMatchObject({
|
||||
ok: true,
|
||||
nodeId: pairedNode.nodeId,
|
||||
command: "ollama.models",
|
||||
payload: {
|
||||
provider: "ollama",
|
||||
models: [
|
||||
expect.objectContaining({ name: LOADED_NODE_MODEL, loaded: true }),
|
||||
expect.objectContaining({ name: NODE_MODEL, loaded: false }),
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const chat = await operator.request("node.invoke", {
|
||||
nodeId: pairedNode.nodeId,
|
||||
command: "ollama.chat",
|
||||
params: {
|
||||
model: NODE_MODEL,
|
||||
prompt: "Reply exactly with PAIRED_NODE_OK",
|
||||
maxTokens: 16,
|
||||
timeoutMs: 10_000,
|
||||
},
|
||||
timeoutMs: 15_000,
|
||||
idempotencyKey: randomUUID(),
|
||||
});
|
||||
expect(chat).toMatchObject({
|
||||
ok: true,
|
||||
nodeId: pairedNode.nodeId,
|
||||
command: "ollama.chat",
|
||||
payload: {
|
||||
provider: "ollama",
|
||||
model: NODE_MODEL,
|
||||
response: "PAIRED_NODE_OK",
|
||||
usage: { promptTokens: 8, completionTokens: 3 },
|
||||
},
|
||||
});
|
||||
|
||||
expect(handlerErrors).toEqual([]);
|
||||
expect(nodeOllama.chatRequests).toEqual([
|
||||
expect.objectContaining({
|
||||
model: NODE_MODEL,
|
||||
stream: false,
|
||||
think: false,
|
||||
options: expect.objectContaining({ num_predict: 16 }),
|
||||
}),
|
||||
]);
|
||||
expect(gatewayOllama.chatRequests).toEqual([]);
|
||||
expect(nodeOllama.paths).toEqual(
|
||||
expect.arrayContaining(["/api/tags", "/api/ps", "/api/show", "/api/chat"]),
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([
|
||||
...(node ? [node.stopAndWait({ timeoutMs: 1000 })] : []),
|
||||
...(operator ? [operator.stopAndWait({ timeoutMs: 1000 })] : []),
|
||||
]);
|
||||
if (gateway) {
|
||||
await stopGatewayProcess(gateway);
|
||||
}
|
||||
await Promise.allSettled([nodeOllama.close(), gatewayOllama.close()]);
|
||||
await state.cleanup();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
async function reserveLoopbackPort(): Promise<number> {
|
||||
const server = createServer();
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const { port } = server.address() as AddressInfo;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
return port;
|
||||
}
|
||||
|
||||
function appendGatewayLog(logs: string[], chunk: string): void {
|
||||
logs.push(chunk);
|
||||
if (logs.length > 32) {
|
||||
logs.shift();
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForGatewayHealth(
|
||||
gateway: ChildProcessWithoutNullStreams,
|
||||
port: number,
|
||||
logs: string[],
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
|
||||
let lastError: unknown;
|
||||
while (Date.now() < deadline) {
|
||||
if (gateway.exitCode !== null || gateway.signalCode !== null) {
|
||||
throw new Error(
|
||||
`Gateway exited before listening (code=${String(gateway.exitCode)}, signal=${String(gateway.signalCode)}):\n${logs.join("")}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/healthz`, {
|
||||
signal: AbortSignal.timeout(1_000),
|
||||
});
|
||||
if (response.status === 200) {
|
||||
return;
|
||||
}
|
||||
lastError = new Error(`Gateway health returned HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
throw new Error(`Gateway did not become healthy:\n${logs.join("")}`, { cause: lastError });
|
||||
}
|
||||
|
||||
async function connectClient(params: {
|
||||
gatewayPort: number;
|
||||
gatewayToken: string;
|
||||
role: "operator" | "node";
|
||||
clientName: "test" | "node-host";
|
||||
mode: "test" | "node";
|
||||
scopes?: string[];
|
||||
platform?: string;
|
||||
displayName?: string;
|
||||
caps?: string[];
|
||||
commands?: string[];
|
||||
onEvent?: (event: { event: string; payload?: unknown }) => void;
|
||||
}): Promise<GatewayClient> {
|
||||
return await new Promise<GatewayClient>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
if (error) {
|
||||
client.stop();
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(client);
|
||||
}
|
||||
};
|
||||
const client = new GatewayClient({
|
||||
url: `ws://127.0.0.1:${params.gatewayPort}`,
|
||||
token: params.gatewayToken,
|
||||
role: params.role,
|
||||
clientName: params.clientName,
|
||||
clientDisplayName: params.displayName ?? "ollama-paired-node-e2e",
|
||||
clientVersion: "1.0.0",
|
||||
platform: params.platform ?? process.platform,
|
||||
mode: params.mode,
|
||||
scopes: params.scopes ?? [],
|
||||
caps: params.caps,
|
||||
commands: params.commands,
|
||||
requestTimeoutMs: 15_000,
|
||||
onEvent: params.onEvent,
|
||||
onHelloOk: () => finish(),
|
||||
onConnectError: (error) => finish(error),
|
||||
onClose: (code, reason) => finish(new Error(`Gateway closed (${code}): ${reason}`)),
|
||||
});
|
||||
const timeout = setTimeout(
|
||||
() => finish(new Error("Gateway client connection timed out")),
|
||||
5_000,
|
||||
);
|
||||
timeout.unref();
|
||||
client.start();
|
||||
});
|
||||
}
|
||||
|
||||
async function approvePendingNodePairings(operator: GatewayClient): Promise<void> {
|
||||
const devices = await operator.request<{
|
||||
pending?: Array<{ requestId?: string; role?: string }>;
|
||||
}>("device.pair.list", {});
|
||||
for (const request of devices.pending ?? []) {
|
||||
if (request.requestId) {
|
||||
await operator.request("device.pair.approve", { requestId: request.requestId });
|
||||
}
|
||||
}
|
||||
|
||||
const nodes = await operator.request<{
|
||||
pending?: Array<{ requestId?: string; displayName?: string }>;
|
||||
}>("node.pair.list", {});
|
||||
for (const request of nodes.pending ?? []) {
|
||||
if (request.requestId) {
|
||||
await operator.request("node.pair.approve", { requestId: request.requestId });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function connectPairedNode(
|
||||
operator: GatewayClient,
|
||||
params: Parameters<typeof connectClient>[0],
|
||||
): Promise<GatewayClient> {
|
||||
try {
|
||||
return await connectClient(params);
|
||||
} catch (error) {
|
||||
const details = (error as { details?: { code?: string } }).details;
|
||||
if (details?.code !== "PAIRING_REQUIRED") {
|
||||
throw error;
|
||||
}
|
||||
// The operator and node share this isolated device identity. Approving the
|
||||
// Gateway's requested role upgrade preserves the real node pairing boundary.
|
||||
await approvePendingNodePairings(operator);
|
||||
return await connectClient(params);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForPairedInferenceNode(operator: GatewayClient, logs: string[]) {
|
||||
let paired:
|
||||
| { nodeId: string; displayName?: string; connected?: boolean; commands?: string[] }
|
||||
| undefined;
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
await approvePendingNodePairings(operator);
|
||||
const result = await operator.request<{
|
||||
nodes?: Array<{
|
||||
nodeId: string;
|
||||
displayName?: string;
|
||||
connected?: boolean;
|
||||
commands?: string[];
|
||||
}>;
|
||||
}>("node.list", {});
|
||||
paired = result.nodes?.find(
|
||||
(entry) => entry.displayName === NODE_DISPLAY_NAME && entry.connected,
|
||||
);
|
||||
expect(paired, logs.join("")).toBeDefined();
|
||||
expect(paired?.commands).toEqual(expect.arrayContaining(["ollama.models", "ollama.chat"]));
|
||||
},
|
||||
{ timeout: 25_000, interval: 100 },
|
||||
);
|
||||
if (!paired) {
|
||||
throw new Error("Ollama-capable paired node never connected");
|
||||
}
|
||||
return paired;
|
||||
}
|
||||
|
||||
async function respondToNodeInvocation(
|
||||
node: GatewayClient | undefined,
|
||||
payload: unknown,
|
||||
commands: ReadonlyMap<string, OpenClawPluginNodeHostCommand>,
|
||||
): Promise<void> {
|
||||
const frame = payload as NodeInvokeFrame;
|
||||
const command = typeof frame.command === "string" ? commands.get(frame.command) : undefined;
|
||||
if (!node || !frame.id || !frame.nodeId || !command) {
|
||||
throw new Error("Gateway sent an invalid or unauthorized Ollama node invocation");
|
||||
}
|
||||
const paramsJSON =
|
||||
frame.paramsJSON ?? (frame.params === undefined ? null : JSON.stringify(frame.params));
|
||||
try {
|
||||
const payloadJSON = await command.handle(paramsJSON);
|
||||
await node.request("node.invoke.result", {
|
||||
id: frame.id,
|
||||
nodeId: frame.nodeId,
|
||||
ok: true,
|
||||
payloadJSON,
|
||||
});
|
||||
} catch (error) {
|
||||
await node.request("node.invoke.result", {
|
||||
id: frame.id,
|
||||
nodeId: frame.nodeId,
|
||||
ok: false,
|
||||
error: {
|
||||
code: "UNAVAILABLE",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function invokeGatewayTool(params: {
|
||||
port: number;
|
||||
token: string;
|
||||
args: Record<string, unknown>;
|
||||
}): Promise<Response> {
|
||||
return await fetch(`http://127.0.0.1:${params.port}/tools/invoke`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${params.token}`,
|
||||
},
|
||||
body: JSON.stringify({ tool: "node_inference", args: params.args }),
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
}
|
||||
|
||||
async function readRequestJson(request: IncomingMessage): Promise<Record<string, unknown>> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function startFakeOllama(owner: "node" | "gateway"): Promise<FakeOllama> {
|
||||
const paths: string[] = [];
|
||||
const chatRequests: Array<Record<string, unknown>> = [];
|
||||
const server = createServer((request, response) => {
|
||||
void handleFakeOllamaRequest(request, response, owner, paths, chatRequests).catch(
|
||||
(error: unknown) => {
|
||||
response.statusCode = 500;
|
||||
response.end(JSON.stringify({ error: String(error) }));
|
||||
},
|
||||
);
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const { port } = server.address() as AddressInfo;
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${port}`,
|
||||
paths,
|
||||
chatRequests,
|
||||
close: async () => {
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function handleFakeOllamaRequest(
|
||||
request: IncomingMessage,
|
||||
response: ServerResponse,
|
||||
owner: "node" | "gateway",
|
||||
paths: string[],
|
||||
chatRequests: Array<Record<string, unknown>>,
|
||||
): Promise<void> {
|
||||
const requestPath = request.url ?? "/";
|
||||
paths.push(requestPath);
|
||||
response.setHeader("content-type", "application/json");
|
||||
|
||||
if (requestPath === "/api/tags") {
|
||||
const models =
|
||||
owner === "node"
|
||||
? [
|
||||
{ name: "remote:cloud", remote_host: "https://ollama.com" },
|
||||
...Array.from({ length: 200 }, (_, index) => ({ name: `embedding-${index}:latest` })),
|
||||
{ name: NODE_MODEL, size: 600 },
|
||||
{ name: LOADED_NODE_MODEL, size: 1200 },
|
||||
]
|
||||
: [{ name: "gateway-remote:latest" }];
|
||||
response.end(JSON.stringify({ models }));
|
||||
return;
|
||||
}
|
||||
if (requestPath === "/api/ps") {
|
||||
response.end(JSON.stringify({ models: [{ name: LOADED_NODE_MODEL }] }));
|
||||
return;
|
||||
}
|
||||
if (requestPath === "/api/show") {
|
||||
const body = await readRequestJson(request);
|
||||
// Ollama documents `model`; the current provider sends its supported `name` alias.
|
||||
const modelName =
|
||||
typeof body.model === "string"
|
||||
? body.model
|
||||
: typeof body.name === "string"
|
||||
? body.name
|
||||
: undefined;
|
||||
if (!modelName) {
|
||||
response.statusCode = 400;
|
||||
response.end(JSON.stringify({ error: "model is required" }));
|
||||
return;
|
||||
}
|
||||
const isEmbedding = modelName.startsWith("embedding-");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
capabilities: isEmbedding ? ["embedding"] : ["completion", "tools"],
|
||||
model_info: isEmbedding ? {} : { "test.context_length": 32_768 },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (requestPath === "/api/chat") {
|
||||
const body = await readRequestJson(request);
|
||||
chatRequests.push(body);
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
model: body.model,
|
||||
message: { content: owner === "node" ? "PAIRED_NODE_OK" : "WRONG_GATEWAY_ENDPOINT" },
|
||||
done_reason: "stop",
|
||||
prompt_eval_count: 8,
|
||||
eval_count: 3,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
response.statusCode = 404;
|
||||
response.end(JSON.stringify({ error: "not found" }));
|
||||
}
|
||||
|
||||
async function stopGatewayProcess(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return;
|
||||
}
|
||||
const exited = once(child, "exit").then(() => true);
|
||||
child.kill("SIGTERM");
|
||||
if (await Promise.race([exited, delay(2_000).then(() => false)])) {
|
||||
return;
|
||||
}
|
||||
child.kill("SIGKILL");
|
||||
await Promise.race([exited, delay(2_000)]);
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// QA native Vitest scenarios must select the configuration that owns E2E files.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
|
||||
import {
|
||||
runQaTestFileScenarios,
|
||||
type QaScenarioCommandExecution,
|
||||
} from "./test-file-scenario-runner.js";
|
||||
|
||||
describe("QA native Vitest scenario routing", () => {
|
||||
it("runs E2E test scenarios under the existing Gateway E2E configuration", async () => {
|
||||
const repoRoot = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qa-vitest-e2e-routing-")),
|
||||
);
|
||||
const commands: QaScenarioCommandExecution[] = [];
|
||||
const testPath = "extensions/ollama/src/node-inference.paired-node.e2e.test.ts";
|
||||
const scenario: QaSeedScenarioWithSource = {
|
||||
id: "ollama-paired-node-inference",
|
||||
title: "Ollama paired-node inference",
|
||||
surface: "models",
|
||||
category: "agent-runtime.local-and-self-hosted-providers",
|
||||
coverage: { primary: [], secondary: ["gateway.remote-host-commands"] },
|
||||
objective: "Run local inference through an authenticated paired Gateway node.",
|
||||
successCriteria: ["The real Gateway routes inference to its paired node."],
|
||||
docsRefs: ["docs/providers/ollama.md"],
|
||||
codeRefs: [testPath],
|
||||
sourcePath: "qa/scenarios/models/ollama-paired-node-inference.yaml",
|
||||
execution: { kind: "vitest", path: testPath },
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await runQaTestFileScenarios({
|
||||
repoRoot,
|
||||
outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", scenario.id),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
scenarios: [scenario],
|
||||
runCommand: async (command) => {
|
||||
commands.push(command);
|
||||
return { exitCode: 0, stdout: "1 passed\n", stderr: "" };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.executionKind).toBe("vitest");
|
||||
expect(result.results).toMatchObject([{ status: "pass" }]);
|
||||
expect(commands.map((command) => command.args)).toEqual([
|
||||
[
|
||||
"scripts/run-vitest.mjs",
|
||||
"run",
|
||||
"--config",
|
||||
"test/vitest/vitest.e2e.config.ts",
|
||||
testPath,
|
||||
"--reporter=verbose",
|
||||
],
|
||||
]);
|
||||
} finally {
|
||||
await fs.rm(repoRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -98,10 +98,18 @@ export function isQaTestFileScenario(
|
||||
}
|
||||
|
||||
function vitestSteps(scenario: QaTestFileScenario): QaScenarioCommandStep[] {
|
||||
const e2eConfigArgs = scenario.execution.path.endsWith(".e2e.test.ts")
|
||||
? ["run", "--config", "test/vitest/vitest.e2e.config.ts"]
|
||||
: [];
|
||||
return [
|
||||
{
|
||||
command: process.execPath,
|
||||
args: ["scripts/run-vitest.mjs", scenario.execution.path, "--reporter=verbose"],
|
||||
args: [
|
||||
"scripts/run-vitest.mjs",
|
||||
...e2eConfigArgs,
|
||||
scenario.execution.path,
|
||||
"--reporter=verbose",
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -11,8 +11,12 @@ scenario:
|
||||
- agent-runtime.tool-capability-flags
|
||||
- agent-runtime.local-smoke-checks
|
||||
- agent-runtime.local-failure-handling
|
||||
objective: Verify paired-node Ollama discovery and inference with the provider-owned deterministic local test server.
|
||||
objective: Verify Ollama discovery and inference across a real authenticated Gateway, paired node connection, and provider-owned deterministic node-local server.
|
||||
successCriteria:
|
||||
- A real Gateway authenticates and pairs the Ollama-capable node over WebSocket.
|
||||
- Authenticated Gateway node.invoke requests cross the paired node request and response boundary.
|
||||
- Unauthenticated Gateway tool requests are rejected.
|
||||
- The Gateway provider endpoint does not replace the node's own Ollama endpoint.
|
||||
- Embedding and cloud models are excluded from local chat discovery.
|
||||
- A chat model after 200 embedding-only models remains discoverable.
|
||||
- Loaded local chat models sort first.
|
||||
@@ -23,8 +27,9 @@ scenario:
|
||||
- docs/nodes/index.md
|
||||
codeRefs:
|
||||
- extensions/ollama/src/node-inference.ts
|
||||
- extensions/ollama/src/node-inference.test.ts
|
||||
- extensions/ollama/src/node-inference.paired-node.e2e.test.ts
|
||||
- extensions/qa-lab/src/test-file-scenario-runner.ts
|
||||
execution:
|
||||
kind: vitest
|
||||
path: extensions/ollama/src/node-inference.test.ts
|
||||
summary: Run deterministic fake-Ollama paired-node discovery, authorization, model-selection, and bounded-chat regression tests.
|
||||
path: extensions/ollama/src/node-inference.paired-node.e2e.test.ts
|
||||
summary: Run actual Gateway authentication, WebSocket pairing, node command policy, provider-local discovery, and bounded inference against an isolated fake Ollama.
|
||||
|
||||
Reference in New Issue
Block a user