From 5eeaa5603f0c279e0b31ea72bb1efa2913dc0b3f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 27 May 2026 16:17:34 +0200 Subject: [PATCH] fix(e2e): bound Open WebUI control probes --- scripts/e2e/openwebui-probe.mjs | 171 +++++++++++++++------ test/scripts/openwebui-probe.test.ts | 213 +++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 44 deletions(-) create mode 100644 test/scripts/openwebui-probe.test.ts diff --git a/scripts/e2e/openwebui-probe.mjs b/scripts/e2e/openwebui-probe.mjs index edb7eb0e76d..8c7df3a3d49 100644 --- a/scripts/e2e/openwebui-probe.mjs +++ b/scripts/e2e/openwebui-probe.mjs @@ -5,13 +5,23 @@ const email = process.env.OPENWEBUI_ADMIN_EMAIL ?? ""; const password = process.env.OPENWEBUI_ADMIN_PASSWORD ?? ""; const expectedNonce = process.env.OPENWEBUI_EXPECTED_NONCE ?? ""; const prompt = process.env.OPENWEBUI_PROMPT ?? ""; -const modelAttempts = Number.parseInt(process.env.OPENWEBUI_MODEL_ATTEMPTS ?? "72", 10); -const modelRetryMs = Number.parseInt(process.env.OPENWEBUI_MODEL_RETRY_MS ?? "5000", 10); -const fetchTimeoutMs = Number.parseInt(process.env.OPENWEBUI_FETCH_TIMEOUT_MS ?? "720000", 10); +const modelAttempts = readPositiveInt(process.env.OPENWEBUI_MODEL_ATTEMPTS, 72); +const modelRetryMs = readNonNegativeInt(process.env.OPENWEBUI_MODEL_RETRY_MS, 5000); +const fetchTimeoutMs = readPositiveInt(process.env.OPENWEBUI_FETCH_TIMEOUT_MS, 720000); +const controlTimeoutMs = readPositiveInt( + process.env.OPENWEBUI_CONTROL_TIMEOUT_MS, + Math.min(fetchTimeoutMs, 30000), +); +const chatTimeoutMs = readPositiveInt(process.env.OPENWEBUI_CHAT_TIMEOUT_MS, fetchTimeoutMs); const smokeMode = process.env.OPENWEBUI_SMOKE_MODE ?? process.env.OPENCLAW_OPENWEBUI_SMOKE_MODE ?? "chat"; -setGlobalDispatcher(new Agent({ bodyTimeout: fetchTimeoutMs, headersTimeout: fetchTimeoutMs })); +setGlobalDispatcher( + new Agent({ + bodyTimeout: Math.max(controlTimeoutMs, chatTimeoutMs), + headersTimeout: Math.max(controlTimeoutMs, chatTimeoutMs), + }), +); if (!baseUrl || !email || !password || !expectedNonce || !prompt) { throw new Error("Missing required OPENWEBUI_* environment variables"); @@ -20,6 +30,41 @@ if (smokeMode !== "models" && smokeMode !== "chat") { throw new Error(`Unsupported OPENWEBUI_SMOKE_MODE: ${smokeMode}`); } +function readPositiveInt(raw, fallback) { + const parsed = Number.parseInt(String(raw || ""), 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function readNonNegativeInt(raw, fallback) { + const parsed = Number.parseInt(String(raw || ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + +function createTimeoutError(label, timeoutMs) { + const error = new Error(`${label} timed out after ${timeoutMs}ms`); + error.code = "ETIMEDOUT"; + return error; +} + +async function withRequestTimeout(label, timeoutMs, run) { + const controller = new AbortController(); + const timeoutError = createTimeoutError(label, timeoutMs); + const timer = setTimeout(() => { + controller.abort(timeoutError); + }, timeoutMs); + timer.unref?.(); + try { + return await run(controller.signal); + } catch (error) { + if (controller.signal.aborted) { + throw timeoutError; + } + throw error; + } finally { + clearTimeout(timer); + } +} + function getCookieHeader(res) { const raw = res.headers.get("set-cookie"); if (!raw) { @@ -49,6 +94,69 @@ function sleep(ms) { }); } +async function fetchSignin() { + return await withRequestTimeout("Open WebUI signin", controlTimeoutMs, async (signal) => { + const response = await fetch(`${baseUrl}/api/v1/auths/signin`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ email, password }), + signal, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`signin failed: HTTP ${response.status} ${body}`); + } + return { + cookie: getCookieHeader(response), + json: await response.json(), + }; + }); +} + +async function fetchModels(authHeaders, attempt) { + return await withRequestTimeout( + `Open WebUI models attempt ${attempt}`, + controlTimeoutMs, + async (signal) => { + const response = await fetch(`${baseUrl}/api/models`, { headers: authHeaders, signal }); + if (!response.ok) { + return { + ok: false, + status: response.status, + text: await response.text(), + }; + } + return { + json: await response.json(), + ok: true, + }; + }, + ); +} + +async function fetchChatCompletion(authHeaders, targetModel) { + return await withRequestTimeout("Open WebUI chat completion", chatTimeoutMs, async (signal) => { + const response = await fetch(`${baseUrl}/api/chat/completions`, { + method: "POST", + headers: { + ...authHeaders, + "content-type": "application/json", + }, + body: JSON.stringify({ + model: targetModel, + messages: [{ role: "user", content: prompt }], + }), + signal, + }); + if (!response.ok) { + throw new Error( + `/api/chat/completions failed: HTTP ${response.status} ${await response.text()}`, + ); + } + return await response.json(); + }); +} + function extractModelIds(modelsJson) { const models = Array.isArray(modelsJson) ? modelsJson @@ -62,22 +170,12 @@ function extractModelIds(modelsJson) { .filter((value) => typeof value === "string"); } -const signinRes = await fetch(`${baseUrl}/api/v1/auths/signin`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ email, password }), -}); -if (!signinRes.ok) { - const body = await signinRes.text(); - throw new Error(`signin failed: HTTP ${signinRes.status} ${body}`); -} - -const signinJson = await signinRes.json(); +const signin = await fetchSignin(); +const signinJson = signin.json; const token = signinJson?.token ?? signinJson?.access_token ?? signinJson?.jwt ?? signinJson?.data?.token ?? ""; -const cookie = getCookieHeader(signinRes); const authHeaders = { - ...buildAuthHeaders(token, cookie), + ...buildAuthHeaders(token, signin.cookie), accept: "application/json", }; @@ -85,25 +183,24 @@ let modelIds = []; let targetModel = ""; let lastModelsError = ""; for (let attempt = 1; attempt <= modelAttempts; attempt += 1) { - const modelsRes = await fetch(`${baseUrl}/api/models`, { headers: authHeaders }).catch( - (error) => { - lastModelsError = error instanceof Error ? error.message : String(error); - return undefined; - }, - ); - if (modelsRes?.ok) { - const modelsJson = await modelsRes.json(); - modelIds = extractModelIds(modelsJson); + const modelsResult = await fetchModels(authHeaders, attempt).catch((error) => { + lastModelsError = error instanceof Error ? error.message : String(error); + return undefined; + }); + if (modelsResult?.ok) { + modelIds = extractModelIds(modelsResult.json); targetModel = modelIds.find((id) => id === "openclaw/default") ?? modelIds.find((id) => id === "openclaw"); if (targetModel) { break; } lastModelsError = `missing openclaw model: ${JSON.stringify(modelIds)}`; - } else if (modelsRes) { - lastModelsError = `HTTP ${modelsRes.status} ${await modelsRes.text()}`; + } else if (modelsResult) { + lastModelsError = `HTTP ${modelsResult.status} ${modelsResult.text}`; + } + if (attempt < modelAttempts) { + await sleep(modelRetryMs); } - await sleep(modelRetryMs); } if (!targetModel) { throw new Error( @@ -115,21 +212,7 @@ if (smokeMode === "models") { process.exit(0); } -const chatRes = await fetch(`${baseUrl}/api/chat/completions`, { - method: "POST", - headers: { - ...authHeaders, - "content-type": "application/json", - }, - body: JSON.stringify({ - model: targetModel, - messages: [{ role: "user", content: prompt }], - }), -}); -if (!chatRes.ok) { - throw new Error(`/api/chat/completions failed: HTTP ${chatRes.status} ${await chatRes.text()}`); -} -const chatJson = await chatRes.json(); +const chatJson = await fetchChatCompletion(authHeaders, targetModel); const reply = chatJson?.choices?.[0]?.message?.content ?? chatJson?.message?.content ?? chatJson?.content ?? ""; if (typeof reply !== "string" || !reply.includes(expectedNonce)) { diff --git a/test/scripts/openwebui-probe.test.ts b/test/scripts/openwebui-probe.test.ts new file mode 100644 index 00000000000..c9686b5a821 --- /dev/null +++ b/test/scripts/openwebui-probe.test.ts @@ -0,0 +1,213 @@ +import { spawn } from "node:child_process"; +import { createServer, type Server as HttpServer } from "node:http"; +import { createServer as createTcpServer, type Server as TcpServer, type Socket } from "node:net"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const probePath = path.resolve("scripts/e2e/openwebui-probe.mjs"); + +interface ProbeResult { + error?: Error; + signal: NodeJS.Signals | null; + status: number | null; + stderr: string; + stdout: string; +} + +async function listen(server: HttpServer | TcpServer): Promise { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("test server did not expose a TCP port"); + } + return `http://127.0.0.1:${address.port}`; +} + +function runProbe(baseUrl: string, env: Record = {}, timeout = 3_000) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [probePath], { + env: { + ...process.env, + OPENWEBUI_ADMIN_EMAIL: "openwebui-e2e@example.com", + OPENWEBUI_ADMIN_PASSWORD: "test-password", + OPENWEBUI_BASE_URL: baseUrl, + OPENWEBUI_CONTROL_TIMEOUT_MS: "250", + OPENWEBUI_EXPECTED_NONCE: "nonce-123", + OPENWEBUI_MODEL_ATTEMPTS: "1", + OPENWEBUI_MODEL_RETRY_MS: "0", + OPENWEBUI_PROMPT: "reply with nonce-123", + OPENWEBUI_SMOKE_MODE: "models", + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, timeout); + child.on("error", (error) => { + clearTimeout(timer); + resolve({ error, signal: null, status: null, stderr, stdout }); + }); + child.on("exit", (status, signal) => { + clearTimeout(timer); + resolve({ + error: timedOut ? new Error(`probe timed out after ${timeout}ms`) : undefined, + signal, + status, + stderr, + stdout, + }); + }); + }); +} + +describe("scripts/e2e/openwebui-probe.mjs", () => { + it("uses a short control-plane timeout for stalled sign-in requests", async () => { + const sockets = new Set(); + const server = createTcpServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("data", () => {}); + }); + const baseUrl = await listen(server); + const startedAt = Date.now(); + try { + const result = await runProbe( + baseUrl, + { + OPENWEBUI_CONTROL_TIMEOUT_MS: "25", + OPENWEBUI_FETCH_TIMEOUT_MS: "5000", + }, + 2_000, + ); + const elapsedMs = Date.now() - startedAt; + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Open WebUI signin timed out after 25ms"); + expect(elapsedMs).toBeLessThan(1500); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + server.close(); + } + }); + + it("keeps the control-plane timeout active while reading sign-in bodies", async () => { + const server = createServer((request, response) => { + if (request.url === "/api/v1/auths/signin") { + response.writeHead(200, { "content-type": "application/json" }); + response.flushHeaders(); + response.write("{"); + return; + } + response.writeHead(404).end(); + }); + const baseUrl = await listen(server); + try { + const result = await runProbe( + baseUrl, + { + OPENWEBUI_CONTROL_TIMEOUT_MS: "25", + OPENWEBUI_FETCH_TIMEOUT_MS: "5000", + }, + 2_000, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Open WebUI signin timed out after 25ms"); + } finally { + server.close(); + } + }); + + it("does not sleep after the final model-list attempt", async () => { + const server = createServer((request, response) => { + if (request.url === "/api/v1/auths/signin") { + response.writeHead(200, { + "content-type": "application/json", + "set-cookie": "openwebui-session=test; Path=/", + }); + response.end(JSON.stringify({ token: "test-token" })); + return; + } + if (request.url === "/api/models") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ data: [{ id: "other-model" }] })); + return; + } + response.writeHead(404).end(); + }); + const baseUrl = await listen(server); + try { + const result = await runProbe( + baseUrl, + { + OPENWEBUI_MODEL_ATTEMPTS: "1", + OPENWEBUI_MODEL_RETRY_MS: "1500", + }, + 1_000, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("openclaw model missing from Open WebUI model list"); + } finally { + server.close(); + } + }); + + it("passes in models mode when Open WebUI exposes the OpenClaw model", async () => { + const server = createServer((request, response) => { + if (request.url === "/api/v1/auths/signin") { + response.writeHead(200, { + "content-type": "application/json", + "set-cookie": "openwebui-session=test; Path=/", + }); + response.end(JSON.stringify({ token: "test-token" })); + return; + } + if (request.url === "/api/models") { + expect(request.headers.authorization).toBe("Bearer test-token"); + expect(request.headers.cookie).toContain("openwebui-session=test"); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] })); + return; + } + response.writeHead(404).end(); + }); + const baseUrl = await listen(server); + try { + const result = await runProbe(baseUrl); + + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + mode: "models", + model: "openclaw/default", + ok: true, + }); + } finally { + server.close(); + } + }); +});