fix(e2e): bound Open WebUI control probes

2026-05-28 04:46:16 +00:00 · 2026-05-27 16:17:34 +02:00
parent b8cf83aeb3
commit 5eeaa5603f
2 changed files with 340 additions and 44 deletions
--- a/scripts/e2e/openwebui-probe.mjs
+++ b/scripts/e2e/openwebui-probe.mjs
@@ -5,13 +5,23 @@ const email = process.env.OPENWEBUI_ADMIN_EMAIL ?? "";
 const password = process.env.OPENWEBUI_ADMIN_PASSWORD ?? "";
 const expectedNonce = process.env.OPENWEBUI_EXPECTED_NONCE ?? "";
 const prompt = process.env.OPENWEBUI_PROMPT ?? "";
-const modelAttempts = Number.parseInt(process.env.OPENWEBUI_MODEL_ATTEMPTS ?? "72", 10);
-const modelRetryMs = Number.parseInt(process.env.OPENWEBUI_MODEL_RETRY_MS ?? "5000", 10);
-const fetchTimeoutMs = Number.parseInt(process.env.OPENWEBUI_FETCH_TIMEOUT_MS ?? "720000", 10);
+const modelAttempts = readPositiveInt(process.env.OPENWEBUI_MODEL_ATTEMPTS, 72);
+const modelRetryMs = readNonNegativeInt(process.env.OPENWEBUI_MODEL_RETRY_MS, 5000);
+const fetchTimeoutMs = readPositiveInt(process.env.OPENWEBUI_FETCH_TIMEOUT_MS, 720000);
+const controlTimeoutMs = readPositiveInt(
+  process.env.OPENWEBUI_CONTROL_TIMEOUT_MS,
+  Math.min(fetchTimeoutMs, 30000),
+);
+const chatTimeoutMs = readPositiveInt(process.env.OPENWEBUI_CHAT_TIMEOUT_MS, fetchTimeoutMs);
 const smokeMode =
  process.env.OPENWEBUI_SMOKE_MODE ?? process.env.OPENCLAW_OPENWEBUI_SMOKE_MODE ?? "chat";

-setGlobalDispatcher(new Agent({ bodyTimeout: fetchTimeoutMs, headersTimeout: fetchTimeoutMs }));
+setGlobalDispatcher(
+  new Agent({
+    bodyTimeout: Math.max(controlTimeoutMs, chatTimeoutMs),
+    headersTimeout: Math.max(controlTimeoutMs, chatTimeoutMs),
+  }),
+);

 if (!baseUrl || !email || !password || !expectedNonce || !prompt) {
  throw new Error("Missing required OPENWEBUI_* environment variables");
@@ -20,6 +30,41 @@ if (smokeMode !== "models" && smokeMode !== "chat") {
  throw new Error(`Unsupported OPENWEBUI_SMOKE_MODE: ${smokeMode}`);
 }

+function readPositiveInt(raw, fallback) {
+  const parsed = Number.parseInt(String(raw || ""), 10);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function readNonNegativeInt(raw, fallback) {
+  const parsed = Number.parseInt(String(raw || ""), 10);
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
+}
+
+function createTimeoutError(label, timeoutMs) {
+  const error = new Error(`${label} timed out after ${timeoutMs}ms`);
+  error.code = "ETIMEDOUT";
+  return error;
+}
+
+async function withRequestTimeout(label, timeoutMs, run) {
+  const controller = new AbortController();
+  const timeoutError = createTimeoutError(label, timeoutMs);
+  const timer = setTimeout(() => {
+    controller.abort(timeoutError);
+  }, timeoutMs);
+  timer.unref?.();
+  try {
+    return await run(controller.signal);
+  } catch (error) {
+    if (controller.signal.aborted) {
+      throw timeoutError;
+    }
+    throw error;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
 function getCookieHeader(res) {
  const raw = res.headers.get("set-cookie");
  if (!raw) {
@@ -49,6 +94,69 @@ function sleep(ms) {
  });
 }

+async function fetchSignin() {
+  return await withRequestTimeout("Open WebUI signin", controlTimeoutMs, async (signal) => {
+    const response = await fetch(`${baseUrl}/api/v1/auths/signin`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ email, password }),
+      signal,
+    });
+    if (!response.ok) {
+      const body = await response.text();
+      throw new Error(`signin failed: HTTP ${response.status} ${body}`);
+    }
+    return {
+      cookie: getCookieHeader(response),
+      json: await response.json(),
+    };
+  });
+}
+
+async function fetchModels(authHeaders, attempt) {
+  return await withRequestTimeout(
+    `Open WebUI models attempt ${attempt}`,
+    controlTimeoutMs,
+    async (signal) => {
+      const response = await fetch(`${baseUrl}/api/models`, { headers: authHeaders, signal });
+      if (!response.ok) {
+        return {
+          ok: false,
+          status: response.status,
+          text: await response.text(),
+        };
+      }
+      return {
+        json: await response.json(),
+        ok: true,
+      };
+    },
+  );
+}
+
+async function fetchChatCompletion(authHeaders, targetModel) {
+  return await withRequestTimeout("Open WebUI chat completion", chatTimeoutMs, async (signal) => {
+    const response = await fetch(`${baseUrl}/api/chat/completions`, {
+      method: "POST",
+      headers: {
+        ...authHeaders,
+        "content-type": "application/json",
+      },
+      body: JSON.stringify({
+        model: targetModel,
+        messages: [{ role: "user", content: prompt }],
+      }),
+      signal,
+    });
+    if (!response.ok) {
+      throw new Error(
+        `/api/chat/completions failed: HTTP ${response.status} ${await response.text()}`,
+      );
+    }
+    return await response.json();
+  });
+}
+
 function extractModelIds(modelsJson) {
  const models = Array.isArray(modelsJson)
    ? modelsJson
@@ -62,22 +170,12 @@ function extractModelIds(modelsJson) {
    .filter((value) => typeof value === "string");
 }

-const signinRes = await fetch(`${baseUrl}/api/v1/auths/signin`, {
-  method: "POST",
-  headers: { "content-type": "application/json" },
-  body: JSON.stringify({ email, password }),
-});
-if (!signinRes.ok) {
-  const body = await signinRes.text();
-  throw new Error(`signin failed: HTTP ${signinRes.status} ${body}`);
-}
-
-const signinJson = await signinRes.json();
+const signin = await fetchSignin();
+const signinJson = signin.json;
 const token =
  signinJson?.token ?? signinJson?.access_token ?? signinJson?.jwt ?? signinJson?.data?.token ?? "";
-const cookie = getCookieHeader(signinRes);
 const authHeaders = {
-  ...buildAuthHeaders(token, cookie),
+  ...buildAuthHeaders(token, signin.cookie),
  accept: "application/json",
 };

@@ -85,25 +183,24 @@ let modelIds = [];
 let targetModel = "";
 let lastModelsError = "";
 for (let attempt = 1; attempt <= modelAttempts; attempt += 1) {
-  const modelsRes = await fetch(`${baseUrl}/api/models`, { headers: authHeaders }).catch(
-    (error) => {
-      lastModelsError = error instanceof Error ? error.message : String(error);
-      return undefined;
-    },
-  );
-  if (modelsRes?.ok) {
-    const modelsJson = await modelsRes.json();
-    modelIds = extractModelIds(modelsJson);
+  const modelsResult = await fetchModels(authHeaders, attempt).catch((error) => {
+    lastModelsError = error instanceof Error ? error.message : String(error);
+    return undefined;
+  });
+  if (modelsResult?.ok) {
+    modelIds = extractModelIds(modelsResult.json);
    targetModel =
      modelIds.find((id) => id === "openclaw/default") ?? modelIds.find((id) => id === "openclaw");
    if (targetModel) {
      break;
    }
    lastModelsError = `missing openclaw model: ${JSON.stringify(modelIds)}`;
-  } else if (modelsRes) {
-    lastModelsError = `HTTP ${modelsRes.status} ${await modelsRes.text()}`;
+  } else if (modelsResult) {
+    lastModelsError = `HTTP ${modelsResult.status} ${modelsResult.text}`;
+  }
+  if (attempt < modelAttempts) {
+    await sleep(modelRetryMs);
  }
-  await sleep(modelRetryMs);
 }
 if (!targetModel) {
  throw new Error(
@@ -115,21 +212,7 @@ if (smokeMode === "models") {
  process.exit(0);
 }

-const chatRes = await fetch(`${baseUrl}/api/chat/completions`, {
-  method: "POST",
-  headers: {
-    ...authHeaders,
-    "content-type": "application/json",
-  },
-  body: JSON.stringify({
-    model: targetModel,
-    messages: [{ role: "user", content: prompt }],
-  }),
-});
-if (!chatRes.ok) {
-  throw new Error(`/api/chat/completions failed: HTTP ${chatRes.status} ${await chatRes.text()}`);
-}
-const chatJson = await chatRes.json();
+const chatJson = await fetchChatCompletion(authHeaders, targetModel);
 const reply =
  chatJson?.choices?.[0]?.message?.content ?? chatJson?.message?.content ?? chatJson?.content ?? "";
 if (typeof reply !== "string" || !reply.includes(expectedNonce)) {
--- a/test/scripts/openwebui-probe.test.ts
+++ b/test/scripts/openwebui-probe.test.ts
@@ -0,0 +1,213 @@
+import { spawn } from "node:child_process";
+import { createServer, type Server as HttpServer } from "node:http";
+import { createServer as createTcpServer, type Server as TcpServer, type Socket } from "node:net";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+const probePath = path.resolve("scripts/e2e/openwebui-probe.mjs");
+
+interface ProbeResult {
+  error?: Error;
+  signal: NodeJS.Signals | null;
+  status: number | null;
+  stderr: string;
+  stdout: string;
+}
+
+async function listen(server: HttpServer | TcpServer): Promise<string> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      server.off("error", reject);
+      resolve();
+    });
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("test server did not expose a TCP port");
+  }
+  return `http://127.0.0.1:${address.port}`;
+}
+
+function runProbe(baseUrl: string, env: Record<string, string> = {}, timeout = 3_000) {
+  return new Promise<ProbeResult>((resolve) => {
+    const child = spawn(process.execPath, [probePath], {
+      env: {
+        ...process.env,
+        OPENWEBUI_ADMIN_EMAIL: "openwebui-e2e@example.com",
+        OPENWEBUI_ADMIN_PASSWORD: "test-password",
+        OPENWEBUI_BASE_URL: baseUrl,
+        OPENWEBUI_CONTROL_TIMEOUT_MS: "250",
+        OPENWEBUI_EXPECTED_NONCE: "nonce-123",
+        OPENWEBUI_MODEL_ATTEMPTS: "1",
+        OPENWEBUI_MODEL_RETRY_MS: "0",
+        OPENWEBUI_PROMPT: "reply with nonce-123",
+        OPENWEBUI_SMOKE_MODE: "models",
+        ...env,
+      },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGKILL");
+    }, timeout);
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      resolve({ error, signal: null, status: null, stderr, stdout });
+    });
+    child.on("exit", (status, signal) => {
+      clearTimeout(timer);
+      resolve({
+        error: timedOut ? new Error(`probe timed out after ${timeout}ms`) : undefined,
+        signal,
+        status,
+        stderr,
+        stdout,
+      });
+    });
+  });
+}
+
+describe("scripts/e2e/openwebui-probe.mjs", () => {
+  it("uses a short control-plane timeout for stalled sign-in requests", async () => {
+    const sockets = new Set<Socket>();
+    const server = createTcpServer((socket) => {
+      sockets.add(socket);
+      socket.on("close", () => sockets.delete(socket));
+      socket.on("data", () => {});
+    });
+    const baseUrl = await listen(server);
+    const startedAt = Date.now();
+    try {
+      const result = await runProbe(
+        baseUrl,
+        {
+          OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
+          OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
+        },
+        2_000,
+      );
+      const elapsedMs = Date.now() - startedAt;
+
+      expect(result.error).toBeUndefined();
+      expect(result.status).not.toBe(0);
+      expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
+      expect(elapsedMs).toBeLessThan(1500);
+    } finally {
+      for (const socket of sockets) {
+        socket.destroy();
+      }
+      server.close();
+    }
+  });
+
+  it("keeps the control-plane timeout active while reading sign-in bodies", async () => {
+    const server = createServer((request, response) => {
+      if (request.url === "/api/v1/auths/signin") {
+        response.writeHead(200, { "content-type": "application/json" });
+        response.flushHeaders();
+        response.write("{");
+        return;
+      }
+      response.writeHead(404).end();
+    });
+    const baseUrl = await listen(server);
+    try {
+      const result = await runProbe(
+        baseUrl,
+        {
+          OPENWEBUI_CONTROL_TIMEOUT_MS: "25",
+          OPENWEBUI_FETCH_TIMEOUT_MS: "5000",
+        },
+        2_000,
+      );
+
+      expect(result.error).toBeUndefined();
+      expect(result.status).not.toBe(0);
+      expect(result.stderr).toContain("Open WebUI signin timed out after 25ms");
+    } finally {
+      server.close();
+    }
+  });
+
+  it("does not sleep after the final model-list attempt", async () => {
+    const server = createServer((request, response) => {
+      if (request.url === "/api/v1/auths/signin") {
+        response.writeHead(200, {
+          "content-type": "application/json",
+          "set-cookie": "openwebui-session=test; Path=/",
+        });
+        response.end(JSON.stringify({ token: "test-token" }));
+        return;
+      }
+      if (request.url === "/api/models") {
+        response.writeHead(200, { "content-type": "application/json" });
+        response.end(JSON.stringify({ data: [{ id: "other-model" }] }));
+        return;
+      }
+      response.writeHead(404).end();
+    });
+    const baseUrl = await listen(server);
+    try {
+      const result = await runProbe(
+        baseUrl,
+        {
+          OPENWEBUI_MODEL_ATTEMPTS: "1",
+          OPENWEBUI_MODEL_RETRY_MS: "1500",
+        },
+        1_000,
+      );
+
+      expect(result.error).toBeUndefined();
+      expect(result.status).not.toBe(0);
+      expect(result.stderr).toContain("openclaw model missing from Open WebUI model list");
+    } finally {
+      server.close();
+    }
+  });
+
+  it("passes in models mode when Open WebUI exposes the OpenClaw model", async () => {
+    const server = createServer((request, response) => {
+      if (request.url === "/api/v1/auths/signin") {
+        response.writeHead(200, {
+          "content-type": "application/json",
+          "set-cookie": "openwebui-session=test; Path=/",
+        });
+        response.end(JSON.stringify({ token: "test-token" }));
+        return;
+      }
+      if (request.url === "/api/models") {
+        expect(request.headers.authorization).toBe("Bearer test-token");
+        expect(request.headers.cookie).toContain("openwebui-session=test");
+        response.writeHead(200, { "content-type": "application/json" });
+        response.end(JSON.stringify({ data: [{ id: "openclaw/default" }] }));
+        return;
+      }
+      response.writeHead(404).end();
+    });
+    const baseUrl = await listen(server);
+    try {
+      const result = await runProbe(baseUrl);
+
+      expect(result.status).toBe(0);
+      expect(JSON.parse(result.stdout)).toMatchObject({
+        mode: "models",
+        model: "openclaw/default",
+        ok: true,
+      });
+    } finally {
+      server.close();
+    }
+  });
+});