mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-16 19:21:40 +00:00
423 lines
14 KiB
JavaScript
423 lines
14 KiB
JavaScript
// WebSocket client helpers for gateway network E2E scenarios.
|
|
import assert from "node:assert/strict";
|
|
import { readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import { pathToFileURL } from "node:url";
|
|
import { WebSocket } from "ws";
|
|
import { sleep as delay } from "../../../lib/sleep.mjs";
|
|
import { waitForWebSocketOpen } from "../websocket-open.mjs";
|
|
import { readGatewayNetworkClientConnectTimeoutMs } from "./limits.mjs";
|
|
import { onceFrame } from "./ws-frames.mjs";
|
|
|
|
function remainingDeadlineMs(deadline) {
|
|
return Math.max(1, deadline - Date.now());
|
|
}
|
|
|
|
async function openSocket(url, timeoutMs = 10_000) {
|
|
const ws = new WebSocket(url);
|
|
await waitForWebSocketOpen(ws, timeoutMs, "ws open timeout");
|
|
return ws;
|
|
}
|
|
|
|
function isRecord(value) {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function hasGatewayHealthSummaryPayload(response) {
|
|
if (!isRecord(response) || !isRecord(response.payload)) {
|
|
return false;
|
|
}
|
|
const { payload } = response;
|
|
return (
|
|
payload.ok === true &&
|
|
typeof payload.ts === "number" &&
|
|
typeof payload.durationMs === "number" &&
|
|
typeof payload.defaultAgentId === "string" &&
|
|
payload.defaultAgentId.trim() !== "" &&
|
|
Array.isArray(payload.agents) &&
|
|
isRecord(payload.channels) &&
|
|
Array.isArray(payload.channelOrder) &&
|
|
isRecord(payload.sessions)
|
|
);
|
|
}
|
|
|
|
function httpUrl(url, pathname = "/") {
|
|
const target = new URL(url);
|
|
target.protocol = target.protocol === "wss:" ? "https:" : "http:";
|
|
target.pathname = pathname;
|
|
target.search = "";
|
|
target.hash = "";
|
|
return target.toString();
|
|
}
|
|
|
|
async function readJson(response, label) {
|
|
let body;
|
|
try {
|
|
body = await response.json();
|
|
} catch {
|
|
throw new Error(`${label} returned non-JSON HTTP ${response.status}`);
|
|
}
|
|
return { status: response.status, body };
|
|
}
|
|
|
|
async function adminRpc({ token, url }, method, params = {}) {
|
|
const response = await fetch(httpUrl(url, "/api/v1/admin/rpc"), {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${token}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({ id: `e2e-${method}`, method, params }),
|
|
});
|
|
return await readJson(response, `Admin RPC ${method}`);
|
|
}
|
|
|
|
async function readProbe(url, pathname) {
|
|
const response = await fetch(httpUrl(url, pathname));
|
|
return await readJson(response, pathname);
|
|
}
|
|
|
|
async function requestUpgradeRejection(url, timeoutMs) {
|
|
const target = new URL(httpUrl(url));
|
|
return await new Promise((resolve, reject) => {
|
|
const request = httpRequest(
|
|
{
|
|
hostname: target.hostname,
|
|
port: target.port,
|
|
path: target.pathname,
|
|
headers: {
|
|
Connection: "Upgrade",
|
|
Upgrade: "websocket",
|
|
"Sec-WebSocket-Key": Buffer.from("gateway-net-e2e!").toString("base64"),
|
|
"Sec-WebSocket-Version": "13",
|
|
},
|
|
},
|
|
(response) => {
|
|
const chunks = [];
|
|
response.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
|
response.on("end", () => {
|
|
resolve({
|
|
status: response.statusCode,
|
|
body: Buffer.concat(chunks).toString("utf8"),
|
|
});
|
|
});
|
|
},
|
|
);
|
|
request.on("upgrade", (response, socket) => {
|
|
socket.destroy();
|
|
reject(new Error(`expected rejected websocket upgrade, received ${response.statusCode}`));
|
|
});
|
|
request.on("error", reject);
|
|
request.setTimeout(timeoutMs, () => {
|
|
request.destroy(new Error("websocket upgrade rejection timeout"));
|
|
});
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
function emitPhase(phase, startedAt) {
|
|
console.log(
|
|
JSON.stringify({
|
|
event: "gateway-network-phase",
|
|
phase,
|
|
durationMs: Date.now() - startedAt,
|
|
ok: true,
|
|
}),
|
|
);
|
|
}
|
|
|
|
export function assertReadySuspensionResponse(response, now = Date.now()) {
|
|
assert.equal(response?.status, 200, "suspension prepare must return HTTP 200");
|
|
assert.equal(response.body?.ok, true, "suspension prepare must succeed");
|
|
const payload = response.body.payload;
|
|
assert.equal(payload?.status, "ready", "suspension prepare must report ready");
|
|
assert.equal(typeof payload.suspensionId, "string", "suspension prepare must return an id");
|
|
assert(payload.suspensionId.length > 0, "suspension prepare must return an id");
|
|
assert(payload.expiresAtMs > now, "suspension lease must expire in the future");
|
|
assert.equal(payload.activeCount, 0, "suspension prepare must report no active work");
|
|
assert.deepEqual(payload.blockers, [], "suspension prepare must report no blockers");
|
|
return payload;
|
|
}
|
|
|
|
export function assertGatewaySuspendingError(response) {
|
|
assert.equal(response?.ok, false, "normal RPC must fail during suspension");
|
|
assert.equal(response.error?.code, "UNAVAILABLE", "normal RPC must be unavailable");
|
|
assert.equal(response.error?.retryable, true, "suspension error must be retryable");
|
|
assert.equal(
|
|
response.error?.details?.reason,
|
|
"gateway-suspending",
|
|
"normal RPC must identify gateway suspension",
|
|
);
|
|
assert.equal(
|
|
response.error?.details?.phase,
|
|
"prepared",
|
|
"normal RPC must identify the prepared phase",
|
|
);
|
|
}
|
|
|
|
export function assertSuspendedProbes(health, readiness) {
|
|
assert.equal(health.status, 200, "/healthz must remain live during suspension");
|
|
assert.equal(health.body?.status, "live", "/healthz must report live");
|
|
assert.equal(health.body?.ok, true, "/healthz must report live");
|
|
assert.equal(readiness.status, 503, "/readyz must fail during suspension");
|
|
assert.equal(readiness.body?.ready, false, "/readyz must report not ready");
|
|
assert.equal(
|
|
readiness.body?.failing?.includes("gateway-draining"),
|
|
true,
|
|
"/readyz must identify gateway-draining",
|
|
);
|
|
}
|
|
|
|
function assertHealthyProbes(health, readiness) {
|
|
assert.equal(health.status, 200, "/healthz must be live");
|
|
assert.equal(health.body?.status, "live", "/healthz must be live");
|
|
assert.equal(readiness.status, 200, "/readyz must recover");
|
|
assert.equal(readiness.body?.ready, true, "/readyz must recover");
|
|
}
|
|
|
|
function assertRpcSuccess(response, message) {
|
|
assert(response?.ok === true, message);
|
|
return response.payload;
|
|
}
|
|
|
|
function assertAdminSuccess(response, message) {
|
|
assert.equal(response?.status, 200, `${message}: expected HTTP 200`);
|
|
return assertRpcSuccess(response.body, message);
|
|
}
|
|
|
|
async function runGatewaySuspensionPreRestartClient({
|
|
statePath,
|
|
token,
|
|
url,
|
|
timeoutMs = readGatewayNetworkClientConnectTimeoutMs(),
|
|
}) {
|
|
const startedAt = Date.now();
|
|
const rpc = (method, params) => adminRpc({ token, url }, method, params);
|
|
const firstLease = assertReadySuspensionResponse(
|
|
await rpc("gateway.suspend.prepare", { requestId: "gateway-network-live-contract" }),
|
|
);
|
|
|
|
assertSuspendedProbes(await readProbe(url, "/healthz"), await readProbe(url, "/readyz"));
|
|
|
|
const blockedAdminHealth = await rpc("health");
|
|
assert.equal(blockedAdminHealth.status, 503, "Admin health must return HTTP 503");
|
|
assertGatewaySuspendingError(blockedAdminHealth.body);
|
|
|
|
const upgrade = await requestUpgradeRejection(url, timeoutMs);
|
|
assert.equal(upgrade.status, 503, "new websocket upgrade must return HTTP 503");
|
|
assert.equal(
|
|
upgrade.body,
|
|
"Gateway websocket admission closed",
|
|
"new websocket upgrade must return the canonical admission body",
|
|
);
|
|
|
|
const wrongResume = await rpc("gateway.suspend.resume", {
|
|
suspensionId: `${firstLease.suspensionId}-wrong`,
|
|
});
|
|
assert.equal(wrongResume.status, 400, "wrong suspension id must return HTTP 400");
|
|
assert.equal(
|
|
wrongResume.body?.error?.code,
|
|
"INVALID_REQUEST",
|
|
"wrong suspension id must return INVALID_REQUEST",
|
|
);
|
|
const statusAfterMismatch = assertAdminSuccess(
|
|
await rpc("gateway.suspend.status", { suspensionId: firstLease.suspensionId }),
|
|
"status after wrong resume",
|
|
);
|
|
assert.equal(statusAfterMismatch?.status, "ready", "wrong resume must preserve the lease");
|
|
|
|
const resumed = assertAdminSuccess(
|
|
await rpc("gateway.suspend.resume", { suspensionId: firstLease.suspensionId }),
|
|
"resume first lease",
|
|
);
|
|
assert.deepEqual(
|
|
{ status: resumed?.status, resumed: resumed?.resumed },
|
|
{ status: "running", resumed: true },
|
|
"first resume must release the lease",
|
|
);
|
|
const repeatedResume = assertAdminSuccess(
|
|
await rpc("gateway.suspend.resume", { suspensionId: firstLease.suspensionId }),
|
|
"repeat first resume",
|
|
);
|
|
assert.equal(repeatedResume?.resumed, false, "repeat resume must be idempotent");
|
|
|
|
assertHealthyProbes(await readProbe(url, "/healthz"), await readProbe(url, "/readyz"));
|
|
|
|
const requestId = "gateway-network-restart-contract";
|
|
const secondLease = assertReadySuspensionResponse(
|
|
await rpc("gateway.suspend.prepare", { requestId }),
|
|
);
|
|
await writeFile(
|
|
statePath,
|
|
JSON.stringify({
|
|
requestId,
|
|
suspensionId: secondLease.suspensionId,
|
|
expiresAtMs: secondLease.expiresAtMs,
|
|
}),
|
|
);
|
|
emitPhase("pre-restart", startedAt);
|
|
}
|
|
|
|
async function runGatewaySuspensionPostRestartClient({ statePath, token, url }) {
|
|
const startedAt = Date.now();
|
|
const state = JSON.parse(await readFile(statePath, "utf8"));
|
|
assert(Date.now() < state.expiresAtMs, "restart proof exceeded the original lease");
|
|
const rpc = (method, params) => adminRpc({ token, url }, method, params);
|
|
|
|
const oldStatus = assertAdminSuccess(
|
|
await rpc("gateway.suspend.status", { suspensionId: state.suspensionId }),
|
|
"old lease status after restart",
|
|
);
|
|
assert(oldStatus?.status === "running", "old lease must not survive process restart");
|
|
const oldResume = assertAdminSuccess(
|
|
await rpc("gateway.suspend.resume", { suspensionId: state.suspensionId }),
|
|
"old lease resume after restart",
|
|
);
|
|
assert(oldResume?.resumed === false, "old lease resume must be idempotently inactive");
|
|
|
|
assertHealthyProbes(await readProbe(url, "/healthz"), await readProbe(url, "/readyz"));
|
|
assertAdminSuccess(await rpc("health"), "Admin health after restart");
|
|
|
|
const replacement = assertReadySuspensionResponse(
|
|
await rpc("gateway.suspend.prepare", { requestId: state.requestId }),
|
|
);
|
|
assert(
|
|
replacement.suspensionId !== state.suspensionId,
|
|
"reused request id must create a fresh suspension lease after restart",
|
|
);
|
|
const replacementResume = assertAdminSuccess(
|
|
await rpc("gateway.suspend.resume", { suspensionId: replacement.suspensionId }),
|
|
"replacement lease resume",
|
|
);
|
|
assert(replacementResume?.resumed === true, "replacement lease must resume");
|
|
emitPhase("post-restart", startedAt);
|
|
}
|
|
|
|
export function responseError(method, response) {
|
|
const message = response.error?.message ?? "unknown";
|
|
return new Error(`${method} failed: ${message}`);
|
|
}
|
|
|
|
function isRetryableStartupError(message) {
|
|
return (
|
|
message.includes("gateway starting") ||
|
|
message.includes("closed before frame") ||
|
|
message.includes("closed before open") ||
|
|
message.includes("ws open timeout") ||
|
|
message.includes("ECONNREFUSED") ||
|
|
message.includes("ECONNRESET") ||
|
|
message.includes("timeout")
|
|
);
|
|
}
|
|
|
|
async function readProtocolVersion() {
|
|
const protocol = await import("../../../../dist/gateway/protocol/index.js");
|
|
return protocol.PROTOCOL_VERSION;
|
|
}
|
|
|
|
export async function runGatewayNetworkClient(
|
|
{ token, url, timeoutMs = readGatewayNetworkClientConnectTimeoutMs() },
|
|
deps = {},
|
|
) {
|
|
const deadline = Date.now() + timeoutMs;
|
|
const delayImpl = deps.delay ?? delay;
|
|
const onceFrameImpl = deps.onceFrame ?? onceFrame;
|
|
const openSocketImpl = deps.openSocket ?? openSocket;
|
|
const protocolVersion = deps.protocolVersion ?? (await readProtocolVersion());
|
|
const stdout = deps.stdout ?? console.log;
|
|
|
|
let lastError;
|
|
while (Date.now() < deadline) {
|
|
let ws;
|
|
try {
|
|
ws = await openSocketImpl(url, remainingDeadlineMs(deadline));
|
|
ws.send(
|
|
JSON.stringify({
|
|
type: "req",
|
|
id: "c1",
|
|
method: "connect",
|
|
params: {
|
|
minProtocol: protocolVersion,
|
|
maxProtocol: protocolVersion,
|
|
client: {
|
|
id: "test",
|
|
displayName: "docker-net-e2e",
|
|
version: "dev",
|
|
platform: process.platform,
|
|
mode: "test",
|
|
},
|
|
caps: [],
|
|
auth: { token },
|
|
},
|
|
}),
|
|
);
|
|
|
|
const connectRes = await onceFrameImpl(
|
|
ws,
|
|
(frame) => frame?.type === "res" && frame?.id === "c1",
|
|
remainingDeadlineMs(deadline),
|
|
);
|
|
if (!connectRes.ok) {
|
|
lastError = responseError("connect", connectRes);
|
|
if (!isRetryableStartupError(lastError.message)) {
|
|
throw lastError;
|
|
}
|
|
} else {
|
|
ws.send(JSON.stringify({ type: "req", id: "h1", method: "health" }));
|
|
const healthRes = await onceFrameImpl(
|
|
ws,
|
|
(frame) => frame?.type === "res" && frame?.id === "h1",
|
|
remainingDeadlineMs(deadline),
|
|
);
|
|
if (healthRes.ok) {
|
|
if (!hasGatewayHealthSummaryPayload(healthRes)) {
|
|
throw new Error("health failed: missing health summary payload");
|
|
}
|
|
stdout("ok");
|
|
return;
|
|
}
|
|
|
|
throw responseError("health", healthRes);
|
|
}
|
|
} catch (error) {
|
|
lastError = error instanceof Error ? error : new Error(String(error));
|
|
if (!isRetryableStartupError(lastError.message)) {
|
|
throw lastError;
|
|
}
|
|
} finally {
|
|
ws?.close();
|
|
}
|
|
|
|
const retryDelayMs = Math.min(500, deadline - Date.now());
|
|
if (retryDelayMs > 0) {
|
|
await delayImpl(retryDelayMs);
|
|
}
|
|
}
|
|
|
|
throw lastError ?? new Error("connect failed: timeout");
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
const url = process.env.GW_URL;
|
|
const token = process.env.GW_TOKEN;
|
|
if (!url || !token) {
|
|
throw new Error("missing GW_URL/GW_TOKEN");
|
|
}
|
|
const mode = process.env.GW_MODE ?? "network";
|
|
if (mode === "network") {
|
|
await runGatewayNetworkClient({ token, url });
|
|
} else {
|
|
const statePath = process.env.GW_STATE_PATH;
|
|
if (!statePath) {
|
|
throw new Error("missing GW_STATE_PATH");
|
|
}
|
|
if (mode === "suspension-pre-restart") {
|
|
await runGatewaySuspensionPreRestartClient({ statePath, token, url });
|
|
} else if (mode === "suspension-post-restart") {
|
|
await runGatewaySuspensionPostRestartClient({ statePath, token, url });
|
|
} else {
|
|
throw new Error(`unknown GW_MODE: ${mode}`);
|
|
}
|
|
}
|
|
}
|