mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-26 07:19:32 +00:00
1975 lines
63 KiB
TypeScript
1975 lines
63 KiB
TypeScript
// Kitchen Sink Rpc Walk tests cover kitchen sink rpc walk script behavior.
|
|
import { createHash } from "node:crypto";
|
|
import { EventEmitter } from "node:events";
|
|
import fs, {
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { setTimeout as delay } from "node:timers/promises";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
appendBoundedOutput,
|
|
assertChannelAccountRunning,
|
|
assertCommandResourceCeiling,
|
|
assertCreatedKitchenSinkSession,
|
|
assertDiagnosticStabilityClean,
|
|
assertExpectedKitchenSinkToolEntries,
|
|
assertGatewayHealthPayload,
|
|
assertGatewayStatusPayload,
|
|
assertKitchenSinkImageJobInvokeResult,
|
|
assertKitchenSinkUiDescriptors,
|
|
assertKitchenSinkSearchInvokeResult,
|
|
assertKitchenSinkTextInvokeResult,
|
|
assertOperatorRpcDenied,
|
|
assertResourceCeiling,
|
|
assertTtsProviderCoverage,
|
|
cleanupKitchenSinkEnv,
|
|
createGatewayReadyLogScanner,
|
|
createRpcCliRunOptions,
|
|
extractPluginCommandNames,
|
|
extractTtsProviderIds,
|
|
fetchJson,
|
|
findErrorLogFindings,
|
|
findDistCallGatewayModuleFiles,
|
|
hasChildExited,
|
|
listKitchenSinkToolInvokeNames,
|
|
listKitchenSinkAuthorizationRpcProbeNames,
|
|
listKitchenSinkReadOnlyRpcProbeNames,
|
|
makeEnv,
|
|
parseGatewayCliRequestFailure,
|
|
readPositiveInt,
|
|
readBoundedResponseText,
|
|
resolveKitchenSinkRpcPort,
|
|
runCommand,
|
|
sampleProcess,
|
|
sampleWindowsProcessByPort,
|
|
shouldPrintHelp,
|
|
stopGateway,
|
|
summarizeProcessSamples,
|
|
tailFile,
|
|
unwrapRpcPayload,
|
|
usesBuiltOpenClawEntry,
|
|
waitForGatewayReady,
|
|
} from "../../scripts/e2e/kitchen-sink-rpc-walk.mjs";
|
|
import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js";
|
|
|
|
const posixIt = process.platform === "win32" ? it.skip : it;
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
function captureSyncError(action: () => void): Error {
|
|
try {
|
|
action();
|
|
} catch (error) {
|
|
return error as Error;
|
|
}
|
|
throw new Error("expected action to throw");
|
|
}
|
|
|
|
describe("kitchen-sink RPC isolated state", () => {
|
|
it("prints help without creating temp state or installing the plugin", async () => {
|
|
const result = await runCommand(process.execPath, [
|
|
"scripts/e2e/kitchen-sink-rpc-walk.mjs",
|
|
"--help",
|
|
]);
|
|
|
|
expect(result.stderr).toBe("");
|
|
expect(result.stdout).toContain("Usage: node scripts/e2e/kitchen-sink-rpc-walk.mjs");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_NPM_SPEC");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_PERSONALITY");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_RPC_PORT");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_RPC_FETCH_MS");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_RPC_FETCH_BODY_BYTES");
|
|
expect(result.stdout).toContain("OPENCLAW_KITCHEN_SINK_OUTPUT_CAPTURE_CHARS");
|
|
expect(result.stdout).not.toContain("Kitchen Sink RPC walk using");
|
|
expect(result.stdout).not.toContain("temp root preserved");
|
|
});
|
|
|
|
it("prints help before parsing malformed runtime guardrails", async () => {
|
|
const result = await runCommand(
|
|
process.execPath,
|
|
["scripts/e2e/kitchen-sink-rpc-walk.mjs", "--help"],
|
|
{
|
|
env: {
|
|
...process.env,
|
|
OPENCLAW_KITCHEN_SINK_MAX_RSS_MIB: "1e3",
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(result.stderr).toBe("");
|
|
expect(result.stdout).toContain("Usage: node scripts/e2e/kitchen-sink-rpc-walk.mjs");
|
|
});
|
|
|
|
it("detects short and long help flags", () => {
|
|
expect(shouldPrintHelp(["--help"])).toBe(true);
|
|
expect(shouldPrintHelp(["-h"])).toBe(true);
|
|
expect(shouldPrintHelp([])).toBe(false);
|
|
});
|
|
|
|
it("rejects loose numeric env values before they bypass runtime guardrails", () => {
|
|
expect(readPositiveInt(undefined, 60_000)).toBe(60_000);
|
|
expect(readPositiveInt("", 60_000)).toBe(60_000);
|
|
expect(readPositiveInt("1000", 60_000)).toBe(1000);
|
|
expect(readPositiveInt(" 1000 ", 60_000)).toBe(1000);
|
|
expect(() => readPositiveInt("1e3", 60_000, "OPENCLAW_KITCHEN_SINK_MAX_RSS_MIB")).toThrow(
|
|
'OPENCLAW_KITCHEN_SINK_MAX_RSS_MIB must be a positive integer. Got: "1e3"',
|
|
);
|
|
expect(() => readPositiveInt("1000ms", 60_000, "OPENCLAW_KITCHEN_SINK_RPC_READY_MS")).toThrow(
|
|
'OPENCLAW_KITCHEN_SINK_RPC_READY_MS must be a positive integer. Got: "1000ms"',
|
|
);
|
|
expect(() => readPositiveInt("0", 60_000, "OPENCLAW_KITCHEN_SINK_RPC_PORT")).toThrow(
|
|
'OPENCLAW_KITCHEN_SINK_RPC_PORT must be a positive integer. Got: "0"',
|
|
);
|
|
});
|
|
|
|
it("uses an explicit RPC port or asks the OS for an available fallback", async () => {
|
|
await expect(
|
|
resolveKitchenSinkRpcPort({ OPENCLAW_KITCHEN_SINK_RPC_PORT: "19080" }),
|
|
).resolves.toBe(19080);
|
|
await expect(
|
|
resolveKitchenSinkRpcPort({}, { findAvailablePort: async () => 45678 }),
|
|
).resolves.toBe(45678);
|
|
});
|
|
|
|
it("cleans up the generated temporary home tree", async () => {
|
|
const { root, env } = makeEnv();
|
|
|
|
expect(root).toContain("openclaw-kitchen-sink-rpc-");
|
|
expect(env.HOME).toBe(path.join(root, "home"));
|
|
expect(env.USERPROFILE).toBe(env.HOME);
|
|
expect(env.OPENCLAW_HOME).toBe(env.HOME);
|
|
expect(env.OPENCLAW_STATE_DIR).toBe(path.join(env.HOME, ".openclaw"));
|
|
expect(env.OPENCLAW_CONFIG_PATH).toBe(path.join(env.OPENCLAW_STATE_DIR, "openclaw.json"));
|
|
expect(existsSync(env.OPENCLAW_STATE_DIR)).toBe(true);
|
|
|
|
await expect(cleanupKitchenSinkEnv(root)).resolves.toBe(true);
|
|
|
|
expect(existsSync(root)).toBe(false);
|
|
});
|
|
|
|
it("can fail the walk when generated temp cleanup cannot remove the root", async () => {
|
|
const rmSyncSpy = vi.spyOn(fs, "rmSync").mockImplementation(() => {
|
|
throw new Error("device busy");
|
|
});
|
|
|
|
try {
|
|
await expect(
|
|
cleanupKitchenSinkEnv("/tmp/openclaw-kitchen-sink-rpc-stuck", {
|
|
attempts: 3,
|
|
delayMs: 1,
|
|
throwOnFailure: true,
|
|
warn: false,
|
|
}),
|
|
).rejects.toThrow(
|
|
"failed to remove Kitchen Sink RPC temp root: /tmp/openclaw-kitchen-sink-rpc-stuck",
|
|
);
|
|
expect(rmSyncSpy).toHaveBeenCalledTimes(3);
|
|
} finally {
|
|
rmSyncSpy.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC gateway teardown", () => {
|
|
it("treats signaled gateway children as exited", () => {
|
|
expect(hasChildExited({ exitCode: null, signalCode: "SIGTERM" })).toBe(true);
|
|
expect(hasChildExited({ exitCode: 0, signalCode: null })).toBe(true);
|
|
expect(hasChildExited({ exitCode: null, signalCode: null })).toBe(false);
|
|
});
|
|
|
|
it("releases gateway handles when the process ignores teardown signals", async () => {
|
|
const child = new EventEmitter() as EventEmitter & {
|
|
exitCode: number | null;
|
|
kill: ReturnType<typeof vi.fn>;
|
|
signalCode: NodeJS.Signals | null;
|
|
stderr: { destroy: ReturnType<typeof vi.fn> };
|
|
stdin: { destroy: ReturnType<typeof vi.fn> };
|
|
stdout: { destroy: ReturnType<typeof vi.fn> };
|
|
unref: ReturnType<typeof vi.fn>;
|
|
};
|
|
child.exitCode = null;
|
|
child.signalCode = null;
|
|
child.kill = vi.fn(() => true);
|
|
child.stderr = { destroy: vi.fn() };
|
|
child.stdin = { destroy: vi.fn() };
|
|
child.stdout = { destroy: vi.fn() };
|
|
child.unref = vi.fn();
|
|
|
|
await stopGateway(child, { killGraceMs: 1, teardownGraceMs: 1 });
|
|
|
|
expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM");
|
|
expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL");
|
|
expect(child.stdin.destroy).toHaveBeenCalledOnce();
|
|
expect(child.stdout.destroy).toHaveBeenCalledOnce();
|
|
expect(child.stderr.destroy).toHaveBeenCalledOnce();
|
|
expect(child.unref).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("treats ESRCH during gateway teardown as already exited", async () => {
|
|
const child = new EventEmitter() as EventEmitter & {
|
|
exitCode: number | null;
|
|
kill: ReturnType<typeof vi.fn>;
|
|
signalCode: NodeJS.Signals | null;
|
|
};
|
|
child.exitCode = null;
|
|
child.signalCode = null;
|
|
child.kill = vi.fn(() => {
|
|
const error = Object.assign(new Error("process already exited"), { code: "ESRCH" });
|
|
throw error;
|
|
});
|
|
|
|
await expect(
|
|
stopGateway(child, { killGraceMs: 1, teardownGraceMs: 1 }),
|
|
).resolves.toBeUndefined();
|
|
|
|
expect(child.kill).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
it("treats failed gateway kill signals as already exited", async () => {
|
|
const child = new EventEmitter() as EventEmitter & {
|
|
exitCode: number | null;
|
|
kill: ReturnType<typeof vi.fn>;
|
|
signalCode: NodeJS.Signals | null;
|
|
};
|
|
child.exitCode = null;
|
|
child.signalCode = null;
|
|
child.kill = vi.fn(() => false);
|
|
|
|
await expect(
|
|
stopGateway(child, { killGraceMs: 1, teardownGraceMs: 1 }),
|
|
).resolves.toBeUndefined();
|
|
|
|
expect(child.kill).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
posixIt("does not trust an exited wrapper while the gateway process group is alive", async () => {
|
|
const child = Object.assign(new EventEmitter(), {
|
|
exitCode: 0,
|
|
kill: vi.fn(),
|
|
pid: 12347,
|
|
signalCode: null as NodeJS.Signals | null,
|
|
});
|
|
const killProcess = vi.fn(() => true);
|
|
|
|
await stopGateway(child, { killGraceMs: 1, killProcess, teardownGraceMs: 1 });
|
|
|
|
expect(killProcess).toHaveBeenNthCalledWith(1, -12347, 0);
|
|
expect(killProcess).toHaveBeenNthCalledWith(2, -12347, "SIGTERM");
|
|
expect(killProcess).toHaveBeenCalledWith(-12347, "SIGKILL");
|
|
expect(child.kill).not.toHaveBeenCalled();
|
|
});
|
|
|
|
posixIt("rechecks process group liveness after the wrapper exits during teardown", async () => {
|
|
const child = Object.assign(new EventEmitter(), {
|
|
exitCode: null as number | null,
|
|
kill: vi.fn(),
|
|
pid: 12348,
|
|
signalCode: null as NodeJS.Signals | null,
|
|
});
|
|
const killProcess = vi.fn((_pid: number, signal: number | NodeJS.Signals) => {
|
|
if (signal === "SIGTERM") {
|
|
setTimeout(() => {
|
|
child.exitCode = 0;
|
|
child.emit("exit", 0, null);
|
|
}, 0);
|
|
}
|
|
return true;
|
|
});
|
|
|
|
await stopGateway(child, { killGraceMs: 1, killProcess, teardownGraceMs: 100 });
|
|
|
|
expect(killProcess).toHaveBeenNthCalledWith(1, -12348, 0);
|
|
expect(killProcess).toHaveBeenNthCalledWith(2, -12348, "SIGTERM");
|
|
expect(killProcess).toHaveBeenCalledWith(-12348, "SIGKILL");
|
|
expect(child.kill).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("fails readiness waits before polling after signaled gateway exits", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-signal-ready-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "gateway died\n");
|
|
const fetchImpl = vi.fn(() => {
|
|
throw new Error("fetch should not run after process exit");
|
|
});
|
|
|
|
await expect(
|
|
waitForGatewayReady({ exitCode: null, signalCode: "SIGTERM" }, 9, logPath, {
|
|
fetchImpl,
|
|
pollDelayMs: 1,
|
|
timeoutMs: 1,
|
|
}),
|
|
).rejects.toThrow("gateway exited before ready");
|
|
expect(fetchImpl).not.toHaveBeenCalled();
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("aborts stalled readiness probes when the gateway exits mid-probe", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-exit-during-ready-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "gateway died during readiness\n");
|
|
const child = Object.assign(new EventEmitter(), {
|
|
exitCode: null,
|
|
signalCode: null as NodeJS.Signals | null,
|
|
});
|
|
const fetchImpl = vi.fn((_url: string, init?: RequestInit) => {
|
|
return new Promise((_resolve, reject) => {
|
|
init?.signal?.addEventListener(
|
|
"abort",
|
|
() => {
|
|
const reason = init.signal?.reason;
|
|
reject(reason instanceof Error ? reason : new Error("fetch aborted"));
|
|
},
|
|
{ once: true },
|
|
);
|
|
});
|
|
});
|
|
const startedAt = Date.now();
|
|
setTimeout(() => {
|
|
child.signalCode = "SIGTERM";
|
|
child.emit("exit", null, "SIGTERM");
|
|
}, 25);
|
|
|
|
await expect(
|
|
waitForGatewayReady(child, 9, logPath, {
|
|
fetchImpl,
|
|
pollDelayMs: 5_000,
|
|
timeoutMs: 2_000,
|
|
}),
|
|
).rejects.toThrow("gateway exited before ready");
|
|
|
|
expect(fetchImpl).toHaveBeenCalledOnce();
|
|
expect(Date.now() - startedAt).toBeLessThan(500);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("keeps stalled readiness probes inside the caller deadline", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-stalled-ready-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "booting\n");
|
|
let calls = 0;
|
|
const startedAt = Date.now();
|
|
|
|
await expect(
|
|
waitForGatewayReady({ exitCode: null, signalCode: null }, 9, logPath, {
|
|
fetchImpl: () => {
|
|
calls += 1;
|
|
return new Promise(() => {});
|
|
},
|
|
pollDelayMs: 1,
|
|
timeoutMs: 25,
|
|
}),
|
|
).rejects.toThrow("gateway did not become ready");
|
|
|
|
expect(calls).toBe(1);
|
|
expect(Date.now() - startedAt).toBeLessThan(500);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("requires /readyz body.ready before accepting gateway readiness", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-ready-body-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "[gateway] ready\n");
|
|
const fetchImpl = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response('{"ready":false}', { status: 200 }))
|
|
.mockResolvedValueOnce(new Response('{"ready":true}', { status: 200 }));
|
|
|
|
await expect(
|
|
waitForGatewayReady({ exitCode: null, signalCode: null }, 9, logPath, {
|
|
fetchImpl,
|
|
pollDelayMs: 1,
|
|
timeoutMs: 100,
|
|
}),
|
|
).resolves.toBeUndefined();
|
|
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC gateway readiness logs", () => {
|
|
it("scans gateway readiness logs incrementally across appended chunks", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-scan-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "booting\n".repeat(1000));
|
|
const scanner = createGatewayReadyLogScanner(logPath, "[gateway] ready");
|
|
|
|
expect(scanner()).toBe(false);
|
|
|
|
writeFileSync(logPath, "[gateway] rea", { flag: "a" });
|
|
expect(scanner()).toBe(false);
|
|
|
|
writeFileSync(logPath, "dy\n", { flag: "a" });
|
|
expect(scanner()).toBe(true);
|
|
expect(scanner()).toBe(true);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("resets the readiness scanner after log rotation", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-rotate-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "older log contents without readiness\n");
|
|
const scanner = createGatewayReadyLogScanner(logPath, "[gateway] ready");
|
|
|
|
expect(scanner()).toBe(false);
|
|
|
|
writeFileSync(logPath, "[gateway] ready\n");
|
|
expect(scanner()).toBe(true);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("tails large gateway logs without returning older content", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-tail-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, `old fatal marker\n${"noise\n".repeat(2000)}recent ready\n`);
|
|
|
|
const tail = tailFile(logPath, 128);
|
|
|
|
expect(tail).toContain("recent ready");
|
|
expect(tail).not.toContain("old fatal marker");
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("honors short reads when a gateway log shrinks during tailing", () => {
|
|
vi.spyOn(fs, "existsSync").mockReturnValue(true);
|
|
vi.spyOn(fs, "statSync").mockReturnValue({
|
|
isFile: () => true,
|
|
size: 64,
|
|
} as fs.Stats);
|
|
vi.spyOn(fs, "openSync").mockReturnValue(123 as never);
|
|
vi.spyOn(fs, "closeSync").mockImplementation(() => undefined);
|
|
vi.spyOn(fs, "readSync").mockImplementation((_fd, buffer) => {
|
|
if (!Buffer.isBuffer(buffer)) {
|
|
throw new Error("expected buffer read");
|
|
}
|
|
buffer.write("recent ready");
|
|
return 12;
|
|
});
|
|
|
|
expect(tailFile("/tmp/truncated-kitchen-rpc.log", 64)).toBe("recent ready");
|
|
});
|
|
|
|
it("scans gateway error logs incrementally and keeps the latest findings", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-errors-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, `${"ordinary line\n".repeat(2000)}0 errors\n[ERROR] late failure\n`);
|
|
|
|
expect(findErrorLogFindings(logPath)).toEqual([
|
|
{
|
|
line: "[ERROR] late failure",
|
|
lineNumber: 2002,
|
|
},
|
|
]);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("does not allowlist dirty error lines that mention zero errors", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-zero-error-smuggle-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, "[ERROR] 0 errors reported but fatal state remained\n");
|
|
|
|
expect(findErrorLogFindings(logPath)).toEqual([
|
|
{
|
|
line: "[ERROR] 0 errors reported but fatal state remained",
|
|
lineNumber: 1,
|
|
},
|
|
]);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("bounds scanner memory for very long log lines", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-log-long-line-"));
|
|
try {
|
|
const logPath = path.join(root, "gateway.log");
|
|
writeFileSync(logPath, `${"x".repeat(200_000)}[ERROR] giant line\n`);
|
|
|
|
const findings = findErrorLogFindings(logPath);
|
|
|
|
expect(findings).toHaveLength(1);
|
|
expect(findings[0]?.lineNumber).toBe(1);
|
|
expect(findings[0]?.line).toContain("[truncated]");
|
|
expect(findings[0]?.line.length).toBeLessThan(20_000);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC command output capture", () => {
|
|
it("keeps a bounded tail and tracks truncated output", () => {
|
|
const first = appendBoundedOutput({ text: "", truncatedChars: 0 }, "abcdef", 5);
|
|
expect(first).toEqual({ text: "bcdef", truncatedChars: 1 });
|
|
|
|
const second = appendBoundedOutput(first, "ghij", 5);
|
|
expect(second).toEqual({ text: "fghij", truncatedChars: 5 });
|
|
});
|
|
|
|
it("honors the resolved command output capture limit", async () => {
|
|
const result = await runCommand(
|
|
process.execPath,
|
|
["-e", "process.stdout.write('abcdef'); process.stderr.write('UVWXYZ');"],
|
|
{
|
|
outputCaptureChars: 3,
|
|
},
|
|
);
|
|
|
|
expect(result.stdout).toBe("def");
|
|
expect(result.stderr).toBe("XYZ");
|
|
expect(result.stdoutTruncatedChars).toBe(3);
|
|
expect(result.stderrTruncatedChars).toBe(3);
|
|
});
|
|
|
|
posixIt("kills timed command process groups", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-timeout-"));
|
|
const scriptPath = path.join(root, "trap-term.mjs");
|
|
const grandchildPidPath = path.join(root, "grandchild.pid");
|
|
let grandchildPid = 0;
|
|
|
|
writeFileSync(
|
|
scriptPath,
|
|
`
|
|
import { spawn } from "node:child_process";
|
|
import fs from "node:fs";
|
|
|
|
const grandchild = spawn(process.execPath, [
|
|
"-e",
|
|
"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);",
|
|
], { stdio: "ignore" });
|
|
fs.writeFileSync(process.argv[2], String(grandchild.pid));
|
|
process.on("SIGTERM", () => {});
|
|
setInterval(() => {}, 1000);
|
|
`,
|
|
"utf8",
|
|
);
|
|
|
|
const runPromise = runCommand(process.execPath, [scriptPath, grandchildPidPath], {
|
|
detached: undefined,
|
|
timeoutKillGraceMs: 25,
|
|
timeoutMs: 500,
|
|
});
|
|
|
|
try {
|
|
await waitFor(() => existsSync(grandchildPidPath));
|
|
grandchildPid = Number.parseInt(readText(grandchildPidPath), 10);
|
|
expect(Number.isInteger(grandchildPid)).toBe(true);
|
|
expect(isProcessAlive(grandchildPid)).toBe(true);
|
|
|
|
await expect(runPromise).rejects.toThrow("timed out after 500ms");
|
|
await waitFor(() => !isProcessAlive(grandchildPid), 5_000);
|
|
} finally {
|
|
await runPromise.catch(() => {});
|
|
if (grandchildPid && isProcessAlive(grandchildPid)) {
|
|
process.kill(grandchildPid, "SIGKILL");
|
|
}
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
posixIt("rejects timed commands that exit cleanly after SIGTERM", async () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-timeout-zero-"));
|
|
const scriptPath = path.join(root, "term-zero.mjs");
|
|
writeFileSync(
|
|
scriptPath,
|
|
`
|
|
process.on("SIGTERM", () => process.exit(0));
|
|
setInterval(() => {}, 1000);
|
|
`,
|
|
"utf8",
|
|
);
|
|
|
|
try {
|
|
await expect(
|
|
runCommand(process.execPath, [scriptPath], {
|
|
timeoutKillGraceMs: 1000,
|
|
timeoutMs: 100,
|
|
}),
|
|
).rejects.toThrow("timed out after 100ms");
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("records resource samples for command process trees", async () => {
|
|
const samples: Array<{
|
|
aggregateRssMiB?: number;
|
|
elapsedMs?: number;
|
|
label?: string;
|
|
processId?: number;
|
|
rssMiB?: number;
|
|
}> = [];
|
|
const seenPids: number[] = [];
|
|
|
|
const result = await runCommand(process.execPath, ["-e", "setTimeout(() => {}, 50);"], {
|
|
resourceLabel: "plugins install",
|
|
resourceSampleIntervalMs: 1,
|
|
resourceSamples: samples,
|
|
sampleProcessImpl: async (pid: number) => {
|
|
seenPids.push(pid);
|
|
return {
|
|
aggregateRssMiB: 640,
|
|
cpuPercent: 12,
|
|
processId: pid + 1,
|
|
rssMiB: 512,
|
|
};
|
|
},
|
|
});
|
|
|
|
expect(result.stdout).toBe("");
|
|
expect(seenPids.length).toBeGreaterThan(0);
|
|
expect(samples[0]).toMatchObject({
|
|
aggregateRssMiB: 640,
|
|
label: "plugins install",
|
|
processId: seenPids[0] + 1,
|
|
rssMiB: 512,
|
|
});
|
|
expect(samples[0]?.elapsedMs).toBeGreaterThanOrEqual(0);
|
|
});
|
|
|
|
it("rejects required command resource proof when sampling captures nothing", async () => {
|
|
const samples: unknown[] = [];
|
|
|
|
await expect(
|
|
runCommand(process.execPath, ["-e", "setTimeout(() => {}, 50);"], {
|
|
requireResourceSample: true,
|
|
resourceLabel: "plugins install",
|
|
resourceSampleIntervalMs: 1,
|
|
resourceSamples: samples,
|
|
sampleProcessImpl: async () => null,
|
|
}),
|
|
).rejects.toThrow("plugins install RSS sample was not captured");
|
|
|
|
expect(samples).toEqual([]);
|
|
});
|
|
|
|
it("includes sampler errors in required command resource proof failures", async () => {
|
|
await expect(
|
|
runCommand(process.execPath, ["-e", "setTimeout(() => {}, 50);"], {
|
|
requireResourceSample: true,
|
|
resourceLabel: "plugins install",
|
|
resourceSampleIntervalMs: 1,
|
|
resourceSamples: [],
|
|
sampleProcessImpl: async () => {
|
|
throw new Error("ps failed");
|
|
},
|
|
}),
|
|
).rejects.toThrow("plugins install RSS sample was not captured: ps failed");
|
|
});
|
|
|
|
it("rejects command spawn failures as Error objects", async () => {
|
|
await expect(runCommand("openclaw-definitely-missing-command", [])).rejects.toMatchObject({
|
|
code: "ENOENT",
|
|
});
|
|
});
|
|
|
|
it("preserves failed command output for structured consumers", async () => {
|
|
await expect(
|
|
runCommand(process.execPath, [
|
|
"-e",
|
|
'process.stdout.write("request failure"); process.stderr.write("diagnostic"); process.exit(7)',
|
|
]),
|
|
).rejects.toMatchObject({
|
|
status: 7,
|
|
signal: null,
|
|
stdout: "request failure",
|
|
stderr: "diagnostic",
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC caller loading", () => {
|
|
it("samples CLI-backed gateway RPC calls as command work", () => {
|
|
const resourceSamples: unknown[] = [];
|
|
|
|
expect(
|
|
createRpcCliRunOptions("tools.invoke", {
|
|
commandResourceOptions: {
|
|
resourceSampleIntervalMs: 500,
|
|
resourceSamples,
|
|
},
|
|
}),
|
|
).toMatchObject({
|
|
resourceLabel: "gateway call tools.invoke",
|
|
resourceSampleIntervalMs: 500,
|
|
resourceSamples,
|
|
timeoutMs: 90_000,
|
|
});
|
|
});
|
|
|
|
it("uses built callGateway chunks for dist and packaged entries", () => {
|
|
expect(usesBuiltOpenClawEntry({ command: "node", baseArgs: ["dist/index.js"] })).toBe(true);
|
|
expect(
|
|
usesBuiltOpenClawEntry({ command: "node", baseArgs: ["/app/openclaw.mjs"] }, "/repo", {
|
|
OPENCLAW_ENTRY: "/app/openclaw.mjs",
|
|
}),
|
|
).toBe(true);
|
|
});
|
|
|
|
it("does not deep-import gateway TypeScript for source pnpm runners", () => {
|
|
expect(usesBuiltOpenClawEntry({ pnpm: true, baseArgs: ["openclaw"] })).toBe(false);
|
|
expect(usesBuiltOpenClawEntry({ command: "node", baseArgs: ["scripts/dev.mjs"] })).toBe(false);
|
|
});
|
|
|
|
it("finds only built callGateway chunks", () => {
|
|
const root = mkdtempSync(path.join(tmpdir(), "openclaw-rpc-call-chunks-"));
|
|
try {
|
|
mkdirSync(path.join(root, "dist"));
|
|
writeFileSync(path.join(root, "dist", "call-Abc123.js"), "");
|
|
writeFileSync(path.join(root, "dist", "call.runtime-Def456.js"), "");
|
|
writeFileSync(path.join(root, "dist", "index.js"), "");
|
|
|
|
expect(findDistCallGatewayModuleFiles(root)).toEqual([
|
|
"call-Abc123.js",
|
|
"call.runtime-Def456.js",
|
|
]);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
posixIt("kills descendants when timed commands exit cleanly after SIGTERM", async () => {
|
|
const tempDirs: string[] = [];
|
|
const root = makeTempDir(tempDirs, "openclaw-kitchen-rpc-timeout-clean-parent-");
|
|
const scriptPath = path.join(root, "term-zero-grandchild.mjs");
|
|
const grandchildPidPath = path.join(root, "grandchild.pid");
|
|
let grandchildPid = 0;
|
|
|
|
writeFileSync(
|
|
scriptPath,
|
|
`
|
|
import { spawn } from "node:child_process";
|
|
import fs from "node:fs";
|
|
|
|
const grandchild = spawn(process.execPath, [
|
|
"-e",
|
|
"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);",
|
|
], { stdio: "ignore" });
|
|
fs.writeFileSync(process.argv[2], String(grandchild.pid));
|
|
process.on("SIGTERM", () => process.exit(0));
|
|
setInterval(() => {}, 1000);
|
|
`,
|
|
"utf8",
|
|
);
|
|
|
|
const runPromise = runCommand(process.execPath, [scriptPath, grandchildPidPath], {
|
|
timeoutKillGraceMs: 2_000,
|
|
timeoutMs: 100,
|
|
});
|
|
|
|
try {
|
|
await waitFor(() => existsSync(grandchildPidPath));
|
|
grandchildPid = Number.parseInt(readText(grandchildPidPath), 10);
|
|
expect(Number.isInteger(grandchildPid)).toBe(true);
|
|
expect(isProcessAlive(grandchildPid)).toBe(true);
|
|
|
|
await expect(runPromise).rejects.toThrow("timed out after 100ms");
|
|
await waitFor(() => !isProcessAlive(grandchildPid), 5_000);
|
|
} finally {
|
|
await runPromise.catch(() => {});
|
|
if (grandchildPid && isProcessAlive(grandchildPid)) {
|
|
process.kill(grandchildPid, "SIGKILL");
|
|
}
|
|
cleanupTempDirs(tempDirs);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC payload unwrapping", () => {
|
|
it("preserves explicit nullish JSON-RPC result fields", () => {
|
|
expect(unwrapRpcPayload({ jsonrpc: "2.0", result: null })).toBeNull();
|
|
expect(unwrapRpcPayload({ jsonrpc: "2.0", result: undefined })).toBeUndefined();
|
|
});
|
|
|
|
it("prefers result before legacy payload and data envelopes", () => {
|
|
expect(unwrapRpcPayload({ result: false, payload: { stale: true } })).toBe(false);
|
|
expect(unwrapRpcPayload({ payload: null, data: { stale: true } })).toBeNull();
|
|
expect(unwrapRpcPayload({ data: 0 })).toBe(0);
|
|
});
|
|
|
|
it("rejects error envelopes without success payload fields", () => {
|
|
const error = captureSyncError(() =>
|
|
unwrapRpcPayload({ error: { message: "session store unavailable" } }),
|
|
);
|
|
|
|
expect(error.message).toContain("gateway RPC returned error envelope");
|
|
expect(error.message).toContain("session store unavailable");
|
|
expect(unwrapRpcPayload({ error: { message: "ignored" }, payload: { ok: true } })).toEqual({
|
|
ok: true,
|
|
});
|
|
});
|
|
|
|
it("bounds failed RPC payload diagnostics", () => {
|
|
const error = captureSyncError(() =>
|
|
unwrapRpcPayload({
|
|
ok: false,
|
|
error: {
|
|
message: `rpc failed ${"x".repeat(4096)} DO_NOT_DUMP_RPC_MIDDLE ${"y".repeat(4096)} end`,
|
|
},
|
|
}),
|
|
);
|
|
|
|
expect(error.message).toContain("gateway RPC failed");
|
|
expect(error.message).toContain("truncated");
|
|
expect(error.message).not.toContain("DO_NOT_DUMP_RPC_MIDDLE");
|
|
expect(error.message.length).toBeLessThan(1200);
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC command catalog assertions", () => {
|
|
it("keeps plugin commands and deduplicates aliases", () => {
|
|
expect(
|
|
extractPluginCommandNames({
|
|
commands: [
|
|
{
|
|
source: "core",
|
|
name: "/kitchen-sink",
|
|
},
|
|
{
|
|
source: "plugin",
|
|
name: "/kitchen",
|
|
nativeName: "kitchen",
|
|
textAliases: ["/kitchen-sink", "kitchen-sink"],
|
|
},
|
|
],
|
|
}),
|
|
).toEqual(["kitchen", "kitchen-sink"]);
|
|
});
|
|
|
|
it("requires every expected Kitchen Sink plugin tool", () => {
|
|
expect(() =>
|
|
assertExpectedKitchenSinkToolEntries(
|
|
[{ id: "kitchen_sink_text", source: "plugin", pluginId: "openclaw-kitchen-sink-fixture" }],
|
|
"tools.catalog plugin tools",
|
|
{ requirePluginProvenance: true },
|
|
),
|
|
).toThrow("tools.catalog plugin tools missing kitchen_sink_search, kitchen_sink_image_job");
|
|
});
|
|
|
|
it("requires plugin provenance for expected catalog tools", () => {
|
|
expect(() =>
|
|
assertExpectedKitchenSinkToolEntries(
|
|
[
|
|
{ id: "kitchen_sink_text", source: "plugin", pluginId: "openclaw-kitchen-sink-fixture" },
|
|
{ id: "kitchen_sink_search", source: "core", pluginId: "openclaw-kitchen-sink-fixture" },
|
|
{ id: "kitchen_sink_image_job", source: "plugin", pluginId: "other-plugin" },
|
|
],
|
|
"tools.catalog plugin tools",
|
|
{ requirePluginProvenance: true },
|
|
),
|
|
).toThrow("tools.catalog plugin tools plugin provenance mismatch");
|
|
});
|
|
|
|
it("accepts complete expected tool coverage", () => {
|
|
expect(
|
|
assertExpectedKitchenSinkToolEntries(
|
|
[
|
|
{ id: "kitchen_sink_text", source: "plugin", pluginId: "openclaw-kitchen-sink-fixture" },
|
|
{
|
|
id: "kitchen_sink_search",
|
|
source: "plugin",
|
|
pluginId: "openclaw-kitchen-sink-fixture",
|
|
},
|
|
{
|
|
id: "kitchen_sink_image_job",
|
|
source: "plugin",
|
|
pluginId: "openclaw-kitchen-sink-fixture",
|
|
},
|
|
],
|
|
"tools.catalog plugin tools",
|
|
{ requirePluginProvenance: true },
|
|
),
|
|
).toEqual(["kitchen_sink_text", "kitchen_sink_search", "kitchen_sink_image_job"]);
|
|
});
|
|
|
|
it("invokes every advertised Kitchen Sink tool during the RPC walk", () => {
|
|
expect(listKitchenSinkToolInvokeNames().toSorted()).toEqual([
|
|
"kitchen_sink_image_job",
|
|
"kitchen_sink_search",
|
|
"kitchen_sink_text",
|
|
]);
|
|
});
|
|
|
|
it("walks broad read-only gateway RPC surfaces", () => {
|
|
expect(listKitchenSinkReadOnlyRpcProbeNames()).toEqual(
|
|
expect.arrayContaining([
|
|
"gateway.identity.get",
|
|
"config.schema.lookup",
|
|
"models.list",
|
|
"skills.status",
|
|
"agents.list",
|
|
"sessions.list",
|
|
"cron.list",
|
|
"tasks.list",
|
|
"usage.status",
|
|
"voicewake.routing.get",
|
|
"talk.catalog",
|
|
"update.status",
|
|
"node.list",
|
|
"device.pair.list",
|
|
"exec.approvals.get",
|
|
"environments.status",
|
|
]),
|
|
);
|
|
});
|
|
|
|
it("proves node-only RPC authorization boundaries", async () => {
|
|
expect(listKitchenSinkAuthorizationRpcProbeNames()).toEqual(["skills.bins"]);
|
|
await expect(
|
|
assertOperatorRpcDenied({ method: "skills.bins", params: {} }, async () => {
|
|
throw Object.assign(new Error("unauthorized role: operator"), {
|
|
gatewayCode: "INVALID_REQUEST",
|
|
});
|
|
}),
|
|
).resolves.toBeUndefined();
|
|
await expect(
|
|
assertOperatorRpcDenied({ method: "skills.bins", params: {} }, async () => {
|
|
throw new Error(
|
|
"openclaw gateway call skills.bins failed with 1\nGateway call failed: unauthorized role: operator",
|
|
);
|
|
}),
|
|
).rejects.toThrow("Gateway call failed: unauthorized role: operator");
|
|
await expect(
|
|
assertOperatorRpcDenied({ method: "skills.bins", params: {} }, async () => ({})),
|
|
).rejects.toThrow("skills.bins unexpectedly allowed operator access");
|
|
});
|
|
|
|
it("reconstructs typed request failures from gateway CLI JSON", async () => {
|
|
const { formatGatewayClientRequestErrorJson } = await import("../../src/gateway/call.js");
|
|
const payload = formatGatewayClientRequestErrorJson(
|
|
Object.assign(new Error("unauthorized role: operator"), {
|
|
name: "GatewayClientRequestError",
|
|
gatewayCode: "INVALID_REQUEST",
|
|
details: { method: "skills.bins" },
|
|
retryable: false,
|
|
retryAfterMs: 250,
|
|
}),
|
|
);
|
|
|
|
expect(
|
|
parseGatewayCliRequestFailure({
|
|
stdout: JSON.stringify(payload),
|
|
}),
|
|
).toMatchObject({
|
|
name: "GatewayClientRequestError",
|
|
message: "unauthorized role: operator",
|
|
gatewayCode: "INVALID_REQUEST",
|
|
details: { method: "skills.bins" },
|
|
retryable: false,
|
|
retryAfterMs: 250,
|
|
});
|
|
expect(parseGatewayCliRequestFailure(new Error("plain failure"))).toBeNull();
|
|
for (const invalidFields of [{ retryable: "no" }, { retryable: false, retryAfterMs: -1 }]) {
|
|
expect(
|
|
parseGatewayCliRequestFailure({
|
|
stdout: JSON.stringify({
|
|
ok: false,
|
|
error: {
|
|
type: "gateway_request_error",
|
|
code: "INVALID_REQUEST",
|
|
message: "unauthorized role: operator",
|
|
...invalidFields,
|
|
},
|
|
}),
|
|
}),
|
|
).toBeNull();
|
|
}
|
|
});
|
|
|
|
it("requires provenance for effective Kitchen Sink plugin tools too", () => {
|
|
expect(() =>
|
|
assertExpectedKitchenSinkToolEntries(
|
|
[
|
|
{ id: "kitchen_sink_text", source: "plugin", pluginId: "openclaw-kitchen-sink-fixture" },
|
|
{
|
|
id: "kitchen_sink_search",
|
|
source: "plugin",
|
|
pluginId: "openclaw-kitchen-sink-fixture",
|
|
},
|
|
{
|
|
id: "kitchen_sink_image_job",
|
|
source: "core",
|
|
pluginId: "openclaw-kitchen-sink-fixture",
|
|
},
|
|
],
|
|
"tools.effective plugin tools",
|
|
{ requirePluginProvenance: true },
|
|
),
|
|
).toThrow("tools.effective plugin tools plugin provenance mismatch");
|
|
});
|
|
|
|
it("requires the exact Kitchen Sink channel account", () => {
|
|
expect(() =>
|
|
assertChannelAccountRunning({
|
|
channelAccounts: {
|
|
"kitchen-sink-channel": [{ accountId: "other", configured: true, running: true }],
|
|
},
|
|
}),
|
|
).toThrow("Kitchen Sink channel account local was not reported");
|
|
});
|
|
|
|
it("checks TTS providers on the exact response surfaces", () => {
|
|
expect(extractTtsProviderIds({ providers: [{ id: "nested-miss" }] }, "providers")).toEqual([
|
|
"nested-miss",
|
|
]);
|
|
expect(
|
|
extractTtsProviderIds(
|
|
{
|
|
metadata: { id: "kitchen-sink-speech" },
|
|
providers: [{ id: "other", configured: true }],
|
|
},
|
|
"providers",
|
|
),
|
|
).toEqual(["other"]);
|
|
|
|
expect(() =>
|
|
assertTtsProviderCoverage(
|
|
{
|
|
providers: [{ id: "kitchen-sink-speech", configured: true }],
|
|
},
|
|
"providers",
|
|
),
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertTtsProviderCoverage(
|
|
{
|
|
providerStates: [{ id: "kitchen-sink-speech-provider", configured: true }],
|
|
},
|
|
"status",
|
|
),
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertTtsProviderCoverage(
|
|
{
|
|
metadata: { id: "kitchen-sink-speech" },
|
|
providers: [{ id: "other", configured: true }],
|
|
},
|
|
"providers",
|
|
),
|
|
).toThrow("tts.providers missing one of");
|
|
expect(() =>
|
|
assertTtsProviderCoverage(
|
|
{
|
|
providerStates: [{ id: "kitchen-sink-speech", configured: false }],
|
|
},
|
|
"status",
|
|
),
|
|
).toThrow("did not report a configured Kitchen Sink speech provider");
|
|
});
|
|
|
|
it("checks search, text, and image job tool invocation fixtures separately", () => {
|
|
const pngBytes = Buffer.from(
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
|
|
"base64",
|
|
);
|
|
const pngSha256 = createHash("sha256").update(pngBytes).digest("hex");
|
|
expect(() =>
|
|
assertKitchenSinkSearchInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: { results: [{ title: "Kitchen Sink image fixture" }] },
|
|
}),
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertKitchenSinkTextInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: {
|
|
route: "tool:kitchen_sink_text",
|
|
text: "Kitchen Sink text provider produced a deterministic reply.",
|
|
},
|
|
}),
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertKitchenSinkImageJobInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: {
|
|
ok: true,
|
|
route: "tool:kitchen_sink_image_job",
|
|
job: { status: "completed", route: "tool:kitchen_sink_image_job" },
|
|
mediaUrl: `data:image/png;base64,${pngBytes.toString("base64")}`,
|
|
image: {
|
|
mimeType: "image/png",
|
|
metadata: {
|
|
assetName: "kitchen_sink_office.png",
|
|
height: 1024,
|
|
sha256: pngSha256,
|
|
width: 1024,
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
).not.toThrow();
|
|
expect(() =>
|
|
assertKitchenSinkTextInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: { route: "tool:kitchen_sink_search" },
|
|
}),
|
|
).toThrow("Kitchen Sink text tool output missed expected fixture");
|
|
});
|
|
|
|
it("rejects tool invocation canaries outside the structured result fields", () => {
|
|
expect(() =>
|
|
assertKitchenSinkSearchInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: { note: "prompt mentioned Kitchen Sink image fixture" },
|
|
}),
|
|
).toThrow("Kitchen Sink search tool output missed expected fixture");
|
|
|
|
expect(() =>
|
|
assertKitchenSinkTextInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: { text: "Kitchen Sink prompt echoed tool:kitchen_sink_text" },
|
|
}),
|
|
).toThrow("Kitchen Sink text tool output missed expected fixture");
|
|
|
|
expect(() =>
|
|
assertKitchenSinkImageJobInvokeResult({
|
|
ok: true,
|
|
source: "plugin",
|
|
output: {
|
|
ok: true,
|
|
route: "tool:kitchen_sink_image_job",
|
|
job: { status: "completed", route: "tool:kitchen_sink_image_job" },
|
|
mediaUrl: "data:image/png;base64,fixture",
|
|
image: {
|
|
mimeType: "image/png",
|
|
metadata: {
|
|
assetName: "kitchen_sink_office.png",
|
|
height: 1024,
|
|
sha256: "not-a-real-hash",
|
|
width: 1024,
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
).toThrow("Kitchen Sink image job tool output missed expected fixture");
|
|
});
|
|
|
|
it("bounds failed tool invocation diagnostics", () => {
|
|
const error = captureSyncError(() =>
|
|
assertKitchenSinkSearchInvokeResult({
|
|
ok: false,
|
|
source: "plugin",
|
|
output: {
|
|
text: `prefix ${"x".repeat(4096)} DO_NOT_DUMP_TOOL_MIDDLE ${"y".repeat(4096)} suffix`,
|
|
},
|
|
}),
|
|
);
|
|
|
|
expect(error.message).toContain("Kitchen Sink search tool invoke failed");
|
|
expect(error.message).toContain("truncated");
|
|
expect(error.message).not.toContain("DO_NOT_DUMP_TOOL_MIDDLE");
|
|
expect(error.message.length).toBeLessThan(1400);
|
|
});
|
|
|
|
it("requires sessions.create to return the requested Kitchen Sink session", () => {
|
|
expect(() =>
|
|
assertCreatedKitchenSinkSession({
|
|
ok: true,
|
|
key: "agent:main:kitchen-sink-rpc",
|
|
sessionId: "session-1",
|
|
}),
|
|
).not.toThrow();
|
|
|
|
expect(() =>
|
|
assertCreatedKitchenSinkSession({
|
|
ok: true,
|
|
key: "agent:main:stale-session",
|
|
sessionId: "session-1",
|
|
}),
|
|
).toThrow("sessions.create did not return the requested Kitchen Sink session");
|
|
expect(() =>
|
|
assertCreatedKitchenSinkSession({
|
|
ok: true,
|
|
key: "agent:main:kitchen-sink-rpc",
|
|
}),
|
|
).toThrow("sessions.create did not return the requested Kitchen Sink session");
|
|
});
|
|
|
|
it("requires Kitchen Sink UI descriptor coverage", () => {
|
|
expect(() =>
|
|
assertKitchenSinkUiDescriptors({
|
|
ok: true,
|
|
descriptors: [{ pluginId: "openclaw-kitchen-sink-fixture", id: "kitchen-sink-panel" }],
|
|
}),
|
|
).not.toThrow();
|
|
|
|
expect(() => assertKitchenSinkUiDescriptors({})).toThrow(
|
|
"plugins.uiDescriptors returned invalid payload",
|
|
);
|
|
expect(() => assertKitchenSinkUiDescriptors({ ok: true, descriptors: [] })).toThrow(
|
|
"plugins.uiDescriptors did not report Kitchen Sink descriptor",
|
|
);
|
|
});
|
|
|
|
it("allows conformance mode to skip generated Kitchen Sink UI descriptors", () => {
|
|
expect(() =>
|
|
assertKitchenSinkUiDescriptors(
|
|
{
|
|
ok: true,
|
|
descriptors: [],
|
|
},
|
|
{ expectDescriptor: false },
|
|
),
|
|
).not.toThrow();
|
|
|
|
expect(() => assertKitchenSinkUiDescriptors({}, { expectDescriptor: false })).toThrow(
|
|
"plugins.uiDescriptors returned invalid payload",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC diagnostics assertions", () => {
|
|
it("fails when stability reports dropped or rejected payload diagnostics", () => {
|
|
expect(() =>
|
|
assertDiagnosticStabilityClean({
|
|
dropped: 1,
|
|
events: [{ type: "diagnostic.async_queue.dropped" }],
|
|
summary: {
|
|
payloadLarge: {
|
|
rejected: 1,
|
|
truncated: 1,
|
|
},
|
|
},
|
|
}),
|
|
).toThrow("diagnostics.stability reported instability");
|
|
});
|
|
|
|
it("fails when async diagnostic drops only appear in the full summary", () => {
|
|
expect(() =>
|
|
assertDiagnosticStabilityClean({
|
|
dropped: 0,
|
|
events: [],
|
|
summary: {
|
|
byType: {
|
|
"diagnostic.async_queue.dropped": 2,
|
|
},
|
|
},
|
|
}),
|
|
).toThrow("async diagnostic drops=2");
|
|
});
|
|
|
|
it("allows chunked payload diagnostics that did not reject or truncate data", () => {
|
|
expect(() =>
|
|
assertDiagnosticStabilityClean({
|
|
dropped: 0,
|
|
events: [{ type: "payload.large", action: "chunked" }],
|
|
summary: {
|
|
payloadLarge: {
|
|
rejected: 0,
|
|
truncated: 0,
|
|
chunked: 1,
|
|
},
|
|
},
|
|
}),
|
|
).not.toThrow();
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC health/status assertions", () => {
|
|
it("rejects empty health and status RPC payloads", () => {
|
|
expect(() => assertGatewayHealthPayload({})).toThrow("health payload missing");
|
|
expect(() => assertGatewayStatusPayload({})).toThrow("status payload missing");
|
|
});
|
|
|
|
it("accepts minimally valid gateway health and status RPC payloads", () => {
|
|
expect(() =>
|
|
assertGatewayHealthPayload({
|
|
ok: true,
|
|
ts: Date.now(),
|
|
durationMs: 12,
|
|
channels: {},
|
|
channelOrder: [],
|
|
channelLabels: {},
|
|
heartbeatSeconds: 30,
|
|
defaultAgentId: "main",
|
|
agents: [],
|
|
sessions: {
|
|
path: "/tmp/openclaw-sessions.sqlite",
|
|
count: 0,
|
|
recent: [],
|
|
},
|
|
}),
|
|
).not.toThrow();
|
|
|
|
expect(() =>
|
|
assertGatewayStatusPayload({
|
|
heartbeat: {
|
|
defaultAgentId: "main",
|
|
agents: [],
|
|
},
|
|
channelSummary: [],
|
|
queuedSystemEvents: [],
|
|
tasks: {},
|
|
taskAudit: {},
|
|
sessions: {
|
|
paths: [],
|
|
count: 0,
|
|
defaults: {
|
|
model: null,
|
|
contextTokens: null,
|
|
},
|
|
recent: [],
|
|
byAgent: [],
|
|
},
|
|
}),
|
|
).not.toThrow();
|
|
});
|
|
|
|
it("bounds failed health and status payload diagnostics", () => {
|
|
const oversizedValue = `start ${"x".repeat(4096)} DO_NOT_DUMP_STATUS_MIDDLE ${"y".repeat(
|
|
4096,
|
|
)} end`;
|
|
|
|
const healthError = captureSyncError(() =>
|
|
assertGatewayHealthPayload({
|
|
ok: false,
|
|
oversizedValue,
|
|
}),
|
|
);
|
|
const statusError = captureSyncError(() =>
|
|
assertGatewayStatusPayload({
|
|
heartbeat: {},
|
|
oversizedValue,
|
|
}),
|
|
);
|
|
|
|
expect(healthError.message).toContain("health payload missing");
|
|
expect(statusError.message).toContain("status payload missing");
|
|
expect(`${healthError.message}\n${statusError.message}`).toContain("truncated");
|
|
expect(`${healthError.message}\n${statusError.message}`).not.toContain(
|
|
"DO_NOT_DUMP_STATUS_MIDDLE",
|
|
);
|
|
expect(healthError.message.length).toBeLessThan(1600);
|
|
expect(statusError.message.length).toBeLessThan(1600);
|
|
});
|
|
});
|
|
|
|
describe("kitchen-sink RPC process sampling", () => {
|
|
it("samples RSS on Windows instead of silently disabling the resource guard", async () => {
|
|
const calls: Array<{ command: string; args: string[] }> = [];
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "win32",
|
|
runCommand: async (command: string, args: string[]) => {
|
|
calls.push({ command, args });
|
|
return { stdout: `${256 * 1024 * 1024} 1.5 5678 ${288 * 1024 * 1024}`, stderr: "" };
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 288,
|
|
cpuPercent: null,
|
|
cpuSeconds: 1.5,
|
|
processId: 5678,
|
|
rssMiB: 256,
|
|
});
|
|
expect(calls[0]?.command).toBe("powershell.exe");
|
|
expect(calls[0]?.args.join(" ")).toContain("$rootPid = 1234");
|
|
expect(calls[0]?.args.join(" ")).toContain("ParentProcessId");
|
|
});
|
|
|
|
it("can locate a Windows gateway process by command line when the launcher is gone", async () => {
|
|
const calls: Array<{ command: string; args: string[] }> = [];
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "win32",
|
|
runCommand: async (command: string, args: string[]) => {
|
|
calls.push({ command, args });
|
|
return { stdout: `${384 * 1024 * 1024} 2.25 6789 ${512 * 1024 * 1024}`, stderr: "" };
|
|
},
|
|
windowsCommandLineNeedles: ["gateway", "--port", "19080"],
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 512,
|
|
cpuPercent: null,
|
|
cpuSeconds: 2.25,
|
|
processId: 6789,
|
|
rssMiB: 384,
|
|
});
|
|
const command = calls[0]?.args.join(" ") ?? "";
|
|
expect(command).toContain("CommandLine");
|
|
expect(command).toContain("'gateway'");
|
|
expect(command).toContain("'19080'");
|
|
expect(command).toContain("ProcessId -eq $PID");
|
|
expect(command).toContain("ParentProcessId");
|
|
expect(command).toContain("Sort-Object WorkingSet64 -Descending");
|
|
});
|
|
|
|
it("falls back to the legacy powershell command name on Windows", async () => {
|
|
const commands: string[] = [];
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "win32",
|
|
runCommand: async (command: string) => {
|
|
commands.push(command);
|
|
if (command === "powershell.exe") {
|
|
throw new Error("missing powershell.exe");
|
|
}
|
|
return { stdout: `${96 * 1024 * 1024} 0 1234`, stderr: "" };
|
|
},
|
|
});
|
|
|
|
expect(commands).toEqual(["powershell.exe", "powershell"]);
|
|
expect(sample?.rssMiB).toBe(96);
|
|
expect(sample?.aggregateRssMiB).toBe(96);
|
|
});
|
|
|
|
it("does not truncate malformed Windows PowerShell CPU or id samples", async () => {
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "win32",
|
|
runCommand: async () => ({
|
|
stdout: `${256 * 1024 * 1024} 2.25oops 6789x ${512 * 1024 * 1024}oops`,
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 256,
|
|
cpuPercent: null,
|
|
cpuSeconds: null,
|
|
processId: 1234,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("rejects malformed Windows PowerShell RSS samples", async () => {
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "win32",
|
|
runCommand: async () => ({
|
|
stdout: `${256 * 1024 * 1024}oops 2.25 6789 ${512 * 1024 * 1024}`,
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toBeNull();
|
|
});
|
|
|
|
posixIt("rejects malformed POSIX process rows before sampling RSS", async () => {
|
|
const badRows = [
|
|
" 5678 1234 9007199254740993 0.2 child",
|
|
" 5678x 1234 2048 0.2 child",
|
|
" 5678 1234x 2048 0.2 child",
|
|
];
|
|
|
|
for (const badRow of badRows) {
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "linux",
|
|
runCommand: async () => ({
|
|
stdout: [
|
|
" PID PPID RSS %CPU COMMAND",
|
|
" 1234 1 2048 0.1 openclaw-gateway",
|
|
badRow,
|
|
].join("\n"),
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toBeNull();
|
|
}
|
|
});
|
|
|
|
posixIt("ignores malformed POSIX process rows outside the sampled tree", async () => {
|
|
const sample = await sampleProcess(1234, {
|
|
platform: "linux",
|
|
runCommand: async () => ({
|
|
stdout: [
|
|
" PID PPID RSS %CPU COMMAND",
|
|
" 1234 1 2048 0.1 openclaw-gateway",
|
|
" 5678 1234 4096 0.2 child",
|
|
" 9999 9998 9007199254740993 0.2 unrelated",
|
|
].join("\n"),
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toMatchObject({
|
|
aggregateRssMiB: 6,
|
|
processId: 1234,
|
|
rssMiB: 2,
|
|
});
|
|
});
|
|
|
|
it("samples the Windows gateway process by listening port", async () => {
|
|
const calls: Array<{ command: string; args: string[] }> = [];
|
|
const sample = await sampleWindowsProcessByPort(19675, {
|
|
runCommand: async (command: string, args: string[]) => {
|
|
calls.push({ command, args });
|
|
if (command === "netstat.exe") {
|
|
return {
|
|
stdout: [
|
|
" Proto Local Address Foreign Address State PID",
|
|
" TCP 127.0.0.1:196750 0.0.0.0:0 LISTENING 1111",
|
|
" TCP 127.0.0.1:1967 0.0.0.0:0 LISTENING 2222",
|
|
" TCP 127.0.0.1:19675 0.0.0.0:0 LISTENING 6789",
|
|
].join("\r\n"),
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (command === "powershell.exe") {
|
|
return { stdout: `${384 * 1024 * 1024} 2.25 6789 ${512 * 1024 * 1024}`, stderr: "" };
|
|
}
|
|
throw new Error(`unexpected command ${command}`);
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 512,
|
|
cpuPercent: null,
|
|
cpuSeconds: 2.25,
|
|
processId: 6789,
|
|
rssMiB: 384,
|
|
});
|
|
expect(calls).toEqual([
|
|
{ command: "netstat.exe", args: ["-ano", "-p", "tcp"] },
|
|
{
|
|
command: "powershell.exe",
|
|
args: expect.arrayContaining(["-Command", expect.stringContaining("$rootPid = 6789")]),
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("falls back to strict tasklist RSS when Windows PowerShell sampling fails", async () => {
|
|
const calls: string[] = [];
|
|
const sample = await sampleWindowsProcessByPort(19675, {
|
|
runCommand: async (command: string) => {
|
|
calls.push(command);
|
|
if (command === "netstat.exe") {
|
|
return {
|
|
stdout: [
|
|
" Proto Local Address Foreign Address State PID",
|
|
" TCP 127.0.0.1:19675 0.0.0.0:0 LISTENING 6789",
|
|
].join("\r\n"),
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (command === "powershell.exe" || command === "powershell") {
|
|
throw new Error("powershell unavailable");
|
|
}
|
|
if (command === "tasklist.exe") {
|
|
return {
|
|
stdout: '"node.exe","6789","Console","1","262,144 K"',
|
|
stderr: "",
|
|
};
|
|
}
|
|
throw new Error(`unexpected command ${command}`);
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
cpuPercent: null,
|
|
cpuSeconds: null,
|
|
processId: 6789,
|
|
rssMiB: 256,
|
|
});
|
|
expect(calls).toEqual(["netstat.exe", "powershell.exe", "powershell", "tasklist.exe"]);
|
|
});
|
|
|
|
it("falls back to the known Windows pid when tasklist reports malformed pid text", async () => {
|
|
const sample = await sampleWindowsProcessByPort(19675, {
|
|
runCommand: async (command: string) => {
|
|
if (command === "netstat.exe") {
|
|
return {
|
|
stdout: [
|
|
" Proto Local Address Foreign Address State PID",
|
|
" TCP 127.0.0.1:19675 0.0.0.0:0 LISTENING 6789",
|
|
].join("\r\n"),
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (command === "powershell.exe" || command === "powershell") {
|
|
throw new Error("powershell unavailable");
|
|
}
|
|
if (command === "tasklist.exe") {
|
|
return {
|
|
stdout: '"node.exe","9999x","Console","1","262,144 K"',
|
|
stderr: "",
|
|
};
|
|
}
|
|
throw new Error(`unexpected command ${command}`);
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
cpuPercent: null,
|
|
cpuSeconds: null,
|
|
processId: 6789,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("rejects malformed tasklist RSS instead of stripping digits", async () => {
|
|
const sample = await sampleWindowsProcessByPort(19675, {
|
|
runCommand: async (command: string) => {
|
|
if (command === "netstat.exe") {
|
|
return {
|
|
stdout: [
|
|
" Proto Local Address Foreign Address State PID",
|
|
" TCP 127.0.0.1:19675 0.0.0.0:0 LISTENING 6789",
|
|
].join("\r\n"),
|
|
stderr: "",
|
|
};
|
|
}
|
|
if (command === "powershell.exe" || command === "powershell") {
|
|
throw new Error("powershell unavailable");
|
|
}
|
|
if (command === "tasklist.exe") {
|
|
return {
|
|
stdout: '"node.exe","6789","Console","1","262x144 K"',
|
|
stderr: "",
|
|
};
|
|
}
|
|
throw new Error(`unexpected command ${command}`);
|
|
},
|
|
});
|
|
|
|
expect(sample).toBeNull();
|
|
});
|
|
|
|
it("samples direct POSIX gateway RSS with descendants", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "linux",
|
|
runCommand: async (command: string, args: string[]) => {
|
|
expect(command).toBe("ps");
|
|
expect(args).toEqual(["-ww", "-axo", "pid=,ppid=,rss=,pcpu=,command="]);
|
|
return {
|
|
stdout: [
|
|
" 4321 1 262144 12.5 node dist/index.js gateway --port 19080",
|
|
" 4322 4321 131072 1.5 node helper.js",
|
|
].join("\n"),
|
|
stderr: "",
|
|
};
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 384,
|
|
cpuPercent: 12.5,
|
|
processId: 4321,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("does not truncate malformed POSIX CPU samples", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "linux",
|
|
runCommand: async () => ({
|
|
stdout: " 4321 1 262144 12.5.6 node dist/index.js gateway --port 19080",
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 256,
|
|
cpuPercent: null,
|
|
processId: 4321,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("samples the POSIX gateway child instead of the pnpm launcher", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "linux",
|
|
posixCommandLineNeedles: ["gateway", "--port", "19080"],
|
|
runCommand: async (command: string, args: string[]) => {
|
|
expect(command).toBe("ps");
|
|
expect(args).toEqual(["-ww", "-axo", "pid=,ppid=,rss=,pcpu=,command="]);
|
|
return {
|
|
stdout: [
|
|
" 4321 1 16384 0.0 node /usr/local/bin/corepack pnpm openclaw gateway --port 19080",
|
|
" 4322 4321 262144 12.5 node dist/index.js gateway --port 19080 --bind loopback",
|
|
" 4323 4322 32768 1.5 node helper.js",
|
|
].join("\n"),
|
|
stderr: "",
|
|
};
|
|
},
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 288,
|
|
cpuPercent: 12.5,
|
|
processId: 4322,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("falls back to the POSIX gateway process title when the port arg is rewritten", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "darwin",
|
|
posixCommandLineNeedles: ["gateway", "--port", "19080"],
|
|
runCommand: async () => ({
|
|
stdout: [
|
|
" 4321 1 1048576 0.0 node /usr/local/bin/corepack pnpm openclaw gateway --port 19080",
|
|
" 4322 4321 262144 12.5 openclaw-gateway",
|
|
" 4323 4322 32768 1.5 node helper.js",
|
|
].join("\n"),
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 288,
|
|
cpuPercent: 12.5,
|
|
processId: 4322,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("falls back to the largest POSIX child when the gateway command line is unavailable", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "linux",
|
|
posixCommandLineNeedles: ["gateway", "--port", "19080"],
|
|
runCommand: async () => ({
|
|
stdout: [
|
|
" 4321 1 1048576 0.0 node /usr/local/bin/corepack pnpm openclaw gateway --port 19080",
|
|
" 4322 4321 262144 12.5 node",
|
|
" 4323 4322 32768 1.5 node helper.js",
|
|
].join("\n"),
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toEqual({
|
|
aggregateRssMiB: 288,
|
|
cpuPercent: 12.5,
|
|
processId: 4322,
|
|
rssMiB: 256,
|
|
});
|
|
});
|
|
|
|
it("does not accept a POSIX launcher sample when the gateway child is missing", async () => {
|
|
const sample = await sampleProcess(4321, {
|
|
platform: "darwin",
|
|
posixCommandLineNeedles: ["gateway", "--port", "19080"],
|
|
runCommand: async () => ({
|
|
stdout: " 4321 1 16384 0.0 node /usr/local/bin/corepack pnpm openclaw status\n",
|
|
stderr: "",
|
|
}),
|
|
});
|
|
|
|
expect(sample).toBeNull();
|
|
});
|
|
|
|
it("retries transient loopback fetch resets from Windows HTTP probes", async () => {
|
|
const reset = new TypeError("fetch failed", {
|
|
cause: Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
|
|
});
|
|
const fetchImpl = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(reset)
|
|
.mockResolvedValueOnce(new Response('{"status":"live"}', { status: 200 }));
|
|
|
|
await expect(
|
|
fetchJson("http://127.0.0.1:19680/healthz", {
|
|
attempts: 2,
|
|
fetchImpl,
|
|
retryDelayMs: 0,
|
|
}),
|
|
).resolves.toEqual({ ok: true, status: 200, body: { status: "live" } });
|
|
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("bounds HTTP probe response bodies", async () => {
|
|
const fetchImpl = vi.fn().mockResolvedValue(new Response("x".repeat(1025), { status: 200 }));
|
|
|
|
await expect(
|
|
fetchJson("http://127.0.0.1:19680/healthz", {
|
|
attempts: 1,
|
|
fetchImpl,
|
|
maxBodyBytes: 1024,
|
|
}),
|
|
).rejects.toMatchObject({
|
|
code: "ETOOBIG",
|
|
message: "fetch response body exceeded 1024 bytes",
|
|
});
|
|
});
|
|
|
|
it("rejects oversized HTTP probe responses before reading declared large bodies", async () => {
|
|
let canceled = false;
|
|
const response = new Response(
|
|
new ReadableStream({
|
|
cancel() {
|
|
canceled = true;
|
|
},
|
|
}),
|
|
{
|
|
headers: {
|
|
"content-length": "1025",
|
|
},
|
|
},
|
|
);
|
|
|
|
await expect(readBoundedResponseText(response, 1024)).rejects.toMatchObject({
|
|
code: "ETOOBIG",
|
|
message: "fetch response body exceeded 1024 bytes",
|
|
});
|
|
expect(canceled).toBe(true);
|
|
});
|
|
|
|
it("bounds HTTP probe response bodies without a readable stream", async () => {
|
|
const response = {
|
|
headers: new Headers(),
|
|
text: vi.fn(async () => "x".repeat(1025)),
|
|
};
|
|
|
|
await expect(readBoundedResponseText(response, 1024)).rejects.toMatchObject({
|
|
code: "ETOOBIG",
|
|
message: "fetch response body exceeded 1024 bytes",
|
|
});
|
|
expect(response.text).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects declared large HTTP probe responses without a readable stream", async () => {
|
|
const response = {
|
|
headers: new Headers({
|
|
"content-length": "1025",
|
|
}),
|
|
text: vi.fn(async () => "not read"),
|
|
};
|
|
|
|
await expect(readBoundedResponseText(response, 1024)).rejects.toMatchObject({
|
|
code: "ETOOBIG",
|
|
message: "fetch response body exceeded 1024 bytes",
|
|
});
|
|
expect(response.text).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("reads bounded response streams", async () => {
|
|
await expect(readBoundedResponseText(new Response('{"status":"live"}'), 1024)).resolves.toBe(
|
|
'{"status":"live"}',
|
|
);
|
|
});
|
|
|
|
it("times out stalled HTTP probe response bodies", async () => {
|
|
vi.useFakeTimers();
|
|
const fetchImpl = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
text: () => new Promise(() => {}),
|
|
});
|
|
|
|
const result = fetchJson("http://127.0.0.1:19680/readyz", {
|
|
attempts: 1,
|
|
fetchImpl,
|
|
timeoutMs: 100,
|
|
});
|
|
const rejection = expect(result).rejects.toMatchObject({
|
|
code: "ETIMEDOUT",
|
|
message: "fetch http://127.0.0.1:19680/readyz timed out after 100ms",
|
|
});
|
|
|
|
await vi.advanceTimersByTimeAsync(100);
|
|
await rejection;
|
|
expect(fetchImpl.mock.calls[0]?.[1]?.signal.aborted).toBe(true);
|
|
});
|
|
|
|
it("fails when the sampled RSS exceeds the configured ceiling", () => {
|
|
expect(() => assertResourceCeiling({ rssMiB: 2049 })).toThrow(
|
|
"gateway RSS exceeded 2048 MiB: 2049 MiB",
|
|
);
|
|
});
|
|
|
|
it("fails when aggregate RSS exceeds the configured ceiling", () => {
|
|
expect(() => assertResourceCeiling({ aggregateRssMiB: 2049, rssMiB: 1024 })).toThrow(
|
|
"gateway aggregate RSS exceeded 2048 MiB: 2049 MiB",
|
|
);
|
|
});
|
|
|
|
it("summarizes peak RSS across repeated process samples", () => {
|
|
expect(
|
|
summarizeProcessSamples([
|
|
{ aggregateRssMiB: 128, rssMiB: 128, cpuPercent: 2 },
|
|
{ aggregateRssMiB: 768, rssMiB: 512, cpuPercent: 25 },
|
|
{ aggregateRssMiB: 1024, rssMiB: 256, cpuPercent: 8 },
|
|
]),
|
|
).toEqual({
|
|
aggregateRssMiB: 1024,
|
|
rssMiB: 256,
|
|
cpuPercent: 8,
|
|
sampleCount: 3,
|
|
peakCpuPercent: 25,
|
|
});
|
|
});
|
|
|
|
it("fails when process sampling does not capture RSS", () => {
|
|
expect(() => assertResourceCeiling(null)).toThrow("gateway RSS sample was not captured");
|
|
});
|
|
|
|
it("fails zero-valued process RSS samples", () => {
|
|
expect(() => assertResourceCeiling({ rssMiB: 0 })).toThrow(
|
|
"gateway RSS sample was invalid: 0 MiB",
|
|
);
|
|
expect(() => assertCommandResourceCeiling({ aggregateRssMiB: 0, rssMiB: 128 })).toThrow(
|
|
"command aggregate RSS sample was invalid: 0 MiB",
|
|
);
|
|
});
|
|
|
|
it("fails missing command samples and command RSS spikes", () => {
|
|
expect(() => assertCommandResourceCeiling(null)).toThrow("command RSS sample was not captured");
|
|
expect(() => assertCommandResourceCeiling({ aggregateRssMiB: 8193, rssMiB: 1024 })).toThrow(
|
|
"command aggregate RSS exceeded 8192 MiB: 8193 MiB",
|
|
);
|
|
});
|
|
});
|
|
|
|
function readText(file: string) {
|
|
return readFileSync(file, "utf8");
|
|
}
|
|
|
|
async function waitFor(condition: () => boolean, timeoutMs = 3_000) {
|
|
const startedAt = Date.now();
|
|
while (!condition()) {
|
|
if (Date.now() - startedAt > timeoutMs) {
|
|
throw new Error("timed out waiting for condition");
|
|
}
|
|
await delay(25);
|
|
}
|
|
}
|
|
|
|
function isProcessAlive(pid: number) {
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|