vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-23 15:11:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: speed up cli backend live lane

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-06 21:01:39 +01:00
parent 51c6b1c2bc
commit aa4cb43627
2 changed files with 57 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
scripts/gateway-cli-bootstrap-live-probe.ts
View File

@@ -1,243 +0,0 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { clearRuntimeConfigSnapshot, type OpenClawConfig } from "../src/config/config.js";
import { GatewayClient } from "../src/gateway/client.js";
import { startGatewayServer } from "../src/gateway/server.js";
import { extractPayloadText } from "../src/gateway/test-helpers.agent-results.js";
import { getFreePortBlockWithPermissionFallback } from "../src/test-utils/ports.js";
import { GATEWAY_CLIENT_NAMES } from "../src/utils/message-channel.js";
const DEFAULT_CLAUDE_ARGS = [
"-p",
"--output-format",
"stream-json",
"--include-partial-messages",
"--verbose",
"--permission-mode",
"bypassPermissions",
];
const DEFAULT_CLEAR_ENV = ["ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY_OLD"];
const CLI_BOOTSTRAP_TIMEOUT_MS = 300_000;
const GATEWAY_CONNECT_TIMEOUT_MS = 30_000;
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function withMcpConfigOverrides(args: string[], mcpConfigPath: string): string[] {
const next = [...args];
if (!next.includes("--strict-mcp-config")) {
next.push("--strict-mcp-config");
}
if (!next.includes("--mcp-config")) {
next.push("--mcp-config", mcpConfigPath);
}
return next;
}
async function connectClient(params: { url: string; token: string }) {
const startedAt = Date.now();
let attempt = 0;
let lastError: Error | null = null;
while (Date.now() - startedAt < GATEWAY_CONNECT_TIMEOUT_MS) {
attempt += 1;
const remainingMs = GATEWAY_CONNECT_TIMEOUT_MS - (Date.now() - startedAt);
if (remainingMs <= 0) {
break;
}
try {
return await connectClientOnce({
...params,
timeoutMs: Math.min(remainingMs, 10_000),
});
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 2_000) {
throw lastError;
}
await sleep(Math.min(500 * attempt, 2_000));
}
}
throw lastError ?? new Error("gateway connect timeout");
}
async function connectClientOnce(params: { url: string; token: string; timeoutMs: number }) {
return await new Promise<GatewayClient>((resolve, reject) => {
let done = false;
let client: GatewayClient | undefined;
const finish = (result: { client?: GatewayClient; error?: Error }) => {
if (done) {
return;
}
done = true;
clearTimeout(connectTimeout);
if (result.error) {
if (client) {
void client.stopAndWait({ timeoutMs: 1_000 }).catch(() => {});
}
reject(result.error);
return;
}
resolve(result.client as GatewayClient);
};
client = new GatewayClient({
url: params.url,
token: params.token,
clientName: GATEWAY_CLIENT_NAMES.TEST,
clientVersion: "dev",
mode: "test",
requestTimeoutMs: params.timeoutMs,
connectChallengeTimeoutMs: params.timeoutMs,
onHelloOk: () => finish({ client }),
onConnectError: (error) => finish({ error }),
onClose: (code, reason) =>
finish({ error: new Error(`gateway closed during connect (${code}): ${reason}`) }),
});
const connectTimeout = setTimeout(
() => finish({ error: new Error("gateway connect timeout") }),
params.timeoutMs,
);
connectTimeout.unref();
client.start();
});
}
function isRetryableGatewayConnectError(error: Error): boolean {
const message = error.message.toLowerCase();
return (
message.includes("gateway closed during connect (1000)") ||
message.includes("gateway connect timeout") ||
message.includes("gateway connect challenge timeout")
);
}
async function getFreeGatewayPort(): Promise<number> {
return await getFreePortBlockWithPermissionFallback({
offsets: [0, 1, 2, 4],
fallbackBase: 40_000,
});
}
async function main() {
const preservedEnv = new Set(
JSON.parse(process.env.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV ?? "[]") as string[],
);
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-inline-bootstrap-"));
const workspaceRootDir = path.join(tempDir, "workspace");
const workspaceDir = path.join(workspaceRootDir, "dev");
const soulSecret = `SOUL-${randomUUID()}`;
const identitySecret = `IDENTITY-${randomUUID()}`;
const userSecret = `USER-${randomUUID()}`;
await fs.mkdir(workspaceDir, { recursive: true });
await fs.writeFile(
path.join(workspaceDir, "AGENTS.md"),
[
"# AGENTS.md",
"",
"When the user sends a BOOTSTRAP_CHECK token, reply with exactly:",
`BOOTSTRAP_OK ${soulSecret} ${identitySecret} ${userSecret}`,
"Do not add any other words or punctuation.",
].join("\n"),
);
await fs.writeFile(path.join(workspaceDir, "SOUL.md"), `${soulSecret}\n`);
await fs.writeFile(path.join(workspaceDir, "IDENTITY.md"), `${identitySecret}\n`);
await fs.writeFile(path.join(workspaceDir, "USER.md"), `${userSecret}\n`);
const cfg: OpenClawConfig = {};
const existingBackends = cfg.agents?.defaults?.cliBackends ?? {};
const claudeBackend = existingBackends["claude-cli"] ?? {};
const cliCommand =
process.env.OPENCLAW_LIVE_CLI_BACKEND_COMMAND ?? claudeBackend.command ?? "claude";
let cliArgs = claudeBackend.args ?? DEFAULT_CLAUDE_ARGS;
const mcpConfigPath = path.join(tempDir, "claude-mcp.json");
await fs.writeFile(mcpConfigPath, `${JSON.stringify({ mcpServers: {} }, null, 2)}\n`);
cliArgs = withMcpConfigOverrides(cliArgs, mcpConfigPath);
const cliClearEnv = (claudeBackend.clearEnv ?? DEFAULT_CLEAR_ENV).filter(
(name) => !preservedEnv.has(name),
);
const preservedCliEnv = Object.fromEntries(
[...preservedEnv]
.map((name) => [name, process.env[name]])
.filter((entry): entry is [string, string] => typeof entry[1] === "string"),
);
const nextCfg = {
...cfg,
agents: {
...cfg.agents,
defaults: {
...cfg.agents?.defaults,
workspace: workspaceRootDir,
model: { primary: "claude-cli/claude-sonnet-4-6" },
models: { "claude-cli/claude-sonnet-4-6": {} },
cliBackends: {
...existingBackends,
"claude-cli": {
...claudeBackend,
command: cliCommand,
args: cliArgs,
clearEnv: cliClearEnv.length > 0 ? cliClearEnv : undefined,
env: Object.keys(preservedCliEnv).length > 0 ? preservedCliEnv : undefined,
systemPromptWhen: "first",
},
},
sandbox: { mode: "off" },
},
},
};
const tempConfigPath = path.join(tempDir, "openclaw.json");
await fs.writeFile(tempConfigPath, `${JSON.stringify(nextCfg, null, 2)}\n`);
process.env.OPENCLAW_CONFIG_PATH = tempConfigPath;
process.env.OPENCLAW_SKIP_CHANNELS = "1";
process.env.OPENCLAW_SKIP_GMAIL_WATCHER = "1";
process.env.OPENCLAW_SKIP_CRON = "1";
process.env.OPENCLAW_SKIP_CANVAS_HOST = "1";
const port = await getFreeGatewayPort();
const token = `test-${randomUUID()}`;
process.env.OPENCLAW_GATEWAY_TOKEN = token;
const server = await startGatewayServer(port, {
bind: "loopback",
auth: { mode: "token", token },
controlUiEnabled: false,
});
const client = await connectClient({ url: `ws://127.0.0.1:${port}`, token });
try {
const payload = await client.request(
"agent",
{
sessionKey: `agent:dev:inline-cli-bootstrap-${randomUUID()}`,
idempotencyKey: `idem-${randomUUID()}`,
message: `BOOTSTRAP_CHECK ${randomUUID()}`,
deliver: false,
},
{ expectFinal: true, timeoutMs: CLI_BOOTSTRAP_TIMEOUT_MS },
);
const text = extractPayloadText(payload?.result);
process.stdout.write(
`${JSON.stringify({
ok: true,
text,
expectedText: `BOOTSTRAP_OK ${soulSecret} ${identitySecret} ${userSecret}`,
systemPromptReport: payload?.result?.meta?.systemPromptReport ?? null,
})}\n`,
);
} finally {
await client.stopAndWait();
await server.close({ reason: "bootstrap live probe done" });
await fs.rm(tempDir, { recursive: true, force: true });
clearRuntimeConfigSnapshot();
}
}
try {
await main();
process.exit(0);
} catch (error) {
process.stderr.write(`${String(error)}\n`);
process.exit(1);
}

147
src/gateway/gateway-cli-backend.live.test.ts
View File

@@ -1,4 +1,3 @@
import { spawn } from "node:child_process";
import { randomBytes, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
@@ -23,11 +22,7 @@ const describeLive = LIVE && CLI_LIVE ? describe : describe.skip;
const DEFAULT_MODEL = "claude-cli/claude-sonnet-4-6";
const CLI_BACKEND_LIVE_TIMEOUT_MS = 180_000;
const CLI_BOOTSTRAP_LIVE_TIMEOUT_MS = 300_000;
const CLI_GATEWAY_CONNECT_TIMEOUT_MS = 30_000;
const BOOTSTRAP_LIVE_MODEL = process.env.OPENCLAW_LIVE_CLI_BACKEND_MODEL ?? DEFAULT_MODEL;
const describeClaudeBootstrapLive =
LIVE && CLI_LIVE && BOOTSTRAP_LIVE_MODEL.startsWith("claude-cli/") ? describe : describe.skip;
const DEFAULT_CLAUDE_ARGS = [
"-p",
"--output-format",
@@ -139,6 +134,12 @@ function parseImageMode(raw?: string): "list" | "repeat" | undefined {
throw new Error("OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE must be 'list' or 'repeat'.");
}
function matchesCliBackendReply(text: string, expected: string): boolean {
const normalized = text.trim();
const target = expected.trim();
return normalized === target || normalized === target.slice(0, -1);
}
function withMcpConfigOverrides(args: string[], mcpConfigPath: string): string[] {
const next = [...args];
if (!next.includes("--strict-mcp-config")) {
@@ -157,6 +158,35 @@ async function getFreeGatewayPort(): Promise<number> {
});
}
type BootstrapWorkspaceContext = {
expectedInjectedFiles: string[];
workspaceRootDir: string;
};
type SystemPromptReport = {
injectedWorkspaceFiles?: Array<{ name?: string }>;
};
async function createBootstrapWorkspace(tempDir: string): Promise<BootstrapWorkspaceContext> {
const workspaceRootDir = path.join(tempDir, "workspace");
const workspaceDir = path.join(workspaceRootDir, "dev");
const expectedInjectedFiles = ["AGENTS.md", "SOUL.md", "IDENTITY.md", "USER.md"];
await fs.mkdir(workspaceDir, { recursive: true });
await fs.writeFile(
path.join(workspaceDir, "AGENTS.md"),
[
"# AGENTS.md",
"",
"Follow exact reply instructions from the user.",
"Do not add extra punctuation when the user asks for an exact response.",
].join("\n"),
);
await fs.writeFile(path.join(workspaceDir, "SOUL.md"), `SOUL-${randomUUID()}\n`);
await fs.writeFile(path.join(workspaceDir, "IDENTITY.md"), `IDENTITY-${randomUUID()}\n`);
await fs.writeFile(path.join(workspaceDir, "USER.md"), `USER-${randomUUID()}\n`);
return { expectedInjectedFiles, workspaceRootDir };
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
@@ -175,14 +205,14 @@ async function connectClient(params: { url: string; token: string }) {
try {
return await connectClientOnce({
...params,
timeoutMs: Math.min(remainingMs, 10_000),
timeoutMs: Math.min(remainingMs, 35_000),
});
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 2_000) {
if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 5_000) {
throw lastError;
}
await sleep(Math.min(500 * attempt, 2_000));
await sleep(Math.min(1_000 * attempt, 5_000));
}
}
@@ -239,73 +269,12 @@ function isRetryableGatewayConnectError(error: Error): boolean {
return (
message.includes("gateway closed during connect (1000)") ||
message.includes("gateway connect timeout") ||
message.includes("gateway connect challenge timeout")
message.includes("gateway connect challenge timeout") ||
message.includes("gateway request timeout for connect") ||
message.includes("gateway client stopped")
);
}
async function runGatewayCliBootstrapLiveProbe(): Promise<{
ok: boolean;
text: string;
expectedText: string;
systemPromptReport: {
injectedWorkspaceFiles?: Array<{ name?: string }>;
} | null;
}> {
return await new Promise((resolve, reject) => {
const env = { ...process.env };
delete env.VITEST;
const child = spawn(
"pnpm",
["exec", "tsx", path.join("scripts", "gateway-cli-bootstrap-live-probe.ts")],
{
cwd: process.cwd(),
env,
stdio: ["ignore", "pipe", "pipe"],
},
);
let stdout = "";
let stderr = "";
const timeout = setTimeout(() => {
child.kill("SIGTERM");
reject(new Error(`bootstrap probe timed out\nstdout:\n${stdout}\nstderr:\n${stderr}`));
}, CLI_BOOTSTRAP_LIVE_TIMEOUT_MS);
timeout.unref();
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk: string) => {
stdout += chunk;
});
child.stderr.on("data", (chunk: string) => {
stderr += chunk;
});
child.on("error", (error) => {
clearTimeout(timeout);
reject(error);
});
child.on("close", (code) => {
clearTimeout(timeout);
if (code !== 0) {
reject(
new Error(`bootstrap probe exit=${String(code)}\nstdout:\n${stdout}\nstderr:\n${stderr}`),
);
return;
}
const line = stdout
.trim()
.split(/\r?\n/)
.map((entry) => entry.trim())
.findLast((entry) => entry.startsWith("{") && entry.endsWith("}"));
if (!line) {
reject(
new Error(`bootstrap probe missing JSON result\nstdout:\n${stdout}\nstderr:\n${stderr}`),
);
return;
}
resolve(JSON.parse(line) as Awaited<ReturnType<typeof runGatewayCliBootstrapLiveProbe>>);
});
});
}
describeLive("gateway live (cli backend)", () => {
it(
"runs the agent pipeline against the local CLI backend",
@@ -395,6 +364,8 @@ describeLive("gateway live (cli backend)", () => {
}
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-live-cli-"));
const bootstrapWorkspace =
providerId === "claude-cli" ? await createBootstrapWorkspace(tempDir) : null;
const disableMcpConfig = process.env.OPENCLAW_LIVE_CLI_BACKEND_DISABLE_MCP_CONFIG !== "0";
let cliArgs = baseCliArgs;
if (providerId === "claude-cli" && disableMcpConfig) {
@@ -418,6 +389,7 @@ describeLive("gateway live (cli backend)", () => {
...cfg.agents,
defaults: {
...cfg.agents?.defaults,
...(bootstrapWorkspace ? { workspace: bootstrapWorkspace.workspaceRootDir } : {}),
model: { primary: modelKey },
models: {
[modelKey]: {},
@@ -429,7 +401,7 @@ describeLive("gateway live (cli backend)", () => {
args: cliArgs,
clearEnv: filteredCliClearEnv.length > 0 ? filteredCliClearEnv : undefined,
env: Object.keys(preservedCliEnv).length > 0 ? preservedCliEnv : undefined,
systemPromptWhen: "never",
systemPromptWhen: providerId === "claude-cli" ? "first" : "never",
...(cliImageArg ? { imageArg: cliImageArg, imageMode: cliImageMode } : {}),
},
},
@@ -478,7 +450,15 @@ describeLive("gateway live (cli backend)", () => {
if (providerId === "codex-cli") {
expect(text).toContain(`CLI-BACKEND-${nonce}`);
} else {
expect(text).toContain(`CLI backend OK ${nonce}.`);
const resultWithMeta = payload?.result as {
meta?: { systemPromptReport?: SystemPromptReport };
};
expect(matchesCliBackendReply(text, `CLI backend OK ${nonce}.`)).toBe(true);
expect(
resultWithMeta.meta?.systemPromptReport?.injectedWorkspaceFiles?.map(
(entry) => entry.name,
) ?? [],
).toEqual(expect.arrayContaining(bootstrapWorkspace?.expectedInjectedFiles ?? []));
}
if (CLI_RESUME) {
@@ -505,7 +485,9 @@ describeLive("gateway live (cli backend)", () => {
if (providerId === "codex-cli") {
expect(resumeText).toContain(`CLI-RESUME-${resumeNonce}`);
} else {
expect(resumeText).toContain(`CLI backend RESUME OK ${resumeNonce}.`);
expect(
matchesCliBackendReply(resumeText, `CLI backend RESUME OK ${resumeNonce}.`),
).toBe(true);
}
}
@@ -600,21 +582,6 @@ describeLive("gateway live (cli backend)", () => {
}
}
},
CLI_BOOTSTRAP_LIVE_TIMEOUT_MS,
);
});
describeClaudeBootstrapLive("gateway live (claude-cli bootstrap context)", () => {
it(
"injects AGENTS, SOUL, IDENTITY, and USER files into the first Claude CLI turn",
async () => {
const result = await runGatewayCliBootstrapLiveProbe();
expect(result.ok).toBe(true);
expect(result.text).toBe(result.expectedText);
expect(
result.systemPromptReport?.injectedWorkspaceFiles?.map((entry) => entry.name) ?? [],
).toEqual(expect.arrayContaining(["AGENTS.md", "SOUL.md", "IDENTITY.md", "USER.md"]));
},
CLI_BACKEND_LIVE_TIMEOUT_MS,
);
});