import { execFile } from "node:child_process"; import fs from "node:fs/promises"; import { createServer, type Server } from "node:http"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { appendJsonl, assertRequiredEnv, buildRttResult, buildRunId, createHarnessEnv, extractRtt, readTelegramSummary, safeRunLabel, validateOpenClawPackageSpec, } from "../../scripts/lib/rtt-harness.ts"; import { testing as cliTesting } from "../../scripts/rtt.ts"; const TEST_DIR = path.dirname(fileURLToPath(import.meta.url)); const FIXTURE_PATH = path.resolve(TEST_DIR, "../fixtures/telegram-qa-summary-rtt.json"); const DOCKER_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-docker.sh"); const CREDENTIAL_SCRIPT_PATH = path.resolve( TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-credentials.mjs", ); const CONFIG_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-config.mjs"); const execFileAsync = promisify(execFile); const tempDirs: string[] = []; afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); async function listenOnLoopback(server: Server) { await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address(); if (!address || typeof address === "string") { throw new Error("Expected TCP server address."); } return address; } function closeServer(server: Server) { return new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); }); } function credentialBrokerEnv(port: number) { return { ...process.env, OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1", OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "test-secret", OPENCLAW_QA_CONVEX_SITE_URL: `http://127.0.0.1:${port}`, OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "1000", OPENCLAW_QA_CREDENTIAL_OWNER_ID: "test-owner", OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: "maintainer", }; } describe("RTT harness", () => { it("validates OpenClaw package specs", () => { expect(validateOpenClawPackageSpec("openclaw@main")).toBe("openclaw@main"); expect(validateOpenClawPackageSpec("openclaw@alpha")).toBe("openclaw@alpha"); expect(validateOpenClawPackageSpec("openclaw@beta")).toBe("openclaw@beta"); expect(validateOpenClawPackageSpec("openclaw@latest")).toBe("openclaw@latest"); expect(validateOpenClawPackageSpec("openclaw@2026.4.30")).toBe("openclaw@2026.4.30"); expect(validateOpenClawPackageSpec("openclaw@2026.4.30-beta.2")).toBe( "openclaw@2026.4.30-beta.2", ); expect(validateOpenClawPackageSpec("openclaw@2026.4.30-alpha.2")).toBe( "openclaw@2026.4.30-alpha.2", ); expect(() => validateOpenClawPackageSpec("@openclaw/openclaw@beta")).toThrow( /Package spec must be/, ); expect(() => validateOpenClawPackageSpec("openclaw@next")).toThrow(/Package spec must be/); }); it("builds stable run labels", () => { expect(safeRunLabel("openclaw@beta")).toBe("openclaw_beta"); expect( buildRunId({ now: new Date("2026-05-01T03:04:05.678Z"), spec: "openclaw@beta", index: 1, }), ).toBe("2026-05-01T030405678Z-openclaw_beta-2"); }); it("constructs harness env without dropping caller env", () => { const env = createHarnessEnv({ baseEnv: { OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", OPENCLAW_NPM_TELEGRAM_FAST: "0", }, providerMode: "mock-openai", rawOutputDir: ".artifacts/rtt/run/raw", samples: 20, sampleTimeoutMs: 30_000, scenarios: ["telegram-mentioned-message-reply"], spec: "openclaw@beta", timeoutMs: 180_000, version: "2026.4.30-beta.1", }); expect(env.OPENCLAW_QA_TELEGRAM_GROUP_ID).toBe("-100123"); expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC).toBe("openclaw@beta"); expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL).toBe("openclaw@beta (2026.4.30-beta.1)"); expect(env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE).toBe("mock-openai"); expect(env.OPENCLAW_NPM_TELEGRAM_SCENARIOS).toBe("telegram-mentioned-message-reply"); expect(env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR).toBe(".artifacts/rtt/run/raw"); expect(env.OPENCLAW_NPM_TELEGRAM_FAST).toBe("0"); expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("20"); expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("30000"); expect(env.OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS).toBe("180000"); expect(env.OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS).toBe("180000"); }); it("forwards Convex credential controls without dropping RTT sample controls", () => { const env = createHarnessEnv({ baseEnv: { OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-credentials.example.convex.site", OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret", }, credentialRole: "maintainer", credentialSource: "convex", providerMode: "mock-openai", rawOutputDir: ".artifacts/rtt/run/raw", samples: 7, sampleTimeoutMs: 45_000, scenarios: ["telegram-mentioned-message-reply"], spec: "openclaw@beta", timeoutMs: 180_000, version: "2026.4.30-beta.1", }); expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE).toBe("convex"); expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE).toBe("maintainer"); expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("7"); expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("45000"); expect(() => assertRequiredEnv(env, { credentialRole: "maintainer", credentialSource: "convex" }), ).not.toThrow(); }); it("exports the Telegram bot token after Convex credentials are sourced", async () => { const script = await fs.readFile(DOCKER_SCRIPT_PATH, "utf8"); const sourceIndex = script.indexOf('source "$credential_env_file"'); const tokenExportIndex = script.indexOf( 'export TELEGRAM_BOT_TOKEN="${OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN:?missing OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN}"', ); const installEnvSnapshotIndex = script.indexOf('install_env=("${docker_env[@]}")'); const convexSecretForwardIndex = script.indexOf( "OPENCLAW_QA_CONVEX_SECRET_CI", installEnvSnapshotIndex, ); const bodyLimitForwardIndex = script.indexOf( "OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES", installEnvSnapshotIndex, ); const packageInstallIndex = script.indexOf("npm install -g"); const credentialAcquireIndex = script.indexOf( "node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs acquire", ); const heartbeatStartIndex = script.indexOf("start_credential_heartbeat", sourceIndex); const driverIndex = script.indexOf("node /app/scripts/e2e/npm-telegram-rtt-driver.mjs"); expect(sourceIndex).toBeGreaterThanOrEqual(0); expect(tokenExportIndex).toBeGreaterThan(sourceIndex); expect(installEnvSnapshotIndex).toBeGreaterThanOrEqual(0); expect(convexSecretForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); expect(bodyLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); expect(packageInstallIndex).toBeLessThan(credentialAcquireIndex); expect(script).toContain( '-e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"', ); expect(script).toContain( '"$timeout_bin" --kill-after=30s "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit', ); expect(script).toContain("elif command -v gtimeout >/dev/null 2>&1; then"); expect(script).toContain('timeout_bin="gtimeout"'); expect(script).toContain( 'echo "timeout or gtimeout is required for OPENCLAW_E2E_NPM_INSTALL_TIMEOUT=$npm_install_timeout" >&2', ); expect(script).toContain('"$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1'); expect(script).toContain( '"$timeout_bin" "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit', ); expect(script).not.toContain( "running package install without OPENCLAW_E2E_NPM_INSTALL_TIMEOUT", ); expect(script).toContain("run_logged docker_e2e_docker_run_cmd run --rm"); expect(script).not.toContain("run_logged docker run --rm"); expect(heartbeatStartIndex).toBeGreaterThan(sourceIndex); expect(heartbeatStartIndex).toBeLessThan(driverIndex); expect(script).toContain("start_credential_heartbeat() {\n (\n set +e"); expect(script).toContain("Convex credential heartbeat exited with status"); expect(script).toContain('kill -TERM "$rtt_shell_pid"'); expect(script).not.toContain('export TELEGRAM_BOT_TOKEN="$OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"'); }); it("keeps broker helper heartbeat handling aligned with QA leases", async () => { const script = await fs.readFile(CREDENTIAL_SCRIPT_PATH, "utf8"); expect(script).toContain("leaseTtlMs: acquired.leaseTtlMs ?? config.leaseTtlMs"); expect(script).toContain("leaseTtlMs: leaseTtlMsFromLease(config, lease)"); }); it("bounds Convex credential broker response bodies", async () => { const server = createServer((_request, response) => { response.writeHead(500, { "content-type": "application/json" }); response.end(JSON.stringify({ status: "error", message: "x".repeat(128) })); }); const { port } = await listenOnLoopback(server); const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-")); tempDirs.push(tempDir); try { await execFileAsync( process.execPath, [ CREDENTIAL_SCRIPT_PATH, "acquire", "--lease-file", path.join(tempDir, "lease.json"), "--credential-env-file", path.join(tempDir, "credentials.env"), ], { env: { ...credentialBrokerEnv(port), OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES: "16", }, maxBuffer: 128 * 1024, }, ); throw new Error("Expected credential acquire to fail."); } catch (error) { const execError = error as Error & { stderr?: string }; expect(execError.stderr).toContain( "credential broker acquire response body exceeded 16 bytes", ); expect(execError.stderr).not.toContain("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); } finally { await closeServer(server); } }); it("preserves empty broker responses for successful lease release", async () => { const server = createServer((_request, response) => { response.writeHead(204); response.end(); }); const { port } = await listenOnLoopback(server); const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-")); tempDirs.push(tempDir); const leaseFile = path.join(tempDir, "lease.json"); await fs.writeFile( leaseFile, `${JSON.stringify({ kind: "telegram", ownerId: "test-owner", actorRole: "maintainer", credentialId: "credential", leaseToken: "lease", })}\n`, ); try { await execFileAsync( process.execPath, [CREDENTIAL_SCRIPT_PATH, "release", "--lease-file", leaseFile], { env: credentialBrokerEnv(port), }, ); await expect(fs.stat(leaseFile)).rejects.toMatchObject({ code: "ENOENT" }); } finally { await closeServer(server); } }); it("generates final-only Telegram RTT delivery config for release packages", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-config-test-")); tempDirs.push(tempDir); const configPath = path.join(tempDir, "config.json"); await execFileAsync(process.execPath, [ CONFIG_SCRIPT_PATH, configPath, "12345", "-100123", "111:driver-token", "222:sut-token", "2026.5.16-beta.6", ]); const config = JSON.parse(await fs.readFile(configPath, "utf8")); expect(config.channels.telegram.streaming).toEqual({ mode: "off" }); expect(config.messages.groupChat.visibleReplies).toBe("automatic"); }); it("extracts RTT values from Telegram QA summaries", async () => { const summary = await readTelegramSummary(FIXTURE_PATH); expect(extractRtt(summary)).toEqual({ canaryMs: 1234, mentionReplyMs: 5000, warmSamples: [4000, 5000, 7000], avgMs: 5333, p50Ms: 5000, p95Ms: 7000, maxMs: 7000, failedSamples: 0, }); }); it("builds normalized result JSON", async () => { const summary = await readTelegramSummary(FIXTURE_PATH); const result = buildRttResult({ artifacts: { rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", rawReportPath: "runs/run/raw/telegram-qa-report.md", rawSummaryPath: "runs/run/raw/telegram-qa-summary.json", resultPath: "runs/run/result.json", }, finishedAt: new Date("2026-05-01T00:00:12.000Z"), providerMode: "mock-openai", rawSummary: summary, runId: "run", scenarios: ["telegram-mentioned-message-reply"], spec: "openclaw@beta", startedAt: new Date("2026-05-01T00:00:00.000Z"), version: "2026.4.30-beta.1", }); expect(result).toStrictEqual({ artifacts: { rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", rawReportPath: "runs/run/raw/telegram-qa-report.md", rawSummaryPath: "runs/run/raw/telegram-qa-summary.json", resultPath: "runs/run/result.json", }, package: { spec: "openclaw@beta", version: "2026.4.30-beta.1" }, run: { durationMs: 12_000, finishedAt: "2026-05-01T00:00:12.000Z", id: "run", startedAt: "2026-05-01T00:00:00.000Z", status: "pass", }, mode: { providerMode: "mock-openai", scenarios: ["telegram-mentioned-message-reply"], }, rtt: { canaryMs: 1234, mentionReplyMs: 5000, avgMs: 5333, p50Ms: 5000, p95Ms: 7000, maxMs: 7000, failedSamples: 0, warmSamples: [4000, 5000, 7000], }, }); }); it("marks failed scenario summaries as failed results", () => { const result = buildRttResult({ artifacts: { rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", rawReportPath: "runs/run/raw/telegram-qa-report.md", rawSummaryPath: "runs/run/raw/telegram-qa-summary.json", resultPath: "runs/run/result.json", }, finishedAt: new Date("2026-05-01T00:00:12.000Z"), providerMode: "mock-openai", rawSummary: { scenarios: [ { id: "telegram-canary", rttMs: 5948, status: "pass" }, { id: "telegram-mentioned-message-reply", status: "fail" }, ], }, runId: "run", scenarios: ["telegram-mentioned-message-reply"], spec: "openclaw@latest", startedAt: new Date("2026-05-01T00:00:00.000Z"), version: "2026.4.29", }); expect(result.run.status).toBe("fail"); expect(result.rtt).toEqual({ canaryMs: 5948, mentionReplyMs: undefined }); }); it("appends JSONL rows", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-test-")); tempDirs.push(tempDir); const jsonlPath = path.join(tempDir, "data/rtt.jsonl"); await appendJsonl(jsonlPath, { run: 1 }); await appendJsonl(jsonlPath, { run: 2 }); await expect(fs.readFile(jsonlPath, "utf8")).resolves.toBe('{"run":1}\n{"run":2}\n'); }); it("parses CLI options", () => { const parsed = cliTesting.parseArgs([ "openclaw@latest", "--package-tgz", "/tmp/openclaw.tgz", "--provider", "live-frontier", "--credential-source", "convex", "--credential-role", "ci", "--runs", "3", "--samples", "5", "--sample-timeout-ms", "30000", "--timeout-ms", "240000", "--harness-root", "/tmp/openclaw", "--output", "/tmp/runs", ]); expect(parsed.spec).toBe("openclaw@latest"); expect(parsed.options).toStrictEqual({ packageTgz: "/tmp/openclaw.tgz", credentialRole: "ci", credentialSource: "convex", providerMode: "live-frontier", runs: 3, samples: 5, sampleTimeoutMs: 30_000, harnessRoot: "/tmp/openclaw", output: "/tmp/runs", scenarios: ["telegram-mentioned-message-reply"], timeoutMs: 240_000, }); }); });