From 476ac66d804170f670b39c57e061546f014df19b Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Fri, 1 May 2026 21:06:01 +0530 Subject: [PATCH] test(rtt): support main package measurements --- scripts/e2e/npm-telegram-rtt-docker.sh | 4 ++-- scripts/lib/rtt-harness.ts | 21 +++++++++++++++++++-- scripts/rtt.ts | 23 +++++++++++++++++++++-- test/scripts/rtt-harness.test.ts | 6 +++++- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/scripts/e2e/npm-telegram-rtt-docker.sh b/scripts/e2e/npm-telegram-rtt-docker.sh index 5ccfefed915..e730a6e7632 100755 --- a/scripts/e2e/npm-telegram-rtt-docker.sh +++ b/scripts/e2e/npm-telegram-rtt-docker.sh @@ -13,10 +13,10 @@ OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-r validate_openclaw_package_spec() { local spec="$1" - if [[ "$spec" =~ ^openclaw@(beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-beta\.[1-9][0-9]*)?)$ ]]; then + if [[ "$spec" =~ ^openclaw@(main|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-beta\.[1-9][0-9]*)?)$ ]]; then return 0 fi - echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC must be openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: $spec" >&2 + echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC must be openclaw@main, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: $spec" >&2 exit 1 } diff --git a/scripts/lib/rtt-harness.ts b/scripts/lib/rtt-harness.ts index 19b957e8760..058e89f0f0b 100644 --- a/scripts/lib/rtt-harness.ts +++ b/scripts/lib/rtt-harness.ts @@ -8,6 +8,7 @@ const execFileAsync = promisify(execFile); export type RttProviderMode = "mock-openai" | "live-frontier"; export type RttCliOptions = { + packageTgz?: string; providerMode: RttProviderMode; runs: number; samples: number; @@ -75,7 +76,7 @@ export type TelegramQaSummary = { }; const OPENCLAW_PACKAGE_SPEC_RE = - /^openclaw@(beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-beta\.[1-9][0-9]*)?)$/u; + /^openclaw@(main|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-beta\.[1-9][0-9]*)?)$/u; const REQUIRED_TELEGRAM_ENV = [ "OPENCLAW_QA_TELEGRAM_GROUP_ID", @@ -86,7 +87,7 @@ const REQUIRED_TELEGRAM_ENV = [ export function validateOpenClawPackageSpec(spec: string) { if (!OPENCLAW_PACKAGE_SPEC_RE.test(spec)) { throw new Error( - `Package spec must be openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: ${spec}`, + `Package spec must be openclaw@main, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: ${spec}`, ); } return spec; @@ -128,6 +129,7 @@ export function extractRtt(summary: TelegramQaSummary) { export function createHarnessEnv(params: { baseEnv: NodeJS.ProcessEnv; + packageTgz?: string; providerMode: RttProviderMode; scenarios: string[]; spec: string; @@ -140,6 +142,7 @@ export function createHarnessEnv(params: { return { ...params.baseEnv, OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC: params.spec, + ...(params.packageTgz ? { OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ: params.packageTgz } : {}), OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL: `${params.spec} (${params.version})`, OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE: params.providerMode, OPENCLAW_NPM_TELEGRAM_SCENARIOS: params.scenarios.join(","), @@ -189,6 +192,20 @@ export async function resolvePublishedVersion(spec: string) { return parsed.trim(); } +export async function resolveMainVersion(harnessRoot: string) { + const packageJson = JSON.parse( + await fs.readFile(path.join(harnessRoot, "package.json"), "utf8"), + ) as { version?: unknown }; + if (typeof packageJson.version !== "string" || packageJson.version.trim().length === 0) { + throw new Error("OpenClaw package.json must contain a non-empty version."); + } + const { stdout } = await execFileAsync("git", ["rev-parse", "--short=10", "HEAD"], { + cwd: harnessRoot, + timeout: 10_000, + }); + return `${packageJson.version.trim()}+${stdout.trim()}`; +} + export async function readTelegramSummary(summaryPath: string) { return JSON.parse(await fs.readFile(summaryPath, "utf8")) as TelegramQaSummary; } diff --git a/scripts/rtt.ts b/scripts/rtt.ts index c35e5a9e929..708b468e05a 100644 --- a/scripts/rtt.ts +++ b/scripts/rtt.ts @@ -10,6 +10,7 @@ import { buildRunId, createHarnessEnv, readTelegramSummary, + resolveMainVersion, resolvePublishedVersion, runHarness, validateOpenClawPackageSpec, @@ -25,9 +26,10 @@ const DEFAULT_SAMPLE_TIMEOUT_MS = 30_000; function usage() { return [ - "Usage: pnpm rtt [--provider mock-openai|live-frontier] [--runs N] [--samples N] [--sample-timeout-ms N] [--timeout-ms N] [--harness-root PATH] [--output PATH]", + "Usage: pnpm rtt [--package-tgz PATH] [--provider mock-openai|live-frontier] [--runs N] [--samples N] [--sample-timeout-ms N] [--timeout-ms N] [--harness-root PATH] [--output PATH]", "", "Examples:", + " pnpm rtt openclaw@main --package-tgz .artifacts/package/openclaw.tgz", " pnpm rtt openclaw@beta", " pnpm rtt openclaw@2026.4.30", " pnpm rtt openclaw@latest --provider live-frontier", @@ -61,6 +63,7 @@ function resolveHome(input: string) { function parseArgs(argv: string[]) { let spec: string | undefined; + let packageTgz: string | undefined; let providerMode = DEFAULT_PROVIDER_MODE; let runs = 1; let samples = DEFAULT_SAMPLES; @@ -79,6 +82,14 @@ function parseArgs(argv: string[]) { providerMode = parseProviderMode(argv[++index] ?? ""); continue; } + if (arg === "--package-tgz") { + const value = argv[++index] ?? ""; + if (!value.trim()) { + throw new Error("--package-tgz requires a path."); + } + packageTgz = path.resolve(resolveHome(value)); + continue; + } if (arg === "--runs") { runs = parsePositiveInt("--runs", argv[++index] ?? ""); continue; @@ -125,6 +136,7 @@ function parseArgs(argv: string[]) { return { spec: validateOpenClawPackageSpec(spec), options: { + packageTgz, providerMode, runs, samples, @@ -152,6 +164,7 @@ async function runOne(params: { const startedAt = new Date(); const env = createHarnessEnv({ baseEnv: process.env, + packageTgz: params.options.packageTgz, providerMode: params.options.providerMode, rawOutputDir, samples: params.options.samples, @@ -205,7 +218,13 @@ async function main() { assertRequiredEnv(process.env); await assertHarnessRoot(options.harnessRoot); await assertDockerAvailable(); - const version = await resolvePublishedVersion(spec); + if (spec === "openclaw@main" && !options.packageTgz) { + throw new Error("openclaw@main requires --package-tgz."); + } + const version = + spec === "openclaw@main" + ? await resolveMainVersion(options.harnessRoot) + : await resolvePublishedVersion(spec); let failed = false; for (let index = 0; index < options.runs; index += 1) { const run = await runOne({ index, options, spec, version }); diff --git a/test/scripts/rtt-harness.test.ts b/test/scripts/rtt-harness.test.ts index 1f269213f55..7ff01565eff 100644 --- a/test/scripts/rtt-harness.test.ts +++ b/test/scripts/rtt-harness.test.ts @@ -19,7 +19,8 @@ const TEST_DIR = path.dirname(fileURLToPath(import.meta.url)); const FIXTURE_PATH = path.resolve(TEST_DIR, "../fixtures/telegram-qa-summary-rtt.json"); describe("RTT harness", () => { - it("validates published OpenClaw package specs", () => { + it("validates OpenClaw package specs", () => { + expect(validateOpenClawPackageSpec("openclaw@main")).toBe("openclaw@main"); expect(validateOpenClawPackageSpec("openclaw@beta")).toBe("openclaw@beta"); expect(validateOpenClawPackageSpec("openclaw@latest")).toBe("openclaw@latest"); expect(validateOpenClawPackageSpec("openclaw@2026.4.30")).toBe("openclaw@2026.4.30"); @@ -165,6 +166,8 @@ describe("RTT harness", () => { it("parses CLI options", () => { const parsed = cliTesting.parseArgs([ "openclaw@latest", + "--package-tgz", + "/tmp/openclaw.tgz", "--provider", "live-frontier", "--runs", @@ -183,6 +186,7 @@ describe("RTT harness", () => { expect(parsed.spec).toBe("openclaw@latest"); expect(parsed.options).toMatchObject({ + packageTgz: "/tmp/openclaw.tgz", providerMode: "live-frontier", runs: 3, samples: 5,