From 1affe4fcdf5f97f9c4cc6c4a152e51f6226e71de Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 14 Jun 2026 02:02:33 -0700 Subject: [PATCH] Fold Telegram RTT sampling into live QA evidence (#92550) * refactor(qa): fold telegram rtt into live evidence * test: default package telegram rtt samples * refactor(qa-lab): fold telegram rtt into live evidence * fix(qa-lab): keep package telegram rtt optional for focused runs * fix(qa-lab): avoid stale rtt evidence on failed samples * fix(qa-lab): pass telegram live env into credential leasing * fix(qa-lab): update telegram canary remediation artifacts * docs(qa): remove stale telegram observed artifact guidance * fix(qa-lab): clarify telegram empty-reply remediation * fix(qa-lab): honor telegram rtt timeout * ci(qa): drop stale telegram capture env * refactor: align telegram evidence coverage fields * fix: ignore stale telegram observed artifacts * fix: preserve telegram rtt coverage mapping * fix: omit unused telegram rtt catch binding * docs: document telegram rtt check selector --- .github/workflows/mantis-telegram-live.yml | 1 - .github/workflows/npm-telegram-beta-e2e.yml | 1 - .github/workflows/openclaw-release-checks.yml | 1 - .../workflows/qa-live-transports-convex.yml | 1 - docs/concepts/mantis.md | 17 +- docs/concepts/qa-e2e-automation.md | 56 +- docs/help/testing.md | 20 +- .../qa-lab/src/evidence-summary.test.ts | 45 +- extensions/qa-lab/src/evidence-summary.ts | 18 +- .../shared/live-transport-result.ts | 24 + .../shared/live-transport-rtt.ts | 83 ++ .../live-transports/telegram/cli.runtime.ts | 1 - .../telegram/telegram-live.runtime.test.ts | 260 +++--- .../telegram/telegram-live.runtime.ts | 466 +++++----- package.json | 1 - scripts/e2e/npm-telegram-live-docker.sh | 13 +- scripts/e2e/npm-telegram-live-runner.ts | 45 +- scripts/e2e/npm-telegram-rtt-config.mjs | 118 --- scripts/e2e/npm-telegram-rtt-credentials.mjs | 505 ----------- scripts/e2e/npm-telegram-rtt-docker.sh | 454 ---------- scripts/e2e/npm-telegram-rtt-driver.mjs | 592 ------------- scripts/lib/rtt-harness.ts | 362 -------- scripts/mantis/build-telegram-evidence.mjs | 31 +- scripts/rtt.ts | 277 ------ test/fixtures/telegram-qa-summary-rtt.json | 60 -- .../mantis-build-telegram-evidence.test.ts | 33 +- test/scripts/npm-telegram-live.test.ts | 51 +- test/scripts/npm-telegram-rtt-driver.test.ts | 569 ------------ test/scripts/rtt-harness.test.ts | 808 ------------------ 29 files changed, 729 insertions(+), 4184 deletions(-) create mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-result.ts create mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts delete mode 100755 scripts/e2e/npm-telegram-rtt-config.mjs delete mode 100755 scripts/e2e/npm-telegram-rtt-credentials.mjs delete mode 100755 scripts/e2e/npm-telegram-rtt-docker.sh delete mode 100755 scripts/e2e/npm-telegram-rtt-driver.mjs delete mode 100644 scripts/lib/rtt-harness.ts delete mode 100644 scripts/rtt.ts delete mode 100644 test/fixtures/telegram-qa-summary-rtt.json delete mode 100644 test/scripts/npm-telegram-rtt-driver.test.ts delete mode 100644 test/scripts/rtt-harness.test.ts diff --git a/.github/workflows/mantis-telegram-live.yml b/.github/workflows/mantis-telegram-live.yml index 6dc253e5081..0fbb9c8eea4 100644 --- a/.github/workflows/mantis-telegram-live.yml +++ b/.github/workflows/mantis-telegram-live.yml @@ -379,7 +379,6 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR }} CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN }} OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR: ${{ secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }} diff --git a/.github/workflows/npm-telegram-beta-e2e.yml b/.github/workflows/npm-telegram-beta-e2e.yml index 9cbd9fd18e7..e88a128c638 100644 --- a/.github/workflows/npm-telegram-beta-e2e.yml +++ b/.github/workflows/npm-telegram-beta-e2e.yml @@ -220,7 +220,6 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" INPUT_SCENARIO: ${{ inputs.scenario }} PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name || '' }} run: | diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 97ab13a3f6e..efb97f0d62a 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1412,7 +1412,6 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" run: | set -euo pipefail diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index 2bd59685874..8faeccc660e 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -532,7 +532,6 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.scenario || '' }} run: | set -euo pipefail diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md index 825184a86f5..e26f301b2c7 100644 --- a/docs/concepts/mantis.md +++ b/docs/concepts/mantis.md @@ -247,12 +247,13 @@ of only a bot-to-bot Slack transcript. evidence pipeline. It checks out the trusted candidate ref in a separate worktree, runs `pnpm openclaw qa telegram --credential-source convex --credential-role ci`, writes a `mantis-evidence.json` manifest from the -Telegram QA summary and observed-message artifact, renders the redacted -transcript HTML through a Crabbox desktop browser, generates a motion-trimmed GIF -with `crabbox media preview`, and posts the inline PR evidence comment when a PR -number is available. This lane is transcript-visual rather than logged-in -Telegram Web proof: the Telegram Bot API gives stable live message evidence, but -Telegram Web login state is not required for normal Mantis automation. +Telegram QA summary, `qa-evidence.json`, and report artifacts, renders the +redacted evidence HTML through a Crabbox desktop browser, generates a +motion-trimmed GIF with `crabbox media preview`, and posts the inline PR +evidence comment when a PR number is available. This lane is QA-evidence visual +rather than logged-in Telegram Web proof: the Telegram Bot API gives stable live +message evidence, but Telegram Web login state is not required for normal Mantis +automation. `Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop before/after wrapper. A maintainer can trigger it from a PR comment with @@ -494,8 +495,8 @@ zero: - `pnpm openclaw qa discord` already runs a live Discord lane with driver and SUT bots. -- The live transport runner already writes reports and observed-message - artifacts under `.artifacts/qa-e2e/`. +- The live transport runner already writes reports, QA evidence, and + transport-specific artifacts under `.artifacts/qa-e2e/`. - Convex credential leases already provide exclusive access to shared live transport credentials. - The browser control service already supports screenshots, snapshots, diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index ad29fc2fffb..0cac8b4d8f6 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -318,17 +318,17 @@ Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags: -| Flag | Default | Description | -| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `--scenario ` | - | Run only this scenario. Repeatable. | -| `--output-dir ` | `/.artifacts/qa-e2e/-` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. | -| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | -| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | -| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | -| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | -| `--fast` | off | Provider fast mode where supported. | -| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | -| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | +| Flag | Default | Description | +| ------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `--scenario ` | - | Run only this scenario. Repeatable. | +| `--output-dir ` | `/.artifacts/qa-e2e/-` | Where reports, summaries, evidence, transport-specific artifacts, and the output log are written. Relative paths resolve against `--repo-root`. | +| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | +| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | +| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | +| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | +| `--fast` | off | Provider fast mode where supported. | +| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | +| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code. @@ -346,10 +346,6 @@ Required env when `--credential-source env`: - `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN` - `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN` -Optional: - -- `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts (default redacts). - Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`): - `telegram-canary` @@ -375,26 +371,26 @@ Output artifacts: - `telegram-qa-report.md` - `qa-evidence.json` - evidence entries for the live transport checks, including profile, coverage, provider, channel, artifacts, result, and RTT fields. -- `telegram-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1`. -Package RTT comparison uses the same Telegram credential contract while keeping -its RTT sample controls on the RTT harness path: +Package Telegram runs use the same Telegram credential contract. Repeated RTT +measurement is part of the normal package Telegram live lane; the RTT +distribution is folded into `qa-evidence.json` under `result.timing` for the +selected RTT check. ```bash -pnpm rtt openclaw@beta \ - --credential-source convex \ - --credential-role maintainer \ - --samples 20 \ - --sample-timeout-ms 30000 +OPENCLAW_QA_CREDENTIAL_SOURCE=convex \ +pnpm test:docker:npm-telegram-live ``` -When `--credential-source convex` is set, the RTT Docker wrapper leases a -`kind: "telegram"` credential, exports the leased group/driver/SUT bot env into -the installed-package run, heartbeats the lease, and releases it on shutdown. -`--samples` and `--sample-timeout-ms` still feed -`OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES` and -`OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS`, so `result.json` remains comparable -across env-backed and Convex-backed RTT runs. +When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper +leases a `kind: "telegram"` credential, exports the leased group/driver/SUT bot +env into the installed-package run, heartbeats the lease, and releases it on +shutdown. The package wrapper defaults to 20 RTT checks of +`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role +`maintainer` outside CI when Convex is selected. Override +`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, +or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without +creating a separate RTT command or Telegram-specific summary format. ### Discord QA diff --git a/docs/help/testing.md b/docs/help/testing.md index 60d600a291f..82c0abb5911 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -218,17 +218,27 @@ inside every shard. `OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ=/path/to/openclaw-current.tgz` or `OPENCLAW_CURRENT_PACKAGE_TGZ` to test a resolved local tarball instead of installing from the registry. + - Emits repeated RTT timing in `qa-evidence.json` by default with + `OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES=20`. Override + `OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, + `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or + `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the RTT run. + `OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of + Telegram QA check IDs to sample; when unset, the default RTT-capable check + is `telegram-mentioned-message-reply`. - Uses the same Telegram env credentials or Convex credential source as `pnpm openclaw qa telegram`. For CI/release automation, set `OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus - `OPENCLAW_QA_CONVEX_SITE_URL` and the role secret. If + `OPENCLAW_QA_CONVEX_SITE_URL` and a role secret. If `OPENCLAW_QA_CONVEX_SITE_URL` and a Convex role secret are present in CI, the Docker wrapper selects Convex automatically. - The wrapper validates Telegram or Convex credential env on the host before Docker build/install work. Set `OPENCLAW_NPM_TELEGRAM_SKIP_CREDENTIAL_PREFLIGHT=1` only when deliberately debugging pre-credential setup. - `OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE=ci|maintainer` overrides the shared - `OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only. + `OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only. When Convex credentials + are selected and no role is set, the wrapper uses `ci` in CI and + `maintainer` outside CI. - GitHub Actions exposes this lane as the manual maintainer workflow `NPM Telegram Beta E2E`. It does not run on merge. The workflow uses the `qa-live-shared` environment and Convex CI credential leases. @@ -344,11 +354,11 @@ gh workflow run package-acceptance.yml --ref main \ want artifacts without a failing exit code. - Requires two distinct bots in the same private group, with the SUT bot exposing a Telegram username. - For stable bot-to-bot observation, enable Bot-to-Bot Communication Mode in `@BotFather` for both bots and ensure the driver bot can observe group bot traffic. - - Writes a Telegram QA report, summary, and observed-messages artifact under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply. + - Writes a Telegram QA report, summary, and `qa-evidence.json` under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply. `Mantis Telegram Live` is the PR-evidence wrapper around this lane. It runs the -candidate ref with Convex-leased Telegram credentials, renders the redacted -observed-message transcript in a Crabbox desktop browser, records MP4 evidence, +candidate ref with Convex-leased Telegram credentials, renders the redacted QA +report/evidence bundle in a Crabbox desktop browser, records MP4 evidence, generates a motion-trimmed GIF, uploads the artifact bundle, and posts inline PR evidence through the Mantis GitHub App when `pr_number` is set. Maintainers can start it from the Actions UI through `Mantis Scenario` (`scenario_id: diff --git a/extensions/qa-lab/src/evidence-summary.test.ts b/extensions/qa-lab/src/evidence-summary.test.ts index f0bb72e1743..5e74b666e87 100644 --- a/extensions/qa-lab/src/evidence-summary.test.ts +++ b/extensions/qa-lab/src/evidence-summary.test.ts @@ -133,7 +133,6 @@ describe("evidence summary", () => { artifactPaths: [ { kind: "summary", path: QA_EVIDENCE_FILENAME }, { kind: "report", path: "telegram-qa-report.md" }, - { kind: "transport-observations", path: "telegram-qa-observed-messages.json" }, ], env: { OPENCLAW_QA_RUNNER: "crabbox", @@ -206,11 +205,6 @@ describe("evidence summary", () => { path: "telegram-qa-report.md", source: "telegram-live-transport", }, - { - kind: "transport-observations", - path: "telegram-qa-observed-messages.json", - source: "telegram-live-transport", - }, ], }), result: { @@ -226,6 +220,45 @@ describe("evidence summary", () => { ]); }); + it("preserves aggregate live transport timing", () => { + const evidence = buildLiveTransportEvidenceSummary({ + artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }], + generatedAt: "2026-06-07T12:05:00.000Z", + primaryModel: "openai/gpt-5.5", + providerMode: "live-frontier", + checks: [ + { + id: "telegram-mentioned-message-reply", + coverageIds: ["channels.telegram.mention-gating"], + title: "Telegram mentioned message gets a reply", + status: "pass", + details: "5 samples collected.", + rttMs: 2000, + timing: { + rttMs: 1200, + avgMs: 1300, + p50Ms: 1200, + p95Ms: 1800, + maxMs: 2200, + samples: 5, + failedSamples: 1, + }, + }, + ], + transportId: "telegram", + }); + + expect(evidence.entries[0]?.result.timing).toEqual({ + rttMs: 1200, + avgMs: 1300, + p50Ms: 1200, + p95Ms: 1800, + maxMs: 2200, + samples: 5, + failedSamples: 1, + }); + }); + it("builds Vitest runner evidence entries", () => { const evidence = buildVitestEvidenceSummary({ artifactPaths: [ diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts index 301f0c29ac4..3e3605dbd5c 100644 --- a/extensions/qa-lab/src/evidence-summary.ts +++ b/extensions/qa-lab/src/evidence-summary.ts @@ -184,6 +184,7 @@ type QaEvidenceScenarioResultInput = { name: string; status: QaEvidenceStatusInput; details?: string; + timing?: QaEvidenceTiming; rttMs?: number; rttMeasurement?: { finalMatchedReplyRttMs?: number; @@ -195,6 +196,7 @@ type QaEvidenceLiveTransportCheckInput = { title: string; status: QaEvidenceStatusInput; details: string; + timing?: QaEvidenceTiming; rttMs?: number; rttMeasurement?: { finalMatchedReplyRttMs?: number; @@ -203,7 +205,10 @@ type QaEvidenceLiveTransportCheckInput = { artifactPaths?: Readonly>; }; -type QaEvidenceRttInput = Pick; +type QaEvidenceRttInput = Pick< + QaEvidenceScenarioResultInput, + "rttMeasurement" | "rttMs" | "timing" +>; type QaEvidenceTestTargetInput = { id: string; @@ -423,8 +428,17 @@ function failureForResult(result: { } function timingForRttResult(check: QaEvidenceRttInput) { + const timing: QaEvidenceTiming = { ...check.timing }; const rttMs = check.rttMeasurement?.finalMatchedReplyRttMs ?? check.rttMs; - return typeof rttMs === "number" && Number.isFinite(rttMs) && rttMs > 0 ? { rttMs } : undefined; + if ( + timing.rttMs === undefined && + typeof rttMs === "number" && + Number.isFinite(rttMs) && + rttMs > 0 + ) { + timing.rttMs = rttMs; + } + return Object.keys(timing).length > 0 ? timing : undefined; } function timingForTestResult(result: QaEvidenceTestResultInput) { diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts new file mode 100644 index 00000000000..78810cf8e8b --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts @@ -0,0 +1,24 @@ +// Qa Lab plugin module implements shared live-transport result shapes. +import type { QaEvidenceTiming } from "../../evidence-summary.js"; + +export type LiveTransportRttMeasurement = { + finalMatchedReplyRttMs: number; + requestStartedAt: string; + responseObservedAt: string; + source: "request-to-observed-message"; +}; + +export type LiveTransportCheckResult = { + id: string; + title: string; + status: "pass" | "fail"; + details: string; + coverageIds?: readonly string[]; + timing?: QaEvidenceTiming; + rttMs?: number; + requestStartedAt?: string; + responseObservedAt?: string; + rttMeasurement?: LiveTransportRttMeasurement; + sentMessageId?: number; + responseMessageId?: number; +}; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts new file mode 100644 index 00000000000..5f2e0f94972 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts @@ -0,0 +1,83 @@ +// Qa Lab plugin module implements shared live-transport RTT behavior. +import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; +import type { QaEvidenceTiming } from "../../evidence-summary.js"; + +export type LiveTransportRttOptions = { + count: number; + timeoutMs: number; + maxFailures: number; + checkIds: Set; +}; + +export type LiveTransportRttSample = { + rttMs?: number; + status: "pass" | "fail"; +}; + +function normalizePositiveRttInteger(value: number | undefined) { + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value <= 0 || + value > MAX_TIMER_TIMEOUT_MS + ) { + return undefined; + } + return value; +} + +export function normalizeLiveTransportRttOptions(params: { + count?: number; + defaultCheckIds: readonly CheckId[]; + knownCheckIds: ReadonlySet; + maxFailures?: number; + rawCheckIds?: readonly string[]; + timeoutMs?: number; + unknownCheckMessage: (checkId: string) => string; +}): LiveTransportRttOptions | undefined { + const count = normalizePositiveRttInteger(params.count); + if (count === undefined) { + return undefined; + } + const rawCheckIds = + params.rawCheckIds && params.rawCheckIds.length > 0 + ? params.rawCheckIds + : params.defaultCheckIds; + const checkIds = new Set(); + for (const checkId of rawCheckIds) { + if (!params.knownCheckIds.has(checkId as CheckId)) { + throw new Error(params.unknownCheckMessage(checkId)); + } + checkIds.add(checkId as CheckId); + } + return { + count, + maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count, + checkIds, + timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000, + }; +} + +export function percentile(sortedValues: readonly number[], percentileValue: number) { + if (sortedValues.length === 0) { + return undefined; + } + const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1; + return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)]; +} + +export function summarizeLiveTransportRttSamples(samples: readonly LiveTransportRttSample[]) { + const passed = samples.filter((sample) => sample.status === "pass" && sample.rttMs !== undefined); + const sorted = passed.map((sample) => sample.rttMs as number).toSorted((a, b) => a - b); + const sum = sorted.reduce((total, value) => total + value, 0); + const timing: QaEvidenceTiming = { + rttMs: percentile(sorted, 50), + avgMs: sorted.length > 0 ? Math.round(sum / sorted.length) : undefined, + p50Ms: percentile(sorted, 50), + p95Ms: percentile(sorted, 95), + maxMs: sorted.at(-1), + samples: samples.length, + failedSamples: samples.length - passed.length, + }; + return { passed: passed.length, failed: samples.length - passed.length, timing }; +} diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index 30b7820bd8f..eaf9513fcd1 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -22,7 +22,6 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) printLiveTransportQaArtifacts("Telegram QA", { report: result.reportPath, summary: result.summaryPath, - "observed messages": result.observedMessagesPath, }); if (!runOptions.allowFailures) { const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath); diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index 4ca5bacda03..3426c5ddfa7 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { summarizeLiveTransportRttSamples } from "../shared/live-transport-rtt.js"; import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, findMissingLiveTransportStandardScenarios, @@ -159,6 +160,78 @@ describe("telegram live qa runtime", () => { } }); + it("normalizes Telegram RTT options", () => { + expect(testing.normalizeTelegramQaRttOptions({})).toBeUndefined(); + expect( + testing.normalizeTelegramQaRttOptions({ + count: 3, + timeoutMs: 45_000, + }), + ).toEqual({ + count: 3, + maxFailures: 3, + checkIds: new Set(["telegram-mentioned-message-reply"]), + timeoutMs: 45_000, + }); + expect( + testing.normalizeTelegramQaRttOptions({ + checkIds: ["telegram-mentioned-message-reply"], + count: 3, + maxFailures: 1, + }), + ).toEqual({ + count: 3, + maxFailures: 1, + checkIds: new Set(["telegram-mentioned-message-reply"]), + timeoutMs: 30_000, + }); + }); + + it("rejects unknown Telegram RTT checks", () => { + expect(() => + testing.normalizeTelegramQaRttOptions({ + checkIds: ["telegram-rtt-only"], + count: 1, + }), + ).toThrow("unknown Telegram QA RTT check: telegram-rtt-only"); + }); + + it("summarizes live transport RTT timing", () => { + expect( + summarizeLiveTransportRttSamples([ + { status: "pass", rttMs: 1000 }, + { status: "pass", rttMs: 2000 }, + { status: "pass", rttMs: 4000 }, + { status: "fail" }, + ]), + ).toEqual({ + passed: 3, + failed: 1, + timing: { + rttMs: 2000, + avgMs: 2333, + p50Ms: 2000, + p95Ms: 4000, + maxMs: 4000, + samples: 4, + failedSamples: 1, + }, + }); + expect(summarizeLiveTransportRttSamples([{ status: "fail" }, { status: "fail" }])).toEqual({ + passed: 0, + failed: 2, + timing: { + rttMs: undefined, + avgMs: undefined, + p50Ms: undefined, + p95Ms: undefined, + maxMs: undefined, + samples: 2, + failedSamples: 2, + }, + }); + }); + it("sanitizes and truncates Telegram live progress details", () => { expect(testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe( "scenario id value", @@ -580,6 +653,10 @@ describe("telegram live qa runtime", () => { .find((scenario) => scenario.id === "telegram-mentioned-message-reply") ?.buildRun("sut_bot").steps[0].replyToLatestSutMessage, ).toBe(true); + expect( + scenarios.find((scenario) => scenario.id === "telegram-mentioned-message-reply") + ?.evidenceCoverageIds, + ).toEqual(["channels.telegram.mention-gating"]); const replyChainStep = requireScenario(scenarios, "telegram-reply-chain-exact-marker").buildRun( "sut_bot", ).steps[0]; @@ -1140,157 +1217,6 @@ describe("telegram live qa runtime", () => { expect(observedMessages[0]?.scenarioId).toBe("telegram-whoami-command"); }); - it("redacts observed message content by default in artifacts", () => { - expect( - testing.buildObservedMessagesArtifact({ - includeContent: false, - redactMetadata: false, - observedMessages: [ - { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - senderUsername: "driver_bot", - text: "secret text", - caption: "secret caption", - replyToMessageId: 8, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve"], - mediaKinds: ["photo"], - }, - ], - }), - ).toEqual([ - { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - senderUsername: "driver_bot", - replyToMessageId: 8, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve"], - mediaKinds: ["photo"], - }, - ]); - }); - - it("keeps observed message content in public mode when capture is requested", () => { - const redacted = testing.buildObservedMessagesArtifact({ - includeContent: true, - redactMetadata: true, - observedMessages: [ - { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - senderUsername: "driver_bot", - text: "secret text", - caption: "secret caption", - replyToMessageId: 8, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve"], - mediaKinds: ["photo"], - }, - ], - }); - - expect(redacted).toEqual([ - { - senderIsBot: true, - inlineButtonCount: 1, - mediaKinds: ["photo"], - text: "secret text", - caption: "secret caption", - }, - ]); - expect(redacted[0]).not.toHaveProperty("timestamp"); - expect(redacted[0]).not.toHaveProperty("inlineButtons"); - expect(redacted[0]).not.toHaveProperty("senderId"); - expect(redacted[0]).not.toHaveProperty("senderUsername"); - }); - - it("keeps raw timestamp and inline button text when metadata redaction is disabled", () => { - expect( - testing.buildObservedMessagesArtifact({ - includeContent: true, - redactMetadata: false, - observedMessages: [ - { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - senderUsername: "driver_bot", - text: "secret text", - caption: "secret caption", - replyToMessageId: 8, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve"], - mediaKinds: ["photo"], - }, - ], - }), - ).toEqual([ - { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve"], - senderUsername: "driver_bot", - replyToMessageId: 8, - text: "secret text", - caption: "secret caption", - mediaKinds: ["photo"], - }, - ]); - }); - - it("adds scenario context to observed message artifacts", () => { - expect( - testing.buildObservedMessagesArtifact({ - includeContent: false, - redactMetadata: true, - observedMessages: [ - { - updateId: 11, - messageId: 21, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - scenarioId: "telegram-commands-command", - scenarioTitle: "Telegram commands list reply", - matchedScenario: false, - text: "noise from previous turn", - replyToMessageId: 19, - timestamp: 1_700_000_003_000, - inlineButtons: [], - mediaKinds: [], - }, - ], - }), - ).toEqual([ - { - scenarioId: "telegram-commands-command", - scenarioTitle: "Telegram commands list reply", - matchedScenario: false, - senderIsBot: true, - inlineButtonCount: 0, - mediaKinds: [], - }, - ]); - }); - it("prints Telegram scenario RTT in the Markdown report", () => { expect( testing.renderTelegramQaMarkdown({ @@ -1313,6 +1239,38 @@ describe("telegram live qa runtime", () => { ).toContain("- RTT: 4321ms"); }); + it("prints Telegram repeated RTT timing in the Markdown report", () => { + const report = testing.renderTelegramQaMarkdown({ + cleanupIssues: [], + credentialSource: "env", + groupId: "-100123", + redactMetadata: false, + startedAt: "2026-04-23T00:00:00.000Z", + finishedAt: "2026-04-23T00:00:10.000Z", + scenarios: [ + { + id: "telegram-mentioned-message-reply", + title: "Telegram mentioned message gets a reply", + status: "pass", + details: "reply matched; 3/4 RTT checks passed", + rttMs: 2000, + timing: { + avgMs: 2333, + p50Ms: 2000, + p95Ms: 4000, + maxMs: 4000, + samples: 4, + failedSamples: 1, + }, + }, + ], + }); + + expect(report).toContain("- Samples: 3/4"); + expect(report).toContain("- P50: 2000ms"); + expect(report).toContain("- P95: 4000ms"); + }); + it("formats phase-specific canary diagnostics with context", () => { const error = new Error( "SUT bot did not send any group reply after the canary command within 30s.", diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts index bf976346499..bbad16c5c60 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts @@ -1,9 +1,7 @@ // Qa Lab plugin module implements telegram live behavior. -import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; -import { promisify } from "node:util"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { @@ -12,9 +10,12 @@ import { } from "openclaw/plugin-sdk/number-runtime"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { z } from "zod"; -import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js"; +import { + QA_EVIDENCE_FILENAME, + buildLiveTransportEvidenceSummary, + type QaEvidenceTiming, +} from "../../evidence-summary.js"; import { startQaGatewayChild } from "../../gateway-child.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; import { @@ -33,6 +34,13 @@ import { redactQaLiveLaneIssues, } from "../shared/live-artifacts.js"; import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; +import type { LiveTransportCheckResult } from "../shared/live-transport-result.js"; +import { + normalizeLiveTransportRttOptions, + summarizeLiveTransportRttSamples, + type LiveTransportRttOptions, + type LiveTransportRttSample, +} from "../shared/live-transport-rtt.js"; import { collectLiveTransportStandardScenarioCoverage, selectLiveTransportScenarios, @@ -91,8 +99,10 @@ type TelegramQaScenarioRun = { type TelegramQaScenarioDefinition = LiveTransportScenarioDefinition & { buildRun: (sutUsername: string) => TelegramQaScenarioRun; + buildRttRun?: (params: { rttIndex: number; sutUsername: string }) => TelegramQaScenarioRun; defaultEnabled?: boolean; defaultProviderModes?: readonly QaProviderMode[]; + evidenceCoverageIds?: readonly string[]; regressionRefs?: readonly string[]; rationale: string; }; @@ -115,44 +125,26 @@ type TelegramObservedMessage = { mediaKinds: string[]; }; -type TelegramObservedMessageArtifact = { - updateId?: number; - messageId?: number; - chatId?: number; - senderId?: number; - senderIsBot: boolean; - senderUsername?: string; - scenarioId?: string; - scenarioTitle?: string; - matchedScenario?: boolean; - text?: string; - caption?: string; - replyToMessageId?: number; - inlineButtonCount?: number; - timestamp?: number; - inlineButtons?: string[]; - mediaKinds: string[]; -}; - const DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS = 30_000; -type TelegramQaScenarioResult = { - id: string; - standardId?: string; - title: string; - status: "pass" | "fail"; +type TelegramQaScenarioResult = LiveTransportCheckResult; + +function telegramLiveTransportCoverageIds(scenario: TelegramQaScenarioDefinition) { + if (scenario.evidenceCoverageIds) { + return scenario.evidenceCoverageIds; + } + return scenario.standardId ? [`channels.telegram.${scenario.standardId}`] : []; +} + +type TelegramQaRttOptions = LiveTransportRttOptions; + +type TelegramQaRttResult = { details: string; - rttMs?: number; - requestStartedAt?: string; - responseObservedAt?: string; - rttMeasurement?: { - finalMatchedReplyRttMs: number; - requestStartedAt: string; - responseObservedAt: string; - source: "request-to-observed-message"; - }; - sentMessageId?: number; - responseMessageId?: number; + driverOffset: number; + failed: number; + latestSutMessageId?: number; + passed: number; + timing: QaEvidenceTiming; }; type TelegramQaCanaryPhase = "sut_reply_timeout" | "sut_reply_not_threaded" | "sut_reply_empty"; @@ -161,7 +153,6 @@ type TelegramQaRunResult = { outputDir: string; reportPath: string; summaryPath: string; - observedMessagesPath: string; gatewayDebugDirPath?: string; scenarios: TelegramQaScenarioResult[]; }; @@ -420,6 +411,7 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ { id: "telegram-mentioned-message-reply", title: "Telegram mentioned message gets a reply", + evidenceCoverageIds: ["channels.telegram.mention-gating"], rationale: "Bot-to-bot group mention routing must produce a threaded SUT reply.", timeoutMs: 45_000, buildRun: (sutUsername) => @@ -428,6 +420,16 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ input: `@${sutUsername} Telegram QA mention routing check. Reply with a short acknowledgement.`, replyToLatestSutMessage: true, }), + buildRttRun: ({ rttIndex, sutUsername }) => { + const marker = `QA-TELEGRAM-RTT-${rttIndex}-${randomUUID().slice(0, 8).toUpperCase()}`; + return telegramQaStepRun({ + expectReply: true, + input: `@${sutUsername} Telegram RTT check ${rttIndex}. Reply exactly: ${marker}`, + expectedTextIncludes: [marker], + matchText: marker, + replyToLatestSutMessage: true, + }); + }, }, { id: "telegram-reply-chain-exact-marker", @@ -534,12 +536,10 @@ const TELEGRAM_QA_ENV_KEYS = [ "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", ] as const; -const TELEGRAM_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT"; const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; const QA_SUITE_PROGRESS_ENV = "OPENCLAW_QA_SUITE_PROGRESS"; const TELEGRAM_QA_PROGRESS_DETAIL_LIMIT = 240; const TELEGRAM_QA_PROGRESS_PREFIX = "[qa-telegram-live]"; -const execFileAsync = promisify(execFile); const telegramQaCredentialPayloadSchema = z.object({ groupId: z.string().trim().min(1), @@ -617,6 +617,44 @@ function resolveTelegramQaScenarioTimeoutMs( return parsePositiveTelegramQaEnvMs(env, "OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", fallbackMs); } +function normalizeTelegramQaRttOptions(params: { + count?: number; + checkIds?: readonly string[]; + maxFailures?: number; + timeoutMs?: number; +}): TelegramQaRttOptions | undefined { + const knownScenarioIds = new Set(TELEGRAM_QA_SCENARIOS.map((scenario) => scenario.id)); + return normalizeLiveTransportRttOptions({ + count: params.count, + defaultCheckIds: ["telegram-mentioned-message-reply"], + knownCheckIds: knownScenarioIds, + maxFailures: params.maxFailures, + rawCheckIds: params.checkIds, + timeoutMs: params.timeoutMs, + unknownCheckMessage: (checkId) => `unknown Telegram QA RTT check: ${checkId}`, + }); +} + +function assertTelegramQaRttCheckSupport(params: { + rttOptions?: TelegramQaRttOptions; + scenarios: TelegramQaScenarioDefinition[]; +}) { + if (!params.rttOptions) { + return; + } + const selectedScenarioIds = new Set(params.scenarios.map((scenario) => scenario.id)); + for (const scenarioId of params.rttOptions.checkIds) { + if (!selectedScenarioIds.has(scenarioId)) { + throw new Error(`Telegram QA RTT check ${scenarioId} is not selected.`); + } + } + for (const scenario of params.scenarios) { + if (params.rttOptions.checkIds.has(scenario.id) && !scenario.buildRttRun) { + throw new Error(`Telegram QA scenario ${scenario.id} does not support RTT measurement.`); + } + } +} + function formatTelegramQaTimeoutSeconds(timeoutMs: number) { return `${Math.round(timeoutMs / 1_000)}s`; } @@ -1277,6 +1315,23 @@ function renderTelegramQaMarkdown(params: { if (scenario.rttMs !== undefined) { lines.push(`- RTT: ${scenario.rttMs}ms`); } + if (scenario.timing?.samples !== undefined) { + lines.push( + `- Samples: ${scenario.timing.samples - (scenario.timing.failedSamples ?? 0)}/${scenario.timing.samples}`, + ); + if (scenario.timing.avgMs !== undefined) { + lines.push(`- Avg: ${scenario.timing.avgMs}ms`); + } + if (scenario.timing.p50Ms !== undefined) { + lines.push(`- P50: ${scenario.timing.p50Ms}ms`); + } + if (scenario.timing.p95Ms !== undefined) { + lines.push(`- P95: ${scenario.timing.p95Ms}ms`); + } + if (scenario.timing.maxMs !== undefined) { + lines.push(`- Max: ${scenario.timing.maxMs}ms`); + } + } lines.push(""); } if (params.gatewayDebugDirPath) { @@ -1296,50 +1351,6 @@ function renderTelegramQaMarkdown(params: { return lines.join("\n"); } -function buildObservedMessagesArtifact(params: { - observedMessages: TelegramObservedMessage[]; - includeContent: boolean; - redactMetadata: boolean; -}) { - return params.observedMessages.map((message) => { - const scenarioContext = { - ...(message.scenarioId ? { scenarioId: message.scenarioId } : {}), - ...(message.scenarioTitle ? { scenarioTitle: message.scenarioTitle } : {}), - ...(typeof message.matchedScenario === "boolean" - ? { matchedScenario: message.matchedScenario } - : {}), - }; - const base = params.redactMetadata - ? { - ...scenarioContext, - senderIsBot: message.senderIsBot, - inlineButtonCount: message.inlineButtons.length, - mediaKinds: message.mediaKinds, - } - : { - ...scenarioContext, - senderIsBot: message.senderIsBot, - timestamp: message.timestamp, - inlineButtons: message.inlineButtons, - mediaKinds: message.mediaKinds, - updateId: message.updateId, - messageId: message.messageId, - chatId: message.chatId, - senderId: message.senderId, - senderUsername: message.senderUsername, - replyToMessageId: message.replyToMessageId, - }; - if (!params.includeContent) { - return base; - } - return { - ...base, - text: message.text, - caption: message.caption, - }; - }); -} - function shouldRunTelegramScenarioByDefault( scenario: TelegramQaScenarioDefinition, providerMode: QaProviderMode, @@ -1447,16 +1458,19 @@ function resolveTelegramQaScenarioSteps(run: TelegramQaScenarioRun): TelegramQaS async function runTelegramQaScenarioStep(params: { driverOffset: number; driverToken: string; + env: NodeJS.ProcessEnv; groupId: string; latestSutMessageId?: number; observedMessages: TelegramObservedMessage[]; + replyTimeoutMs?: number; scenario: TelegramQaScenarioDefinition; step: TelegramQaScenarioStep; sutBotId: number; }) { + const fallbackTimeoutMs = params.step.timeoutMs ?? params.scenario.timeoutMs; const stepTimeoutMs = params.step.expectReply - ? resolveTelegramQaScenarioTimeoutMs(params.step.timeoutMs ?? params.scenario.timeoutMs) - : (params.step.timeoutMs ?? params.scenario.timeoutMs); + ? (params.replyTimeoutMs ?? resolveTelegramQaScenarioTimeoutMs(fallbackTimeoutMs, params.env)) + : fallbackTimeoutMs; const requestStartedAtMs = Date.now(); const sent = await sendGroupMessage( params.driverToken, @@ -1507,6 +1521,82 @@ async function runTelegramQaScenarioStep(params: { } } +async function runTelegramQaRttChecks(params: { + driverOffset: number; + driverToken: string; + env: NodeJS.ProcessEnv; + groupId: string; + latestSutMessageId?: number; + observedMessages: TelegramObservedMessage[]; + rttOptions: TelegramQaRttOptions; + scenario: TelegramQaScenarioDefinition; + sutBotId: number; + sutUsername: string; +}): Promise { + if (!params.scenario.buildRttRun) { + throw new Error(`Telegram QA scenario ${params.scenario.id} does not support RTT measurement.`); + } + let driverOffset = params.driverOffset; + let latestSutMessageId = params.latestSutMessageId; + const samples: LiveTransportRttSample[] = []; + let failures = 0; + let passed = 0; + for (let index = 1; passed < params.rttOptions.count; index += 1) { + const run = params.scenario.buildRttRun({ + rttIndex: index, + sutUsername: params.sutUsername, + }); + const steps = resolveTelegramQaScenarioSteps(run); + if (steps.length !== 1) { + throw new Error(`Telegram QA RTT check ${params.scenario.id} must have one step.`); + } + try { + driverOffset = await flushTelegramUpdates(params.driverToken); + const stepResult = await runTelegramQaScenarioStep({ + driverOffset, + driverToken: params.driverToken, + env: params.env, + groupId: params.groupId, + latestSutMessageId, + observedMessages: params.observedMessages, + replyTimeoutMs: params.rttOptions.timeoutMs, + scenario: params.scenario, + step: steps[0], + sutBotId: params.sutBotId, + }); + if (!stepResult.matched) { + throw new Error("RTT check did not expect a reply"); + } + driverOffset = stepResult.matched.nextOffset; + latestSutMessageId = stepResult.matched.message.messageId; + const rttMs = stepResult.matched.observedAtMs - stepResult.requestStartedAtMs; + samples.push({ + status: "pass", + rttMs, + }); + passed += 1; + } catch { + failures += 1; + samples.push({ + status: "fail", + }); + } + if (failures >= params.rttOptions.maxFailures) { + break; + } + } + + const summary = summarizeLiveTransportRttSamples(samples); + return { + details: `${summary.passed}/${samples.length} RTT checks passed`, + driverOffset, + failed: summary.failed, + latestSutMessageId, + passed: summary.passed, + timing: summary.timing, + }; +} + function classifyCanaryReply(params: { message: TelegramObservedMessage; groupId: string; @@ -1646,13 +1736,13 @@ function canaryFailureMessage(params: { return [ "1. Check whether the SUT bot is replying in the group without threading to the driver message.", "2. Confirm the Telegram native command path preserves reply-to behavior for group commands.", - "3. Inspect the observed messages artifact for the mismatched SUT message id and reply target.", + "3. Inspect telegram-qa-report.md and gateway debug logs for the mismatched SUT message id and reply target.", ]; case "sut_reply_empty": return [ - "1. Inspect the observed messages artifact to confirm whether the SUT sent media-only or blank text.", - "2. Check whether the Telegram native command response path produced an empty or suppressed reply.", - "3. Confirm the SUT command completed successfully in gateway logs.", + "1. Check whether the Telegram native command response path produced an empty or suppressed reply.", + "2. Confirm the SUT command completed successfully in gateway logs.", + "3. Inspect telegram-qa-report.md for the matched message ids and phase context.", ]; default: return [ @@ -1679,80 +1769,28 @@ function canaryFailureMessage(params: { ].join("\n"); } -async function runInstalledOpenClawTelegramOnboardingPreflight(params: { - openClawCommand: string; - providerMode: ReturnType; - sutToken: string; -}) { - const tempRoot = await fs.mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-npm-telegram-"), - ); - const homeDir = path.join(tempRoot, "home"); - const stateDir = path.join(homeDir, ".openclaw"); - await fs.mkdir(stateDir, { recursive: true }); - const tokenPath = path.join(tempRoot, "sut-token.txt"); - await fs.writeFile(tokenPath, params.sutToken, { encoding: "utf8", mode: 0o600 }); - const env = { - ...process.env, - HOME: homeDir, - OPENCLAW_HOME: stateDir, - OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"), - OPENCLAW_STATE_DIR: stateDir, - OPENCLAW_GATEWAY_TOKEN: "npm-telegram-live-onboard", - ...(params.providerMode === "live-frontier" - ? {} - : { OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? "sk-openclaw-npm-telegram-preflight" }), - }; - try { - await execFileAsync( - params.openClawCommand, - [ - "onboard", - "--non-interactive", - "--accept-risk", - "--mode", - "local", - "--auth-choice", - "openai-api-key", - "--secret-input-mode", - "ref", - "--gateway-port", - "18789", - "--gateway-bind", - "loopback", - "--skip-daemon", - "--skip-ui", - "--skip-skills", - "--skip-health", - "--json", - ], - { env }, - ); - await execFileAsync( - params.openClawCommand, - ["channels", "add", "--channel", "telegram", "--token-file", tokenPath], - { env }, - ); - await execFileAsync(params.openClawCommand, ["doctor", "--non-interactive"], { env }); - } finally { - await fs.rm(tempRoot, { recursive: true, force: true }).catch(() => {}); - } -} - export async function runTelegramQaLive(params: { + env?: NodeJS.ProcessEnv; repoRoot?: string; outputDir?: string; sutOpenClawCommand?: string; - preflightInstalledOnboarding?: boolean; providerMode?: QaProviderModeInput; primaryModel?: string; alternateModel?: string; fastMode?: boolean; scenarioIds?: string[]; + rttCount?: number; + rttTimeoutMs?: number; + maxRttFailures?: number; + rttCheckIds?: string[]; sutAccountId?: string; credentialSource?: string; credentialRole?: string; + redactPublicMetadata?: boolean; + progressEnabled?: boolean; + canaryTimeoutMs?: number; }): Promise { + const env = params.env ?? process.env; const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); const outputDir = params.outputDir ?? @@ -1766,17 +1804,25 @@ export async function runTelegramQaLive(params: { const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); const sutAccountId = params.sutAccountId?.trim() || "sut"; const scenarios = findScenario(params.scenarioIds, providerMode); - const progressEnabled = shouldLogTelegramQaLiveProgress(); + const rttOptions = normalizeTelegramQaRttOptions({ + checkIds: params.rttCheckIds, + count: params.rttCount, + maxFailures: params.maxRttFailures, + timeoutMs: params.rttTimeoutMs, + }); + assertTelegramQaRttCheckSupport({ rttOptions, scenarios }); + const progressEnabled = params.progressEnabled ?? shouldLogTelegramQaLiveProgress(env); writeTelegramQaProgress( progressEnabled, - `run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"}`, + `run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"} rttChecks=${rttOptions?.count ?? 0}`, ); const credentialLease = await acquireQaCredentialLease({ + env, kind: "telegram", source: params.credentialSource, role: params.credentialRole, - resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(), + resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(env), parsePayload: parseTelegramQaCredentialPayload, }); const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); @@ -1790,11 +1836,11 @@ export async function runTelegramQaLive(params: { const runtimeEnv = credentialLease.payload; const observedMessages: TelegramObservedMessage[] = []; - const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); - const includeObservedMessageContent = isTruthyOptIn(process.env[TELEGRAM_QA_CAPTURE_CONTENT_ENV]); + const redactPublicMetadata = + params.redactPublicMetadata ?? isTruthyOptIn(env[QA_REDACT_PUBLIC_METADATA_ENV]); writeTelegramQaProgress( progressEnabled, - `runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"} captureContent=${includeObservedMessageContent ? "on" : "off"}`, + `runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"}`, ); const startedAt = new Date().toISOString(); const scenarioResults: TelegramQaScenarioResult[] = []; @@ -1803,16 +1849,6 @@ export async function runTelegramQaLive(params: { let preservedGatewayDebugArtifacts = false; let canaryFailure: string | null = null; try { - if (params.sutOpenClawCommand && params.preflightInstalledOnboarding === true) { - writeTelegramQaProgress(progressEnabled, "installed package onboarding preflight start"); - await runInstalledOpenClawTelegramOnboardingPreflight({ - openClawCommand: params.sutOpenClawCommand, - providerMode, - sutToken: runtimeEnv.sutToken, - }); - writeTelegramQaProgress(progressEnabled, "installed package onboarding preflight pass"); - } - const driverIdentity = await getBotIdentity(runtimeEnv.driverToken); const sutIdentity = await getBotIdentity(runtimeEnv.sutToken); const sutUsername = sutIdentity.username?.trim(); @@ -1866,13 +1902,13 @@ export async function runTelegramQaLive(params: { groupId: runtimeEnv.groupId, sutUsername, sutBotId: sutIdentity.id, - timeoutMs: resolveTelegramQaCanaryTimeoutMs(), + timeoutMs: params.canaryTimeoutMs ?? resolveTelegramQaCanaryTimeoutMs(env), observedMessages, }); latestSutMessageId = canaryTiming.responseMessageId; scenarioResults.push({ id: "telegram-canary", - standardId: "canary", + coverageIds: ["channels.telegram.canary"], title: "Telegram canary", status: "pass", details: redactPublicMetadata @@ -1903,7 +1939,7 @@ export async function runTelegramQaLive(params: { }); scenarioResults.push({ id: "telegram-canary", - standardId: "canary", + coverageIds: ["channels.telegram.canary"], title: "Telegram canary", status: "fail", details: canaryFailure, @@ -1946,6 +1982,7 @@ export async function runTelegramQaLive(params: { const stepResult = await runTelegramQaScenarioStep({ driverOffset, driverToken: runtimeEnv.driverToken, + env, groupId: runtimeEnv.groupId, latestSutMessageId, observedMessages, @@ -1999,7 +2036,7 @@ export async function runTelegramQaLive(params: { if (!lastMatched || !firstRequestStartedAt || lastSentMessageId === undefined) { const result = { id: scenario.id, - standardId: scenario.standardId, + coverageIds: telegramLiveTransportCoverageIds(scenario), title: scenario.title, status: "pass", details: "no reply", @@ -2021,35 +2058,64 @@ export async function runTelegramQaLive(params: { : `; observed ${lastStep.expectedSutMessageCountRange[0]}-${lastStep.expectedSutMessageCountRange[1]} SUT message(s)` : `; observed ${lastStep.expectedSutMessageCount} SUT message(s)` : `; ${scenarioSteps.filter((step) => step.expectReply).length} command replies matched`; + let resultStatus: "pass" | "fail" = "pass"; + let details = redactPublicMetadata + ? `reply matched in ${rttMs}ms${suffix}` + : `reply message ${lastMatched.message.messageId} matched in ${rttMs}ms${suffix}`; + let resultRttMs: number | undefined = rttMs; + let timing: QaEvidenceTiming | undefined; + if (rttOptions?.checkIds.has(scenario.id)) { + const rttResult = await runTelegramQaRttChecks({ + driverOffset, + driverToken: runtimeEnv.driverToken, + env, + groupId: runtimeEnv.groupId, + latestSutMessageId, + observedMessages, + rttOptions, + scenario, + sutBotId: sutIdentity.id, + sutUsername, + }); + driverOffset = rttResult.driverOffset; + latestSutMessageId = rttResult.latestSutMessageId ?? latestSutMessageId; + timing = rttResult.timing; + resultRttMs = rttResult.timing.p50Ms; + details = `${details}; ${rttResult.details}`; + if (rttResult.passed < rttOptions.count) { + resultStatus = "fail"; + } + } const result = { id: scenario.id, - standardId: scenario.standardId, + coverageIds: telegramLiveTransportCoverageIds(scenario), title: scenario.title, - status: "pass", - details: redactPublicMetadata - ? `reply matched in ${rttMs}ms${suffix}` - : `reply message ${lastMatched.message.messageId} matched in ${rttMs}ms${suffix}`, - rttMs, + status: resultStatus, + details, + rttMs: resultRttMs, + timing, requestStartedAt: firstRequestStartedAt, responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(), - rttMeasurement: { - finalMatchedReplyRttMs: rttMs, - requestStartedAt: new Date(lastRequestStartedAtMs).toISOString(), - responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(), - source: "request-to-observed-message", - }, + rttMeasurement: timing + ? undefined + : { + finalMatchedReplyRttMs: rttMs, + requestStartedAt: new Date(lastRequestStartedAtMs).toISOString(), + responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(), + source: "request-to-observed-message", + }, sentMessageId: redactPublicMetadata ? undefined : lastSentMessageId, responseMessageId: redactPublicMetadata ? undefined : lastMatched.message.messageId, } satisfies TelegramQaScenarioResult; scenarioResults.push(result); writeTelegramQaProgress( progressEnabled, - `scenario pass ${scenarioIndexLabel}: ${scenarioIdForLog}`, + `scenario ${resultStatus} ${scenarioIndexLabel}: ${scenarioIdForLog}`, ); } catch (error) { const result = { id: scenario.id, - standardId: scenario.standardId, + coverageIds: telegramLiveTransportCoverageIds(scenario), title: scenario.title, status: "fail", details: formatErrorMessage(error), @@ -2100,21 +2166,16 @@ export async function runTelegramQaLive(params: { } const reportPath = path.join(outputDir, "telegram-qa-report.md"); const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME); - const observedMessagesPath = path.join(outputDir, "telegram-qa-observed-messages.json"); const evidence = buildLiveTransportEvidenceSummary({ artifactPaths: [ { kind: "summary", path: path.basename(summaryPath) }, { kind: "report", path: path.basename(reportPath) }, - { kind: "transport-observations", path: path.basename(observedMessagesPath) }, ], - env: process.env, + env, generatedAt: finishedAt, primaryModel, providerMode, - checks: scenarioResults.map(({ standardId, ...check }) => ({ - ...check, - coverageIds: standardId ? [`channels.telegram.${standardId}`] : undefined, - })), + checks: scenarioResults, transportId: "telegram", }); await fs.writeFile( @@ -2135,23 +2196,9 @@ export async function runTelegramQaLive(params: { encoding: "utf8", mode: 0o600, }); - await fs.writeFile( - observedMessagesPath, - `${JSON.stringify( - buildObservedMessagesArtifact({ - observedMessages, - includeContent: includeObservedMessageContent, - redactMetadata: redactPublicMetadata, - }), - null, - 2, - )}\n`, - { encoding: "utf8", mode: 0o600 }, - ); const artifactPaths = { report: reportPath, summary: summaryPath, - observedMessages: observedMessagesPath, ...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}), }; if (canaryFailure) { @@ -2176,7 +2223,6 @@ export async function runTelegramQaLive(params: { outputDir, reportPath, summaryPath, - observedMessagesPath, ...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}), scenarios: scenarioResults, }; @@ -2186,7 +2232,6 @@ export const testing = { TELEGRAM_QA_SCENARIOS, TELEGRAM_QA_STANDARD_SCENARIO_IDS, buildTelegramQaConfig, - buildObservedMessagesArtifact, canaryFailureMessage, callTelegramApi, assertTelegramCanaryPresenceReply, @@ -2201,6 +2246,7 @@ export const testing = { normalizeTelegramObservedMessage, parseTelegramQaProgressBooleanEnv, parseTelegramQaCredentialPayload, + normalizeTelegramQaRttOptions, resolveTelegramQaCanaryTimeoutMs, resolveTelegramQaScenarioTimeoutMs, resolveTelegramQaRuntimeEnv, diff --git a/package.json b/package.json index de808937fcc..3db96871d78 100644 --- a/package.json +++ b/package.json @@ -1666,7 +1666,6 @@ "release:plugins:npm:plan": "node --import tsx scripts/plugin-npm-release-plan.ts", "release:verify-beta": "node --import tsx scripts/release-verify-beta.ts", "release:prep": "node scripts/release-preflight.mjs --fix", - "rtt": "node --import tsx scripts/rtt.ts", "runtime-sidecars:check": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --check", "runtime-sidecars:gen": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --write", "start": "node openclaw.mjs", diff --git a/scripts/e2e/npm-telegram-live-docker.sh b/scripts/e2e/npm-telegram-live-docker.sh index 264b3ca5851..df2c09850c6 100755 --- a/scripts/e2e/npm-telegram-live-docker.sh +++ b/scripts/e2e/npm-telegram-live-docker.sh @@ -93,8 +93,12 @@ fi credential_source="$(resolve_credential_source)" credential_role="$(resolve_credential_role)" -if [ -z "$credential_role" ] && [ -n "${CI:-}" ] && [ "$credential_source" = "convex" ]; then - credential_role="ci" +if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then + if [ -n "${CI:-}" ]; then + credential_role="ci" + else + credential_role="maintainer" + fi fi validate_credential_preflight() { @@ -205,7 +209,6 @@ for key in \ OPENCLAW_QA_ALLOW_INSECURE_HTTP \ OPENCLAW_QA_REDACT_PUBLIC_METADATA \ OPENCLAW_QA_PACKAGE_SOURCE_SHA \ - OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT \ OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS \ OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS \ OPENCLAW_QA_SUITE_PROGRESS \ @@ -213,6 +216,10 @@ for key in \ OPENCLAW_NPM_TELEGRAM_MODEL \ OPENCLAW_NPM_TELEGRAM_ALT_MODEL \ OPENCLAW_NPM_TELEGRAM_SCENARIOS \ + OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES \ + OPENCLAW_NPM_TELEGRAM_RTT_CHECKS \ + OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS \ + OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES \ OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH \ OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT \ OPENCLAW_NPM_TELEGRAM_ALLOW_FAILURES; do diff --git a/scripts/e2e/npm-telegram-live-runner.ts b/scripts/e2e/npm-telegram-live-runner.ts index cc0b3c66808..63e4f8fec8d 100644 --- a/scripts/e2e/npm-telegram-live-runner.ts +++ b/scripts/e2e/npm-telegram-live-runner.ts @@ -17,6 +17,21 @@ function splitCsv(value: string | undefined) { .filter((entry) => entry.length > 0); } +function parsePositiveIntegerEnv(env: NodeJS.ProcessEnv, name: string) { + const raw = env[name]?.trim(); + if (!raw) { + return undefined; + } + if (!/^\d+$/u.test(raw)) { + throw new Error(`invalid ${name}: ${raw}`); + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`invalid ${name}: ${raw}`); + } + return value; +} + function resolveCredentialSource(env: NodeJS.ProcessEnv) { return env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE; } @@ -25,6 +40,27 @@ function resolveCredentialRole(env: NodeJS.ProcessEnv) { return env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE ?? env.OPENCLAW_QA_CREDENTIAL_ROLE; } +const DEFAULT_RTT_CHECK_ID = "telegram-mentioned-message-reply"; + +function resolveRttOptions(env: NodeJS.ProcessEnv, selectedScenarioIds: readonly string[] = []) { + const explicitCheckIds = splitCsv(env.OPENCLAW_NPM_TELEGRAM_RTT_CHECKS); + if ( + explicitCheckIds.length === 0 && + selectedScenarioIds.length > 0 && + !selectedScenarioIds.includes(DEFAULT_RTT_CHECK_ID) + ) { + return {}; + } + const rttCount = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20; + return { + rttCount, + rttTimeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"), + maxRttFailures: + parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? rttCount, + rttCheckIds: explicitCheckIds, + }; +} + async function shouldFailPackageTelegramRun( result: { summaryPath: string }, env: NodeJS.ProcessEnv = process.env, @@ -74,16 +110,18 @@ async function main() { const outputDir = process.env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR?.trim() || path.join(repoRoot, ".artifacts", "qa-e2e", `npm-telegram-live-${Date.now().toString(36)}`); + const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS); const result = await runTelegramQaLive({ + env: process.env, repoRoot, outputDir, sutOpenClawCommand, - preflightInstalledOnboarding: true, providerMode: process.env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE, primaryModel: process.env.OPENCLAW_NPM_TELEGRAM_MODEL, alternateModel: process.env.OPENCLAW_NPM_TELEGRAM_ALT_MODEL, fastMode: parseBoolean(process.env.OPENCLAW_NPM_TELEGRAM_FAST), - scenarioIds: splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS), + scenarioIds, + ...resolveRttOptions(process.env, scenarioIds), sutAccountId: process.env.OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT, credentialSource: resolveCredentialSource(process.env), credentialRole: resolveCredentialRole(process.env), @@ -91,7 +129,6 @@ async function main() { process.stdout.write(`Package Telegram QA report: ${result.reportPath}\n`); process.stdout.write(`Package Telegram QA summary: ${result.summaryPath}\n`); - process.stdout.write(`Package Telegram QA observed messages: ${result.observedMessagesPath}\n`); if (await shouldFailPackageTelegramRun(result)) { process.exitCode = 1; } @@ -116,8 +153,10 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) } export const testing = { + parsePositiveIntegerEnv, resolveCredentialRole, resolveCredentialSource, + resolveRttOptions, shouldFailPackageTelegramRun, }; export { testing as __testing }; diff --git a/scripts/e2e/npm-telegram-rtt-config.mjs b/scripts/e2e/npm-telegram-rtt-config.mjs deleted file mode 100755 index 6713e6addf1..00000000000 --- a/scripts/e2e/npm-telegram-rtt-config.mjs +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env node -// Writes npm Telegram RTT config fixtures. -import fs from "node:fs"; - -const [configPath, mockPort, groupId, driverToken, sutToken, packageVersion] = - process.argv.slice(2); - -if (!configPath || !mockPort || !groupId || !driverToken || !sutToken || !packageVersion) { - throw new Error( - "usage: npm-telegram-rtt-config.mjs ", - ); -} - -const driverId = driverToken.split(":", 1)[0]; -const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, "utf8")) : {}; - -function supportsVisibleReplies(version) { - const match = /(\d{4})\.(\d+)\.(\d+)/u.exec(version); - if (!match) { - return false; - } - const [, year, month, day] = match.map(Number); - return year > 2026 || (year === 2026 && (month > 4 || (month === 4 && day >= 27))); -} - -config.gateway = { - mode: "local", - port: 18789, - bind: "loopback", - auth: { mode: "none" }, -}; - -config.models = config.models ?? {}; -config.models.providers = config.models.providers ?? {}; -config.models.providers.openai = { - api: "openai-responses", - agentRuntime: { id: "openclaw" }, - apiKey: { - source: "env", - provider: "default", - id: "OPENAI_API_KEY", - }, - baseUrl: `http://127.0.0.1:${mockPort}/v1`, - request: { allowPrivateNetwork: true }, - models: [ - { - id: "gpt-5.5", - name: "gpt-5.5", - api: "openai-responses", - contextWindow: 128000, - }, - ], -}; - -config.agents = config.agents ?? {}; -config.agents.defaults = config.agents.defaults ?? {}; -config.agents.defaults.model = { primary: "openai/gpt-5.5" }; -config.agents.defaults.models = { - "openai/gpt-5.5": { - agentRuntime: { id: "openclaw" }, - params: { - transport: "sse", - openaiWsWarmup: false, - }, - }, -}; -config.agents.list = [ - { - id: "main", - default: true, - name: "Main", - workspace: "~/workspace", - model: { primary: "openai/gpt-5.5" }, - }, -]; - -config.plugins = config.plugins ?? {}; -config.plugins.enabled = true; -config.plugins.allow = ["telegram", "openai"]; -config.plugins.entries = { - telegram: { enabled: true }, - openai: { enabled: true }, -}; - -config.channels = config.channels ?? {}; -config.channels.telegram = { - enabled: true, - botToken: { - source: "env", - provider: "default", - id: "TELEGRAM_BOT_TOKEN", - }, - streaming: { mode: "off" }, - replyToMode: "first", - dmPolicy: "allowlist", - allowFrom: [driverId], - defaultTo: driverId, - groupPolicy: "allowlist", - groupAllowFrom: [driverId], - groups: { - [groupId]: { - requireMention: false, - allowFrom: [driverId], - }, - }, -}; - -if (supportsVisibleReplies(packageVersion)) { - config.messages = { - ...config.messages, - groupChat: { - ...config.messages?.groupChat, - visibleReplies: "automatic", - }, - }; -} - -fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); diff --git a/scripts/e2e/npm-telegram-rtt-credentials.mjs b/scripts/e2e/npm-telegram-rtt-credentials.mjs deleted file mode 100755 index 3248914fdd4..00000000000 --- a/scripts/e2e/npm-telegram-rtt-credentials.mjs +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env node -// Issues and writes npm Telegram RTT credential fixtures. -import fs from "node:fs/promises"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; -import { readBoundedResponseText } from "./lib/bounded-response-text.mjs"; - -const DEFAULT_ENDPOINT_PREFIX = "/qa-credentials/v1"; -const DEFAULT_ACQUIRE_TIMEOUT_MS = 90_000; -const DEFAULT_HTTP_BODY_MAX_BYTES = 1024 * 1024; -const DEFAULT_HTTP_TIMEOUT_MS = 15_000; -const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000; -const DEFAULT_LEASE_TTL_MS = 20 * 60 * 1_000; -const DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES = 64 * 1024 * 1024; -const DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS = 4096; -const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1"; -const RETRY_BACKOFF_MS = [500, 1_000, 2_000, 4_000, 5_000]; -const RETRYABLE_ACQUIRE_CODES = new Set(["POOL_EXHAUSTED", "NO_CREDENTIAL_AVAILABLE"]); - -function parseArgs(argv) { - const command = argv[2]; - const opts = new Map(); - for (let index = 3; index < argv.length; index += 1) { - const arg = argv[index]; - if (!arg.startsWith("--")) { - throw new Error(`Unexpected argument: ${arg}`); - } - const value = argv[++index]; - if (!value) { - throw new Error(`${arg} requires a value.`); - } - opts.set(arg.slice(2), value); - } - if (!command || !["acquire", "heartbeat", "release"].includes(command)) { - throw new Error( - "Usage: npm-telegram-rtt-credentials.mjs acquire|heartbeat|release --lease-file PATH [--credential-env-file PATH]", - ); - } - return { command, opts }; -} - -function requireOption(opts, key) { - const value = opts.get(key)?.trim(); - if (!value) { - throw new Error(`Missing --${key}.`); - } - return value; -} - -function requireString(record, key) { - const value = record[key]; - if (typeof value !== "string" || value.trim().length === 0) { - throw new Error(`Credential payload is missing ${key}.`); - } - return value.trim(); -} - -class BrokerError extends Error { - constructor(message, options = {}) { - super(message); - this.name = "BrokerError"; - this.code = options.code; - this.retryAfterMs = options.retryAfterMs; - } -} - -function taggedError(message, code) { - return Object.assign(new Error(message), { code }); -} - -function parsePositiveInteger(value, fallback, label) { - const raw = value?.trim(); - if (!raw) { - return fallback; - } - const parsed = Number(raw); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`${label} must be a positive integer; got: ${value}`); - } - return parsed; -} - -const CHUNKED_PAYLOAD_MAX_BYTES = parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES, - DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES, - "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES", -); -const CHUNKED_PAYLOAD_MAX_CHUNKS = parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS, - DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS, - "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS", -); - -function normalizeCredentialRole() { - const raw = - process.env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE?.trim() || - process.env.OPENCLAW_QA_CREDENTIAL_ROLE?.trim() || - (process.env.CI ? "ci" : "maintainer"); - const normalized = raw.toLowerCase(); - if (normalized === "ci" || normalized === "maintainer") { - return normalized; - } - throw new Error(`Credential role must be maintainer or ci; got: ${raw}`); -} - -function normalizeEndpointPrefix() { - const raw = process.env.OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX?.trim() || DEFAULT_ENDPOINT_PREFIX; - if (!raw.startsWith("/") || raw.startsWith("//") || raw.includes("\\") || raw.includes("..")) { - throw new Error( - "OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path like /qa-credentials/v1.", - ); - } - return raw.replace(/\/+$/u, "") || "/"; -} - -function resolveConfig() { - const siteUrl = process.env.OPENCLAW_QA_CONVEX_SITE_URL?.trim(); - if (!siteUrl) { - throw new Error("Missing OPENCLAW_QA_CONVEX_SITE_URL for --credential-source convex."); - } - const parsed = new URL(siteUrl); - const allowInsecure = /^(1|true|yes)$/iu.test(process.env.OPENCLAW_QA_ALLOW_INSECURE_HTTP ?? ""); - const isLoopback = parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1"; - if ( - parsed.protocol !== "https:" && - !(parsed.protocol === "http:" && allowInsecure && isLoopback) - ) { - throw new Error("OPENCLAW_QA_CONVEX_SITE_URL must use https://."); - } - const role = normalizeCredentialRole(); - const authToken = - role === "ci" - ? process.env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim() - : process.env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim(); - if (!authToken) { - throw new Error( - role === "ci" - ? "Missing OPENCLAW_QA_CONVEX_SECRET_CI for CI credential access." - : "Missing OPENCLAW_QA_CONVEX_SECRET_MAINTAINER for maintainer credential access.", - ); - } - const endpointPrefix = normalizeEndpointPrefix(); - const ownerId = - process.env.OPENCLAW_QA_CREDENTIAL_OWNER_ID?.trim() || - `npm-telegram-rtt-${process.pid}-${Date.now()}`; - const joinEndpoint = (endpoint) => - `${siteUrl.replace(/\/+$/u, "")}${endpointPrefix}/${endpoint.replace(/^\/+/u, "")}`; - return { - acquireUrl: joinEndpoint("acquire"), - acquireTimeoutMs: parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS, - DEFAULT_ACQUIRE_TIMEOUT_MS, - "OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS", - ), - heartbeatIntervalMs: parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS, - DEFAULT_HEARTBEAT_INTERVAL_MS, - "OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS", - ), - heartbeatUrl: joinEndpoint("heartbeat"), - httpBodyMaxBytes: parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES, - DEFAULT_HTTP_BODY_MAX_BYTES, - "OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES", - ), - httpTimeoutMs: parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS, - DEFAULT_HTTP_TIMEOUT_MS, - "OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS", - ), - leaseTtlMs: parsePositiveInteger( - process.env.OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS, - DEFAULT_LEASE_TTL_MS, - "OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS", - ), - ownerId, - payloadChunkUrl: joinEndpoint("payload-chunk"), - releaseUrl: joinEndpoint("release"), - role, - siteUrl, - authToken, - }; -} - -function parseBrokerPayload(rawPayload, response) { - if (!rawPayload.trim()) { - return response.ok ? { status: "ok" } : {}; - } - try { - return JSON.parse(rawPayload); - } catch (error) { - throw new Error("Convex credential broker returned invalid JSON.", { cause: error }); - } -} - -async function postBroker(params) { - const controller = new AbortController(); - const timeoutMs = Math.max(1, params.timeoutMs); - const timeoutError = taggedError(`${params.label} timed out after ${timeoutMs}ms`, "ETIMEDOUT"); - let timeout; - const timeoutPromise = new Promise((_, reject) => { - timeout = setTimeout(() => { - controller.abort(timeoutError); - reject(timeoutError); - }, timeoutMs); - timeout.unref?.(); - }); - try { - const response = await Promise.race([ - fetch(params.url, { - method: "POST", - headers: { - authorization: `Bearer ${params.authToken}`, - "content-type": "application/json", - }, - body: JSON.stringify(params.body), - signal: controller.signal, - }), - timeoutPromise, - ]); - const rawPayload = await readBoundedResponseText( - response, - params.label, - params.bodyMaxBytes, - timeoutPromise, - ); - const payload = parseBrokerPayload(rawPayload, response); - if (!response.ok || payload?.status === "error") { - const message = - typeof payload?.message === "string" && payload.message.trim() - ? payload.message.trim() - : `HTTP ${response.status}`; - throw new BrokerError(message, { - code: typeof payload?.code === "string" ? payload.code : undefined, - retryAfterMs: Number.isInteger(payload?.retryAfterMs) ? payload.retryAfterMs : undefined, - }); - } - if (payload?.status !== "ok") { - throw new Error("Convex credential broker returned an invalid response."); - } - return payload; - } finally { - clearTimeout(timeout); - } -} - -function parseChunkedPayloadMarker(payload) { - if (!payload || typeof payload !== "object" || Array.isArray(payload)) { - return undefined; - } - if (payload[CHUNKED_PAYLOAD_MARKER] !== true) { - return undefined; - } - if (!Number.isInteger(payload.chunkCount) || payload.chunkCount < 1) { - throw new Error("Chunked credential payload has invalid chunkCount."); - } - if (payload.chunkCount > CHUNKED_PAYLOAD_MAX_CHUNKS) { - throw new Error(`Chunked credential payload exceeds ${CHUNKED_PAYLOAD_MAX_CHUNKS} chunks.`); - } - if (!Number.isInteger(payload.byteLength) || payload.byteLength < 0) { - throw new Error("Chunked credential payload has invalid byteLength."); - } - if (payload.byteLength > CHUNKED_PAYLOAD_MAX_BYTES) { - throw new Error(`Chunked credential payload exceeds ${CHUNKED_PAYLOAD_MAX_BYTES} bytes.`); - } - return { byteLength: payload.byteLength, chunkCount: payload.chunkCount }; -} - -function parseTelegramCredentialPayload(payload) { - if (!payload || typeof payload !== "object" || Array.isArray(payload)) { - throw new Error("Telegram credential payload must be an object."); - } - const groupId = requireString(payload, "groupId"); - if (!/^-?\d+$/u.test(groupId)) { - throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id."); - } - return { - groupId, - driverToken: requireString(payload, "driverToken"), - sutToken: requireString(payload, "sutToken"), - }; -} - -async function resolveCredentialPayload(config, acquired) { - const marker = parseChunkedPayloadMarker(acquired.payload); - if (!marker) { - return parseTelegramCredentialPayload(acquired.payload); - } - const chunks = []; - let serializedLength = 0; - for (let index = 0; index < marker.chunkCount; index += 1) { - const chunk = await postBroker({ - authToken: config.authToken, - bodyMaxBytes: config.httpBodyMaxBytes, - label: "credential broker payload-chunk", - timeoutMs: config.httpTimeoutMs, - url: config.payloadChunkUrl, - body: { - kind: "telegram", - ownerId: config.ownerId, - actorRole: config.role, - credentialId: requireString(acquired, "credentialId"), - leaseToken: requireString(acquired, "leaseToken"), - index, - }, - }); - const data = requireString(chunk, "data"); - serializedLength += data.length; - if (serializedLength > marker.byteLength) { - throw new Error("Chunked credential payload exceeded declared byteLength."); - } - chunks.push(data); - } - const serialized = chunks.join(""); - if (serializedLength !== marker.byteLength) { - throw new Error("Chunked credential payload length mismatch."); - } - return parseTelegramCredentialPayload(JSON.parse(serialized)); -} - -function shellQuote(value) { - return `'${value.replaceAll("'", "'\\''")}'`; -} - -async function writeCredentialEnv(pathname, payload) { - await fs.writeFile( - pathname, - [ - `export OPENCLAW_QA_TELEGRAM_GROUP_ID=${shellQuote(payload.groupId)}`, - `export OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN=${shellQuote(payload.driverToken)}`, - `export OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN=${shellQuote(payload.sutToken)}`, - "", - ].join("\n"), - { mode: 0o600 }, - ); -} - -async function readLease(pathname) { - return JSON.parse(await fs.readFile(pathname, "utf8")); -} - -function leaseTtlMsFromLease(config, lease) { - const value = lease.leaseTtlMs; - if (value === undefined || value === null) { - return config.leaseTtlMs; - } - return parsePositiveInteger(String(value), config.leaseTtlMs, "leaseTtlMs"); -} - -async function acquire(opts) { - const config = resolveConfig(); - const leaseFile = requireOption(opts, "lease-file"); - const envFile = requireOption(opts, "credential-env-file"); - const acquired = await acquireWithRetry(config); - const lease = { - kind: "telegram", - ownerId: config.ownerId, - actorRole: config.role, - credentialId: requireString(acquired, "credentialId"), - leaseToken: requireString(acquired, "leaseToken"), - heartbeatIntervalMs: acquired.heartbeatIntervalMs ?? config.heartbeatIntervalMs, - leaseTtlMs: acquired.leaseTtlMs ?? config.leaseTtlMs, - }; - try { - await writeCredentialEnv(envFile, await resolveCredentialPayload(config, acquired)); - await fs.writeFile(leaseFile, `${JSON.stringify(lease, null, 2)}\n`, { mode: 0o600 }); - } catch (error) { - await releaseLease(config, lease).catch(() => {}); - throw error; - } - process.stdout.write( - `${JSON.stringify({ status: "ok", credentialId: lease.credentialId, ownerId: lease.ownerId }, null, 2)}\n`, - ); -} - -async function acquireWithRetry(config) { - const startedAt = Date.now(); - let attempt = 0; - while (true) { - attempt += 1; - const attemptElapsedMs = Date.now() - startedAt; - const attemptRemainingMs = config.acquireTimeoutMs - attemptElapsedMs; - if (attemptRemainingMs <= 0) { - throw taggedError( - `credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`, - "ETIMEDOUT", - ); - } - try { - return await postBroker({ - authToken: config.authToken, - bodyMaxBytes: config.httpBodyMaxBytes, - label: "credential broker acquire", - timeoutMs: Math.min(config.httpTimeoutMs, attemptRemainingMs), - url: config.acquireUrl, - body: { - kind: "telegram", - ownerId: config.ownerId, - actorRole: config.role, - leaseTtlMs: config.leaseTtlMs, - heartbeatIntervalMs: config.heartbeatIntervalMs, - }, - }); - } catch (error) { - const code = error instanceof BrokerError ? error.code : undefined; - const retryable = code ? RETRYABLE_ACQUIRE_CODES.has(code) : false; - const elapsedMs = Date.now() - startedAt; - if (!retryable) { - throw error; - } - if (elapsedMs >= config.acquireTimeoutMs) { - throw taggedError( - `credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`, - "ETIMEDOUT", - ); - } - const fallbackDelay = RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)]; - const retryAfterMs = error instanceof BrokerError ? error.retryAfterMs : undefined; - const delayMs = retryAfterMs ?? fallbackDelay; - const remainingMs = config.acquireTimeoutMs - elapsedMs; - if (delayMs >= remainingMs) { - throw taggedError( - `credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`, - "ETIMEDOUT", - ); - } - await new Promise((resolve) => { - setTimeout(resolve, delayMs); - }); - } - } -} - -async function releaseLease(config, lease) { - await postBroker({ - authToken: config.authToken, - bodyMaxBytes: config.httpBodyMaxBytes, - label: "credential broker release", - timeoutMs: config.httpTimeoutMs, - url: config.releaseUrl, - body: { - kind: requireString(lease, "kind"), - ownerId: requireString(lease, "ownerId"), - actorRole: requireString(lease, "actorRole"), - credentialId: requireString(lease, "credentialId"), - leaseToken: requireString(lease, "leaseToken"), - }, - }); -} - -async function release(opts) { - const config = resolveConfig(); - const leaseFile = requireOption(opts, "lease-file"); - const lease = await readLease(leaseFile); - await releaseLease(config, lease); - await fs.rm(leaseFile, { force: true }); -} - -async function heartbeat(opts) { - const config = resolveConfig(); - const leaseFile = requireOption(opts, "lease-file"); - while (true) { - const lease = await readLease(leaseFile); - await postBroker({ - authToken: config.authToken, - bodyMaxBytes: config.httpBodyMaxBytes, - label: "credential broker heartbeat", - timeoutMs: config.httpTimeoutMs, - url: config.heartbeatUrl, - body: { - kind: requireString(lease, "kind"), - ownerId: requireString(lease, "ownerId"), - actorRole: requireString(lease, "actorRole"), - credentialId: requireString(lease, "credentialId"), - leaseTtlMs: leaseTtlMsFromLease(config, lease), - leaseToken: requireString(lease, "leaseToken"), - }, - }); - const intervalMs = parsePositiveInteger( - String(lease.heartbeatIntervalMs ?? config.heartbeatIntervalMs), - config.heartbeatIntervalMs, - "heartbeatIntervalMs", - ); - await new Promise((resolve) => { - setTimeout(resolve, intervalMs); - }); - } -} - -async function main(argv = process.argv) { - const { command, opts } = parseArgs(argv); - if (command === "acquire") { - await acquire(opts); - } else if (command === "heartbeat") { - await heartbeat(opts); - } else { - await release(opts); - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { - await main(); -} - -export { parseChunkedPayloadMarker }; diff --git a/scripts/e2e/npm-telegram-rtt-docker.sh b/scripts/e2e/npm-telegram-rtt-docker.sh deleted file mode 100755 index 68fe8d5d90a..00000000000 --- a/scripts/e2e/npm-telegram-rtt-docker.sh +++ /dev/null @@ -1,454 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" - -IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-npm-telegram-rtt-e2e" OPENCLAW_NPM_TELEGRAM_RTT_E2E_IMAGE)" -DOCKER_TARGET="${OPENCLAW_NPM_TELEGRAM_DOCKER_TARGET:-build}" -PACKAGE_SPEC="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC:-openclaw@beta}" -PACKAGE_TGZ="${OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ:-${OPENCLAW_CURRENT_PACKAGE_TGZ:-}}" -PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-}" -RUN_ID="${OPENCLAW_NPM_TELEGRAM_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}" -OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt/$RUN_ID}" - -resolve_credential_source() { - if [ -n "${OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE:-}" ]; then - printf "%s" "$OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE" - return 0 - fi - if [ -n "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" ]; then - printf "%s" "$OPENCLAW_QA_CREDENTIAL_SOURCE" - return 0 - fi - if [ -n "${CI:-}" ] && [ -n "${OPENCLAW_QA_CONVEX_SITE_URL:-}" ]; then - if [ -n "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ] || [ -n "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then - printf "convex" - fi - fi -} - -resolve_credential_role() { - if [ -n "${OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE:-}" ]; then - printf "%s" "$OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE" - return 0 - fi - if [ -n "${OPENCLAW_QA_CREDENTIAL_ROLE:-}" ]; then - printf "%s" "$OPENCLAW_QA_CREDENTIAL_ROLE" - fi -} - -validate_openclaw_package_spec() { - local spec="$1" - if [[ "$spec" =~ ^openclaw@(main|alpha|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$ ]]; then - return 0 - fi - echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC must be openclaw@main, openclaw@alpha, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: $spec" >&2 - exit 1 -} - -resolve_package_tgz() { - local candidate="$1" - if [ -z "$candidate" ]; then - return 0 - fi - if [ ! -f "$candidate" ]; then - echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ must point to an existing .tgz file; got: $candidate" >&2 - exit 1 - fi - case "$candidate" in - *.tgz) ;; - *) - echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ must point to a .tgz file; got: $candidate" >&2 - exit 1 - ;; - esac - local dir - local base - dir="$(cd "$(dirname "$candidate")" && pwd)" - base="$(basename "$candidate")" - printf "%s/%s" "$dir" "$base" -} - -package_mount_args=() -package_install_source="$PACKAGE_SPEC" -package_source_kind="npm-package" -resolved_package_tgz="$(resolve_package_tgz "$PACKAGE_TGZ")" -if [ -n "$resolved_package_tgz" ]; then - package_install_source="/package-under-test/$(basename "$resolved_package_tgz")" - package_source_kind="packed-tarball" - package_mount_args=(-v "$resolved_package_tgz:$package_install_source:ro") -else - validate_openclaw_package_spec "$PACKAGE_SPEC" -fi -if [ -z "$PACKAGE_LABEL" ]; then - if [ -n "$resolved_package_tgz" ]; then - PACKAGE_LABEL="$(basename "$resolved_package_tgz")" - else - PACKAGE_LABEL="$PACKAGE_SPEC" - fi -fi - -credential_source="$(resolve_credential_source)" -credential_role="$(resolve_credential_role)" -if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then - if [ -n "${CI:-}" ]; then - credential_role="ci" - else - credential_role="maintainer" - fi -fi - -validate_credential_source() { - case "$credential_source" in - "" | env | convex) ;; - *) - echo "OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE must be env or convex; got: $credential_source" >&2 - exit 1 - ;; - esac -} - -validate_credential_role() { - case "$credential_role" in - "" | maintainer | ci) ;; - *) - echo "OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE must be maintainer or ci; got: $credential_role" >&2 - exit 1 - ;; - esac -} - -validate_credential_source -validate_credential_role - -validate_credential_preflight() { - if [ "$credential_source" = "convex" ]; then - if [ -z "${OPENCLAW_QA_CONVEX_SITE_URL:-}" ]; then - echo "Missing required env for Convex credential mode: OPENCLAW_QA_CONVEX_SITE_URL" >&2 - exit 1 - fi - if [ "$credential_role" = "ci" ]; then - if [ -z "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ]; then - echo "Missing required env for Convex ci credential mode: OPENCLAW_QA_CONVEX_SECRET_CI" >&2 - exit 1 - fi - return 0 - fi - if [ "$credential_role" = "maintainer" ]; then - if [ -z "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then - echo "Missing required env for Convex maintainer credential mode: OPENCLAW_QA_CONVEX_SECRET_MAINTAINER" >&2 - exit 1 - fi - return 0 - fi - if [ -z "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ] && [ -z "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then - echo "Missing required env for Convex credential mode: OPENCLAW_QA_CONVEX_SECRET_CI or OPENCLAW_QA_CONVEX_SECRET_MAINTAINER" >&2 - exit 1 - fi - return 0 - fi - - for key in \ - OPENCLAW_QA_TELEGRAM_GROUP_ID \ - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN \ - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN; do - if [ -z "${!key:-}" ]; then - echo "Missing required env: $key" >&2 - exit 1 - fi - done -} - -validate_credential_preflight - -if [ -n "$credential_source" ]; then - export OPENCLAW_QA_CREDENTIAL_SOURCE="$credential_source" -fi -if [ -n "$credential_role" ]; then - export OPENCLAW_QA_CREDENTIAL_ROLE="$credential_role" -fi - -if [ -z "$credential_source" ] || [ "$credential_source" = "env" ]; then - for key in \ - OPENCLAW_QA_TELEGRAM_GROUP_ID \ - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN \ - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN; do - if [ -z "${!key:-}" ]; then - echo "Missing required env: $key" >&2 - exit 1 - fi - done -fi - -for value in "$credential_source" "$credential_role"; do - if [[ "$value" == *[$'\n\r']* ]]; then - echo "Credential source and role must be single-line values." >&2 - exit 1 - fi -done - -docker_e2e_build_or_reuse "$IMAGE_NAME" npm-telegram-rtt "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "$DOCKER_TARGET" - -mkdir -p "$ROOT_DIR/.artifacts/qa-e2e" -run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-npm-telegram-rtt.XXXXXX")" -npm_prefix_host="$(mktemp -d "$ROOT_DIR/.artifacts/qa-e2e/npm-telegram-rtt-prefix.XXXXXX")" -trap 'rm -f "$run_log"; rm -rf "$npm_prefix_host"' EXIT - -docker_env=( - -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 - -e OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE="$package_install_source" - -e OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$PACKAGE_LABEL" - -e OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR="$OUTPUT_DIR" - -e OPENCLAW_QA_PACKAGE_SOURCE="$package_install_source" - -e OPENCLAW_QA_PACKAGE_SOURCE_KIND="$package_source_kind" - -e OPENCLAW_QA_TELEGRAM_GROUP_ID - -e OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN - -e OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN - -e OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS="${OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS:-180000}" - -e OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS="${OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS:-180000}" - -e OPENCLAW_NPM_TELEGRAM_SCENARIOS="${OPENCLAW_NPM_TELEGRAM_SCENARIOS:-telegram-mentioned-message-reply}" - -e OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE="${OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE:-mock-openai}" - -e OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES="${OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES:-20}" - -e OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS="${OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS:-30000}" - -e OPENCLAW_NPM_TELEGRAM_MAX_FAILURES="${OPENCLAW_NPM_TELEGRAM_MAX_FAILURES:-${OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES:-20}}" - -e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" -) - -forward_env_if_set() { - local key="$1" - if [ -n "${!key:-}" ]; then - docker_env+=(-e "$key") - fi -} - -if [ -n "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" ]; then - docker_env+=(-e OPENCLAW_QA_CREDENTIAL_SOURCE="$OPENCLAW_QA_CREDENTIAL_SOURCE") -fi -if [ -n "${OPENCLAW_QA_CREDENTIAL_ROLE:-}" ]; then - docker_env+=(-e OPENCLAW_QA_CREDENTIAL_ROLE="$OPENCLAW_QA_CREDENTIAL_ROLE") -fi - -install_env=("${docker_env[@]}") - -for key in \ - OPENCLAW_QA_CONVEX_SITE_URL \ - OPENCLAW_QA_CONVEX_SECRET_CI \ - OPENCLAW_QA_CONVEX_SECRET_MAINTAINER \ - OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS \ - OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS \ - OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS \ - OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS \ - OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES \ - OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES \ - OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS \ - OPENCLAW_QA_PACKAGE_SOURCE_SHA \ - OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX \ - OPENCLAW_QA_CREDENTIAL_OWNER_ID \ - OPENCLAW_QA_ALLOW_INSECURE_HTTP; do - forward_env_if_set "$key" -done - -run_logged() { - if ! "$@" >"$run_log" 2>&1; then - docker_e2e_print_log "$run_log" - exit 1 - fi - docker_e2e_print_log "$run_log" - >"$run_log" -} - -echo "Installing ${PACKAGE_LABEL} from ${package_install_source}..." -run_logged docker_e2e_docker_run_cmd run --rm \ - "${install_env[@]}" \ - ${package_mount_args[@]+"${package_mount_args[@]}"} \ - -v "$npm_prefix_host:/npm-global" \ - -i "$IMAGE_NAME" bash -s <<'EOF' -set -euo pipefail - -export NPM_CONFIG_PREFIX="/npm-global" -export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" - -install_source="${OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE:?missing OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE}" -package_label="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-$install_source}" - -npm_install_timeout="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}" -run_npm_install() { - if [ -z "$npm_install_timeout" ] || [ "$npm_install_timeout" = "0" ]; then - npm install -g "$install_source" --no-fund --no-audit - return - fi - - local timeout_bin="" - if command -v timeout >/dev/null 2>&1; then - timeout_bin="timeout" - elif command -v gtimeout >/dev/null 2>&1; then - timeout_bin="gtimeout" - fi - if [ -z "$timeout_bin" ]; then - echo "timeout or gtimeout is required for OPENCLAW_E2E_NPM_INSTALL_TIMEOUT=$npm_install_timeout" >&2 - return 127 - fi - - if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then - "$timeout_bin" --kill-after=30s "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit - else - "$timeout_bin" "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit - fi -} -run_npm_install -command -v openclaw -openclaw --version -node -p "require('/npm-global/lib/node_modules/openclaw/package.json').version" -EOF - -echo "Running package Telegram RTT Docker E2E ($PACKAGE_LABEL)..." -run_logged docker_e2e_docker_run_cmd run --rm \ - "${docker_env[@]}" \ - -v "$ROOT_DIR/scripts:/app/scripts:ro" \ - -v "$ROOT_DIR/.artifacts:/app/.artifacts" \ - -v "$npm_prefix_host:/npm-global" \ - -i "$IMAGE_NAME" bash -s <<'EOF' -set -euo pipefail -source scripts/lib/openclaw-e2e-instance.sh - -export HOME="$(mktemp -d "/tmp/openclaw-npm-telegram-rtt.XXXXXX")" -export NPM_CONFIG_PREFIX="/npm-global" -export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" -export OPENAI_API_KEY="sk-openclaw-rtt" -export GATEWAY_AUTH_TOKEN_REF="openclaw-rtt" -export OPENCLAW_DISABLE_BONJOUR="1" - -install_source="${OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE:?missing OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE}" -package_label="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-$install_source}" -mock_port="${OPENCLAW_NPM_TELEGRAM_MOCK_PORT:-44080}" -config_path="$HOME/.openclaw/openclaw.json" -gateway_log="/tmp/openclaw-npm-telegram-rtt-gateway.log" -mock_log="/tmp/openclaw-npm-telegram-rtt-mock.log" -export MOCK_PORT="$mock_port" -credential_env_file="" -credential_lease_file="" -credential_heartbeat_pid="" -rtt_shell_pid="$$" - -dump_logs() { - local status="$1" - if [ "$status" -eq 0 ]; then - return - fi - echo "package Telegram RTT failed with exit code $status" >&2 - for file in \ - "$mock_log" \ - "$gateway_log"; do - if [ -f "$file" ]; then - echo "--- $file ---" >&2 - openclaw_e2e_print_log "$file" >&2 - fi - done -} - -cleanup() { - local status="$?" - kill ${gateway_pid:-} ${mock_pid:-} ${credential_heartbeat_pid:-} 2>/dev/null || true - if [ -n "$credential_lease_file" ] && [ -f "$credential_lease_file" ]; then - node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs release --lease-file "$credential_lease_file" >/dev/null 2>&1 || true - fi - rm -f "$credential_env_file" "$credential_lease_file" - dump_logs "$status" - exit "$status" -} - -start_credential_heartbeat() { - ( - set +e - node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs heartbeat --lease-file "$credential_lease_file" & - local heartbeat_child_pid="$!" - trap 'kill "$heartbeat_child_pid" 2>/dev/null || true; wait "$heartbeat_child_pid" 2>/dev/null || true; exit 0' TERM INT - wait "$heartbeat_child_pid" - local heartbeat_status="$?" - echo "Convex credential heartbeat exited with status $heartbeat_status" >&2 - kill -TERM "$rtt_shell_pid" 2>/dev/null || true - exit "$heartbeat_status" - ) & - credential_heartbeat_pid="$!" -} - -trap cleanup EXIT -trap 'exit 1' TERM INT - -if [ "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" = "convex" ]; then - credential_env_file="$(mktemp "/tmp/openclaw-npm-telegram-rtt-credential-env.XXXXXX")" - credential_lease_file="$(mktemp "/tmp/openclaw-npm-telegram-rtt-credential-lease.XXXXXX")" - rm -f "$credential_env_file" "$credential_lease_file" - node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs acquire \ - --credential-env-file "$credential_env_file" \ - --lease-file "$credential_lease_file" - # shellcheck source=/dev/null - source "$credential_env_file" - start_credential_heartbeat -fi - -export TELEGRAM_BOT_TOKEN="${OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN:?missing OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN}" - -command -v openclaw -openclaw --version -installed_version="$(node -p "require('/npm-global/lib/node_modules/openclaw/package.json').version")" - -node /app/scripts/e2e/mock-openai-server.mjs >"$mock_log" 2>&1 & -mock_pid="$!" -mock_ready=0 -for _ in $(seq 1 60); do - if node --input-type=module -e ' - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), 1000); - try { - const response = await fetch(process.argv[1], { signal: controller.signal }); - process.exit(response.ok ? 0 : 1); - } catch { - process.exit(1); - } finally { - clearTimeout(timer); - } - ' "http://127.0.0.1:${mock_port}/health"; then - mock_ready=1 - break - fi - sleep 1 -done -if [ "$mock_ready" != "1" ]; then - echo "Mock OpenAI server did not become ready" >&2 - openclaw_e2e_print_log "$mock_log" >&2 - exit 1 -fi - -mkdir -p "$(dirname "$config_path")" "$HOME/.openclaw/workspace" "$HOME/.openclaw/agents/main/sessions" "$HOME/workspace" - -node /app/scripts/e2e/npm-telegram-rtt-config.mjs \ - "$config_path" \ - "$mock_port" \ - "$OPENCLAW_QA_TELEGRAM_GROUP_ID" \ - "$OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN" \ - "$OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN" \ - "$installed_version" - -openclaw gateway run --verbose >"$gateway_log" 2>&1 & -gateway_pid="$!" -for _ in $(seq 1 120); do - if ! kill -0 "$gateway_pid" 2>/dev/null; then - echo "gateway exited before readiness" >&2 - exit 1 - fi - if bash -c ":/dev/null 2>&1; then - break - fi - sleep 1 -done -if ! bash -c ":/dev/null 2>&1; then - echo "gateway did not open port 18789" >&2 - exit 1 -fi - -node /app/scripts/e2e/npm-telegram-rtt-driver.mjs -EOF - -echo "package Telegram RTT Docker E2E passed ($PACKAGE_LABEL)" diff --git a/scripts/e2e/npm-telegram-rtt-driver.mjs b/scripts/e2e/npm-telegram-rtt-driver.mjs deleted file mode 100755 index b6ba9084797..00000000000 --- a/scripts/e2e/npm-telegram-rtt-driver.mjs +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env node -// Drives npm Telegram RTT test messages through the fixture server. -import fs from "node:fs/promises"; -import path from "node:path"; -import { readBoundedResponseText } from "./lib/bounded-response-text.mjs"; - -const groupId = process.env.OPENCLAW_QA_TELEGRAM_GROUP_ID; -const driverToken = process.env.OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN; -const sutToken = process.env.OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN; -const outputDir = process.env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR ?? ".artifacts/rtt/raw"; -const providerMode = process.env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE?.trim() || "mock-openai"; -const primaryModel = process.env.OPENCLAW_NPM_TELEGRAM_MODEL?.trim() || null; -const telegramApiBaseUrl = ( - process.env.OPENCLAW_QA_TELEGRAM_API_BASE_URL ?? "https://api.telegram.org" -).replace(/\/+$/u, ""); -const timeoutMs = readPositiveIntEnv("OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", 180000); -const canaryTimeoutMs = readPositiveIntEnv("OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", timeoutMs); -const warmSampleCount = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES", 20); -const sampleTimeoutMs = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS", 30000); -const botApiTimeoutMs = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS", 30000); -const botApiBodyMaxBytes = readPositiveIntEnv( - "OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES", - 1024 * 1024, -); -const maxWarmFailures = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_MAX_FAILURES", warmSampleCount); -const successMarker = process.env.OPENCLAW_NPM_TELEGRAM_SUCCESS_MARKER ?? "OPENCLAW_E2E_OK"; -const supportedScenarioIds = new Set(["telegram-mentioned-message-reply"]); -const requestedScenarioIds = ( - process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS ?? "telegram-mentioned-message-reply" -) - .split(",") - .map((value) => value.trim()) - .filter(Boolean); - -if (requestedScenarioIds.length === 0) { - throw new Error("OPENCLAW_NPM_TELEGRAM_SCENARIOS must include at least one RTT scenario"); -} - -const unknownScenarioIds = requestedScenarioIds.filter( - (scenarioId) => !supportedScenarioIds.has(scenarioId), -); -if (unknownScenarioIds.length > 0) { - throw new Error(`unknown OPENCLAW_NPM_TELEGRAM_SCENARIOS: ${unknownScenarioIds.join(", ")}`); -} - -const scenarioIds = new Set(requestedScenarioIds); - -if (!groupId || !driverToken || !sutToken) { - throw new Error( - "missing Telegram env: OPENCLAW_QA_TELEGRAM_GROUP_ID, OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN, OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", - ); -} -function readPositiveIntEnv(name, fallback) { - const text = String(process.env[name] ?? fallback).trim(); - if (!/^\d+$/u.test(text)) { - throw new Error(`invalid ${name}: ${text}`); - } - const value = Number(text); - if (!Number.isSafeInteger(value) || value <= 0) { - throw new Error(`invalid ${name}: ${text}`); - } - return value; -} - -function taggedError(message, code) { - return Object.assign(new Error(message), { code }); -} - -function parseJsonPayload(rawPayload, label) { - try { - return JSON.parse(rawPayload); - } catch (error) { - throw new Error(`${label} returned invalid JSON`, { cause: error }); - } -} - -async function fetchTelegramJson(url, init, label) { - const controller = new AbortController(); - const timeoutError = taggedError(`${label} timed out after ${botApiTimeoutMs}ms`, "ETIMEDOUT"); - let timeout; - const timeoutPromise = new Promise((_, reject) => { - timeout = setTimeout(() => { - controller.abort(timeoutError); - reject(timeoutError); - }, botApiTimeoutMs); - timeout.unref?.(); - }); - try { - const response = await Promise.race([ - fetch(url, { - ...init, - signal: controller.signal, - }), - timeoutPromise, - ]); - const rawPayload = await readBoundedResponseText( - response, - label, - botApiBodyMaxBytes, - timeoutPromise, - ); - const payload = parseJsonPayload(rawPayload, label); - return { payload, response }; - } finally { - if (timeout) { - clearTimeout(timeout); - } - } -} - -class TelegramBot { - constructor(token) { - this.baseUrl = `${telegramApiBaseUrl}/bot${token}`; - } - - async call(method, body) { - const { payload, response } = await fetchTelegramJson( - `${this.baseUrl}/${method}`, - { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(body), - }, - `Telegram Bot API ${method}`, - ); - if (!response.ok || payload.ok !== true) { - throw new Error(`${method} failed: ${JSON.stringify(payload)}`); - } - return payload.result; - } - - getMe() { - return this.call("getMe", {}); - } - - sendMessage(params) { - return this.call("sendMessage", params); - } - - getUpdates(params) { - return this.call("getUpdates", params); - } -} - -const driver = new TelegramBot(driverToken); -const sut = new TelegramBot(sutToken); -const observedMessages = []; -let driverUpdateOffset = 0; - -function sleep(ms) { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -function messageText(message) { - return message.text ?? message.caption ?? ""; -} - -async function flushUpdates(bot) { - let updates = await bot.getUpdates({ - timeout: 0, - allowed_updates: ["message", "edited_message"], - }); - let nextOffset; - while (updates.length > 0) { - const lastUpdateId = updates.at(-1).update_id; - nextOffset = lastUpdateId + 1; - updates = await bot.getUpdates({ - offset: nextOffset, - timeout: 0, - allowed_updates: ["message", "edited_message"], - }); - } - return nextOffset; -} - -async function waitForSutReply(params) { - const deadline = Date.now() + params.timeoutMs; - while (Date.now() < deadline) { - const updates = await driver.getUpdates({ - offset: driverUpdateOffset, - timeout: 5, - allowed_updates: ["message", "edited_message"], - }); - for (const update of updates) { - driverUpdateOffset = Math.max(driverUpdateOffset, update.update_id + 1); - const message = update.message ?? update.edited_message; - if (!message || String(message.chat?.id) !== String(groupId)) { - continue; - } - observedMessages.push({ - updateType: update.edited_message ? "edited_message" : "message", - updateId: update.update_id, - messageId: message.message_id, - fromId: message.from?.id, - fromUsername: message.from?.username, - replyToMessageId: message.reply_to_message?.message_id, - text: messageText(message), - scenarioId: params.scenarioId, - scenarioTitle: params.scenarioTitle, - sampleIndex: params.sampleIndex, - }); - if (message.from?.id !== params.sutId) { - continue; - } - if (message.date < params.startedUnixSeconds) { - continue; - } - const text = messageText(message); - if (params.matchText && !text.includes(params.matchText)) { - continue; - } - const replyMatches = message.reply_to_message?.message_id === params.requestMessageId; - const textMatches = params.matchText ? text.includes(params.matchText) : false; - if (replyMatches || textMatches) { - return message; - } - } - } - - throw new Error(`timed out after ${params.timeoutMs}ms waiting for Telegram message`); -} - -async function runScenario(params) { - const startedAt = new Date(); - const startedUnixSeconds = Math.floor(startedAt.getTime() / 1000); - const sendParams = { - chat_id: groupId, - text: params.input, - disable_notification: true, - }; - if (params.replyToMessageId) { - sendParams.reply_parameters = { message_id: params.replyToMessageId }; - } - const request = await driver.sendMessage(sendParams); - - try { - const reply = await waitForSutReply({ - allowAnySutReply: params.allowAnySutReply, - matchText: params.matchText, - requestMessageId: request.message_id, - scenarioId: params.id, - scenarioTitle: params.title, - sampleIndex: params.sampleIndex, - startedUnixSeconds, - sutId: params.sutId, - timeoutMs: params.timeoutMs, - }); - const rttMs = Date.now() - startedAt.getTime(); - return { - id: params.id, - title: params.title, - status: "pass", - details: `observed SUT message ${reply.message_id}`, - messageId: reply.message_id, - rttMs, - }; - } catch (error) { - return { - id: params.id, - title: params.title, - status: "fail", - details: error instanceof Error ? error.message : String(error), - }; - } -} - -function percentile(sortedValues, percentileValue) { - if (sortedValues.length === 0) { - return undefined; - } - const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1; - return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)]; -} - -function summarizeSamples(samples) { - const passed = samples.filter((sample) => sample.status === "pass" && sample.rttMs !== undefined); - const sorted = passed.map((sample) => sample.rttMs).toSorted((a, b) => a - b); - const sum = sorted.reduce((total, value) => total + value, 0); - return { - total: samples.length, - passed: passed.length, - failed: samples.length - passed.length, - avgMs: sorted.length > 0 ? Math.round(sum / sorted.length) : undefined, - p50Ms: percentile(sorted, 50), - p95Ms: percentile(sorted, 95), - maxMs: sorted.at(-1), - }; -} - -function splitModelRef(modelRef) { - if (!modelRef) { - return { provider: "openai", model: null, ref: null }; - } - const slashIndex = modelRef.indexOf("/"); - if (slashIndex <= 0 || slashIndex === modelRef.length - 1) { - return { provider: "openai", model: modelRef, ref: modelRef }; - } - return { - provider: modelRef.slice(0, slashIndex), - model: modelRef.slice(slashIndex + 1), - ref: modelRef, - }; -} - -function buildProviderEvidence() { - const split = splitModelRef(primaryModel); - const live = providerMode !== "mock-openai"; - return { - id: split.provider || "openai", - live, - model: { - name: split.model, - ref: split.ref, - }, - ...(live ? { auth: providerMode } : { fixture: providerMode }), - }; -} - -function buildPackageSourceEvidence() { - const spec = process.env.OPENCLAW_QA_PACKAGE_SOURCE?.trim() || undefined; - const sha = process.env.OPENCLAW_QA_PACKAGE_SOURCE_SHA?.trim() || undefined; - const kind = - process.env.OPENCLAW_QA_PACKAGE_SOURCE_KIND?.trim() || - (spec?.endsWith(".tgz") ? "packed-tarball" : spec ? "npm-package" : "source-checkout"); - return { - kind, - ...(spec ? { spec } : {}), - ...(sha ? { sha } : {}), - }; -} - -function coverageIdForScenario(scenarioId) { - if (scenarioId === "telegram-canary") { - return "channels.telegram.canary"; - } - if (scenarioId === "telegram-mentioned-message-reply") { - return "channels.telegram.mention-gating"; - } - return undefined; -} - -function timingForScenario(scenario) { - const timing = {}; - if (typeof scenario.rttMs === "number" && Number.isFinite(scenario.rttMs) && scenario.rttMs > 0) { - timing.rttMs = scenario.rttMs; - } - if (scenario.stats) { - for (const key of ["avgMs", "p50Ms", "p95Ms", "maxMs"]) { - const value = scenario.stats[key]; - if (typeof value === "number" && Number.isFinite(value) && value > 0) { - timing[key] = value; - } - } - if ( - typeof scenario.stats.total === "number" && - Number.isFinite(scenario.stats.total) && - scenario.stats.total > 0 - ) { - timing.samples = scenario.stats.total; - } - if ( - typeof scenario.stats.failed === "number" && - Number.isFinite(scenario.stats.failed) && - scenario.stats.failed >= 0 - ) { - timing.failedSamples = scenario.stats.failed; - } - } - return Object.keys(timing).length > 0 ? timing : undefined; -} - -function buildScenarioCoverage(scenarioId) { - const liveCoverage = { - id: "channels.telegram.live", - role: "live-transport", - surfaceIds: ["channels.telegram"], - categoryIds: ["channels.telegram.live"], - }; - const coverageId = coverageIdForScenario(scenarioId); - if (!coverageId) { - return [liveCoverage]; - } - return [ - liveCoverage, - { - id: coverageId, - role: "live-transport-coverage", - surfaceIds: ["channels.telegram"], - categoryIds: ["channels.telegram.live"], - }, - ]; -} - -function buildEvidenceSummary(params) { - const provider = buildProviderEvidence(); - const generatedAt = new Date().toISOString(); - return { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt, - entries: params.scenarios.map((scenario) => { - const timing = timingForScenario(scenario); - return { - test: { - kind: "live-transport-check", - id: scenario.id, - title: scenario.title, - }, - mapping: { - profile: "release", - coverage: buildScenarioCoverage(scenario.id), - }, - execution: { - runner: "docker", - environment: { - ref: process.env.OPENCLAW_QA_REF?.trim() || process.env.GITHUB_SHA?.trim() || null, - os: process.platform, - nodeVersion: process.version, - }, - provider, - channel: { - id: "telegram", - live: true, - driver: "native", - }, - packageSource: buildPackageSourceEvidence(), - artifacts: [ - { - kind: "summary", - path: "qa-evidence.json", - source: "telegram-rtt", - }, - { - kind: "report", - path: "telegram-qa-report.md", - source: "telegram-rtt", - }, - { - kind: "transport-observations", - path: "telegram-qa-observed-messages.json", - source: "telegram-rtt", - }, - ], - }, - result: { - status: scenario.status, - ...(scenario.status === "pass" ? {} : { failure: { reason: scenario.details } }), - ...(timing ? { timing } : {}), - }, - }; - }), - }; -} - -async function runWarmScenario(params) { - const samples = []; - let failures = 0; - let passed = 0; - for (let index = 0; passed < params.sampleCount; index += 1) { - const sampleMarker = `${successMarker}_${index + 1}`; - const sample = await runScenario({ - allowAnySutReply: false, - id: params.id, - input: `@${params.sutUsername} RTT sample ${index + 1}. Reply with exactly ${sampleMarker}.`, - matchText: sampleMarker, - replyToMessageId: params.replyToMessageId, - sampleIndex: index + 1, - sutId: params.sutId, - timeoutMs: params.sampleTimeoutMs, - title: params.title, - }); - if (sample.status === "fail") { - failures += 1; - } else { - passed += 1; - } - samples.push({ - index: index + 1, - status: sample.status, - details: sample.details, - ...(sample.rttMs === undefined ? {} : { rttMs: sample.rttMs }), - }); - if (failures >= params.maxFailures) { - break; - } - if (passed < params.sampleCount) { - await sleep(500); - } - } - - const stats = summarizeSamples(samples); - return { - id: params.id, - title: params.title, - status: stats.passed >= params.sampleCount ? "pass" : "fail", - details: `${stats.passed}/${stats.total} warm samples passed`, - rttMs: stats.p50Ms, - samples, - stats, - }; -} - -function reportMarkdown(summary) { - const lines = ["# Telegram RTT", ""]; - for (const scenario of summary.scenarios) { - lines.push(`## ${scenario.title}`, ""); - lines.push(`- Status: ${scenario.status}`); - lines.push(`- Details: ${scenario.details}`); - if (scenario.rttMs !== undefined) { - lines.push(`- RTT: ${scenario.rttMs}ms`); - } - if (scenario.stats) { - lines.push(`- Samples: ${scenario.stats.passed}/${scenario.stats.total}`); - if (scenario.stats.avgMs !== undefined) { - lines.push(`- Avg: ${scenario.stats.avgMs}ms`); - } - if (scenario.stats.p50Ms !== undefined) { - lines.push(`- P50: ${scenario.stats.p50Ms}ms`); - } - if (scenario.stats.p95Ms !== undefined) { - lines.push(`- P95: ${scenario.stats.p95Ms}ms`); - } - if (scenario.stats.maxMs !== undefined) { - lines.push(`- Max: ${scenario.stats.maxMs}ms`); - } - } - lines.push(""); - } - return lines.join("\n"); -} - -async function main() { - await fs.mkdir(outputDir, { recursive: true }); - const [driverMe, sutMe] = await Promise.all([driver.getMe(), sut.getMe()]); - driverUpdateOffset = (await flushUpdates(driver)) ?? driverUpdateOffset; - - const scenarios = []; - const canary = await runScenario({ - allowAnySutReply: false, - id: "telegram-canary", - input: `/status@${sutMe.username}`, - sutId: sutMe.id, - timeoutMs: canaryTimeoutMs, - title: "Telegram canary", - }); - scenarios.push(canary); - - if (scenarioIds.has("telegram-mentioned-message-reply")) { - scenarios.push( - await runWarmScenario({ - id: "telegram-mentioned-message-reply", - maxFailures: maxWarmFailures, - replyToMessageId: canary.messageId, - sampleCount: warmSampleCount, - sampleTimeoutMs, - sutId: sutMe.id, - sutUsername: sutMe.username, - title: "Telegram normal reply", - }), - ); - } - - const failed = scenarios.filter((scenario) => scenario.status === "fail").length; - const reportSummary = { - provider: "telegram", - driver: { id: driverMe.id, username: driverMe.username }, - sut: { id: sutMe.id, username: sutMe.username }, - startedAt: new Date().toISOString(), - status: failed > 0 ? "fail" : "pass", - totals: { total: scenarios.length, failed, passed: scenarios.length - failed }, - scenarios, - }; - const evidenceSummary = buildEvidenceSummary({ scenarios }); - - await fs.writeFile( - path.join(outputDir, "qa-evidence.json"), - `${JSON.stringify(evidenceSummary, null, 2)}\n`, - ); - await fs.writeFile(path.join(outputDir, "telegram-qa-report.md"), reportMarkdown(reportSummary)); - await fs.writeFile( - path.join(outputDir, "telegram-qa-observed-messages.json"), - `${JSON.stringify(observedMessages, null, 2)}\n`, - ); - - if (failed > 0) { - process.exitCode = 1; - } -} - -await main(); diff --git a/scripts/lib/rtt-harness.ts b/scripts/lib/rtt-harness.ts deleted file mode 100644 index 00d9a682c65..00000000000 --- a/scripts/lib/rtt-harness.ts +++ /dev/null @@ -1,362 +0,0 @@ -// Rtt Harness script supports OpenClaw repository automation. -import { execFile, spawn } from "node:child_process"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { promisify } from "node:util"; -import { - QA_EVIDENCE_FILENAME, - validateQaEvidenceSummaryJson, - type QaEvidenceSummaryJson, - type QaEvidenceTiming, -} from "../../extensions/qa-lab/src/evidence-summary.ts"; - -const execFileAsync = promisify(execFile); - -export type RttProviderMode = "mock-openai" | "live-frontier"; -export type RttCredentialSource = "env" | "convex"; -export type RttCredentialRole = "maintainer" | "ci"; - -type RttResult = { - package: { - spec: string; - version: string; - }; - run: { - id: string; - startedAt: string; - finishedAt: string; - durationMs: number; - status: "pass" | "fail"; - }; - mode: { - providerMode: RttProviderMode; - scenarios: string[]; - }; - rtt: { - canaryMs?: number; - mentionReplyMs?: number; - warmSamples?: number[]; - avgMs?: number; - p50Ms?: number; - p95Ms?: number; - maxMs?: number; - failedSamples?: number; - }; - artifacts: { - rawSummaryPath: string; - rawReportPath: string; - rawObservedMessagesPath: string; - resultPath: string; - }; -}; - -const OPENCLAW_PACKAGE_SPEC_RE = - /^openclaw@(main|alpha|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$/u; - -const REQUIRED_TELEGRAM_ENV = [ - "OPENCLAW_QA_TELEGRAM_GROUP_ID", - "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", - "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", -] as const; - -export function parseRttCredentialSource(value: string): RttCredentialSource { - const normalized = value.trim().toLowerCase(); - if (normalized === "env" || normalized === "convex") { - return normalized; - } - throw new Error(`--credential-source must be env or convex; got: ${value}`); -} - -export function parseRttCredentialRole(value: string): RttCredentialRole { - const normalized = value.trim().toLowerCase(); - if (normalized === "maintainer" || normalized === "ci") { - return normalized; - } - throw new Error(`--credential-role must be maintainer or ci; got: ${value}`); -} - -function resolveRttCredentialSource( - env: NodeJS.ProcessEnv, - credentialSource?: RttCredentialSource, -): RttCredentialSource { - if (credentialSource) { - return credentialSource; - } - const rawSource = - env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE; - if (rawSource?.trim()) { - return parseRttCredentialSource(rawSource); - } - if ( - env.CI && - env.OPENCLAW_QA_CONVEX_SITE_URL?.trim() && - (env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim() || env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim()) - ) { - return "convex"; - } - return "env"; -} - -function resolveRttCredentialRole( - env: NodeJS.ProcessEnv, - credentialRole?: RttCredentialRole, -): RttCredentialRole { - if (credentialRole) { - return credentialRole; - } - const rawRole = env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE ?? env.OPENCLAW_QA_CREDENTIAL_ROLE; - if (rawRole?.trim()) { - return parseRttCredentialRole(rawRole); - } - return env.CI ? "ci" : "maintainer"; -} - -export function validateOpenClawPackageSpec(spec: string) { - if (!OPENCLAW_PACKAGE_SPEC_RE.test(spec)) { - throw new Error( - `Package spec must be openclaw@main, openclaw@alpha, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: ${spec}`, - ); - } - return spec; -} - -export function safeRunLabel(input: string) { - return input.replace(/[^a-zA-Z0-9.-]+/gu, "_").replace(/^_+|_+$/gu, ""); -} - -export function buildRunId(params: { now: Date; spec: string; index?: number }) { - const stamp = params.now.toISOString().replaceAll(":", "").replaceAll(".", ""); - const suffix = params.index === undefined ? "" : `-${params.index + 1}`; - return `${stamp}-${safeRunLabel(params.spec)}${suffix}`; -} - -export function extractRtt(summary: QaEvidenceSummaryJson) { - const entries = summary.entries ?? []; - const findEntry = (id: string) => entries.find((entry) => entry.test?.id === id); - const canary = findEntry("telegram-canary")?.result?.timing; - const mention = findEntry("telegram-mentioned-message-reply")?.result?.timing; - const rtt: RttResult["rtt"] = { - canaryMs: canary?.rttMs, - mentionReplyMs: mention?.p50Ms ?? mention?.rttMs, - }; - appendRttTiming(rtt, mention); - return rtt; -} - -function appendRttTiming(rtt: RttResult["rtt"], timing: QaEvidenceTiming | undefined) { - if (timing?.avgMs !== undefined) { - rtt.avgMs = timing.avgMs; - } - if (timing?.p50Ms !== undefined) { - rtt.p50Ms = timing.p50Ms; - } - if (timing?.p95Ms !== undefined) { - rtt.p95Ms = timing.p95Ms; - } - if (timing?.maxMs !== undefined) { - rtt.maxMs = timing.maxMs; - } - if (timing?.failedSamples !== undefined) { - rtt.failedSamples = timing.failedSamples; - } -} - -export function createHarnessEnv(params: { - baseEnv: NodeJS.ProcessEnv; - credentialRole?: RttCredentialRole; - credentialSource?: RttCredentialSource; - packageTgz?: string; - providerMode: RttProviderMode; - scenarios: string[]; - spec: string; - version: string; - rawOutputDir: string; - samples: number; - sampleTimeoutMs: number; - timeoutMs: number; -}) { - const packageSourceSpec = params.packageTgz ?? params.spec; - return { - ...params.baseEnv, - OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC: params.spec, - ...(params.packageTgz ? { OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ: params.packageTgz } : {}), - OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL: `${params.spec} (${params.version})`, - OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE: params.providerMode, - OPENCLAW_QA_PACKAGE_SOURCE: packageSourceSpec, - OPENCLAW_QA_PACKAGE_SOURCE_KIND: params.packageTgz ? "packed-tarball" : "npm-package", - ...(params.credentialSource - ? { OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE: params.credentialSource } - : {}), - ...(params.credentialRole - ? { OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: params.credentialRole } - : {}), - OPENCLAW_NPM_TELEGRAM_SCENARIOS: params.scenarios.join(","), - OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: params.rawOutputDir, - OPENCLAW_NPM_TELEGRAM_FAST: params.baseEnv.OPENCLAW_NPM_TELEGRAM_FAST ?? "1", - OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: String(params.samples), - OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS: String(params.sampleTimeoutMs), - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: String(params.timeoutMs), - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: String(params.timeoutMs), - }; -} - -export function assertRequiredEnv( - env: NodeJS.ProcessEnv, - options: { - credentialRole?: RttCredentialRole; - credentialSource?: RttCredentialSource; - } = {}, -) { - const credentialSource = resolveRttCredentialSource(env, options.credentialSource); - if (credentialSource === "convex") { - const missing: string[] = []; - const credentialRole = resolveRttCredentialRole(env, options.credentialRole); - if (!env.OPENCLAW_QA_CONVEX_SITE_URL?.trim()) { - missing.push("OPENCLAW_QA_CONVEX_SITE_URL"); - } - if (credentialRole === "ci" && !env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim()) { - missing.push("OPENCLAW_QA_CONVEX_SECRET_CI"); - } - if (credentialRole === "maintainer" && !env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim()) { - missing.push("OPENCLAW_QA_CONVEX_SECRET_MAINTAINER"); - } - if (missing.length > 0) { - throw new Error(`Missing Convex Telegram QA credential env: ${missing.join(", ")}`); - } - return; - } - - const missing = REQUIRED_TELEGRAM_ENV.filter((key) => !env[key]?.trim()); - if (missing.length > 0) { - throw new Error(`Missing Telegram QA env: ${missing.join(", ")}`); - } -} - -export async function assertHarnessRoot(harnessRoot: string) { - const scriptPath = path.join(harnessRoot, "scripts/e2e/npm-telegram-rtt-docker.sh"); - try { - await fs.access(scriptPath); - } catch { - throw new Error(`Missing OpenClaw Telegram npm harness: ${scriptPath}`); - } -} - -export async function assertDockerAvailable() { - try { - await execFileAsync("docker", ["version", "--format", "{{.Server.Version}}"], { - timeout: 10_000, - }); - } catch { - throw new Error("Docker is required for RTT runs; install/start Docker and retry."); - } -} - -export async function resolvePublishedVersion(spec: string) { - const { stdout } = await execFileAsync("npm", ["view", spec, "version", "--json"], { - timeout: 30_000, - }); - const parsed = JSON.parse(stdout.trim()) as unknown; - if (typeof parsed !== "string" || parsed.trim().length === 0) { - throw new Error(`npm did not return a version for ${spec}.`); - } - return parsed.trim(); -} - -export async function resolveMainVersion(harnessRoot: string) { - const packageJson = JSON.parse( - await fs.readFile(path.join(harnessRoot, "package.json"), "utf8"), - ) as { version?: unknown }; - if (typeof packageJson.version !== "string" || packageJson.version.trim().length === 0) { - throw new Error("OpenClaw package.json must contain a non-empty version."); - } - const { stdout } = await execFileAsync("git", ["rev-parse", "--short=10", "HEAD"], { - cwd: harnessRoot, - timeout: 10_000, - }); - return `${packageJson.version.trim()}+${stdout.trim()}`; -} - -export async function readTelegramSummary(summaryPath: string) { - return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(summaryPath, "utf8"))); -} - -export async function resolveTelegramSummaryPath(outputDir: string) { - return path.join(outputDir, QA_EVIDENCE_FILENAME); -} - -export async function writeJson(pathname: string, value: unknown) { - await fs.mkdir(path.dirname(pathname), { recursive: true }); - await fs.writeFile(pathname, `${JSON.stringify(value, null, 2)}\n`); -} - -export async function appendJsonl(pathname: string, value: unknown) { - await fs.mkdir(path.dirname(pathname), { recursive: true }); - await fs.appendFile(pathname, `${JSON.stringify(value)}\n`); -} - -export async function runHarness(params: { env: NodeJS.ProcessEnv; harnessRoot: string }) { - const scriptPath = path.join(params.harnessRoot, "scripts/e2e/npm-telegram-rtt-docker.sh"); - const child = spawn("bash", [scriptPath], { - cwd: params.harnessRoot, - env: params.env, - stdio: "inherit", - }); - const exitCode = await new Promise((resolve, reject) => { - child.once("error", reject); - child.once("exit", resolve); - }); - return exitCode ?? 1; -} - -function rttSummaryFailed(summary: QaEvidenceSummaryJson, requestedScenarios: string[]) { - const entries = summary.entries ?? []; - const requiredScenarioIds = ["telegram-canary", ...requestedScenarios]; - for (const scenarioId of requiredScenarioIds) { - const entry = entries.find((candidate) => candidate.test?.id === scenarioId); - if (!entry || entry.result?.status !== "pass") { - return true; - } - const timing = entry.result.timing; - const rttMs = - scenarioId === "telegram-mentioned-message-reply" - ? (timing?.p50Ms ?? timing?.rttMs) - : timing?.rttMs; - if (typeof rttMs !== "number" || !Number.isFinite(rttMs)) { - return true; - } - } - return entries.some((entry) => entry.result?.status !== "pass"); -} - -export function buildRttResult(params: { - artifacts: RttResult["artifacts"]; - finishedAt: Date; - providerMode: RttProviderMode; - rawSummary: QaEvidenceSummaryJson; - runId: string; - scenarios: string[]; - spec: string; - startedAt: Date; - version: string; -}): RttResult { - const failed = rttSummaryFailed(params.rawSummary, params.scenarios); - return { - package: { - spec: params.spec, - version: params.version, - }, - run: { - id: params.runId, - startedAt: params.startedAt.toISOString(), - finishedAt: params.finishedAt.toISOString(), - durationMs: params.finishedAt.getTime() - params.startedAt.getTime(), - status: failed ? "fail" : "pass", - }, - mode: { - providerMode: params.providerMode, - scenarios: params.scenarios, - }, - rtt: extractRtt(params.rawSummary), - artifacts: params.artifacts, - }; -} diff --git a/scripts/mantis/build-telegram-evidence.mjs b/scripts/mantis/build-telegram-evidence.mjs index 11b518f6eae..05c70ff2cbc 100644 --- a/scripts/mantis/build-telegram-evidence.mjs +++ b/scripts/mantis/build-telegram-evidence.mjs @@ -324,6 +324,7 @@ export function renderTelegramEvidenceHtml({ observedMessages, summary }) { export function buildTelegramEvidenceManifest({ candidateRef, candidateSha, + hasObservedMessages = true, scenarioLabel, summary, summaryArtifactPath = "qa-evidence.json", @@ -381,13 +382,17 @@ export function buildTelegramEvidenceManifest({ path: summaryArtifactPath, targetPath: "summary.json", }, - { - kind: "metadata", - lane: "run", - label: "Telegram observed messages", - path: "telegram-qa-observed-messages.json", - targetPath: "observed-messages.json", - }, + ...(hasObservedMessages + ? [ + { + kind: "metadata", + lane: "run", + label: "Telegram observed messages", + path: "telegram-qa-observed-messages.json", + targetPath: "observed-messages.json", + }, + ] + : []), { kind: "metadata", lane: "run", @@ -449,16 +454,14 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) { mkdirSync(outputDir, { recursive: true }); const evidenceSummaryPath = path.join(outputDir, "qa-evidence.json"); const legacySummaryPath = path.join(outputDir, "telegram-qa-summary.json"); - const summaryPath = existsSync(evidenceSummaryPath) ? evidenceSummaryPath : legacySummaryPath; + const usesCurrentEvidenceSummary = existsSync(evidenceSummaryPath); + const summaryPath = usesCurrentEvidenceSummary ? evidenceSummaryPath : legacySummaryPath; const observedPath = path.join(outputDir, "telegram-qa-observed-messages.json"); const reportPath = path.join(outputDir, "telegram-qa-report.md"); if (!existsSync(summaryPath)) { throw new Error(`Missing Telegram QA evidence summary: ${evidenceSummaryPath}`); } - if (!existsSync(observedPath)) { - throw new Error(`Missing Telegram observed messages: ${observedPath}`); - } - const summary = existsSync(evidenceSummaryPath) + const summary = usesCurrentEvidenceSummary ? readJson(evidenceSummaryPath) : legacyTelegramSummaryToEvidenceSummary(readJson(legacySummaryPath)); const counts = evidenceCounts(summary); @@ -469,12 +472,14 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) { } writeFileSync(reportPath, "# Mantis Telegram Live QA\n\nTelegram QA report was unavailable.\n"); } - const observedMessages = readJson(observedPath); + const hasLegacyObservedMessages = !usesCurrentEvidenceSummary && existsSync(observedPath); + const observedMessages = hasLegacyObservedMessages ? readJson(observedPath) : []; const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages, summary }); writeFileSync(path.join(outputDir, "telegram-live-transcript.html"), transcriptHtml, "utf8"); const manifest = buildTelegramEvidenceManifest({ candidateRef: args.candidate_ref, candidateSha: args.candidate_sha, + hasObservedMessages: hasLegacyObservedMessages, scenarioLabel: args.scenario_label, summary, summaryArtifactPath: path.basename(summaryPath), diff --git a/scripts/rtt.ts b/scripts/rtt.ts deleted file mode 100644 index bdaf2f8c5ed..00000000000 --- a/scripts/rtt.ts +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env -S node --import tsx -// Rtt script supports OpenClaw repository automation. -import fs from "node:fs/promises"; -import path from "node:path"; -import { - appendJsonl, - assertDockerAvailable, - assertHarnessRoot, - assertRequiredEnv, - buildRttResult, - buildRunId, - createHarnessEnv, - readTelegramSummary, - resolveTelegramSummaryPath, - resolveMainVersion, - resolvePublishedVersion, - runHarness, - validateOpenClawPackageSpec, - writeJson, - parseRttCredentialRole, - parseRttCredentialSource, - type RttCredentialRole, - type RttCredentialSource, - type RttProviderMode, -} from "./lib/rtt-harness.ts"; - -const DEFAULT_SCENARIOS = ["telegram-mentioned-message-reply"]; -const DEFAULT_PROVIDER_MODE = "mock-openai" satisfies RttProviderMode; -const DEFAULT_TIMEOUT_MS = 180_000; -const DEFAULT_SAMPLES = 20; -const DEFAULT_SAMPLE_TIMEOUT_MS = 30_000; - -function usage() { - return [ - "Usage: pnpm rtt [--package-tgz PATH] [--provider mock-openai|live-frontier] [--credential-source env|convex] [--credential-role maintainer|ci] [--runs N] [--samples N] [--sample-timeout-ms N] [--timeout-ms N] [--harness-root PATH] [--output PATH]", - "", - "Examples:", - " pnpm rtt openclaw@main --package-tgz .artifacts/package/openclaw.tgz", - " pnpm rtt openclaw@beta", - " pnpm rtt openclaw@2026.4.30", - " pnpm rtt openclaw@latest --provider live-frontier", - ].join("\n"); -} - -function parseProviderMode(value: string): RttProviderMode { - if (value === "mock-openai" || value === "live-frontier") { - return value; - } - throw new Error(`--provider must be mock-openai or live-frontier; got: ${value}`); -} - -function parsePositiveInt(label: string, value: string) { - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`${label} must be a positive integer; got: ${value}`); - } - return parsed; -} - -function resolveHome(input: string) { - if (input === "~") { - return process.env.HOME ?? input; - } - if (input.startsWith("~/")) { - return path.join(process.env.HOME ?? "~", input.slice(2)); - } - return input; -} - -function readRequiredPathArg(argv: string[], index: number, flag: string) { - const value = argv[index + 1] ?? ""; - if (!value.trim() || value.startsWith("--")) { - throw new Error(`${flag} requires a path.`); - } - return value; -} - -function parseArgs(argv: string[]) { - let spec: string | undefined; - let credentialRole: RttCredentialRole | undefined; - let credentialSource: RttCredentialSource | undefined; - let packageTgz: string | undefined; - let providerMode = DEFAULT_PROVIDER_MODE; - let runs = 1; - let samples = DEFAULT_SAMPLES; - let sampleTimeoutMs = DEFAULT_SAMPLE_TIMEOUT_MS; - let harnessRoot = "~/Developer/clawdbot"; - let output = "runs"; - let timeoutMs = DEFAULT_TIMEOUT_MS; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--help" || arg === "-h") { - process.stdout.write(`${usage()}\n`); - process.exit(0); - } - if (arg === "--provider") { - providerMode = parseProviderMode(argv[++index] ?? ""); - continue; - } - if (arg === "--credential-source") { - credentialSource = parseRttCredentialSource(argv[++index] ?? ""); - continue; - } - if (arg === "--credential-role") { - credentialRole = parseRttCredentialRole(argv[++index] ?? ""); - continue; - } - if (arg === "--package-tgz") { - const value = readRequiredPathArg(argv, index, "--package-tgz"); - index += 1; - packageTgz = path.resolve(resolveHome(value)); - continue; - } - if (arg === "--runs") { - runs = parsePositiveInt("--runs", argv[++index] ?? ""); - continue; - } - if (arg === "--samples") { - samples = parsePositiveInt("--samples", argv[++index] ?? ""); - continue; - } - if (arg === "--sample-timeout-ms") { - sampleTimeoutMs = parsePositiveInt("--sample-timeout-ms", argv[++index] ?? ""); - continue; - } - if (arg === "--harness-root") { - harnessRoot = readRequiredPathArg(argv, index, "--harness-root"); - index += 1; - continue; - } - if (arg === "--timeout-ms") { - timeoutMs = parsePositiveInt("--timeout-ms", argv[++index] ?? ""); - continue; - } - if (arg === "--output") { - output = readRequiredPathArg(argv, index, "--output"); - index += 1; - continue; - } - if (arg.startsWith("--")) { - throw new Error(`Unknown option: ${arg}`); - } - if (spec) { - throw new Error(`Unexpected extra argument: ${arg}`); - } - spec = arg; - } - - if (!spec) { - throw new Error(`Missing package spec.\n${usage()}`); - } - - return { - spec: validateOpenClawPackageSpec(spec), - options: { - packageTgz, - credentialRole, - credentialSource, - providerMode, - runs, - samples, - sampleTimeoutMs, - harnessRoot: path.resolve(resolveHome(harnessRoot)), - output: path.resolve(resolveHome(output)), - scenarios: DEFAULT_SCENARIOS, - timeoutMs, - }, - }; -} - -async function runOne(params: { - index: number; - options: ReturnType["options"]; - spec: string; - version: string; -}) { - const runId = buildRunId({ now: new Date(), spec: params.spec, index: params.index }); - const runDir = path.join(params.options.output, runId); - const rawDir = path.join(runDir, "raw"); - const resultPath = path.join(runDir, "result.json"); - const harnessRawDir = path.join(params.options.harnessRoot, ".artifacts/rtt", runId, "raw"); - const rawOutputDir = path.relative(params.options.harnessRoot, harnessRawDir); - const startedAt = new Date(); - const env = createHarnessEnv({ - baseEnv: process.env, - credentialRole: params.options.credentialRole, - credentialSource: params.options.credentialSource, - packageTgz: params.options.packageTgz, - providerMode: params.options.providerMode, - rawOutputDir, - samples: params.options.samples, - sampleTimeoutMs: params.options.sampleTimeoutMs, - scenarios: params.options.scenarios, - spec: params.spec, - timeoutMs: params.options.timeoutMs, - version: params.version, - }); - - process.stderr.write(`[rtt] run ${params.index + 1}/${params.options.runs}: ${params.spec}\n`); - const harnessExitCode = await runHarness({ env, harnessRoot: params.options.harnessRoot }); - await readTelegramSummary(await resolveTelegramSummaryPath(harnessRawDir)); - await fs.rm(rawDir, { recursive: true, force: true }); - await fs.mkdir(path.dirname(rawDir), { recursive: true }); - await fs.cp(harnessRawDir, rawDir, { recursive: true }); - - const rawSummaryPath = await resolveTelegramSummaryPath(rawDir); - const rawReportPath = path.join(rawDir, "telegram-qa-report.md"); - const rawObservedMessagesPath = path.join(rawDir, "telegram-qa-observed-messages.json"); - const rawSummary = await readTelegramSummary(rawSummaryPath); - const finishedAt = new Date(); - const result = buildRttResult({ - artifacts: { - rawSummaryPath, - rawReportPath, - rawObservedMessagesPath, - resultPath, - }, - finishedAt, - providerMode: params.options.providerMode, - rawSummary, - runId, - scenarios: params.options.scenarios, - spec: params.spec, - startedAt, - version: params.version, - }); - - await writeJson(resultPath, result); - await appendJsonl(path.resolve("data/rtt.jsonl"), result); - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - return { - harnessExitCode, - result, - }; -} - -async function main() { - const { spec, options } = parseArgs(process.argv.slice(2)); - assertRequiredEnv(process.env, { - credentialRole: options.credentialRole, - credentialSource: options.credentialSource, - }); - await assertHarnessRoot(options.harnessRoot); - await assertDockerAvailable(); - if (spec === "openclaw@main" && !options.packageTgz) { - throw new Error("openclaw@main requires --package-tgz."); - } - const version = - spec === "openclaw@main" - ? await resolveMainVersion(options.harnessRoot) - : await resolvePublishedVersion(spec); - let failed = false; - for (let index = 0; index < options.runs; index += 1) { - const run = await runOne({ index, options, spec, version }); - failed = failed || run.harnessExitCode !== 0 || run.result.run.status === "fail"; - } - if (failed) { - process.exitCode = 1; - } -} - -if (import.meta.url === `file://${process.argv[1]}`) { - main().catch((error: unknown) => { - const message = error instanceof Error ? error.message : String(error); - process.stderr.write(`[rtt] ${message}\n`); - process.exitCode = 1; - }); -} - -export const testing = { - parseArgs, - parseProviderMode, - parsePositiveInt, - resolveHome, -}; -export { testing as __testing }; diff --git a/test/fixtures/telegram-qa-summary-rtt.json b/test/fixtures/telegram-qa-summary-rtt.json deleted file mode 100644 index 11f67ba30f5..00000000000 --- a/test/fixtures/telegram-qa-summary-rtt.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "credentials": { - "kind": "telegram", - "source": "env" - }, - "groupId": "-100123", - "startedAt": "2026-05-01T00:00:00.000Z", - "finishedAt": "2026-05-01T00:00:10.000Z", - "cleanupIssues": [], - "counts": { - "total": 2, - "passed": 2, - "failed": 0 - }, - "scenarios": [ - { - "id": "telegram-canary", - "title": "Telegram canary", - "status": "pass", - "details": "reply matched in 1234ms", - "rttMs": 1234 - }, - { - "id": "telegram-mentioned-message-reply", - "title": "Telegram mentioned message gets a reply", - "status": "pass", - "details": "3/3 warm samples passed", - "rttMs": 5000, - "samples": [ - { - "index": 1, - "status": "pass", - "details": "observed SUT message 101", - "rttMs": 4000 - }, - { - "index": 2, - "status": "pass", - "details": "observed SUT message 102", - "rttMs": 5000 - }, - { - "index": 3, - "status": "pass", - "details": "observed SUT message 103", - "rttMs": 7000 - } - ], - "stats": { - "total": 3, - "passed": 3, - "failed": 0, - "avgMs": 5333, - "p50Ms": 5000, - "p95Ms": 7000, - "maxMs": 7000 - } - } - ] -} diff --git a/test/scripts/mantis-build-telegram-evidence.test.ts b/test/scripts/mantis-build-telegram-evidence.test.ts index dd920ce26a7..6fa018acd46 100644 --- a/test/scripts/mantis-build-telegram-evidence.test.ts +++ b/test/scripts/mantis-build-telegram-evidence.test.ts @@ -167,7 +167,6 @@ describe("scripts/mantis/build-telegram-evidence", () => { expect(manifest.comparison.candidate.sha).toBe("abc123"); expect(manifest.artifacts.map((artifact) => artifact.targetPath)).toEqual([ "summary.json", - "observed-messages.json", "telegram-live-transcript.html", "report.md", "mantis-evidence.json", @@ -177,6 +176,35 @@ describe("scripts/mantis/build-telegram-evidence", () => { ); }); + it("does not require observed-message artifacts for current evidence summaries", () => { + const dir = makeTelegramOutput(); + rmSync(path.join(dir, "telegram-qa-observed-messages.json"), { force: true }); + + const result = writeTelegramEvidence(["--output-dir", dir]); + + expect(readFileSync(result.transcriptPath, "utf8")).toContain( + "No observed Telegram messages were recorded.", + ); + const targetPaths = result.manifest.artifacts.map((artifact) => artifact.targetPath); + expect(targetPaths).toContain("summary.json"); + expect(targetPaths).toContain("telegram-live-transcript.html"); + expect(targetPaths).toContain("report.md"); + expect(targetPaths).not.toContain("observed-messages.json"); + }); + + it("ignores stale observed-message files beside current evidence summaries", () => { + const dir = makeTelegramOutput(); + + const result = writeTelegramEvidence(["--output-dir", dir]); + + expect(readFileSync(result.transcriptPath, "utf8")).toContain( + "No observed Telegram messages were recorded.", + ); + expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).not.toContain( + "observed-messages.json", + ); + }); + it("renders historical Telegram summaries when evidence summaries are absent", () => { const dir = makeLegacyTelegramOutput(); @@ -187,6 +215,9 @@ describe("scripts/mantis/build-telegram-evidence", () => { expect( result.manifest.artifacts.find((artifact) => artifact.targetPath === "summary.json"), ).toMatchObject({ path: "telegram-qa-summary.json" }); + expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).toContain( + "observed-messages.json", + ); }); it("does not fabricate a required report artifact for passing Telegram summaries", () => { diff --git a/test/scripts/npm-telegram-live.test.ts b/test/scripts/npm-telegram-live.test.ts index 030ee8d85db..8ac1213f4a2 100644 --- a/test/scripts/npm-telegram-live.test.ts +++ b/test/scripts/npm-telegram-live.test.ts @@ -82,8 +82,11 @@ describe("package Telegram live Docker E2E", () => { expect(script).toContain('docker_e2e_print_log "$run_log"'); expect(script).not.toContain('cat "$run_log"'); expect(script).toContain('"${docker_env[@]}"'); - expect(script).toContain('if [ -z "$credential_role" ] && [ -n "${CI:-}" ]'); + expect(script).toContain( + 'if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then', + ); expect(script).toContain('credential_role="ci"'); + expect(script).toContain('credential_role="maintainer"'); }); it("bounds installed-package hot path OpenClaw commands", () => { @@ -142,6 +145,15 @@ describe("package Telegram live Docker E2E", () => { ); }); + it("forwards repeated RTT controls to the package Telegram live lane", () => { + const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); + + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES"); + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"); + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES"); + expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_CHECKS"); + }); + it("keeps private QA harness imports local while using the installed package dist", () => { const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); const preparePackage = readFileSync(PREPARE_PACKAGE_PATH, "utf8"); @@ -195,6 +207,43 @@ describe("package Telegram live Docker E2E", () => { ).toBe("ci"); }); + it("defaults package Telegram RTT for the normal package live lane", () => { + expect(testing.resolveRttOptions({})).toEqual({ + rttCount: 20, + rttTimeoutMs: undefined, + maxRttFailures: 20, + rttCheckIds: [], + }); + }); + + it("does not force default RTT onto focused non-RTT scenario runs", () => { + expect(testing.resolveRttOptions({}, ["telegram-canary"])).toEqual({}); + }); + + it("maps repeated RTT env onto package Telegram live options", () => { + expect( + testing.resolveRttOptions({ + OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7", + OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS: "45000", + OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES: "2", + OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply", + }), + ).toEqual({ + rttCount: 7, + rttTimeoutMs: 45_000, + maxRttFailures: 2, + rttCheckIds: ["telegram-mentioned-message-reply"], + }); + }); + + it("rejects invalid repeated RTT env", () => { + expect(() => + testing.resolveRttOptions({ + OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7samples", + }), + ).toThrow("invalid OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: 7samples"); + }); + it("gates package Telegram status on the summary artifact", async () => { const summaryPath = path.join(mkTempRoot(), "qa-evidence.json"); writeFileSync( diff --git a/test/scripts/npm-telegram-rtt-driver.test.ts b/test/scripts/npm-telegram-rtt-driver.test.ts deleted file mode 100644 index dc023503a36..00000000000 --- a/test/scripts/npm-telegram-rtt-driver.test.ts +++ /dev/null @@ -1,569 +0,0 @@ -// Npm Telegram Rtt Driver tests cover npm telegram rtt driver script behavior. -import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; -import { beforeAll, describe, expect, it } from "vitest"; -import { createBoundedChildOutput } from "../helpers/bounded-child-output.js"; - -const DRIVER_SCRIPT = "scripts/e2e/npm-telegram-rtt-driver.mjs"; -const QA_EVIDENCE_FILENAME = "qa-evidence.json"; - -type EvidenceSummaryForTest = { - kind?: string; - entries: Array<{ - test: { id: string }; - mapping?: { - coverage?: Array<{ id?: string }>; - }; - result: { - status?: string; - failure?: { reason?: string }; - }; - }>; -}; - -function runDriver(env: Record) { - return spawnSync(process.execPath, [DRIVER_SCRIPT], { - encoding: "utf8", - env: { - ...process.env, - OPENCLAW_QA_TELEGRAM_API_BASE_URL: "http://127.0.0.1:9", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token", - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token", - ...env, - }, - }); -} - -async function waitForFile(filePath: string, timeoutMs = 3000): Promise { - const startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { - if (existsSync(filePath)) { - return readFileSync(filePath, "utf8"); - } - await delay(25); - } - throw new Error(`timed out waiting for ${filePath}`); -} - -async function stopChild(child: ChildProcessWithoutNullStreams): Promise { - if (child.exitCode !== null) { - return; - } - child.kill("SIGTERM"); - const startedAt = Date.now(); - while (child.exitCode === null && Date.now() - startedAt < 1000) { - await delay(25); - } - if (child.exitCode === null) { - child.kill("SIGKILL"); - } -} - -function startStalledJsonServer(portPath: string) { - return spawn( - process.execPath, - [ - "--input-type=module", - "--eval", - [ - 'import net from "node:net";', - 'import fs from "node:fs";', - 'const server = net.createServer((socket) => socket.write("HTTP/1.1 200 OK\\r\\nContent-Type: application/json\\r\\n\\r\\n"));', - 'server.listen(0, "127.0.0.1", () => {', - " const address = server.address();", - " fs.writeFileSync(process.env.PORT_FILE, String(address.port));", - "});", - "setInterval(() => {}, 1000);", - ].join("\n"), - ], - { - env: { ...process.env, PORT_FILE: portPath }, - stdio: "pipe", - }, - ); -} - -function startOversizedJsonServer(portPath: string) { - return spawn( - process.execPath, - [ - "--input-type=module", - "--eval", - [ - 'import net from "node:net";', - 'import fs from "node:fs";', - "const server = net.createServer((socket) => {", - ' const body = JSON.stringify({ ok: true, result: { id: 1, username: "sut" }, padding: "x".repeat(128) });', - " socket.end(`HTTP/1.1 200 OK\\r\\nContent-Type: application/json\\r\\nContent-Length: ${Buffer.byteLength(body)}\\r\\n\\r\\n${body}`);", - "});", - 'server.listen(0, "127.0.0.1", () => {', - " const address = server.address();", - " fs.writeFileSync(process.env.PORT_FILE, String(address.port));", - "});", - "setInterval(() => {}, 1000);", - ].join("\n"), - ], - { - env: { ...process.env, PORT_FILE: portPath }, - stdio: "pipe", - }, - ); -} - -type DriverCaseResult = { - result: ReturnType; - elapsedMs?: number; -}; - -type AsyncDriverResult = { - status: number | null; - signal: NodeJS.Signals | null; - stdout: string; - stderr: string; - timedOut: boolean; -}; - -type TelegramUpdate = { - update_id: number; - message: { - message_id: number; - date: number; - chat: { id: number }; - from: { id: number; username: string }; - reply_to_message?: { message_id: number }; - text: string; - }; -}; - -function runDriverAsync(env: Record, timeoutMs = 5000): Promise { - return new Promise((resolve) => { - const child = spawn(process.execPath, [DRIVER_SCRIPT], { - env: { - ...process.env, - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token", - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token", - ...env, - }, - stdio: "pipe", - }); - const stdout = createBoundedChildOutput(); - const stderr = createBoundedChildOutput(); - let settled = false; - const timeout = setTimeout(() => { - if (settled) { - return; - } - settled = true; - child.kill("SIGKILL"); - resolve({ - status: null, - signal: "SIGKILL", - stdout: stdout.text(), - stderr: stderr.text(), - timedOut: true, - }); - }, timeoutMs); - timeout.unref?.(); - child.stdout.on("data", (chunk) => { - stdout.append(chunk); - }); - child.stderr.on("data", (chunk) => { - stderr.append(chunk); - }); - child.on("close", (status, signal) => { - if (settled) { - return; - } - settled = true; - clearTimeout(timeout); - resolve({ status, signal, stdout: stdout.text(), stderr: stderr.text(), timedOut: false }); - }); - }); -} - -function readRequestJson(req: IncomingMessage): Promise> { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - req.on("data", (chunk) => { - chunks.push(Buffer.from(chunk)); - }); - req.on("error", reject); - req.on("end", () => { - const raw = Buffer.concat(chunks).toString("utf8").trim(); - if (!raw) { - resolve({}); - return; - } - try { - resolve(JSON.parse(raw) as Record); - } catch (error) { - reject(error instanceof Error ? error : new Error(String(error))); - } - }); - }); -} - -function writeTelegramJson(res: ServerResponse, payload: unknown): void { - const body = `${JSON.stringify(payload)}\n`; - res.writeHead(200, { - "content-length": Buffer.byteLength(body), - "content-type": "application/json", - }); - res.end(body); -} - -function closeServer(server: Server): Promise { - return new Promise((resolve, reject) => { - server.close((error) => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); -} - -function readEvidenceSummary(outputDir: string) { - const summary = JSON.parse( - readFileSync(path.join(outputDir, QA_EVIDENCE_FILENAME), "utf8"), - ) as EvidenceSummaryForTest; - expect(Array.isArray(summary.entries)).toBe(true); - return summary; -} - -async function startTelegramApiServer(options: { - canaryReplyToRequest: boolean; -}): Promise<{ baseUrl: string; close: () => Promise }> { - const pendingUpdates: TelegramUpdate[] = []; - let nextMessageId = 100; - let nextUpdateId = 1; - const chatId = -100123; - const nowUnixSeconds = () => Math.floor(Date.now() / 1000); - const pushSutUpdate = (params: { replyToMessageId?: number; text: string }) => { - pendingUpdates.push({ - update_id: nextUpdateId++, - message: { - message_id: nextMessageId++, - date: nowUnixSeconds(), - chat: { id: chatId }, - from: { id: 222, username: "sut_bot" }, - ...(params.replyToMessageId === undefined - ? {} - : { reply_to_message: { message_id: params.replyToMessageId } }), - text: params.text, - }, - }); - }; - - const server = createServer((req, res) => { - void (async () => { - const match = req.url?.match(/^\/bot([^/]+)\/([^/?]+)/u); - if (!match) { - res.writeHead(404).end(); - return; - } - const [, token, method] = match; - const body = await readRequestJson(req); - - if (method === "getMe") { - writeTelegramJson(res, { - ok: true, - result: - token === "sut-token" - ? { id: 222, username: "sut_bot" } - : { id: 111, username: "driver_bot" }, - }); - return; - } - - if (method === "sendMessage") { - const messageId = nextMessageId++; - const text = typeof body.text === "string" ? body.text : ""; - if (token === "driver-token" && text.startsWith("/status@")) { - pushSutUpdate({ - replyToMessageId: options.canaryReplyToRequest ? messageId : undefined, - text: "status ok", - }); - } else if (token === "driver-token" && text.includes("Reply with exactly ")) { - const marker = text.match(/Reply with exactly ([^.]+)\./u)?.[1] ?? "OPENCLAW_E2E_OK_1"; - pushSutUpdate({ replyToMessageId: messageId, text: marker }); - } - writeTelegramJson(res, { - ok: true, - result: { - message_id: messageId, - date: nowUnixSeconds(), - chat: { id: chatId }, - text, - }, - }); - return; - } - - if (method === "getUpdates") { - const offset = typeof body.offset === "number" ? body.offset : 0; - const updates = pendingUpdates.filter((update) => update.update_id >= offset); - if (updates.length === 0) { - await delay(25); - } - writeTelegramJson(res, { ok: true, result: updates }); - return; - } - - writeTelegramJson(res, { ok: false, description: `unexpected method ${method}` }); - })().catch((error: unknown) => { - res.writeHead(500, { "content-type": "text/plain" }).end(String(error)); - }); - }); - - await new Promise((resolve) => { - server.listen(0, "127.0.0.1", resolve); - }); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("fake Telegram server did not bind a TCP port"); - } - return { - baseUrl: `http://127.0.0.1:${address.port}`, - close: () => closeServer(server), - }; -} - -async function runStalledTelegramBodyCase(): Promise { - const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-driver-")); - const portPath = path.join(root, "port.txt"); - const outputDir = path.join(root, "out"); - const server = startStalledJsonServer(portPath); - - try { - const port = Number.parseInt((await waitForFile(portPath)).trim(), 10); - const startedAt = Date.now(); - const result = spawnSync(process.execPath, [DRIVER_SCRIPT], { - encoding: "utf8", - env: { - ...process.env, - OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS: "100", - OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir, - OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1", - OPENCLAW_QA_TELEGRAM_API_BASE_URL: `http://127.0.0.1:${port}`, - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token", - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token", - }, - killSignal: "SIGKILL", - timeout: 2500, - }); - return { result, elapsedMs: Date.now() - startedAt }; - } finally { - await stopChild(server); - rmSync(root, { force: true, recursive: true }); - } -} - -async function runOversizedTelegramBodyCase(): Promise { - const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-driver-")); - const portPath = path.join(root, "port.txt"); - const outputDir = path.join(root, "out"); - const server = startOversizedJsonServer(portPath); - - try { - const port = Number.parseInt((await waitForFile(portPath)).trim(), 10); - const result = spawnSync(process.execPath, [DRIVER_SCRIPT], { - encoding: "utf8", - env: { - ...process.env, - OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES: "16", - OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS: "1000", - OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir, - OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1", - OPENCLAW_QA_TELEGRAM_API_BASE_URL: `http://127.0.0.1:${port}`, - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token", - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token", - }, - killSignal: "SIGKILL", - timeout: 2500, - }); - return { result }; - } finally { - await stopChild(server); - rmSync(root, { force: true, recursive: true }); - } -} - -describe("npm Telegram RTT driver", () => { - let stalledBodyCase: DriverCaseResult; - let oversizedBodyCase: DriverCaseResult; - - beforeAll(async () => { - [stalledBodyCase, oversizedBodyCase] = await Promise.all([ - runStalledTelegramBodyCase(), - runOversizedTelegramBodyCase(), - ]); - }); - - it("rejects loose numeric env values instead of parsing prefixes", () => { - for (const [name, value] of [ - ["OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", "180000ms"], - ["OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", "1e3"], - ["OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES", "20samples"], - ["OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS", "30000ms"], - ["OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS", "100ms"], - ["OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES", "1mb"], - ["OPENCLAW_NPM_TELEGRAM_MAX_FAILURES", "2failures"], - ]) { - const result = runDriver({ [name]: value }); - - expect(result.status).not.toBe(0); - expect(result.stderr).toContain(`invalid ${name}: ${value}`); - } - }); - - it("rejects zero where positive numeric env values are required", () => { - const result = runDriver({ OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "0" }); - - expect(result.status).not.toBe(0); - expect(result.stderr).toContain("invalid OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: 0"); - }); - - it("rejects empty scenario selections before live Telegram calls", () => { - const result = runDriver({ OPENCLAW_NPM_TELEGRAM_SCENARIOS: "," }); - - expect(result.status).not.toBe(0); - expect(result.stderr).toContain( - "OPENCLAW_NPM_TELEGRAM_SCENARIOS must include at least one RTT scenario", - ); - }); - - it("rejects unknown scenario selections before live Telegram calls", () => { - const result = runDriver({ OPENCLAW_NPM_TELEGRAM_SCENARIOS: "does-not-exist" }); - - expect(result.status).not.toBe(0); - expect(result.stderr).toContain("unknown OPENCLAW_NPM_TELEGRAM_SCENARIOS: does-not-exist"); - }); - - it("bounds stalled Telegram Bot API response bodies", async () => { - const { result, elapsedMs } = stalledBodyCase; - - expect(result.error).toBeUndefined(); - expect(result.signal).not.toBe("SIGKILL"); - expect(result.status).not.toBe(0); - expect(result.stderr).toMatch(/abort|timed out|terminated/iu); - expect(elapsedMs).toBeLessThan(2500); - }); - - it("bounds oversized Telegram Bot API response bodies", async () => { - const { result } = oversizedBodyCase; - - expect(result.error).toBeUndefined(); - expect(result.signal).not.toBe("SIGKILL"); - expect(result.status).not.toBe(0); - expect(result.stderr).toContain("Telegram Bot API getMe response body exceeded 16 bytes"); - expect(result.stderr).not.toContain("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - }); - - it("rejects unrelated SUT messages during the Telegram canary", async () => { - const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-causal-")); - const server = await startTelegramApiServer({ canaryReplyToRequest: false }); - - try { - const outputDir = path.join(root, "out"); - const result = await runDriverAsync({ - OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir, - OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1", - OPENCLAW_QA_TELEGRAM_API_BASE_URL: server.baseUrl, - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "250", - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "250", - }); - const summary = readEvidenceSummary(outputDir); - const canary = summary.entries.find((entry) => entry.test.id === "telegram-canary"); - - expect(result.timedOut).toBe(false); - expect(result.status).not.toBe(0); - expect(canary).toMatchObject({ - test: { id: "telegram-canary" }, - result: { status: "fail" }, - }); - expect(canary?.result.failure?.reason).toContain("timed out"); - expect(existsSync(path.join(outputDir, "telegram-qa-summary.json"))).toBe(false); - } finally { - await server.close(); - rmSync(root, { force: true, recursive: true }); - } - }); - - it("accepts reply-threaded SUT messages during the Telegram canary", async () => { - const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-causal-")); - const server = await startTelegramApiServer({ canaryReplyToRequest: true }); - - try { - const outputDir = path.join(root, "out"); - const result = await runDriverAsync({ - OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir, - OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1", - OPENCLAW_QA_PACKAGE_SOURCE: "/package-under-test/openclaw.tgz", - OPENCLAW_QA_PACKAGE_SOURCE_KIND: "packed-tarball", - OPENCLAW_QA_PACKAGE_SOURCE_SHA: "abc123", - OPENCLAW_QA_TELEGRAM_API_BASE_URL: server.baseUrl, - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000", - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000", - }); - const summary = readEvidenceSummary(outputDir); - - expect(result).toMatchObject({ - signal: null, - status: 0, - timedOut: false, - }); - expect(summary.kind).toBe("openclaw.qa.evidence-summary"); - expect(summary.entries).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - test: expect.objectContaining({ id: "telegram-canary" }), - mapping: expect.objectContaining({ - coverage: expect.arrayContaining([ - expect.objectContaining({ id: "channels.telegram.canary" }), - ]), - }), - result: expect.objectContaining({ status: "pass" }), - }), - expect.objectContaining({ - test: expect.objectContaining({ id: "telegram-mentioned-message-reply" }), - mapping: expect.objectContaining({ - coverage: expect.arrayContaining([ - expect.objectContaining({ id: "channels.telegram.mention-gating" }), - ]), - }), - result: expect.objectContaining({ - status: "pass", - timing: expect.objectContaining({ - failedSamples: 0, - samples: 1, - }), - }), - }), - ]), - ); - expect(summary.entries[0]?.execution.packageSource).toEqual({ - kind: "packed-tarball", - spec: "/package-under-test/openclaw.tgz", - sha: "abc123", - }); - expect(existsSync(path.join(outputDir, "telegram-qa-summary.json"))).toBe(false); - } finally { - await server.close(); - rmSync(root, { force: true, recursive: true }); - } - }); -}); diff --git a/test/scripts/rtt-harness.test.ts b/test/scripts/rtt-harness.test.ts deleted file mode 100644 index 15f24ad970d..00000000000 --- a/test/scripts/rtt-harness.test.ts +++ /dev/null @@ -1,808 +0,0 @@ -// Rtt Harness tests cover rtt harness script behavior. -import { execFile } from "node:child_process"; -import fs from "node:fs/promises"; -import { createServer, type Server } from "node:http"; -import os from "node:os"; -import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { promisify } from "node:util"; -import { afterEach, describe, expect, it } from "vitest"; -import { - appendJsonl, - assertRequiredEnv, - buildRttResult, - buildRunId, - createHarnessEnv, - extractRtt, - resolveTelegramSummaryPath, - safeRunLabel, - validateOpenClawPackageSpec, -} from "../../scripts/lib/rtt-harness.ts"; -import { testing as cliTesting } from "../../scripts/rtt.ts"; - -const TEST_DIR = path.dirname(fileURLToPath(import.meta.url)); -const DOCKER_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-docker.sh"); -const CREDENTIAL_SCRIPT_PATH = path.resolve( - TEST_DIR, - "../../scripts/e2e/npm-telegram-rtt-credentials.mjs", -); -const CONFIG_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-config.mjs"); -const QA_EVIDENCE_FILENAME = "qa-evidence.json"; -const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1"; -const execFileAsync = promisify(execFile); -const tempDirs: string[] = []; - -type EvidenceStatus = "pass" | "fail" | "blocked" | "skipped"; -type EvidenceTiming = { - rttMs?: number; - avgMs?: number; - p50Ms?: number; - p95Ms?: number; - maxMs?: number; - samples?: number; - failedSamples?: number; -}; - -type EvidenceSummaryForTest = { - kind: "openclaw.qa.evidence-summary"; - schemaVersion: 2; - generatedAt: string; - entries: Array<{ - test: { - kind: string; - id: string; - title: string; - }; - mapping: { - profile: string; - coverage: []; - }; - execution: { - runner: string; - environment: { ref: null; os: string; nodeVersion: string }; - provider: { - id: string; - live: boolean; - model: { name: null; ref: null }; - fixture: string; - }; - packageSource: { kind: string; spec: string }; - artifacts: []; - }; - result: { - status: EvidenceStatus; - timing?: EvidenceTiming; - }; - }>; -}; - -function makeTelegramRttEvidenceSummary( - options: { - canaryStatus?: EvidenceStatus; - canaryTiming?: EvidenceTiming; - mentionStatus?: EvidenceStatus; - mentionTiming?: EvidenceTiming; - } = {}, -): EvidenceSummaryForTest { - const canaryStatus = options.canaryStatus ?? "pass"; - const canaryTiming = Object.hasOwn(options, "canaryTiming") - ? options.canaryTiming - : { rttMs: 1234 }; - const mentionStatus = options.mentionStatus ?? "pass"; - const mentionTiming = Object.hasOwn(options, "mentionTiming") - ? options.mentionTiming - : { - rttMs: 6000, - avgMs: 5333, - p50Ms: 5000, - p95Ms: 7000, - maxMs: 7000, - samples: 3, - failedSamples: 0, - }; - const entry = ( - id: string, - title: string, - status: EvidenceStatus, - timing: EvidenceTiming | undefined, - ): EvidenceSummaryForTest["entries"][number] => { - const result = timing === undefined ? { status } : { status, timing }; - return { - test: { - kind: "live-transport-check", - id, - title, - }, - mapping: { profile: "release", coverage: [] }, - execution: { - runner: "docker", - environment: { ref: null, os: "linux", nodeVersion: "v24.0.0" }, - provider: { - id: "openai", - live: false, - model: { name: null, ref: null }, - fixture: "mock-openai", - }, - packageSource: { kind: "npm-package", spec: "openclaw@beta" }, - artifacts: [], - }, - result, - }; - }; - return { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-05-01T00:00:00.000Z", - entries: [ - entry("telegram-canary", "Telegram canary", canaryStatus, canaryTiming), - entry( - "telegram-mentioned-message-reply", - "Telegram normal reply", - mentionStatus, - mentionTiming, - ), - ], - }; -} - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); -}); - -async function listenOnLoopback(server: Server) { - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - server.off("error", reject); - resolve(); - }); - }); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected TCP server address."); - } - return address; -} - -function closeServer(server: Server) { - return new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); -} - -function credentialBrokerEnv(port: number) { - return { - ...process.env, - OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1", - OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "test-secret", - OPENCLAW_QA_CONVEX_SITE_URL: `http://127.0.0.1:${port}`, - OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "1000", - OPENCLAW_QA_CREDENTIAL_OWNER_ID: "test-owner", - OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: "maintainer", - }; -} - -describe("RTT harness", () => { - it("validates OpenClaw package specs", () => { - expect(validateOpenClawPackageSpec("openclaw@main")).toBe("openclaw@main"); - expect(validateOpenClawPackageSpec("openclaw@alpha")).toBe("openclaw@alpha"); - expect(validateOpenClawPackageSpec("openclaw@beta")).toBe("openclaw@beta"); - expect(validateOpenClawPackageSpec("openclaw@latest")).toBe("openclaw@latest"); - expect(validateOpenClawPackageSpec("openclaw@2026.4.30")).toBe("openclaw@2026.4.30"); - expect(validateOpenClawPackageSpec("openclaw@2026.4.30-beta.2")).toBe( - "openclaw@2026.4.30-beta.2", - ); - expect(validateOpenClawPackageSpec("openclaw@2026.4.30-alpha.2")).toBe( - "openclaw@2026.4.30-alpha.2", - ); - - expect(() => validateOpenClawPackageSpec("@openclaw/openclaw@beta")).toThrow( - /Package spec must be/, - ); - expect(() => validateOpenClawPackageSpec("openclaw@next")).toThrow(/Package spec must be/); - }); - - it("builds stable run labels", () => { - expect(safeRunLabel("openclaw@beta")).toBe("openclaw_beta"); - expect( - buildRunId({ - now: new Date("2026-05-01T03:04:05.678Z"), - spec: "openclaw@beta", - index: 1, - }), - ).toBe("2026-05-01T030405678Z-openclaw_beta-2"); - }); - - it("constructs harness env without dropping caller env", () => { - const env = createHarnessEnv({ - baseEnv: { - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_NPM_TELEGRAM_FAST: "0", - }, - providerMode: "mock-openai", - rawOutputDir: ".artifacts/rtt/run/raw", - samples: 20, - sampleTimeoutMs: 30_000, - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@beta", - timeoutMs: 180_000, - version: "2026.4.30-beta.1", - }); - - expect(env.OPENCLAW_QA_TELEGRAM_GROUP_ID).toBe("-100123"); - expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC).toBe("openclaw@beta"); - expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL).toBe("openclaw@beta (2026.4.30-beta.1)"); - expect(env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE).toBe("mock-openai"); - expect(env.OPENCLAW_QA_PACKAGE_SOURCE).toBe("openclaw@beta"); - expect(env.OPENCLAW_QA_PACKAGE_SOURCE_KIND).toBe("npm-package"); - expect(env.OPENCLAW_NPM_TELEGRAM_SCENARIOS).toBe("telegram-mentioned-message-reply"); - expect(env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR).toBe(".artifacts/rtt/run/raw"); - expect(env.OPENCLAW_NPM_TELEGRAM_FAST).toBe("0"); - expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("20"); - expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("30000"); - expect(env.OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS).toBe("180000"); - expect(env.OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS).toBe("180000"); - }); - - it("marks package tarball provenance in RTT evidence env", () => { - const env = createHarnessEnv({ - baseEnv: {}, - packageTgz: "/tmp/openclaw.tgz", - providerMode: "mock-openai", - rawOutputDir: ".artifacts/rtt/run/raw", - samples: 20, - sampleTimeoutMs: 30_000, - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@main", - timeoutMs: 180_000, - version: "2026.4.30+abc123", - }); - - expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC).toBe("openclaw@main"); - expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ).toBe("/tmp/openclaw.tgz"); - expect(env.OPENCLAW_QA_PACKAGE_SOURCE).toBe("/tmp/openclaw.tgz"); - expect(env.OPENCLAW_QA_PACKAGE_SOURCE_KIND).toBe("packed-tarball"); - }); - - it("forwards Convex credential controls without dropping RTT sample controls", () => { - const env = createHarnessEnv({ - baseEnv: { - OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-credentials.example.convex.site", - OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret", - }, - credentialRole: "maintainer", - credentialSource: "convex", - providerMode: "mock-openai", - rawOutputDir: ".artifacts/rtt/run/raw", - samples: 7, - sampleTimeoutMs: 45_000, - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@beta", - timeoutMs: 180_000, - version: "2026.4.30-beta.1", - }); - - expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE).toBe("convex"); - expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE).toBe("maintainer"); - expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("7"); - expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("45000"); - expect(() => - assertRequiredEnv(env, { credentialRole: "maintainer", credentialSource: "convex" }), - ).not.toThrow(); - }); - - it("exports the Telegram bot token after Convex credentials are sourced", async () => { - const script = await fs.readFile(DOCKER_SCRIPT_PATH, "utf8"); - const sourceIndex = script.indexOf('source "$credential_env_file"'); - const tokenExportIndex = script.indexOf( - 'export TELEGRAM_BOT_TOKEN="${OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN:?missing OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN}"', - ); - const installEnvSnapshotIndex = script.indexOf('install_env=("${docker_env[@]}")'); - const convexSecretForwardIndex = script.indexOf( - "OPENCLAW_QA_CONVEX_SECRET_CI", - installEnvSnapshotIndex, - ); - const bodyLimitForwardIndex = script.indexOf( - "OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES", - installEnvSnapshotIndex, - ); - const payloadByteLimitForwardIndex = script.indexOf( - "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES", - installEnvSnapshotIndex, - ); - const payloadChunkLimitForwardIndex = script.indexOf( - "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS", - installEnvSnapshotIndex, - ); - const packageInstallIndex = script.indexOf("npm install -g"); - const credentialAcquireIndex = script.indexOf( - "node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs acquire", - ); - const heartbeatStartIndex = script.indexOf("start_credential_heartbeat", sourceIndex); - const driverIndex = script.indexOf("node /app/scripts/e2e/npm-telegram-rtt-driver.mjs"); - - expect(sourceIndex).toBeGreaterThanOrEqual(0); - expect(tokenExportIndex).toBeGreaterThan(sourceIndex); - expect(installEnvSnapshotIndex).toBeGreaterThanOrEqual(0); - expect(convexSecretForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); - expect(bodyLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); - expect(payloadByteLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); - expect(payloadChunkLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex); - expect(packageInstallIndex).toBeLessThan(credentialAcquireIndex); - expect(script).toContain( - '-e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"', - ); - expect(script).toContain('-e OPENCLAW_QA_PACKAGE_SOURCE="$package_install_source"'); - expect(script).toContain('-e OPENCLAW_QA_PACKAGE_SOURCE_KIND="$package_source_kind"'); - expect(script).toContain("OPENCLAW_QA_PACKAGE_SOURCE_SHA"); - expect(script).toContain( - '"$timeout_bin" --kill-after=30s "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit', - ); - expect(script).toContain("elif command -v gtimeout >/dev/null 2>&1; then"); - expect(script).toContain('timeout_bin="gtimeout"'); - expect(script).toContain( - 'echo "timeout or gtimeout is required for OPENCLAW_E2E_NPM_INSTALL_TIMEOUT=$npm_install_timeout" >&2', - ); - expect(script).toContain('"$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1'); - expect(script).toContain( - '"$timeout_bin" "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit', - ); - expect(script).not.toContain( - "running package install without OPENCLAW_E2E_NPM_INSTALL_TIMEOUT", - ); - expect(script).toContain("run_logged docker_e2e_docker_run_cmd run --rm"); - expect(script).not.toContain("run_logged docker run --rm"); - expect(script).toContain("source scripts/lib/openclaw-e2e-instance.sh"); - expect(script).toContain('docker_e2e_print_log "$run_log"'); - expect(script).not.toContain('cat "$run_log"'); - expect(heartbeatStartIndex).toBeGreaterThan(sourceIndex); - expect(heartbeatStartIndex).toBeLessThan(driverIndex); - expect(script).toContain("start_credential_heartbeat() {\n (\n set +e"); - expect(script).toContain("Convex credential heartbeat exited with status"); - expect(script).toContain('kill -TERM "$rtt_shell_pid"'); - expect(script).toContain("const controller = new AbortController();"); - expect(script).toContain("const timer = setTimeout(() => controller.abort(), 1000);"); - expect(script).toContain('if [ "$mock_ready" != "1" ]; then'); - expect(script).toContain("Mock OpenAI server did not become ready"); - expect(script).toContain('openclaw_e2e_print_log "$mock_log"'); - expect(script).toContain('openclaw_e2e_print_log "$file"'); - expect(script).not.toContain('cat "$mock_log"'); - expect(script).not.toContain("sed -n '1,260p'"); - expect(script).not.toContain("fetch('http://127.0.0.1:${mock_port}/health')"); - expect(script).not.toContain('export TELEGRAM_BOT_TOKEN="$OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"'); - }); - - it("rejects oversized chunked RTT credential markers before hydration", async () => { - const credentialModule = (await import( - `${pathToFileURL(CREDENTIAL_SCRIPT_PATH).href}?case=chunk-marker-${Date.now()}` - )) as { - parseChunkedPayloadMarker(payload: unknown): unknown; - }; - - expect(() => - credentialModule.parseChunkedPayloadMarker({ - [CHUNKED_PAYLOAD_MARKER]: true, - byteLength: 1, - chunkCount: 4097, - }), - ).toThrow("Chunked credential payload exceeds 4096 chunks."); - expect(() => - credentialModule.parseChunkedPayloadMarker({ - [CHUNKED_PAYLOAD_MARKER]: true, - byteLength: 64 * 1024 * 1024 + 1, - chunkCount: 1, - }), - ).toThrow("Chunked credential payload exceeds 67108864 bytes."); - }); - - it("keeps RTT Docker artifacts isolated by default", async () => { - const script = await fs.readFile(DOCKER_SCRIPT_PATH, "utf8"); - - expect(script).toContain( - 'RUN_ID="${OPENCLAW_NPM_TELEGRAM_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"', - ); - expect(script).toContain( - 'OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt/$RUN_ID}"', - ); - expect(script).toContain('-e OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR="$OUTPUT_DIR"'); - expect(script).not.toContain( - 'OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt}"', - ); - }); - - it("keeps broker helper heartbeat handling aligned with QA leases", async () => { - const script = await fs.readFile(CREDENTIAL_SCRIPT_PATH, "utf8"); - - expect(script).toContain("leaseTtlMs: acquired.leaseTtlMs ?? config.leaseTtlMs"); - expect(script).toContain("leaseTtlMs: leaseTtlMsFromLease(config, lease)"); - }); - - it("bounds Convex credential broker response bodies", async () => { - const server = createServer((_request, response) => { - response.writeHead(500, { "content-type": "application/json" }); - response.end(JSON.stringify({ status: "error", message: "x".repeat(128) })); - }); - const { port } = await listenOnLoopback(server); - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-")); - tempDirs.push(tempDir); - - try { - await execFileAsync( - process.execPath, - [ - CREDENTIAL_SCRIPT_PATH, - "acquire", - "--lease-file", - path.join(tempDir, "lease.json"), - "--credential-env-file", - path.join(tempDir, "credentials.env"), - ], - { - env: { - ...credentialBrokerEnv(port), - OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES: "16", - }, - maxBuffer: 128 * 1024, - }, - ); - throw new Error("Expected credential acquire to fail."); - } catch (error) { - const execError = error as Error & { stderr?: string }; - expect(execError.stderr).toContain( - "credential broker acquire response body exceeded 16 bytes", - ); - expect(execError.stderr).not.toContain("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); - } finally { - await closeServer(server); - } - }); - - it("does not start another credential acquire after retry delay exhausts the deadline", async () => { - let requests = 0; - const server = createServer((_request, response) => { - requests += 1; - response.writeHead(503, { "content-type": "application/json" }); - response.end( - JSON.stringify({ - status: "error", - code: "POOL_EXHAUSTED", - message: "credential pool exhausted", - retryAfterMs: 1_000, - }), - ); - }); - const { port } = await listenOnLoopback(server); - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-retry-")); - tempDirs.push(tempDir); - const startedAt = Date.now(); - - try { - await execFileAsync( - process.execPath, - [ - CREDENTIAL_SCRIPT_PATH, - "acquire", - "--lease-file", - path.join(tempDir, "lease.json"), - "--credential-env-file", - path.join(tempDir, "credentials.env"), - ], - { - env: { - ...credentialBrokerEnv(port), - OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "75", - OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "250", - }, - maxBuffer: 128 * 1024, - }, - ); - throw new Error("Expected credential acquire to fail."); - } catch (error) { - const execError = error as Error & { stderr?: string }; - expect(execError.stderr).toContain("credential broker acquire timed out after 75ms"); - expect(Date.now() - startedAt).toBeLessThan(500); - expect(requests).toBe(1); - } finally { - await closeServer(server); - } - }); - - it("caps credential acquire HTTP retries to the remaining acquire deadline", async () => { - let requests = 0; - const server = createServer((_request, response) => { - requests += 1; - if (requests === 1) { - response.writeHead(503, { "content-type": "application/json" }); - response.end( - JSON.stringify({ - status: "error", - code: "POOL_EXHAUSTED", - message: "credential pool exhausted", - retryAfterMs: 1, - }), - ); - } - }); - const { port } = await listenOnLoopback(server); - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-cap-")); - tempDirs.push(tempDir); - const startedAt = Date.now(); - - try { - await execFileAsync( - process.execPath, - [ - CREDENTIAL_SCRIPT_PATH, - "acquire", - "--lease-file", - path.join(tempDir, "lease.json"), - "--credential-env-file", - path.join(tempDir, "credentials.env"), - ], - { - env: { - ...credentialBrokerEnv(port), - OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "100", - OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "900", - }, - maxBuffer: 128 * 1024, - }, - ); - throw new Error("Expected credential acquire to fail."); - } catch (error) { - const execError = error as Error & { stderr?: string }; - expect(execError.stderr).toContain("credential broker acquire timed out after"); - expect(Date.now() - startedAt).toBeLessThan(500); - expect(requests).toBe(2); - } finally { - await closeServer(server); - } - }); - - it("preserves empty broker responses for successful lease release", async () => { - const server = createServer((_request, response) => { - response.writeHead(204); - response.end(); - }); - const { port } = await listenOnLoopback(server); - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-")); - tempDirs.push(tempDir); - const leaseFile = path.join(tempDir, "lease.json"); - await fs.writeFile( - leaseFile, - `${JSON.stringify({ - kind: "telegram", - ownerId: "test-owner", - actorRole: "maintainer", - credentialId: "credential", - leaseToken: "lease", - })}\n`, - ); - - try { - await execFileAsync( - process.execPath, - [CREDENTIAL_SCRIPT_PATH, "release", "--lease-file", leaseFile], - { - env: credentialBrokerEnv(port), - }, - ); - await expect(fs.stat(leaseFile)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - await closeServer(server); - } - }); - - it("generates final-only Telegram RTT delivery config for release packages", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-config-test-")); - tempDirs.push(tempDir); - const configPath = path.join(tempDir, "config.json"); - - await execFileAsync(process.execPath, [ - CONFIG_SCRIPT_PATH, - configPath, - "12345", - "-100123", - "111:driver-token", - "222:sut-token", - "2026.5.16-beta.6", - ]); - - const config = JSON.parse(await fs.readFile(configPath, "utf8")); - expect(config.channels.telegram.replyToMode).toBe("first"); - expect(config.channels.telegram.streaming).toEqual({ mode: "off" }); - expect(config.messages.groupChat.visibleReplies).toBe("automatic"); - }); - - it("extracts RTT values from evidence summaries", () => { - const summary = makeTelegramRttEvidenceSummary(); - - expect(extractRtt(summary)).toEqual({ - canaryMs: 1234, - mentionReplyMs: 5000, - avgMs: 5333, - p50Ms: 5000, - p95Ms: 7000, - maxMs: 7000, - failedSamples: 0, - }); - }); - - it("resolves the evidence summary path for Telegram RTT artifacts", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-summary-test-")); - tempDirs.push(tempDir); - - await expect(resolveTelegramSummaryPath(tempDir)).resolves.toBe( - path.join(tempDir, QA_EVIDENCE_FILENAME), - ); - }); - - it("builds RTT result JSON", async () => { - const summary = makeTelegramRttEvidenceSummary(); - const result = buildRttResult({ - artifacts: { - rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", - rawReportPath: "runs/run/raw/telegram-qa-report.md", - rawSummaryPath: "runs/run/raw/qa-evidence.json", - resultPath: "runs/run/result.json", - }, - finishedAt: new Date("2026-05-01T00:00:12.000Z"), - providerMode: "mock-openai", - rawSummary: summary, - runId: "run", - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@beta", - startedAt: new Date("2026-05-01T00:00:00.000Z"), - version: "2026.4.30-beta.1", - }); - - expect(result).toStrictEqual({ - artifacts: { - rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", - rawReportPath: "runs/run/raw/telegram-qa-report.md", - rawSummaryPath: "runs/run/raw/qa-evidence.json", - resultPath: "runs/run/result.json", - }, - package: { spec: "openclaw@beta", version: "2026.4.30-beta.1" }, - run: { - durationMs: 12_000, - finishedAt: "2026-05-01T00:00:12.000Z", - id: "run", - startedAt: "2026-05-01T00:00:00.000Z", - status: "pass", - }, - mode: { - providerMode: "mock-openai", - scenarios: ["telegram-mentioned-message-reply"], - }, - rtt: { - canaryMs: 1234, - mentionReplyMs: 5000, - avgMs: 5333, - p50Ms: 5000, - p95Ms: 7000, - maxMs: 7000, - failedSamples: 0, - }, - }); - }); - - it("marks failed scenario summaries as failed results", () => { - const result = buildRttResult({ - artifacts: { - rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", - rawReportPath: "runs/run/raw/telegram-qa-report.md", - rawSummaryPath: "runs/run/raw/qa-evidence.json", - resultPath: "runs/run/result.json", - }, - finishedAt: new Date("2026-05-01T00:00:12.000Z"), - providerMode: "mock-openai", - rawSummary: makeTelegramRttEvidenceSummary({ - canaryTiming: { rttMs: 5948 }, - mentionStatus: "fail", - mentionTiming: undefined, - }), - runId: "run", - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@latest", - startedAt: new Date("2026-05-01T00:00:00.000Z"), - version: "2026.4.29", - }); - - expect(result.run.status).toBe("fail"); - expect(result.rtt).toEqual({ canaryMs: 5948, mentionReplyMs: undefined }); - }); - - it("marks incomplete RTT summaries as failed results", () => { - const baseParams = { - artifacts: { - rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json", - rawReportPath: "runs/run/raw/telegram-qa-report.md", - rawSummaryPath: "runs/run/raw/qa-evidence.json", - resultPath: "runs/run/result.json", - }, - finishedAt: new Date("2026-05-01T00:00:12.000Z"), - providerMode: "mock-openai" as const, - runId: "run", - scenarios: ["telegram-mentioned-message-reply"], - spec: "openclaw@latest", - startedAt: new Date("2026-05-01T00:00:00.000Z"), - version: "2026.4.29", - }; - const emptySummary = { ...makeTelegramRttEvidenceSummary(), entries: [] }; - const canaryOnlySummary = makeTelegramRttEvidenceSummary(); - canaryOnlySummary.entries = canaryOnlySummary.entries.slice(0, 1); - - for (const rawSummary of [ - emptySummary, - canaryOnlySummary, - makeTelegramRttEvidenceSummary({ mentionStatus: "skipped" }), - makeTelegramRttEvidenceSummary({ mentionTiming: undefined }), - ]) { - expect(buildRttResult({ ...baseParams, rawSummary }).run.status).toBe("fail"); - } - }); - - it("appends JSONL rows", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-test-")); - tempDirs.push(tempDir); - const jsonlPath = path.join(tempDir, "data/rtt.jsonl"); - await appendJsonl(jsonlPath, { run: 1 }); - await appendJsonl(jsonlPath, { run: 2 }); - - await expect(fs.readFile(jsonlPath, "utf8")).resolves.toBe('{"run":1}\n{"run":2}\n'); - }); - - it("parses CLI options", () => { - const parsed = cliTesting.parseArgs([ - "openclaw@latest", - "--package-tgz", - "/tmp/openclaw.tgz", - "--provider", - "live-frontier", - "--credential-source", - "convex", - "--credential-role", - "ci", - "--runs", - "3", - "--samples", - "5", - "--sample-timeout-ms", - "30000", - "--timeout-ms", - "240000", - "--harness-root", - "/tmp/openclaw", - "--output", - "/tmp/runs", - ]); - - expect(parsed.spec).toBe("openclaw@latest"); - expect(parsed.options).toStrictEqual({ - packageTgz: "/tmp/openclaw.tgz", - credentialRole: "ci", - credentialSource: "convex", - providerMode: "live-frontier", - runs: 3, - samples: 5, - sampleTimeoutMs: 30_000, - harnessRoot: "/tmp/openclaw", - output: "/tmp/runs", - scenarios: ["telegram-mentioned-message-reply"], - timeoutMs: 240_000, - }); - }); - - it("rejects missing CLI path option values", () => { - for (const [flag, next] of [ - ["--package-tgz", "--runs"], - ["--harness-root", "--output"], - ["--output", "--samples"], - ] as const) { - expect(() => cliTesting.parseArgs(["openclaw@latest", flag, next])).toThrow( - `${flag} requires a path.`, - ); - } - }); -});