From 77d07dc3e98755f2ae280b4949443da90282ff01 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 15 Jul 2026 12:55:00 -0700 Subject: [PATCH] refactor(qa): migrate Telegram scenarios into QA Lab (#108430) * refactor(qa): migrate Telegram scenarios into QA Lab * refactor(qa): remove retired Telegram runner exports --- .github/workflows/mantis-telegram-live.yml | 4 +- .github/workflows/npm-telegram-beta-e2e.yml | 2 +- config/max-lines-baseline.txt | 2 - docs/concepts/mantis.md | 2 +- docs/concepts/qa-e2e-automation.md | 23 +- docs/help/testing.md | 7 +- extensions/qa-lab/src/cli.runtime.test.ts | 121 +- extensions/qa-lab/src/coverage-report.test.ts | 6 +- .../qa-lab/src/evidence-summary.test.ts | 8 +- .../shared/canonical-scenarios.test.ts | 48 +- .../shared/canonical-scenarios.ts | 45 - .../shared/live-transport-rtt.ts | 52 - .../shared/live-transport-scenarios.test.ts | 7 +- .../shared/live-transport-scenarios.ts | 11 +- .../telegram/adapter.runtime.test.ts | 181 ++ .../telegram/adapter.runtime.ts | 118 +- .../live-transports/telegram/cli.runtime.ts | 127 +- .../src/live-transports/telegram/cli.test.ts | 13 + .../src/live-transports/telegram/cli.ts | 5 +- .../telegram/legacy-scenario-parity.test.ts | 93 + .../live-transports/telegram/profiles.test.ts | 59 + .../src/live-transports/telegram/profiles.ts | 77 + .../telegram/telegram-api.runtime.test.ts | 125 + .../telegram/telegram-api.runtime.ts | 433 ++++ .../telegram/telegram-live.runtime.test.ts | 1379 ----------- .../telegram/telegram-live.runtime.ts | 2064 ----------------- .../qa-lab/src/suite-round-trip.test.ts | 66 + extensions/qa-lab/src/suite-round-trip.ts | 66 + extensions/qa-lab/src/suite-summary.ts | 3 +- extensions/qa-lab/src/suite.ts | 51 +- qa/scenarios/channels/channel-canary.yaml | 3 + .../telegram-long-final-reuses-preview.yaml | 56 + .../telegram-long-final-three-chunks.yaml | 56 + .../telegram-other-bot-command-gating.yaml | 39 + .../telegram-stream-final-single-message.yaml | 55 + scripts/e2e/npm-telegram-live-runner.ts | 54 +- scripts/mantis/build-telegram-evidence.mjs | 78 +- scripts/release-beta-smoke.ts | 2 +- .../mantis-build-telegram-evidence.test.ts | 43 +- test/scripts/npm-telegram-live.test.ts | 44 +- 40 files changed, 1673 insertions(+), 3955 deletions(-) create mode 100644 extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/cli.test.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/legacy-scenario-parity.test.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/profiles.test.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/profiles.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts create mode 100644 extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts delete mode 100644 extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts create mode 100644 extensions/qa-lab/src/suite-round-trip.test.ts create mode 100644 extensions/qa-lab/src/suite-round-trip.ts create mode 100644 qa/scenarios/channels/telegram-long-final-reuses-preview.yaml create mode 100644 qa/scenarios/channels/telegram-long-final-three-chunks.yaml create mode 100644 qa/scenarios/channels/telegram-other-bot-command-gating.yaml create mode 100644 qa/scenarios/channels/telegram-stream-final-single-message.yaml diff --git a/.github/workflows/mantis-telegram-live.yml b/.github/workflows/mantis-telegram-live.yml index c6bd6bb802b0..eeef9904d7d5 100644 --- a/.github/workflows/mantis-telegram-live.yml +++ b/.github/workflows/mantis-telegram-live.yml @@ -444,7 +444,7 @@ jobs: telegram_exit=$? set -e - if [[ ! -f "$root/qa-evidence.json" && ! -f "$root/telegram-qa-summary.json" ]]; then + if [[ ! -f "$root/qa-evidence.json" ]]; then echo "Telegram live QA did not produce an evidence summary." >&2 exit "$telegram_exit" fi @@ -496,7 +496,7 @@ jobs: fi fi - cat "$root/telegram-qa-report.md" >> "$GITHUB_STEP_SUMMARY" + cat "$root/qa-suite-report.md" >> "$GITHUB_STEP_SUMMARY" - name: Upload Mantis Telegram artifacts id: upload_artifact diff --git a/.github/workflows/npm-telegram-beta-e2e.yml b/.github/workflows/npm-telegram-beta-e2e.yml index 266d0b3cbed8..64ea2397a19a 100644 --- a/.github/workflows/npm-telegram-beta-e2e.yml +++ b/.github/workflows/npm-telegram-beta-e2e.yml @@ -423,7 +423,7 @@ jobs: append_telegram_summary() { local status=$? - local report="${output_dir}/telegram-qa-report.md" + local report="${output_dir}/qa-suite-report.md" if [[ -n "${GITHUB_STEP_SUMMARY:-}" && -f "${report}" ]]; then { echo "## Package Telegram E2E" diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 32e0ad6326ed..0d68779308cf 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -249,8 +249,6 @@ extensions/qa-lab/src/lab-server.ts extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-e2ee-destructive.ts extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts -extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts -extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md index cf6d46dfd030..d941770f83f1 100644 --- a/docs/concepts/mantis.md +++ b/docs/concepts/mantis.md @@ -325,7 +325,7 @@ Comment triggers, from a PR with write/maintain/admin access: @openclaw-mantis discord status reactions baseline=origin/main candidate=HEAD @openclaw-mantis telegram @openclaw-mantis telegram scenario=telegram-status-command -@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply +@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary @openclaw-mantis web ui chat @openclaw-mantis web-ui-chat candidate=HEAD ``` diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 13b8618dc24a..b913ccf0342e 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -491,11 +491,12 @@ Required env when `--credential-source env`: - `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN` - `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN` -Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`): +The `release` profile selects the maintained Telegram YAML scenarios; `all` +adds opt-in session, usage, reply-chain, and streaming stress checks. Explicit +`--scenario` values override the profile. -- `telegram-canary` -- `telegram-mention-gating` -- `telegram-mentioned-message-reply` +- `channel-canary` +- `channel-mention-gating` - `telegram-help-command` - `telegram-commands-command` - `telegram-tools-compact-command` @@ -511,19 +512,21 @@ Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime - `telegram-long-final-reuses-preview` - `telegram-long-final-three-chunks` -The implicit default set always covers canary, mention gating, native command +The `release` profile always covers canary, mention gating, native command replies, command addressing, and bot-to-bot group replies. `mock-openai` -defaults also include deterministic reply-chain and final-message streaming -checks. `telegram-current-session-status-tool` and +also includes the deterministic long-final preview check. +`telegram-current-session-status-tool` and `telegram-tool-only-usage-footer` remain opt-in: the former is only stable when threaded directly after canary, and the latter is a real-Telegram proof of the `/usage` footer on tool-only replies. Use `pnpm openclaw qa telegram --list-scenarios --provider-mode mock-openai` to print the current -default/optional split with regression refs. +default/optional split with regression refs. Use `--profile all` for every +Telegram scenario and `--fail-fast` to stop after the first failed scenario. Output artifacts: -- `telegram-qa-report.md` +- `qa-suite-report.md` +- `qa-suite-summary.json` - `qa-evidence.json` - evidence entries for the live transport checks, including profile, coverage, provider, channel, artifacts, result, and RTT fields. @@ -542,7 +545,7 @@ When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper leases a `kind: "telegram"` credential, exports the leased group/driver/SUT bot env into the installed-package run, heartbeats the lease, and releases it on shutdown. The package wrapper defaults to 20 RTT checks of -`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role +`channel-canary`, a 30s RTT timeout, and Convex role `maintainer` outside CI when Convex is selected. Override `OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without diff --git a/docs/help/testing.md b/docs/help/testing.md index 64b028444549..5e1ede06e298 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -276,9 +276,8 @@ inside every shard. `OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the run. - `OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of - Telegram QA check IDs to sample; when unset, the default RTT-capable - check is `telegram-mentioned-message-reply`. + `OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` selects the Telegram QA scenario to + sample; the supported RTT target is `channel-canary`. - Uses the same Telegram env credentials or Convex credential source as `pnpm openclaw qa telegram`. For CI/release automation, set `OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus @@ -441,7 +440,7 @@ set. Maintainers can start it from the Actions UI through `Mantis Scenario` ```text @openclaw-mantis telegram @openclaw-mantis telegram scenario=telegram-status-command -@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply +@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary ``` `Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index b66fd9324983..806f474682d1 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { QaScenarioPack } from "./scenario-catalog.js"; const { @@ -11,10 +11,7 @@ const { runQaSuite, runQaCharacterEval, runQaMultipass, - runCanonicalLiveScenarios, listLiveTransportQaAdapterFactories, - listTelegramQaScenarioCatalog, - runTelegramQaLive, startQaLabServer, writeQaDockerHarnessFiles, buildQaDockerHarnessImage, @@ -27,10 +24,7 @@ const { runQaSuite: vi.fn(), runQaCharacterEval: vi.fn(), runQaMultipass: vi.fn(), - runCanonicalLiveScenarios: vi.fn(), listLiveTransportQaAdapterFactories: vi.fn(), - listTelegramQaScenarioCatalog: vi.fn(), - runTelegramQaLive: vi.fn(), startQaLabServer: vi.fn(), writeQaDockerHarnessFiles: vi.fn(), buildQaDockerHarnessImage: vi.fn(), @@ -61,20 +55,10 @@ vi.mock("./live-transports/cli.js", () => ({ listLiveTransportQaAdapterFactories, })); -vi.mock("./live-transports/shared/canonical-scenarios.js", async (importOriginal) => ({ - ...(await importOriginal()), - runCanonicalLiveScenarios, -})); - vi.mock("./live-transports/telegram/adapter.runtime.js", () => ({ createTelegramQaTransportAdapter: vi.fn(), })); -vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({ - listTelegramQaScenarioCatalog, - runTelegramQaLive, -})); - vi.mock("./lab-server.js", () => ({ startQaLabServer, })); @@ -118,7 +102,6 @@ import { } from "./cli.runtime.js"; import { QaSuiteInfraError } from "./errors.js"; import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js"; -import { loadNonYamlScenarioRefs } from "./live-transports/shared/live-transport-scenarios.js"; import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js"; import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js"; import type { QaProviderModeInput } from "./run-config.js"; @@ -253,10 +236,7 @@ describe("qa cli runtime", () => { runQaCharacterEval.mockReset(); runQaManualLane.mockReset(); runQaMultipass.mockReset(); - runCanonicalLiveScenarios.mockReset(); listLiveTransportQaAdapterFactories.mockReset(); - listTelegramQaScenarioCatalog.mockReset(); - runTelegramQaLive.mockReset(); startQaLabServer.mockReset(); writeQaDockerHarnessFiles.mockReset(); buildQaDockerHarnessImage.mockReset(); @@ -300,28 +280,6 @@ describe("qa cli runtime", () => { vmName: "openclaw-qa-test", scenarioIds: ["channel-chat-baseline"], }); - runTelegramQaLive.mockResolvedValue({ - outputDir: telegramArtifactsDir, - reportPath: path.join(telegramArtifactsDir, "report.md"), - summaryPath: telegramSummaryPath, - observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"), - scenarios: [], - }); - runCanonicalLiveScenarios.mockResolvedValue({ - outputDir: telegramArtifactsDir, - reportPath: path.join(telegramArtifactsDir, "canonical-report.md"), - summaryPath: telegramSummaryPath, - scenarios: [], - }); - listTelegramQaScenarioCatalog.mockReturnValue([ - { - id: "telegram-stream-final-single-message", - title: "Telegram streamed final reply", - defaultEnabled: false, - rationale: "stream rationale", - regressionRefs: [], - }, - ]); listLiveTransportQaAdapterFactories.mockReturnValue([ { id: "telegram", @@ -998,22 +956,20 @@ describe("qa cli runtime", () => { sutAccountId: "sut-live", }); - expect(runCanonicalLiveScenarios).toHaveBeenCalledWith( + expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( expect.objectContaining({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"), + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-luna", + fastMode: true, + channelDriver: "live", channelId: "telegram", - options: expect.objectContaining({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"), - providerMode: "live-frontier", - primaryModel: "openai/gpt-5.6-luna", - alternateModel: "openai/gpt-5.6-luna", - fastMode: true, - sutAccountId: "sut-live", - }), + adapterOptions: expect.objectContaining({ sutAccountId: "sut-live" }), scenarioIds: ["telegram-help-command"], }), ); - expect(runTelegramQaLive).not.toHaveBeenCalled(); }); it("rejects output dirs that escape the repo root", () => { @@ -1031,31 +987,33 @@ describe("qa cli runtime", () => { scenarioIds: ["telegram-help-command"], }); - expect(runCanonicalLiveScenarios).toHaveBeenCalledWith( + expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( expect.objectContaining({ - options: expect.objectContaining({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - providerMode: "live-frontier", - allowFailures: undefined, - }), + repoRoot: path.resolve("/tmp/openclaw-repo"), + providerMode: "live-frontier", + scenarioIds: ["telegram-help-command"], }), ); - expect(runTelegramQaLive).not.toHaveBeenCalled(); }); - it("keeps remaining Telegram defaults when Commander supplies an empty scenario list", async () => { + it("resolves the Telegram release profile when Commander supplies an empty scenario list", async () => { await runQaTelegramCommand({ repoRoot: "/tmp/openclaw-repo", scenarioIds: [], }); - expect(runCanonicalLiveScenarios).toHaveBeenCalled(); - expect(runTelegramQaLive).toHaveBeenCalledWith( - expect.objectContaining({ scenarioIds: undefined }), + expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( + expect.objectContaining({ + scenarioIds: expect.arrayContaining([ + "channel-canary", + "channel-mention-gating", + "telegram-other-bot-command-gating", + ]), + }), ); }); - it("uses the trusted Telegram launcher for canonical and legacy gateways", async () => { + it("uses the trusted Telegram launcher for the shared suite gateway", async () => { const candidateRoot = path.join(telegramArtifactsDir, "candidate"); const boundaryDir = path.join(telegramArtifactsDir, "boundary"); const launcherPath = path.join(telegramArtifactsDir, "openclaw-telegram-sut-launcher"); @@ -1097,12 +1055,9 @@ describe("qa cli runtime", () => { terminationRetryTimeoutMs: 60_000, }, }; - expect(runCanonicalLiveScenarios).toHaveBeenCalledWith( - expect.objectContaining({ - options: expect.objectContaining({ sutOpenClawCommand }), - }), + expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( + expect.objectContaining({ sutOpenClawCommand }), ); - expect(runTelegramQaLive).toHaveBeenCalledWith(expect.objectContaining({ sutOpenClawCommand })); }); it("rejects relative Telegram launcher paths before starting a gateway", async () => { @@ -1114,8 +1069,7 @@ describe("qa cli runtime", () => { }), ).rejects.toThrow("OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must be an absolute file path."); - expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); - expect(runTelegramQaLive).not.toHaveBeenCalled(); + expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled(); }); it("rejects non-executable Telegram launcher files before starting a gateway", async () => { @@ -1131,8 +1085,7 @@ describe("qa cli runtime", () => { `OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must point to an executable regular file: ${launcherPath}`, ); - expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); - expect(runTelegramQaLive).not.toHaveBeenCalled(); + expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled(); }); it("rejects unknown mixed Telegram selections before resolving the SUT launcher", async () => { @@ -1144,8 +1097,7 @@ describe("qa cli runtime", () => { }), ).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario"); - expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); - expect(runTelegramQaLive).not.toHaveBeenCalled(); + expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled(); }); it("prints telegram scenario catalog without resolving the SUT launcher", async () => { @@ -1156,9 +1108,7 @@ describe("qa cli runtime", () => { listScenarios: true, }); - expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai"); - expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); - expect(runTelegramQaLive).not.toHaveBeenCalled(); + expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled(); expectWriteContains( stdoutWrite, "telegram-status-command\tdefault\tTelegram status command reply\tVerify Telegram status returns model, session, and activation details. refs=openclaw/openclaw#74698", @@ -1176,11 +1126,10 @@ describe("qa cli runtime", () => { }), "utf8", ); - runTelegramQaLive.mockResolvedValueOnce({ + runQaFlowSuiteFromRuntime.mockResolvedValueOnce({ outputDir: telegramArtifactsDir, reportPath: path.join(telegramArtifactsDir, "report.md"), summaryPath: telegramSummaryPath, - observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"), scenarios: [], }); @@ -1205,11 +1154,10 @@ describe("qa cli runtime", () => { }), "utf8", ); - runTelegramQaLive.mockResolvedValueOnce({ + runQaFlowSuiteFromRuntime.mockResolvedValueOnce({ outputDir: telegramArtifactsDir, reportPath: path.join(telegramArtifactsDir, "report.md"), summaryPath: telegramSummaryPath, - observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"), scenarios: [ { id: "telegram-help-command", @@ -1999,11 +1947,6 @@ describe("qa cli runtime", () => { }); describe("coverage inventory command", () => { - beforeAll(async () => { - listTelegramQaScenarioCatalog.mockReturnValue([]); - await loadNonYamlScenarioRefs(); - }); - it("prints a markdown report from scenario metadata", async () => { await runQaCoverageReportCommand({ repoRoot: process.cwd() }); diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index 3befd5943727..68b011898ea2 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -277,12 +277,12 @@ describe("qa coverage report", () => { nonYamlScenarios: [ { id: scenario.id, - sourcePath: "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts", + sourcePath: "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts", }, ], }), ).toThrow( - "duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts)", + "duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts)", ); }); @@ -308,7 +308,7 @@ describe("qa coverage report", () => { expect(report).toContain("personal-share-safe-diagnostics-artifact"); expect(report).toContain("## Live Transport Lanes"); expect(report).toContain( - "- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume", + "- telegram (telegram): canary: channel-canary, help-command: telegram-help-command, mention-gating: channel-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume", ); expect(report).toContain("## Scorecard Taxonomy"); expect(report).toContain("- Taxonomy: taxonomy.yaml"); diff --git a/extensions/qa-lab/src/evidence-summary.test.ts b/extensions/qa-lab/src/evidence-summary.test.ts index f3f60e5ca051..96cdd07400df 100644 --- a/extensions/qa-lab/src/evidence-summary.test.ts +++ b/extensions/qa-lab/src/evidence-summary.test.ts @@ -152,7 +152,7 @@ describe("evidence summary", () => { const evidence = buildLiveTransportEvidenceSummary({ artifactPaths: [ { kind: "summary", path: QA_EVIDENCE_FILENAME }, - { kind: "report", path: "telegram-qa-report.md" }, + { kind: "report", path: "qa-suite-report.md" }, ], env: { OPENCLAW_QA_RUNNER: "crabbox", @@ -218,7 +218,7 @@ describe("evidence summary", () => { }, { kind: "report", - path: "telegram-qa-report.md", + path: "qa-suite-report.md", source: "telegram-live-transport", }, ], @@ -244,9 +244,9 @@ describe("evidence summary", () => { providerMode: "live-frontier", checks: [ { - id: "telegram-mentioned-message-reply", + id: "channel-canary", coverageIds: ["channels.telegram.mention-gating"], - title: "Telegram mentioned message gets a reply", + title: "Channel transport canary", status: "pass", details: "5 samples collected.", rttMs: 2000, diff --git a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts index dc2a9631988a..63276a2c37b4 100644 --- a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts @@ -8,14 +8,12 @@ const { runQaFlowSuiteFromRuntime } = vi.hoisted(() => ({ vi.mock("../../suite-launch.runtime.js", () => ({ runQaFlowSuiteFromRuntime, })); +import { readQaScenarioPack } from "../../scenario-catalog.js"; import { assertKnownScenarioIds, partitionCanonicalScenarioIds, - TELEGRAM_CANONICAL_SCENARIO_IDS, - TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, WHATSAPP_CANONICAL_SCENARIO_IDS, whatsappDefaultCanonicalScenarioIds, - listCanonicalScenarios, runCanonicalLiveScenarios, } from "./canonical-scenarios.js"; import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js"; @@ -23,42 +21,27 @@ import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js"; describe("canonical live-transport scenarios", () => { it("loads every migrated routing, command, and session-context scenario from YAML", () => { const whatsAppMockDefaultIds = whatsappDefaultCanonicalScenarioIds("mock-openai"); - const telegram = listCanonicalScenarios({ - ids: TELEGRAM_CANONICAL_SCENARIO_IDS, - defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, - }); - const whatsapp = listCanonicalScenarios({ - ids: WHATSAPP_CANONICAL_SCENARIO_IDS, - defaultIds: whatsAppMockDefaultIds, - }); + const expectedIds = new Set(WHATSAPP_CANONICAL_SCENARIO_IDS); + const whatsappIds = new Set( + readQaScenarioPack() + .scenarios.filter((scenario) => expectedIds.has(scenario.id)) + .map((scenario) => scenario.id), + ); - expect(telegram.map(({ id }) => id).toSorted()).toEqual( - [...TELEGRAM_CANONICAL_SCENARIO_IDS].toSorted(), - ); - expect(whatsapp.map(({ id }) => id).toSorted()).toEqual( - [...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted(), - ); - expect(telegram.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual( - expect.arrayContaining([...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]), - ); - expect(whatsapp.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual( - expect.arrayContaining([...whatsAppMockDefaultIds]), - ); + expect([...whatsappIds].toSorted()).toEqual([...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted()); + expect([...whatsAppMockDefaultIds].every((id) => whatsappIds.has(id))).toBe(true); expect(whatsappDefaultCanonicalScenarioIds("live-frontier")).toEqual(["whatsapp-help-command"]); - expect(telegram.find(({ id }) => id === "telegram-status-command")?.regressionRefs).toEqual([ - "openclaw/openclaw#74698", - ]); }); it("partitions canonical aliases from remaining imperative scenarios", () => { expect( partitionCanonicalScenarioIds( - ["telegram-help-command", "telegram-mentioned-message-reply"], - TELEGRAM_CANONICAL_SCENARIO_IDS, + ["whatsapp-help-command", "whatsapp-canary"], + WHATSAPP_CANONICAL_SCENARIO_IDS, ), ).toEqual({ - canonical: ["telegram-help-command"], - legacy: ["telegram-mentioned-message-reply"], + canonical: ["whatsapp-help-command"], + legacy: ["whatsapp-canary"], }); }); @@ -107,10 +90,7 @@ describe("canonical live-transport scenarios", () => { it("removes migrated ids from non-YAML scenario ownership", async () => { const nonYamlIds = new Set((await loadNonYamlScenarioRefs()).map(({ id }) => id)); - for (const scenarioId of [ - ...TELEGRAM_CANONICAL_SCENARIO_IDS, - ...WHATSAPP_CANONICAL_SCENARIO_IDS, - ]) { + for (const scenarioId of WHATSAPP_CANONICAL_SCENARIO_IDS) { expect(nonYamlIds.has(scenarioId), scenarioId).toBe(false); } }); diff --git a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts index dd15f60beb6e..b87c18265d45 100644 --- a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts @@ -2,37 +2,9 @@ import path from "node:path"; import type { QaGatewayChildCommand } from "../../gateway-child.js"; import type { QaTransportAdapterFactory } from "../../qa-transport-registry.js"; -import { readQaScenarioPack } from "../../scenario-catalog.js"; import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js"; import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js"; -export const TELEGRAM_CANONICAL_SCENARIO_IDS = [ - "channel-canary", - "channel-mention-gating", - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", - "telegram-context-command", - "telegram-current-session-status-tool", - "telegram-tool-only-usage-footer", - "telegram-reply-chain-exact-marker", -] as const; - -export const TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS = [ - "channel-canary", - "channel-mention-gating", - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", - "telegram-context-command", -] as const; - export const WHATSAPP_ROUTING_CANONICAL_SCENARIO_IDS = [ "channel-canary", "channel-dm-group-routing", @@ -100,23 +72,6 @@ export function partitionCanonicalScenarioIds( return { canonical, legacy }; } -export function listCanonicalScenarios(params: { - ids: readonly string[]; - defaultIds: readonly string[]; -}) { - const requestedIds = new Set(params.ids); - const defaultIds = new Set(params.defaultIds); - return readQaScenarioPack() - .scenarios.filter((scenario) => requestedIds.has(scenario.id)) - .map((scenario) => ({ - id: scenario.id, - title: scenario.title, - rationale: scenario.objective, - regressionRefs: scenario.regressionRefs ?? [], - defaultEnabled: defaultIds.has(scenario.id), - })); -} - export async function runCanonicalLiveScenarios(params: { channelId: string; factory: QaTransportAdapterFactory; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts index e2dcd64ea8e1..55eb11522a9a 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts @@ -1,63 +1,11 @@ // Qa Lab plugin module implements shared live-transport RTT behavior. -import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import type { QaEvidenceTiming } from "../../evidence-summary.js"; -export type LiveTransportRttOptions = { - count: number; - timeoutMs: number; - maxFailures: number; - checkIds: Set; -}; - export type LiveTransportRttSample = { rttMs?: number; status: "pass" | "fail"; }; -function normalizePositiveRttInteger(value: number | undefined) { - if ( - typeof value !== "number" || - !Number.isSafeInteger(value) || - value <= 0 || - value > MAX_TIMER_TIMEOUT_MS - ) { - return undefined; - } - return value; -} - -export function normalizeLiveTransportRttOptions(params: { - count?: number; - defaultCheckIds: readonly CheckId[]; - knownCheckIds: ReadonlySet; - maxFailures?: number; - rawCheckIds?: readonly string[]; - timeoutMs?: number; - unknownCheckMessage: (checkId: string) => string; -}): LiveTransportRttOptions | undefined { - const count = normalizePositiveRttInteger(params.count); - if (count === undefined) { - return undefined; - } - const rawCheckIds = - params.rawCheckIds && params.rawCheckIds.length > 0 - ? params.rawCheckIds - : params.defaultCheckIds; - const checkIds = new Set(); - for (const checkId of rawCheckIds) { - if (!params.knownCheckIds.has(checkId as CheckId)) { - throw new Error(params.unknownCheckMessage(checkId)); - } - checkIds.add(checkId as CheckId); - } - return { - count, - maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count, - checkIds, - timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000, - }; -} - function percentile(sortedValues: readonly number[], percentileValue: number) { if (sortedValues.length === 0) { return undefined; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index 785b9059dcc9..3a5b4788af3c 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -8,7 +8,6 @@ import { import { describe, expect, it } from "vitest"; import { discordQaLiveRuntime } from "../discord/discord-live.runtime.js"; import { __testing as slackTesting } from "../slack/slack-live.runtime.js"; -import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js"; import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js"; import { buildLiveTransportCoverageLaneSummaries, @@ -115,6 +114,7 @@ describe("live transport scenario helpers", () => { ]); expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({ standardId: "canary", + scenarioId: "channel-canary", }); expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({ standardId: "restart-resume", @@ -151,10 +151,7 @@ describe("live transport scenario helpers", () => { expect.arrayContaining(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS), ); expect(lanes.get("telegram")).toEqual( - expect.arrayContaining([ - "help-command", - ...telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS, - ]), + expect.arrayContaining(["canary", "help-command", "mention-gating"]), ); expect(lanes.get("whatsapp")).toEqual( expect.arrayContaining([ diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index 0f00e91c8ab7..4ae3caaf0282 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -57,9 +57,9 @@ const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [ transportId: "telegram", commandName: "telegram", members: [ - { standardId: "canary" }, + { standardId: "canary", scenarioId: "channel-canary" }, { standardId: "help-command", scenarioId: "telegram-help-command" }, - { standardId: "mention-gating", scenarioId: "telegram-mention-gating" }, + { standardId: "mention-gating", scenarioId: "channel-mention-gating" }, ], }, { @@ -107,10 +107,9 @@ export function buildLiveTransportCoverageLaneSummaries( } export async function loadNonYamlScenarioRefs() { - const [discord, slack, telegram, whatsapp] = await Promise.all([ + const [discord, slack, whatsapp] = await Promise.all([ import("../discord/discord-live.runtime.js"), import("../slack/slack-live.runtime.js"), - import("../telegram/telegram-live.runtime.js"), import("../whatsapp/whatsapp-live.runtime.js"), ]); const refs = (sourcePath: string, scenarios: readonly { id: string }[]) => @@ -124,10 +123,6 @@ export async function loadNonYamlScenarioRefs() { "extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts", slack.listSlackQaScenarioCatalog(), ), - ...refs( - "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts", - telegram.listTelegramQaScenarioCatalog(), - ), ...refs( "extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts", whatsapp.listWhatsAppQaScenarioCatalog(), diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts new file mode 100644 index 000000000000..a09a91cbc16c --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts @@ -0,0 +1,181 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + acquireQaCredentialLease: vi.fn(), + assertQaGatewayCredentialLeaseQuarantine: vi.fn(), + callTelegramApi: vi.fn(), + flushTelegramUpdates: vi.fn(), + heartbeatStop: vi.fn(), + heartbeatThrowIfFailed: vi.fn(), + leaseHeartbeat: vi.fn(), + leaseRelease: vi.fn(), + shouldRetainQaGatewayCredentialLease: vi.fn(), + waitForTelegramChannelRunning: vi.fn(), +})); + +vi.mock("../shared/credential-lease.runtime.js", () => ({ + acquireQaCredentialLease: mocks.acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat: () => ({ + stop: mocks.heartbeatStop, + throwIfFailed: mocks.heartbeatThrowIfFailed, + }), +})); + +vi.mock("../../gateway-process-boundary.js", () => ({ + assertQaGatewayCredentialLeaseQuarantine: mocks.assertQaGatewayCredentialLeaseQuarantine, + shouldRetainQaGatewayCredentialLease: mocks.shouldRetainQaGatewayCredentialLease, +})); + +vi.mock("./telegram-api.runtime.js", async (importOriginal) => ({ + ...(await importOriginal()), + callTelegramApi: mocks.callTelegramApi, + flushTelegramUpdates: mocks.flushTelegramUpdates, + waitForTelegramChannelRunning: mocks.waitForTelegramChannelRunning, +})); + +import { createTelegramQaTransportAdapter } from "./adapter.runtime.js"; + +describe("Telegram QA transport adapter", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.acquireQaCredentialLease.mockResolvedValue({ + payload: { + groupId: "-100123", + driverToken: "placeholder", + sutToken: "placeholder", + }, + source: "env", + heartbeat: mocks.leaseHeartbeat, + release: mocks.leaseRelease, + }); + mocks.flushTelegramUpdates.mockResolvedValue(0); + mocks.shouldRetainQaGatewayCredentialLease.mockResolvedValue(false); + }); + + it("rejects credentials that do not identify two distinct bots", async () => { + mocks.callTelegramApi.mockResolvedValue({ + id: 1, + is_bot: true, + first_name: "bot", + username: "same_bot", + }); + + await expect( + createTelegramQaTransportAdapter({ adapterOptions: {}, messages: {} } as never), + ).rejects.toThrow("requires two distinct bots"); + expect(mocks.heartbeatStop).toHaveBeenCalledOnce(); + expect(mocks.leaseRelease).toHaveBeenCalledOnce(); + expect(mocks.flushTelegramUpdates).not.toHaveBeenCalled(); + }); + + it("maps native sends, replies, edits, and cleanup inside the adapter", async () => { + const pollResolvers: Array<(updates: unknown[]) => void> = []; + let getMeCalls = 0; + let sendMessageCalls = 0; + mocks.callTelegramApi.mockImplementation( + async (_token: string, method: string): Promise => { + if (method === "getMe") { + getMeCalls += 1; + return getMeCalls === 1 + ? { id: 1, is_bot: true, first_name: "driver", username: "driver_bot" } + : { id: 2, is_bot: true, first_name: "sut", username: "sut_bot" }; + } + if (method === "sendMessage") { + sendMessageCalls += 1; + return { message_id: sendMessageCalls === 1 ? 10 : 12 }; + } + if (method === "getUpdates") { + return await new Promise((resolve) => { + pollResolvers.push(resolve); + }); + } + throw new Error(`unexpected Telegram API method: ${method}`); + }, + ); + const addInboundMessage = vi.fn().mockResolvedValue({ id: "in-1" }); + const addOutboundMessage = vi.fn().mockResolvedValue({ id: "out-1" }); + const editMessage = vi.fn().mockResolvedValue({ id: "out-1" }); + const adapter = await createTelegramQaTransportAdapter({ + adapterOptions: { sutAccountId: "sut" }, + messages: { addInboundMessage, addOutboundMessage, editMessage }, + } as never); + + await vi.waitFor(() => expect(pollResolvers).toHaveLength(1)); + await adapter.sendInbound?.({ + conversation: { id: "logical-room", kind: "group" }, + senderId: "driver", + text: "@openclaw reply exactly: QA-MARKER", + }); + expect(mocks.callTelegramApi).toHaveBeenCalledWith( + "placeholder", + "sendMessage", + expect.objectContaining({ + chat_id: "-100123", + text: "@sut_bot reply exactly: QA-MARKER", + }), + ); + + pollResolvers[0]?.([ + { + update_id: 1, + message: { + message_id: 11, + date: 100, + chat: { id: -100123 }, + from: { id: 2, is_bot: true, username: "sut_bot" }, + text: "preview", + reply_to_message: { message_id: 10 }, + }, + }, + ]); + await vi.waitFor(() => expect(addOutboundMessage).toHaveBeenCalledOnce()); + expect(addOutboundMessage).toHaveBeenCalledWith( + expect.objectContaining({ + to: "group:logical-room", + text: "preview", + replyToId: "in-1", + }), + ); + await adapter.sendInbound?.({ + conversation: { id: "logical-room", kind: "group" }, + senderId: "driver", + text: "follow-up", + replyToId: "out-1", + }); + expect(mocks.callTelegramApi).toHaveBeenCalledWith( + "placeholder", + "sendMessage", + expect.objectContaining({ + reply_parameters: { + message_id: 11, + allow_sending_without_reply: true, + }, + }), + ); + + await vi.waitFor(() => expect(pollResolvers).toHaveLength(2)); + pollResolvers[1]?.([ + { + update_id: 2, + edited_message: { + message_id: 11, + date: 101, + chat: { id: -100123 }, + from: { id: 2, is_bot: true, username: "sut_bot" }, + text: "final", + }, + }, + ]); + await vi.waitFor(() => expect(editMessage).toHaveBeenCalledOnce()); + expect(editMessage).toHaveBeenCalledWith( + expect.objectContaining({ messageId: "out-1", text: "final", timestamp: 101_000 }), + ); + + await vi.waitFor(() => expect(pollResolvers).toHaveLength(3)); + const cleanup = adapter.cleanup?.(); + pollResolvers[2]?.([]); + await cleanup; + expect(mocks.heartbeatStop).toHaveBeenCalledOnce(); + expect(mocks.leaseRelease).toHaveBeenCalledOnce(); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts index 48b8df8411f8..cb52c79b379a 100644 --- a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts @@ -1,5 +1,4 @@ // Qa Lab plugin module implements Telegram live transport adapter behavior. -import type { TelegramBotUpdate } from "@openclaw/telegram/api.js"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime"; import { @@ -10,23 +9,34 @@ import { acquireQaCredentialLease, startQaCredentialLeaseHeartbeat, } from "../shared/credential-lease.runtime.js"; -import { __testing as telegramLive } from "./telegram-live.runtime.js"; +import { + buildTelegramQaConfig, + callTelegramApi, + flushTelegramUpdates, + isRecoverableTelegramQaPollError, + normalizeTelegramObservedMessage, + parseTelegramQaCredentialPayload, + resolveTelegramQaRuntimeEnv, + waitForTelegramChannelRunning, + waitForTelegramPollRetryDelay, + type TelegramBotIdentity, + type TelegramQaRuntimeEnv, + type TelegramUpdate, +} from "./telegram-api.runtime.js"; type AdapterFactory = NonNullable; type FactoryContext = Parameters[0]; type AdapterDefinition = Awaited>; -type TelegramRuntimeEnv = ReturnType; - export async function createTelegramQaTransportAdapter( context: FactoryContext, ): Promise { const options = context.adapterOptions ?? {}; - const credentialLease = await acquireQaCredentialLease({ + const credentialLease = await acquireQaCredentialLease({ kind: "telegram", source: options.credentialSource, role: options.credentialRole, - resolveEnvPayload: () => telegramLive.resolveTelegramQaRuntimeEnv(), - parsePayload: telegramLive.parseTelegramQaCredentialPayload, + resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(), + parsePayload: parseTelegramQaCredentialPayload, }); try { assertQaGatewayCredentialLeaseQuarantine(credentialLease); @@ -36,17 +46,26 @@ export async function createTelegramQaTransportAdapter( } const heartbeat = startQaCredentialLeaseHeartbeat(credentialLease); const runtimeEnv = credentialLease.payload; - let driverIdentity: { id: number; username?: string }; - let sutIdentity: { id: number; username?: string }; + let driverIdentity: TelegramBotIdentity; + let sutIdentity: TelegramBotIdentity; let offset: number; try { - [driverIdentity, sutIdentity, offset] = await Promise.all([ - telegramLive.callTelegramApi<{ id: number; username?: string }>( - runtimeEnv.driverToken, - "getMe", - ), - telegramLive.callTelegramApi<{ id: number; username?: string }>(runtimeEnv.sutToken, "getMe"), - telegramLive.flushTelegramUpdates(runtimeEnv.driverToken), + [driverIdentity, sutIdentity] = await Promise.all([ + callTelegramApi(runtimeEnv.driverToken, "getMe"), + callTelegramApi(runtimeEnv.sutToken, "getMe"), + ]); + if (!driverIdentity.is_bot || !sutIdentity.is_bot) { + throw new Error("Telegram QA credentials must belong to bots."); + } + if (driverIdentity.id === sutIdentity.id) { + throw new Error("Telegram QA requires two distinct bots for driver and SUT."); + } + if (!sutIdentity.username?.trim()) { + throw new Error("Telegram QA requires the SUT bot to have a Telegram username."); + } + [offset] = await Promise.all([ + flushTelegramUpdates(runtimeEnv.driverToken), + flushTelegramUpdates(runtimeEnv.sutToken), ]); } catch (error) { await heartbeat.stop(); @@ -64,39 +83,56 @@ export async function createTelegramQaTransportAdapter( if (stopped) { return; } - const updates = await telegramLive.callTelegramApi( - runtimeEnv.driverToken, - "getUpdates", - { offset, timeout: 1, allowed_updates: ["message", "edited_message"] }, - 6_000, - ); + let updates: TelegramUpdate[]; + try { + updates = await callTelegramApi( + runtimeEnv.driverToken, + "getUpdates", + { offset, timeout: 1, allowed_updates: ["message", "edited_message"] }, + 6_000, + ); + } catch (error) { + if (!isRecoverableTelegramQaPollError(error)) { + throw error; + } + await waitForTelegramPollRetryDelay(); + continue; + } for (const update of updates) { offset = Math.max(offset, update.update_id + 1); - const message = update.edited_message ?? update.message; - if (!message?.from?.id || message.from.id !== sutIdentity.id) { + const message = normalizeTelegramObservedMessage(update); + if ( + !message || + message.chatId !== Number(runtimeEnv.groupId) || + message.senderId !== sutIdentity.id + ) { continue; } - const existingMessageId = busMessageIds.get(message.message_id); - if (update.edited_message && existingMessageId) { - await context.messages.editMessage({ - accountId: options.sutAccountId?.trim() || "sut", - messageId: existingMessageId, - text: message.text ?? message.caption ?? "", - }); + const existingMessageId = busMessageIds.get(message.messageId); + if (update.edited_message) { + if (existingMessageId) { + await context.messages.editMessage({ + accountId: options.sutAccountId?.trim() || "sut", + messageId: existingMessageId, + text: message.text, + timestamp: message.timestamp, + }); + } continue; } const outbound = await context.messages.addOutboundMessage({ accountId: options.sutAccountId?.trim() || "sut", to: `${logicalConversationKind}:${logicalConversationId}`, - senderId: String(message.from.id), - senderName: message.from.username, - text: message.text ?? message.caption ?? "", - timestamp: message.date * 1_000, - replyToId: message.reply_to_message?.message_id - ? busMessageIds.get(message.reply_to_message.message_id) + senderId: String(message.senderId), + senderName: message.senderUsername, + text: message.text, + timestamp: message.timestamp, + replyToId: message.replyToMessageId + ? busMessageIds.get(message.replyToMessageId) : undefined, }); - busMessageIds.set(message.message_id, outbound.id); + nativeMessageIds.set(outbound.id, message.messageId); + busMessageIds.set(message.messageId, outbound.id); } } }; @@ -127,7 +163,7 @@ export async function createTelegramQaTransportAdapter( ? input.text.replaceAll("@openclaw", `@${sutIdentity.username}`) : input.text; const nativeReplyToId = input.replyToId ? nativeMessageIds.get(input.replyToId) : undefined; - const sent = await telegramLive.callTelegramApi<{ message_id: number }>( + const sent = await callTelegramApi<{ message_id: number }>( runtimeEnv.driverToken, "sendMessage", { @@ -161,14 +197,14 @@ export async function createTelegramQaTransportAdapter( busMessageIds.clear(); }, createGatewayConfig: () => - telegramLive.buildTelegramQaConfig({} as OpenClawConfig, { + buildTelegramQaConfig({} as OpenClawConfig, { groupId: runtimeEnv.groupId, sutToken: runtimeEnv.sutToken, driverBotId: driverIdentity.id, sutAccountId: accountId, }), waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) => - await telegramLive.waitForTelegramChannelRunning(gateway as never, accountId, { + await waitForTelegramChannelRunning(gateway, accountId, { timeoutMs, pollMs: pollIntervalMs, }), diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index 6977390aeffb..bc6702193da1 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -2,22 +2,15 @@ import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { QaGatewayChildCommand } from "../../gateway-child.js"; +import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js"; +import type { QaSuiteRoundTripProbe } from "../../suite-round-trip.js"; import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; -import { - assertKnownScenarioIds, - canonicalScenarioOutputDir, - listCanonicalScenarios, - partitionCanonicalScenarioIds, - runCanonicalLiveScenarios, - TELEGRAM_CANONICAL_SCENARIO_IDS, - TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, -} from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; import { createTelegramQaTransportAdapter } from "./adapter.runtime.js"; -import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js"; +import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js"; const TELEGRAM_QA_SUT_OPENCLAW_COMMAND_ENV = "OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND"; const TELEGRAM_QA_SUT_UID_ENV = "OPENCLAW_QA_TELEGRAM_SUT_UID"; @@ -142,17 +135,15 @@ async function resolveTelegramQaSutOpenClawCommand( }; } -export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) { +type TelegramQaSuiteOptions = LiveTransportQaCommandOptions & { + roundTripProbe?: QaSuiteRoundTripProbe; + sutOpenClawCommand?: QaGatewayChildCommand; +}; + +export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) { const runOptions = resolveLiveTransportQaRunOptions(opts); if (runOptions.listScenarios) { - const scenarios = [ - ...listCanonicalScenarios({ - ids: TELEGRAM_CANONICAL_SCENARIO_IDS, - defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, - }), - ...listTelegramQaScenarioCatalog(runOptions.providerMode), - ]; - for (const scenario of scenarios) { + for (const scenario of listTelegramQaScenarios(runOptions.providerMode)) { const defaultLabel = scenario.defaultEnabled ? "default" : "optional"; const refs = scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : ""; @@ -160,62 +151,40 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) `${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`, ); } - return; + return undefined; } - const selected = partitionCanonicalScenarioIds( - runOptions.scenarioIds, - TELEGRAM_CANONICAL_SCENARIO_IDS, - ); - const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0; - if (hasExplicitScenarioIds) { - assertKnownScenarioIds({ - ids: selected.legacy, - knownIds: listTelegramQaScenarioCatalog(runOptions.providerMode).map(({ id }) => id), - laneLabel: "Telegram", - }); - } - const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot); - const executionOptions = { - ...runOptions, - ...(sutOpenClawCommand ? { sutOpenClawCommand } : {}), - }; - const canonicalScenarioIds = hasExplicitScenarioIds - ? selected.canonical - : [...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]; - const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0; - if (canonicalScenarioIds.length > 0) { - const canonical = await runCanonicalLiveScenarios({ - channelId: "telegram", - factory: { + const scenarioIds = resolveTelegramQaScenarioIds({ + profile: opts.profile, + providerMode: runOptions.providerMode, + scenarioIds: runOptions.scenarioIds, + }); + const result = await runQaFlowSuiteFromRuntime({ + adapterFactories: [ + { id: "telegram", matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", create: createTelegramQaTransportAdapter, }, - options: { - ...executionOptions, - outputDir: canonicalScenarioOutputDir(executionOptions, runsLegacyScenarios), - }, - scenarioIds: canonicalScenarioIds, - }); - printLiveTransportQaArtifacts("Telegram canonical QA", { - report: canonical.reportPath, - summary: canonical.summaryPath, - }); - if (!runOptions.allowFailures) { - const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile( - canonical.summaryPath, - ); - if (failedScenarioCount > 0) { - process.exitCode = 1; - } - } - } - if (!runsLegacyScenarios) { - return; - } - const result = await runTelegramQaLive({ - ...executionOptions, - scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined, + ], + adapterOptions: { + repoRoot: runOptions.repoRoot, + ...(runOptions.credentialRole ? { credentialRole: runOptions.credentialRole } : {}), + ...(runOptions.credentialSource ? { credentialSource: runOptions.credentialSource } : {}), + ...(runOptions.sutAccountId ? { sutAccountId: runOptions.sutAccountId } : {}), + }, + channelDriver: "live", + channelId: "telegram", + concurrency: 1, + ...(runOptions.alternateModel ? { alternateModel: runOptions.alternateModel } : {}), + ...(runOptions.fastMode !== undefined ? { fastMode: runOptions.fastMode } : {}), + ...(runOptions.failFast !== undefined ? { failFast: runOptions.failFast } : {}), + ...(runOptions.outputDir ? { outputDir: runOptions.outputDir } : {}), + ...(runOptions.primaryModel ? { primaryModel: runOptions.primaryModel } : {}), + providerMode: runOptions.providerMode, + repoRoot: runOptions.repoRoot, + roundTripProbe: opts.roundTripProbe, + scenarioIds, + sutOpenClawCommand: opts.sutOpenClawCommand, }); printLiveTransportQaArtifacts("Telegram QA", { report: result.reportPath, @@ -227,4 +196,22 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) process.exitCode = 1; } } + return result; +} + +export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) { + const runOptions = resolveLiveTransportQaRunOptions(opts); + if (runOptions.listScenarios) { + return await runQaTelegramSuite(opts); + } + resolveTelegramQaScenarioIds({ + profile: opts.profile, + providerMode: runOptions.providerMode, + scenarioIds: runOptions.scenarioIds, + }); + const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot); + return await runQaTelegramSuite({ + ...opts, + ...(sutOpenClawCommand ? { sutOpenClawCommand } : {}), + }); } diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.test.ts b/extensions/qa-lab/src/live-transports/telegram/cli.test.ts new file mode 100644 index 000000000000..6acf708979bf --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/cli.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { telegramQaCliRegistration } from "./cli.js"; +import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js"; + +describe("Telegram QA CLI registration", () => { + it("keeps the generic live baseline outside command profile selection", () => { + expect(telegramQaCliRegistration.adapterFactory?.scenarioIds).toEqual([ + "channel-chat-baseline", + ...TELEGRAM_QA_ALL_SCENARIO_IDS, + ]); + expect(TELEGRAM_QA_ALL_SCENARIO_IDS).not.toContain("channel-chat-baseline"); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index beab355b996b..84281567783d 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -1,4 +1,3 @@ -import { TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { createLazyCliRuntimeLoader, @@ -6,6 +5,7 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; +import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js"; type TelegramQaAdapterRuntime = typeof import("./adapter.runtime.js"); type TelegramQaCliRuntime = typeof import("./cli.runtime.js"); @@ -23,7 +23,7 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) { const telegramQaAdapterFactory: NonNullable = { id: "telegram", - scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS], + scenarioIds: ["channel-chat-baseline", ...TELEGRAM_QA_ALL_SCENARIO_IDS], matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", async create(context) { return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context); @@ -42,6 +42,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration = description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness", listScenariosHelp: "Print available Telegram scenario ids and exit", outputDirHelp: "Telegram QA artifact directory", + profileHelp: "QA Lab Telegram profile: release or all (default: release)", run: runQaTelegram, scenarioHelp: "Run only the named Telegram QA scenario (repeatable)", sutAccountHelp: "Temporary Telegram account id inside the QA gateway config", diff --git a/extensions/qa-lab/src/live-transports/telegram/legacy-scenario-parity.test.ts b/extensions/qa-lab/src/live-transports/telegram/legacy-scenario-parity.test.ts new file mode 100644 index 000000000000..315e23a5660c --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/legacy-scenario-parity.test.ts @@ -0,0 +1,93 @@ +import fs from "node:fs"; +import { describe, expect, it } from "vitest"; +import { readQaScenarioById } from "../../scenario-catalog.js"; + +const LEGACY_TELEGRAM_BEHAVIOR_LEDGER = [ + { + id: "telegram-other-bot-command-gating", + owner: "qa/scenarios/channels/telegram-other-bot-command-gating.yaml", + scenarioId: "telegram-other-bot-command-gating", + }, + { + id: "telegram-mentioned-message-reply", + owner: "qa/scenarios/channels/channel-canary.yaml", + scenarioId: "channel-canary", + }, + { + id: "telegram-stream-final-single-message", + owner: "qa/scenarios/channels/telegram-stream-final-single-message.yaml", + scenarioId: "telegram-stream-final-single-message", + }, + { + id: "telegram-long-final-reuses-preview", + owner: "qa/scenarios/channels/telegram-long-final-reuses-preview.yaml", + scenarioId: "telegram-long-final-reuses-preview", + }, + { + id: "telegram-long-final-three-chunks", + owner: "qa/scenarios/channels/telegram-long-final-three-chunks.yaml", + scenarioId: "telegram-long-final-three-chunks", + }, + { + id: "telegram-mention-gating", + owner: "qa/scenarios/channels/channel-mention-gating.yaml", + scenarioId: "channel-mention-gating", + }, +] as const; + +const LEGACY_TELEGRAM_CAPABILITY_LEDGER = [ + { + id: "credentials-and-lease-cleanup", + owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts", + proof: "acquireQaCredentialLease", + }, + { + id: "bot-api-and-rich-message-observation", + owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts", + proof: "normalizeTelegramObservedMessage", + }, + { + id: "native-send-reply-edit-mapping", + owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts", + proof: "reply_parameters", + }, + { + id: "gateway-readiness-and-config", + owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts", + proof: "waitForTelegramChannelRunning", + }, + { + id: "selection-and-standard-artifacts", + owner: "extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts", + proof: "runQaFlowSuiteFromRuntime", + }, + { + id: "package-rtt-sampling", + owner: "scripts/e2e/npm-telegram-live-runner.ts", + proof: "createRoundTripProbe", + }, +] as const; + +describe("legacy Telegram runner parity", () => { + it("maps every retired scenario to one maintained YAML owner", () => { + expect(LEGACY_TELEGRAM_BEHAVIOR_LEDGER).toHaveLength(6); + expect(new Set(LEGACY_TELEGRAM_BEHAVIOR_LEDGER.map(({ id }) => id)).size).toBe(6); + + for (const entry of LEGACY_TELEGRAM_BEHAVIOR_LEDGER) { + const scenario = readQaScenarioById(entry.scenarioId); + expect(scenario.sourcePath, entry.id).toBe(entry.owner); + } + }); + + it("keeps every runner capability at its intended owner boundary", () => { + for (const entry of LEGACY_TELEGRAM_CAPABILITY_LEDGER) { + expect(fs.readFileSync(entry.owner, "utf8"), entry.id).toContain(entry.proof); + } + }); + + it("has no unresolved legacy runner owner", () => { + expect( + fs.existsSync("extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts"), + ).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts new file mode 100644 index 000000000000..e9d142bceebe --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { + listTelegramQaScenarios, + resolveTelegramQaScenarioIds, + TELEGRAM_QA_ALL_SCENARIO_IDS, +} from "./profiles.js"; + +describe("Telegram QA profiles", () => { + it("keeps release focused and adds the scripted long-final check for mock runs", () => { + const live = resolveTelegramQaScenarioIds({ providerMode: "live-frontier" }); + const mock = resolveTelegramQaScenarioIds({ providerMode: "mock-openai" }); + + expect(live).toContain("telegram-other-bot-command-gating"); + expect(live).not.toContain("telegram-long-final-reuses-preview"); + expect(mock).toEqual([...live, "telegram-long-final-reuses-preview"]); + }); + + it("selects every migrated Telegram scenario through all", () => { + expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual([ + ...TELEGRAM_QA_ALL_SCENARIO_IDS, + ]); + }); + + it("lets explicit scenarios override profile selection", () => { + expect( + resolveTelegramQaScenarioIds({ + profile: "all", + providerMode: "live-frontier", + scenarioIds: ["telegram-status-command"], + }), + ).toEqual(["telegram-status-command"]); + }); + + it("rejects unknown profiles and scenarios before gateway startup", () => { + expect(() => + resolveTelegramQaScenarioIds({ providerMode: "live-frontier", profile: "transport" }), + ).toThrow('Unknown QA Lab Telegram profile "transport"'); + expect(() => + resolveTelegramQaScenarioIds({ + providerMode: "live-frontier", + scenarioIds: ["telegram-missing"], + }), + ).toThrow("unknown Telegram QA scenario id(s): telegram-missing"); + }); + + it("lists the YAML catalog with provider-specific release defaults", () => { + const scenarios = listTelegramQaScenarios("mock-openai"); + + expect(scenarios.map(({ id }) => id).toSorted()).toEqual( + [...TELEGRAM_QA_ALL_SCENARIO_IDS].toSorted(), + ); + expect( + scenarios.find(({ id }) => id === "telegram-long-final-reuses-preview")?.defaultEnabled, + ).toBe(true); + expect( + scenarios.find(({ id }) => id === "telegram-long-final-three-chunks")?.defaultEnabled, + ).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.ts new file mode 100644 index 000000000000..5f95f774fba8 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.ts @@ -0,0 +1,77 @@ +import { readQaScenarioPack } from "../../scenario-catalog.js"; + +const TELEGRAM_QA_RELEASE_SCENARIO_IDS = [ + "channel-canary", + "channel-mention-gating", + "telegram-help-command", + "telegram-commands-command", + "telegram-tools-compact-command", + "telegram-whoami-command", + "telegram-status-command", + "telegram-repeated-command-authorization", + "telegram-context-command", + "telegram-other-bot-command-gating", +] as const; + +const TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS = [ + ...TELEGRAM_QA_RELEASE_SCENARIO_IDS, + "telegram-long-final-reuses-preview", +] as const; + +export const TELEGRAM_QA_ALL_SCENARIO_IDS = [ + ...TELEGRAM_QA_RELEASE_SCENARIO_IDS, + "telegram-current-session-status-tool", + "telegram-tool-only-usage-footer", + "telegram-reply-chain-exact-marker", + "telegram-stream-final-single-message", + "telegram-long-final-reuses-preview", + "telegram-long-final-three-chunks", +] as const; + +type TelegramQaProfile = "all" | "release"; + +function resolveTelegramQaProfile(profile: string | undefined): TelegramQaProfile { + const normalized = profile?.trim() || "release"; + if (normalized === "all" || normalized === "release") { + return normalized; + } + throw new Error( + `Unknown QA Lab Telegram profile "${normalized}". Expected one of: all, release.`, + ); +} + +export function resolveTelegramQaScenarioIds(params: { + profile?: string; + providerMode: string; + scenarioIds?: readonly string[]; +}): string[] { + const knownIds = new Set(TELEGRAM_QA_ALL_SCENARIO_IDS); + if (params.scenarioIds && params.scenarioIds.length > 0) { + const unknownIds = params.scenarioIds.filter((id) => !knownIds.has(id)); + if (unknownIds.length > 0) { + throw new Error(`unknown Telegram QA scenario id(s): ${unknownIds.join(", ")}`); + } + return [...params.scenarioIds]; + } + const profile = resolveTelegramQaProfile(params.profile); + if (profile === "all") { + return [...TELEGRAM_QA_ALL_SCENARIO_IDS]; + } + return params.providerMode === "mock-openai" + ? [...TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS] + : [...TELEGRAM_QA_RELEASE_SCENARIO_IDS]; +} + +export function listTelegramQaScenarios(providerMode: string) { + const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode, profile: "release" })); + const allIds = new Set(TELEGRAM_QA_ALL_SCENARIO_IDS); + return readQaScenarioPack() + .scenarios.filter((scenario) => allIds.has(scenario.id)) + .map((scenario) => ({ + id: scenario.id, + title: scenario.title, + rationale: scenario.objective, + regressionRefs: scenario.regressionRefs ?? [], + defaultEnabled: defaultIds.has(scenario.id), + })); +} diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts new file mode 100644 index 000000000000..d691b9d725d0 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it, vi } from "vitest"; +import { + buildTelegramQaConfig, + isRecoverableTelegramQaPollError, + normalizeTelegramObservedMessage, + parseTelegramQaCredentialPayload, + resolveTelegramQaRuntimeEnv, + waitForTelegramChannelRunning, +} from "./telegram-api.runtime.js"; + +describe("Telegram QA API boundary", () => { + it("parses env and leased credential payloads", () => { + expect( + resolveTelegramQaRuntimeEnv({ + OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", + OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "placeholder", + OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "placeholder", + }), + ).toEqual({ + groupId: "-100123", + driverToken: "placeholder", + sutToken: "placeholder", + }); + expect( + parseTelegramQaCredentialPayload({ + groupId: "-100456", + driverToken: "placeholder", + sutToken: "placeholder", + }), + ).toEqual({ + groupId: "-100456", + driverToken: "placeholder", + sutToken: "placeholder", + }); + expect(() => + parseTelegramQaCredentialPayload({ + groupId: "group-name", + driverToken: "placeholder", + sutToken: "placeholder", + }), + ).toThrow("numeric Telegram chat id"); + }); + + it("normalizes rich edited messages and native reply metadata", () => { + expect( + normalizeTelegramObservedMessage({ + update_id: 9, + edited_message: { + message_id: 42, + date: 123, + chat: { id: -100123 }, + from: { id: 2, is_bot: true, username: "sut_bot" }, + rich_message: { + blocks: [{ text: "final " }, { text: [{ text: "reply" }] }], + }, + reply_to_message: { message_id: 41 }, + }, + }), + ).toMatchObject({ + updateId: 9, + messageId: 42, + chatId: -100123, + senderId: 2, + senderIsBot: true, + text: "final \nreply", + replyToMessageId: 41, + timestamp: 123_000, + }); + }); + + it("builds the isolated Telegram gateway config", () => { + const config = buildTelegramQaConfig( + { plugins: { allow: ["qa-lab"] } }, + { + groupId: "-100123", + sutToken: "placeholder", + driverBotId: 1, + sutAccountId: "sut", + }, + ); + + expect(config.plugins?.allow).toEqual(["qa-lab", "telegram"]); + expect(config.channels?.telegram).toMatchObject({ + enabled: true, + defaultAccount: "sut", + accounts: { + sut: { + botToken: "placeholder", + dmPolicy: "disabled", + replyToMode: "first", + groups: { + "-100123": { + groupPolicy: "allowlist", + allowFrom: ["1"], + requireMention: true, + }, + }, + }, + }, + }); + }); + + it("waits for the selected Telegram account to become connected", async () => { + const call = vi + .fn() + .mockResolvedValueOnce({ + channelAccounts: { + telegram: [{ accountId: "sut", running: true, connected: false }], + }, + }) + .mockResolvedValueOnce({ + channelAccounts: { + telegram: [{ accountId: "sut", running: true, connected: true }], + }, + }); + + await waitForTelegramChannelRunning({ call }, "sut", { timeoutMs: 100, pollMs: 1 }); + expect(call).toHaveBeenCalledTimes(2); + }); + + it("classifies transient polling failures", () => { + expect(isRecoverableTelegramQaPollError(new Error("socket hang up"))).toBe(true); + expect(isRecoverableTelegramQaPollError(new Error("Telegram unauthorized"))).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts new file mode 100644 index 000000000000..c152523f7b94 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts @@ -0,0 +1,433 @@ +import { setTimeout as sleep } from "node:timers/promises"; +// Qa Lab plugin module owns Telegram live adapter API and credential behavior. +import type { TelegramBotMessage, TelegramBotUpdate } from "@openclaw/telegram/api.js"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + parseStrictPositiveInteger, + resolveTimerTimeoutMs, +} from "openclaw/plugin-sdk/number-runtime"; +import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http"; +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; +import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { z } from "zod"; + +export type TelegramQaRuntimeEnv = { + groupId: string; + driverToken: string; + sutToken: string; +}; + +export type TelegramBotIdentity = { + id: number; + is_bot: boolean; + first_name: string; + username?: string; +}; + +type TelegramApiEnvelope = { + ok: boolean; + result?: T; + description?: string; +}; + +type TelegramReplyMarkup = { + inline_keyboard?: Array>; +}; + +type TelegramRichMessage = { + markdown?: string; + html?: string; + blocks?: unknown[]; +}; + +type TelegramMessage = Pick & + Partial> & { + audio?: unknown; + chat: { id: number }; + document?: unknown; + from?: Pick, "id" | "is_bot" | "username">; + photo?: unknown[]; + rich_message?: TelegramRichMessage; + reply_markup?: TelegramReplyMarkup; + reply_to_message?: { message_id?: number }; + sticker?: unknown; + video?: unknown; + voice?: unknown; + }; + +export type TelegramUpdate = Pick & { + edited_message?: TelegramMessage; + message?: TelegramMessage; +}; + +type TelegramObservedMessage = { + updateId: number; + messageId: number; + chatId: number; + senderId: number; + senderIsBot: boolean; + senderUsername?: string; + text: string; + caption?: string; + replyToMessageId?: number; + timestamp: number; + inlineButtons: string[]; + mediaKinds: string[]; +}; + +type TelegramChannelStatus = { + accountId?: string; + connected?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string | null; + restartPending?: boolean; + running?: boolean; +}; + +type TelegramGatewayClient = { + call: (method: string, params?: unknown, options?: { timeoutMs?: number }) => Promise; +}; + +const TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS = 45_000; +const TELEGRAM_QA_ENV_FIELDS = [ + { field: "groupId", envKey: "OPENCLAW_QA_TELEGRAM_GROUP_ID" }, + { field: "driverToken", envKey: "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN" }, + { field: "sutToken", envKey: "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN" }, +] as const; + +const telegramQaCredentialPayloadSchema = z.object({ + groupId: z.string().trim().min(1), + driverToken: z.string().trim().min(1), + sutToken: z.string().trim().min(1), +}); + +function resolveEnvValue( + env: NodeJS.ProcessEnv, + key: (typeof TELEGRAM_QA_ENV_FIELDS)[number]["envKey"], +) { + const value = env[key]?.trim(); + if (!value) { + throw new Error(`Missing ${key}.`); + } + return value; +} + +export function resolveTelegramQaRuntimeEnv( + env: NodeJS.ProcessEnv = process.env, +): TelegramQaRuntimeEnv { + return parseTelegramQaCredentialPayload( + Object.fromEntries( + TELEGRAM_QA_ENV_FIELDS.map(({ envKey, field }) => [field, resolveEnvValue(env, envKey)]), + ), + ); +} + +export function parseTelegramQaCredentialPayload(payload: unknown): TelegramQaRuntimeEnv { + const parsed = telegramQaCredentialPayloadSchema.parse(payload); + if (!/^-?\d+$/u.test(parsed.groupId)) { + throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id."); + } + return parsed; +} + +function flattenInlineButtons(replyMarkup?: TelegramReplyMarkup) { + return (replyMarkup?.inline_keyboard ?? []) + .flat() + .map((button) => button.text?.trim()) + .filter((text): text is string => Boolean(text)); +} + +function detectMediaKinds(message: TelegramMessage) { + const kinds: string[] = []; + if (Array.isArray(message.photo) && message.photo.length > 0) { + kinds.push("photo"); + } + for (const kind of ["document", "audio", "video", "voice", "sticker"] as const) { + if (message[kind]) { + kinds.push(kind); + } + } + return kinds; +} + +function flattenTelegramRichText(value: unknown): string { + if (typeof value === "string") { + return value; + } + if (Array.isArray(value)) { + return value.map((part) => flattenTelegramRichText(part)).join(""); + } + if (!isRecord(value)) { + return ""; + } + if ("text" in value) { + return flattenTelegramRichText(value.text); + } + if (typeof value.alternative_text === "string") { + return value.alternative_text; + } + return typeof value.expression === "string" ? value.expression : ""; +} + +function flattenTelegramRichTableCells(value: unknown): string { + if (!Array.isArray(value)) { + return flattenTelegramRichBlock(value); + } + return value + .map((row) => { + const cells = Array.isArray(row) ? row : [row]; + return cells + .map((cell) => flattenTelegramRichBlock(cell)) + .filter((cell) => cell.trim()) + .join("\t"); + }) + .filter((row) => row.trim()) + .join("\n"); +} + +function flattenTelegramRichBlocks(value: unknown): string { + const blocks = Array.isArray(value) ? value : [value]; + return blocks + .map((block) => flattenTelegramRichBlock(block)) + .filter((part) => part.trim()) + .join("\n"); +} + +function flattenTelegramRichBlock(value: unknown): string { + if (typeof value === "string" || Array.isArray(value)) { + return flattenTelegramRichText(value); + } + if (!isRecord(value)) { + return ""; + } + const parts = [ + "text" in value ? flattenTelegramRichText(value.text) : "", + "summary" in value ? flattenTelegramRichText(value.summary) : "", + typeof value.label === "string" ? value.label : "", + typeof value.expression === "string" ? value.expression : "", + "blocks" in value ? flattenTelegramRichBlocks(value.blocks) : "", + "items" in value ? flattenTelegramRichBlocks(value.items) : "", + "cells" in value ? flattenTelegramRichTableCells(value.cells) : "", + "caption" in value ? flattenTelegramRichBlock(value.caption) : "", + "credit" in value ? flattenTelegramRichText(value.credit) : "", + ]; + return parts.filter((part) => part.trim()).join("\n"); +} + +function selectTelegramObservedText(message: TelegramMessage) { + return ( + message.text || + message.caption || + message.rich_message?.markdown || + message.rich_message?.html || + flattenTelegramRichBlocks(message.rich_message?.blocks) || + "" + ); +} + +export function normalizeTelegramObservedMessage( + update: TelegramUpdate, +): TelegramObservedMessage | null { + const message = update.edited_message ?? update.message; + if (!message?.from?.id) { + return null; + } + return { + updateId: update.update_id, + messageId: message.message_id, + chatId: message.chat.id, + senderId: message.from.id, + senderIsBot: message.from.is_bot, + senderUsername: message.from.username, + text: selectTelegramObservedText(message), + caption: message.caption, + replyToMessageId: message.reply_to_message?.message_id, + timestamp: message.date * 1_000, + inlineButtons: flattenInlineButtons(message.reply_markup), + mediaKinds: detectMediaKinds(message), + }; +} + +export function buildTelegramQaConfig( + baseCfg: OpenClawConfig, + params: { + groupId: string; + sutToken: string; + driverBotId: number; + sutAccountId: string; + }, +): OpenClawConfig { + return { + ...baseCfg, + agents: { + ...baseCfg.agents, + defaults: { + ...baseCfg.agents?.defaults, + models: { + ...baseCfg.agents?.defaults?.models, + "openai/gpt-5.6-luna": { + ...baseCfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"], + agentRuntime: { id: "openclaw" }, + }, + }, + skipBootstrap: true, + }, + }, + plugins: { + ...baseCfg.plugins, + allow: uniqueStrings([...(baseCfg.plugins?.allow ?? []), "telegram"]), + entries: { + ...baseCfg.plugins?.entries, + telegram: { enabled: true }, + }, + }, + messages: { + ...baseCfg.messages, + groupChat: { + ...baseCfg.messages?.groupChat, + visibleReplies: "automatic", + }, + }, + channels: { + ...baseCfg.channels, + telegram: { + enabled: true, + defaultAccount: params.sutAccountId, + accounts: { + [params.sutAccountId]: { + enabled: true, + botToken: params.sutToken, + dmPolicy: "disabled", + replyToMode: "first", + groups: { + [params.groupId]: { + groupPolicy: "allowlist", + allowFrom: [String(params.driverBotId)], + requireMention: true, + }, + }, + }, + }, + }, + }, + }; +} + +export async function callTelegramApi( + token: string, + method: string, + body?: Record, + timeoutMs = 15_000, +): Promise { + const requestTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 15_000); + const { response, release } = await fetchWithSsrFGuard({ + url: `https://api.telegram.org/bot${token}/${method}`, + init: { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body ?? {}), + }, + timeoutMs: requestTimeoutMs, + policy: { hostnameAllowlist: ["api.telegram.org"] }, + auditContext: "qa-lab-telegram-live", + capture: false, + }); + try { + const payload = await readProviderJsonResponse>( + response, + `qa-lab-telegram-live.${method}`, + ); + if (!response.ok || !payload.ok || payload.result === undefined) { + throw new Error( + payload.description?.trim() || `${method} failed with status ${response.status}`, + ); + } + return payload.result; + } finally { + await release(); + } +} + +export function isRecoverableTelegramQaPollError(error: unknown): boolean { + const message = formatErrorMessage(error).toLowerCase(); + return [ + "fetch failed", + "aborted due to timeout", + "operation was aborted", + "request timed out", + "aborterror", + "econnreset", + "etimedout", + "socket hang up", + "terminated", + ].some((fragment) => message.includes(fragment)); +} + +export async function waitForTelegramPollRetryDelay(remainingMs = 250) { + await sleep(Math.min(250, Math.max(100, remainingMs))); +} + +export async function flushTelegramUpdates(token: string) { + const startedAt = Date.now(); + let offset = 0; + while (Date.now() - startedAt < 15_000) { + const updates = await callTelegramApi( + token, + "getUpdates", + { offset, timeout: 0, allowed_updates: ["message", "edited_message"] }, + 15_000, + ); + if (updates.length === 0) { + return offset; + } + offset = (updates.at(-1)?.update_id ?? offset) + 1; + } + throw new Error("timed out after 15000ms draining Telegram updates"); +} + +function resolveTelegramQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS; + return raw + ? (parseStrictPositiveInteger(raw) ?? TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS) + : TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS; +} + +export async function waitForTelegramChannelRunning( + gateway: TelegramGatewayClient, + accountId: string, + options?: { env?: NodeJS.ProcessEnv; pollMs?: number; timeoutMs?: number }, +) { + const startedAt = Date.now(); + const timeoutMs = options?.timeoutMs ?? resolveTelegramQaReadyTimeoutMs(options?.env); + const pollMs = options?.pollMs ?? 500; + let lastProbeError: string | undefined; + let lastStatus: TelegramChannelStatus | undefined; + while (Date.now() - startedAt < timeoutMs) { + try { + const payload = (await gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { channelAccounts?: Record }; + const match = (payload.channelAccounts?.telegram ?? []).find( + (entry) => entry.accountId === accountId, + ); + lastProbeError = undefined; + lastStatus = match; + if (match?.running && match.connected === true && match.restartPending !== true) { + return; + } + } catch (error) { + lastProbeError = formatErrorMessage(error); + } + await sleep(pollMs); + } + const details = lastStatus + ? `; last status: ${JSON.stringify(lastStatus)}` + : lastProbeError + ? `; last probe error: ${lastProbeError}` + : ""; + throw new Error(`telegram account "${accountId}" did not become ready${details}`); +} diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts deleted file mode 100644 index 0dcb3c342efe..000000000000 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ /dev/null @@ -1,1379 +0,0 @@ -// Qa Lab tests cover telegram live plugin behavior. -import { expectDefined } from "@openclaw/normalization-core"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - findMissingLiveTransportStandardScenarios, -} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { summarizeLiveTransportRttSamples } from "../shared/live-transport-rtt.js"; -import { __testing as testing } from "./telegram-live.runtime.js"; - -const fetchWithSsrFGuardMock = vi.hoisted(() => - vi.fn( - async (params: { - url: string; - init?: RequestInit; - signal?: AbortSignal; - timeoutMs?: number; - }) => ({ - response: await fetch(params.url, { - ...params.init, - signal: params.signal ?? AbortSignal.timeout(params.timeoutMs ?? 0), - }), - release: async () => {}, - }), - ), -); - -vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => { - const actual = await vi.importActual( - "openclaw/plugin-sdk/ssrf-runtime", - ); - return { - ...actual, - fetchWithSsrFGuard: fetchWithSsrFGuardMock, - }; -}); - -function requireScenario(scenarios: T[], id: string): T { - const scenario = scenarios.find((candidate) => candidate.id === id); - if (!scenario) { - throw new Error(`Expected scenario ${id}`); - } - return scenario; -} - -describe("telegram live qa runtime", () => { - afterEach(() => { - fetchWithSsrFGuardMock.mockClear(); - vi.restoreAllMocks(); - vi.unstubAllGlobals(); - }); - - it("resolves required Telegram QA env vars", () => { - expect( - testing.resolveTelegramQaRuntimeEnv({ - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut", - }), - ).toEqual({ - groupId: "-100123", - driverToken: "driver", - sutToken: "sut", - }); - }); - - it("fails when a required Telegram QA env var is missing", () => { - expect(() => - testing.resolveTelegramQaRuntimeEnv({ - OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", - }), - ).toThrow("OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"); - }); - - it("fails when the Telegram group id is not numeric", () => { - expect(() => - testing.resolveTelegramQaRuntimeEnv({ - OPENCLAW_QA_TELEGRAM_GROUP_ID: "qa-group", - OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", - OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut", - }), - ).toThrow("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id."); - }); - - it("parses Telegram live progress env booleans", () => { - expect(testing.parseTelegramQaProgressBooleanEnv("true")).toBe(true); - expect(testing.parseTelegramQaProgressBooleanEnv("on")).toBe(true); - expect(testing.parseTelegramQaProgressBooleanEnv("false")).toBe(false); - expect(testing.parseTelegramQaProgressBooleanEnv("off")).toBe(false); - expect(testing.parseTelegramQaProgressBooleanEnv("maybe")).toBeUndefined(); - }); - - it("defaults Telegram live progress logging from CI when no override is set", () => { - expect(testing.shouldLogTelegramQaLiveProgress({ CI: "true" })).toBe(true); - expect(testing.shouldLogTelegramQaLiveProgress({ CI: "false" })).toBe(false); - }); - - it("applies OPENCLAW_QA_SUITE_PROGRESS override to Telegram live logging", () => { - expect( - testing.shouldLogTelegramQaLiveProgress({ - CI: "false", - OPENCLAW_QA_SUITE_PROGRESS: "true", - }), - ).toBe(true); - expect( - testing.shouldLogTelegramQaLiveProgress({ - CI: "true", - OPENCLAW_QA_SUITE_PROGRESS: "false", - }), - ).toBe(false); - expect( - testing.shouldLogTelegramQaLiveProgress({ - CI: "true", - OPENCLAW_QA_SUITE_PROGRESS: "definitely", - }), - ).toBe(true); - }); - - it("waits until the Telegram channel account is connected", async () => { - const gateway = { - call: vi - .fn() - .mockResolvedValueOnce({ - channelAccounts: { - telegram: [ - { - accountId: "sut", - connected: false, - restartPending: false, - running: true, - }, - ], - }, - }) - .mockResolvedValueOnce({ - channelAccounts: { - telegram: [ - { - accountId: "sut", - connected: true, - restartPending: false, - running: true, - }, - ], - }, - }), - }; - - await testing.waitForTelegramChannelRunning(gateway as never, "sut", { - pollMs: 1, - timeoutMs: 100, - }); - - expect(gateway.call).toHaveBeenCalledTimes(2); - }); - - it("normalizes the Telegram QA transport ready timeout env", () => { - expect(testing.resolveTelegramQaReadyTimeoutMs({})).toBe(45_000); - expect( - testing.resolveTelegramQaReadyTimeoutMs({ - OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000", - }), - ).toBe(180_000); - expect( - testing.resolveTelegramQaReadyTimeoutMs({ - OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "bad", - }), - ).toBe(45_000); - for (const value of ["0x10", "1e3", "10.5"]) { - expect( - testing.resolveTelegramQaReadyTimeoutMs({ - OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: value, - }), - ).toBe(45_000); - } - }); - - it("includes the last Telegram readiness status when the account stays unavailable", async () => { - const gateway = { - call: vi.fn().mockResolvedValue({ - channelAccounts: { - telegram: [ - { - accountId: "sut", - connected: false, - lastError: "Telegram getUpdates conflict", - restartPending: true, - running: true, - }, - ], - }, - }), - }; - - await expect( - testing.waitForTelegramChannelRunning(gateway as never, "sut", { - pollMs: 1, - timeoutMs: 5, - }), - ).rejects.toThrow( - 'telegram account "sut" did not become ready; last status: {"connected":false,"lastError":"Telegram getUpdates conflict","restartPending":true,"running":true}', - ); - }); - - it("normalizes the Telegram QA canary timeout env", () => { - expect(testing.resolveTelegramQaCanaryTimeoutMs({})).toBe(30_000); - expect( - testing.resolveTelegramQaCanaryTimeoutMs({ - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "90000", - }), - ).toBe(90_000); - expect( - testing.resolveTelegramQaCanaryTimeoutMs({ - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "nope", - }), - ).toBe(30_000); - for (const value of ["0x10", "1e3", "10.5"]) { - expect( - testing.resolveTelegramQaCanaryTimeoutMs({ - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: value, - }), - ).toBe(30_000); - } - }); - - it("normalizes the Telegram QA scenario timeout env", () => { - expect(testing.resolveTelegramQaScenarioTimeoutMs(45_000, {})).toBe(45_000); - expect( - testing.resolveTelegramQaScenarioTimeoutMs(45_000, { - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "180000", - }), - ).toBe(180_000); - expect( - testing.resolveTelegramQaScenarioTimeoutMs(45_000, { - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "nope", - }), - ).toBe(45_000); - for (const value of ["0x10", "1e3", "10.5"]) { - expect( - testing.resolveTelegramQaScenarioTimeoutMs(45_000, { - OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: value, - }), - ).toBe(45_000); - } - }); - - it("normalizes Telegram RTT options", () => { - expect(testing.normalizeTelegramQaRttOptions({})).toBeUndefined(); - expect( - testing.normalizeTelegramQaRttOptions({ - count: 3, - timeoutMs: 45_000, - }), - ).toEqual({ - count: 3, - maxFailures: 3, - checkIds: new Set(["telegram-mentioned-message-reply"]), - timeoutMs: 45_000, - }); - expect( - testing.normalizeTelegramQaRttOptions({ - checkIds: ["telegram-mentioned-message-reply"], - count: 3, - maxFailures: 1, - }), - ).toEqual({ - count: 3, - maxFailures: 1, - checkIds: new Set(["telegram-mentioned-message-reply"]), - timeoutMs: 30_000, - }); - }); - - it("rejects unknown Telegram RTT checks", () => { - expect(() => - testing.normalizeTelegramQaRttOptions({ - checkIds: ["telegram-rtt-only"], - count: 1, - }), - ).toThrow("unknown Telegram QA RTT check: telegram-rtt-only"); - }); - - it("summarizes live transport RTT timing", () => { - expect( - summarizeLiveTransportRttSamples([ - { status: "pass", rttMs: 1000 }, - { status: "pass", rttMs: 2000 }, - { status: "pass", rttMs: 4000 }, - { status: "fail" }, - ]), - ).toEqual({ - passed: 3, - failed: 1, - timing: { - rttMs: 2000, - avgMs: 2333, - p50Ms: 2000, - p95Ms: 4000, - maxMs: 4000, - samples: 4, - failedSamples: 1, - }, - }); - expect(summarizeLiveTransportRttSamples([{ status: "fail" }, { status: "fail" }])).toEqual({ - passed: 0, - failed: 2, - timing: { - rttMs: undefined, - avgMs: undefined, - p50Ms: undefined, - p95Ms: undefined, - maxMs: undefined, - samples: 2, - failedSamples: 2, - }, - }); - }); - - it("sanitizes and truncates Telegram live progress details", () => { - expect(testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe( - "scenario id value", - ); - expect(testing.sanitizeTelegramQaProgressValue("\u0000\u0001")).toBe(""); - const details = testing.formatTelegramQaProgressDetails(`header\n${"x".repeat(500)}`); - expect(details.startsWith("header ")).toBe(true); - expect(details.length).toBeLessThanOrEqual(240); - expect(details.endsWith("...")).toBe(true); - }); - - it("parses Telegram pooled credential payloads", () => { - expect( - testing.parseTelegramQaCredentialPayload({ - groupId: "-100123", - driverToken: "driver", - sutToken: "sut", - }), - ).toEqual({ - groupId: "-100123", - driverToken: "driver", - sutToken: "sut", - }); - }); - - it("rejects Telegram pooled credential payloads with non-numeric group ids", () => { - expect(() => - testing.parseTelegramQaCredentialPayload({ - groupId: "qa-group", - driverToken: "driver", - sutToken: "sut", - }), - ).toThrow("Telegram credential payload groupId must be a numeric Telegram chat id."); - }); - - it("injects a temporary Telegram account into the QA gateway config", () => { - const baseCfg: OpenClawConfig = { - plugins: { - allow: ["memory-core", "qa-channel"], - entries: { - "memory-core": { enabled: true }, - "qa-channel": { enabled: true }, - }, - }, - channels: { - "qa-channel": { - enabled: true, - baseUrl: "http://127.0.0.1:43123", - botUserId: "openclaw", - botDisplayName: "OpenClaw QA", - allowFrom: ["*"], - }, - }, - }; - - const next = testing.buildTelegramQaConfig(baseCfg, { - groupId: "-100123", - sutToken: "sut-token", - driverBotId: 42, - sutAccountId: "sut", - }); - - expect(next.agents?.defaults?.skipBootstrap).toBe(true); - expect(next.agents?.defaults?.models?.["openai/gpt-5.6-luna"]?.agentRuntime).toEqual({ - id: "openclaw", - }); - expect(next.plugins?.allow).toContain("telegram"); - expect(next.plugins?.entries?.telegram).toEqual({ enabled: true }); - expect(next.messages?.groupChat?.visibleReplies).toBe("automatic"); - expect(next.channels?.telegram).toEqual({ - enabled: true, - defaultAccount: "sut", - accounts: { - sut: { - enabled: true, - botToken: "sut-token", - dmPolicy: "disabled", - replyToMode: "first", - groups: { - "-100123": { - groupPolicy: "allowlist", - allowFrom: ["42"], - requireMention: true, - }, - }, - }, - }, - }); - }); - - it("normalizes observed Telegram messages", () => { - expect( - testing.normalizeTelegramObservedMessage({ - update_id: 7, - message: { - message_id: 9, - date: 1_700_000_000, - text: "hello", - chat: { id: -100123 }, - from: { - id: 42, - is_bot: true, - username: "driver_bot", - }, - reply_to_message: { message_id: 8 }, - reply_markup: { - inline_keyboard: [[{ text: "Approve" }, { text: "Deny" }]], - }, - photo: [{}], - }, - }), - ).toEqual({ - updateId: 7, - messageId: 9, - chatId: -100123, - senderId: 42, - senderIsBot: true, - senderUsername: "driver_bot", - text: "hello", - caption: undefined, - replyToMessageId: 8, - timestamp: 1_700_000_000_000, - inlineButtons: ["Approve", "Deny"], - mediaKinds: ["photo"], - }); - }); - - it("normalizes Telegram rich messages as observed text", () => { - expect( - testing.normalizeTelegramObservedMessage({ - update_id: 8, - message: { - message_id: 10, - date: 1_700_000_001, - rich_message: { markdown: "**hello** rich" }, - chat: { id: -100123 }, - from: { - id: 88, - is_bot: true, - username: "sut_bot", - }, - reply_to_message: { message_id: 9 }, - }, - })?.text, - ).toBe("**hello** rich"); - - expect( - testing.normalizeTelegramObservedMessage({ - update_id: 9, - message: { - message_id: 11, - date: 1_700_000_002, - rich_message: { html: "hello rich" }, - chat: { id: -100123 }, - from: { - id: 88, - is_bot: true, - username: "sut_bot", - }, - }, - })?.text, - ).toBe("hello rich"); - - expect( - testing.normalizeTelegramObservedMessage({ - update_id: 10, - message: { - message_id: 12, - date: 1_700_000_003, - rich_message: { - blocks: [ - { - type: "paragraph", - text: ["hello ", { type: "bold", text: "rich" }], - }, - { - type: "footer", - text: { type: "italic", text: "footer" }, - }, - ], - }, - chat: { id: -100123 }, - from: { - id: 88, - is_bot: true, - username: "sut_bot", - }, - }, - })?.text, - ).toBe("hello rich\nfooter"); - - expect( - testing.normalizeTelegramObservedMessage({ - update_id: 11, - message: { - message_id: 13, - date: 1_700_000_004, - rich_message: { - blocks: [ - { - type: "table", - cells: [[{ text: "Open" }, { text: "Claw" }], [{ text: "release" }]], - }, - ], - }, - chat: { id: -100123 }, - from: { - id: 88, - is_bot: true, - username: "sut_bot", - }, - }, - })?.text, - ).toBe("Open\tClaw\nrelease"); - }); - - it("ignores unrelated sut replies when matching the canary response", () => { - expect( - testing.classifyCanaryReply({ - groupId: "-100123", - sutBotId: 88, - driverMessageId: 55, - message: { - updateId: 1, - messageId: 9, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "other reply", - replyToMessageId: 999, - timestamp: 1_700_000_000_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toBe("unthreaded"); - expect( - testing.classifyCanaryReply({ - groupId: "-100123", - sutBotId: 88, - driverMessageId: 55, - message: { - updateId: 2, - messageId: 10, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "canary reply", - replyToMessageId: 55, - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toBe("match"); - }); - - it("classifies threaded blank sut replies as matches", () => { - expect( - testing.classifyCanaryReply({ - groupId: "-100123", - sutBotId: 88, - driverMessageId: 55, - message: { - updateId: 3, - messageId: 11, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "", - replyToMessageId: 55, - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: ["photo"], - }, - }), - ).toBe("match"); - }); - - it("keeps scenario text checks strict while allowing canary presence replies", () => { - const message = { - updateId: 3, - messageId: 11, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "", - replyToMessageId: 55, - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: [], - }; - - expect(() => testing.assertTelegramScenarioReply({ message })).toThrow( - "reply message 11 was empty", - ); - expect(() => testing.assertTelegramCanaryPresenceReply(message)).not.toThrow(); - }); - - it("fails when any requested Telegram scenario id is unknown", () => { - expect(() => - testing.findScenario(["telegram-mentioned-message-reply", "typo-scenario"]), - ).toThrow("unknown Telegram QA scenario id(s): typo-scenario"); - }); - - it("recognizes Telegram observation timeouts with retry details", () => { - expect( - testing.isTelegramObservedMessageTimeoutError( - new Error( - "timed out after 8000ms waiting for Telegram message; last polling error: The operation was aborted due to timeout", - ), - 8000, - ), - ).toBe(true); - expect( - testing.isTelegramObservedMessageTimeoutError( - new Error("timed out after 9000ms waiting for Telegram message"), - 8000, - ), - ).toBe(false); - }); - - it("includes mention gating in the Telegram live scenario catalog", () => { - const scenarios = testing.findScenario([ - "telegram-other-bot-command-gating", - "telegram-mentioned-message-reply", - "telegram-stream-final-single-message", - "telegram-long-final-reuses-preview", - "telegram-long-final-three-chunks", - "telegram-mention-gating", - ]); - expect(scenarios.map((scenario) => scenario.id)).toEqual([ - "telegram-other-bot-command-gating", - "telegram-mentioned-message-reply", - "telegram-stream-final-single-message", - "telegram-long-final-reuses-preview", - "telegram-long-final-three-chunks", - "telegram-mention-gating", - ]); - const otherBotStep = requireScenario(scenarios, "telegram-other-bot-command-gating").buildRun( - "sut_bot", - ).steps[0]; - expect(otherBotStep?.expectReply).toBe(false); - expect(otherBotStep?.input).toBe("/status@OpenClawQaOtherBot"); - const mentionedReplyRun = requireScenario( - scenarios, - "telegram-mentioned-message-reply", - ).buildRun("sut_bot"); - expect( - expectDefined(mentionedReplyRun.steps[0], "mentioned-message reply step") - .replyToLatestSutMessage, - ).toBe(true); - expect( - scenarios.find((scenario) => scenario.id === "telegram-mentioned-message-reply") - ?.evidenceCoverageIds, - ).toEqual(["channels.telegram.mention-gating"]); - const streamSingleStep = requireScenario( - scenarios, - "telegram-stream-final-single-message", - ).buildRun("sut_bot").steps[0]; - expect(streamSingleStep?.expectedJoinedSutTextIncludes).toEqual([ - "QA-TELEGRAM-STREAM-SINGLE-OK", - ]); - expect(streamSingleStep?.expectedSutMessageCount).toBe(1); - expect(streamSingleStep?.replyToLatestSutMessage).toBeUndefined(); - const longReusesStep = requireScenario( - scenarios, - "telegram-long-final-reuses-preview", - ).buildRun("sut_bot").steps[0]; - expect(longReusesStep?.expectedJoinedSutTextIncludes).toEqual([ - "TELEGRAM-LONG-FINAL-BEGIN", - "TELEGRAM-LONG-FINAL-END", - ]); - expect(longReusesStep?.expectedSutMessageCountRange).toEqual([1, 2]); - expect(longReusesStep?.replyToLatestSutMessage).toBe(true); - const longThreeChunksStep = requireScenario( - scenarios, - "telegram-long-final-three-chunks", - ).buildRun("sut_bot").steps[0]; - expect(longThreeChunksStep?.expectedJoinedSutTextIncludes).toEqual([ - "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN", - "TELEGRAM-LONG-FINAL-3CHUNK-END", - ]); - expect(longThreeChunksStep?.expectedSutMessageCount).toBe(3); - expect(longThreeChunksStep?.replyToLatestSutMessage).toBe(true); - }); - - it("keeps mock-scripted Telegram checks out of the default live-frontier set", () => { - expect(testing.findScenario(undefined, "live-frontier").map((scenario) => scenario.id)).toEqual( - [ - "telegram-other-bot-command-gating", - "telegram-mentioned-message-reply", - "telegram-mention-gating", - ], - ); - }); - - it("adds deterministic model-scripted checks to the default mock-openai set", () => { - expect(testing.findScenario(undefined, "mock-openai").map((scenario) => scenario.id)).toEqual([ - "telegram-other-bot-command-gating", - "telegram-mentioned-message-reply", - "telegram-long-final-reuses-preview", - "telegram-mention-gating", - ]); - }); - - it("lists remaining optional Telegram scenario metadata", () => { - const catalog = testing.listTelegramQaScenarioCatalog("mock-openai"); - const streamSingle = requireScenario(catalog, "telegram-stream-final-single-message"); - expect(streamSingle.defaultEnabled).toBe(false); - expect(streamSingle.regressionRefs).toEqual(["openclaw/openclaw#39905"]); - }); - - it("tracks Telegram live coverage against the shared transport contract", () => { - expect(testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]); - expect( - findMissingLiveTransportStandardScenarios({ - coveredStandardScenarioIds: testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS, - expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - }), - ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); - }); - - it("asserts long Telegram final replies reuse the streamed preview message", () => { - expect( - testing.assertTelegramScenarioMessageSet({ - expectedJoinedSutTextIncludes: ["TELEGRAM-LONG-FINAL-BEGIN", "TELEGRAM-LONG-FINAL-END"], - expectedSutMessageCountRange: [1, 2], - groupId: "-100123", - scenarioId: "telegram-long-final-reuses-preview", - sutBotId: 99, - observedMessages: [ - { - updateId: 1, - messageId: 10, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "TELEGRAM-LONG-FINAL-BEGIN part one part two TELEGRAM-LONG-FINAL-END", - timestamp: 1_700_000_000_000, - inlineButtons: [], - mediaKinds: [], - }, - ], - }), - ).toBeUndefined(); - - expect( - testing.assertTelegramScenarioMessageSet({ - expectedJoinedSutTextIncludes: ["TELEGRAM-LONG-FINAL-BEGIN", "TELEGRAM-LONG-FINAL-END"], - expectedSutMessageCountRange: [1, 2], - groupId: "-100123", - scenarioId: "telegram-long-final-reuses-preview", - sutBotId: 99, - observedMessages: [ - { - updateId: 1, - messageId: 10, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "TELEGRAM-LONG-FINAL-BEGIN part one ", - timestamp: 1_700_000_000_000, - inlineButtons: [], - mediaKinds: [], - }, - { - updateId: 2, - messageId: 11, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "part two TELEGRAM-LONG-FINAL-END", - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - ], - }), - ).toBeUndefined(); - - expect(() => - testing.assertTelegramScenarioMessageSet({ - expectedSutMessageCountRange: [1, 2], - groupId: "-100123", - scenarioId: "telegram-long-final-reuses-preview", - sutBotId: 99, - observedMessages: [ - { - updateId: 1, - messageId: 10, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "preview", - timestamp: 1_700_000_000_000, - inlineButtons: [], - mediaKinds: [], - }, - { - updateId: 2, - messageId: 11, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "final chunk one", - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - { - updateId: 3, - messageId: 12, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-reuses-preview", - scenarioTitle: "Telegram long final reuses the preview message", - matchedScenario: true, - text: "final chunk two", - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: [], - }, - ], - }), - ).toThrow("expected 1-2 SUT message(s), observed 3"); - }); - - it("accepts legitimate three-chunk Telegram final replies", () => { - expect( - testing.assertTelegramScenarioMessageSet({ - expectedJoinedSutTextIncludes: [ - "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN", - "TELEGRAM-LONG-FINAL-3CHUNK-END", - ], - expectedSutMessageCount: 3, - groupId: "-100123", - scenarioId: "telegram-long-final-three-chunks", - sutBotId: 99, - observedMessages: [ - { - updateId: 1, - messageId: 10, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-three-chunks", - scenarioTitle: "Telegram three-chunk final keeps only final chunks", - matchedScenario: true, - text: "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN part one ", - timestamp: 1_700_000_000_000, - inlineButtons: [], - mediaKinds: [], - }, - { - updateId: 2, - messageId: 11, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-three-chunks", - scenarioTitle: "Telegram three-chunk final keeps only final chunks", - matchedScenario: true, - text: "part two ", - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - { - updateId: 3, - messageId: 12, - chatId: -100123, - senderId: 99, - senderIsBot: true, - scenarioId: "telegram-long-final-three-chunks", - scenarioTitle: "Telegram three-chunk final keeps only final chunks", - matchedScenario: true, - text: "part three TELEGRAM-LONG-FINAL-3CHUNK-END", - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: [], - }, - ], - }), - ).toBeUndefined(); - }); - - it("matches scenario replies by thread or exact marker", () => { - expect( - testing.matchesTelegramScenarioReply({ - groupId: "-100123", - sentMessageId: 55, - sutBotId: 88, - message: { - updateId: 1, - messageId: 56, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "reply with TELEGRAM_QA_NOMENTION_TOKEN", - replyToMessageId: undefined, - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - matchText: "TELEGRAM_QA_NOMENTION_TOKEN", - }), - ).toBe(true); - expect( - testing.matchesTelegramScenarioReply({ - groupId: "-100123", - sentMessageId: 55, - sutBotId: 88, - message: { - updateId: 5, - messageId: 54, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "reply with TELEGRAM_QA_NOMENTION_TOKEN", - replyToMessageId: undefined, - timestamp: 1_700_000_005_000, - inlineButtons: [], - mediaKinds: [], - }, - matchText: "TELEGRAM_QA_NOMENTION_TOKEN", - }), - ).toBe(false); - expect( - testing.matchesTelegramScenarioReply({ - groupId: "-100123", - sentMessageId: 55, - sutBotId: 88, - message: { - updateId: 2, - messageId: 11, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "unrelated chatter", - replyToMessageId: undefined, - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: [], - }, - matchText: "TELEGRAM_QA_NOMENTION_TOKEN", - }), - ).toBe(false); - expect( - testing.matchesTelegramScenarioReply({ - allowAnySutReply: true, - groupId: "-100123", - sentMessageId: 55, - sutBotId: 88, - message: { - updateId: 3, - messageId: 56, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "Protocol note: acknowledged.", - replyToMessageId: undefined, - timestamp: 1_700_000_003_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toBe(true); - expect( - testing.matchesTelegramScenarioReply({ - allowAnySutReply: true, - groupId: "-100123", - sentMessageId: 55, - sutBotId: 88, - message: { - updateId: 4, - messageId: 54, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "stale reply from a previous scenario", - replyToMessageId: undefined, - timestamp: 1_700_000_004_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toBe(false); - }); - - it("validates expected Telegram reply markers", () => { - expect( - testing.assertTelegramScenarioReply({ - expectedTextIncludes: ["🧭 Identity", "Channel: telegram"], - message: { - updateId: 1, - messageId: 10, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "🧭 Identity\nChannel: telegram\nUser id: 42", - replyToMessageId: 55, - timestamp: 1_700_000_001_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toBeUndefined(); - expect(() => - testing.assertTelegramScenarioReply({ - expectedTextIncludes: ["Use /tools verbose for descriptions."], - message: { - updateId: 2, - messageId: 11, - chatId: -100123, - senderId: 88, - senderIsBot: true, - senderUsername: "sut_bot", - text: "exec\nbash", - replyToMessageId: 55, - timestamp: 1_700_000_002_000, - inlineButtons: [], - mediaKinds: [], - }, - }), - ).toThrow("reply message 11 missing expected text: Use /tools verbose for descriptions."); - }); - - it("adds an abort deadline to Telegram API requests", async () => { - const controller = new AbortController(); - const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); - let signal: AbortSignal | undefined; - vi.stubGlobal( - "fetch", - vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => { - signal = init?.signal as AbortSignal | undefined; - return new Response(JSON.stringify({ ok: true, result: { id: 42 } }), { - status: 200, - headers: { - "content-type": "application/json", - }, - }); - }), - ); - - await expect(testing.callTelegramApi("token", "getMe", undefined, 25)).resolves.toEqual({ - id: 42, - }); - expect(timeoutSpy).toHaveBeenCalledWith(25); - expect(fetchWithSsrFGuardMock.mock.calls.at(-1)?.[0]).toMatchObject({ - capture: false, - }); - expect(signal).toBe(controller.signal); - expect(signal?.aborted).toBe(false); - controller.abort(); - expect(signal?.aborted).toBe(true); - }); - - it("caps oversized Telegram API request deadlines", async () => { - const controller = new AbortController(); - const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); - vi.stubGlobal( - "fetch", - vi.fn( - async () => - new Response(JSON.stringify({ ok: true, result: { id: 42 } }), { - status: 200, - headers: { - "content-type": "application/json", - }, - }), - ), - ); - - await expect( - testing.callTelegramApi("token", "getMe", undefined, Number.MAX_SAFE_INTEGER), - ).resolves.toEqual({ - id: 42, - }); - - expect(timeoutSpy).toHaveBeenCalledWith(MAX_TIMER_TIMEOUT_MS); - expect(fetchWithSsrFGuardMock.mock.calls.at(-1)?.[0]).toMatchObject({ - timeoutMs: MAX_TIMER_TIMEOUT_MS, - }); - }); - - it("rejects oversized Telegram API success bodies", async () => { - vi.stubGlobal( - "fetch", - vi.fn( - async () => - new Response( - JSON.stringify({ - ok: true, - result: { - blob: "x".repeat(16 * 1024 * 1024), - }, - }), - { - status: 200, - headers: { - "content-type": "application/json", - }, - }, - ), - ), - ); - - await expect(testing.callTelegramApi("token", "getMe")).rejects.toThrow( - "qa-lab-telegram-live.getMe: JSON response exceeds 16777216 bytes", - ); - }); - - it("treats transient Telegram getUpdates network errors as recoverable", () => { - expect(testing.isRecoverableTelegramQaPollError(new TypeError("fetch failed"))).toBe(true); - expect(testing.isRecoverableTelegramQaPollError(new Error("socket hang up"))).toBe(true); - expect( - testing.isRecoverableTelegramQaPollError( - new Error("The operation was aborted due to timeout"), - ), - ).toBe(true); - expect(testing.isRecoverableTelegramQaPollError(new Error("request timed out"))).toBe(true); - expect(testing.isRecoverableTelegramQaPollError(new Error("AbortError"))).toBe(true); - expect(testing.isRecoverableTelegramQaPollError(new Error("Bad Request: chat not found"))).toBe( - false, - ); - }); - - it("retries transient Telegram polling fetch failures while waiting for scenario replies", async () => { - const fetchMock = vi - .fn() - .mockRejectedValueOnce(new TypeError("fetch failed")) - .mockResolvedValueOnce( - new Response( - JSON.stringify({ - ok: true, - result: [ - { - update_id: 10, - message: { - message_id: 99, - chat: { id: -100123 }, - from: { id: 88, is_bot: true, username: "sut_bot" }, - text: "Identity\nChannel: telegram", - date: 1_700_000_000, - reply_to_message: { message_id: 55 }, - }, - }, - ], - }), - { - status: 200, - headers: { - "content-type": "application/json", - }, - }, - ), - ); - vi.stubGlobal("fetch", fetchMock); - const observedMessages: Parameters< - typeof testing.waitForObservedMessage - >[0]["observedMessages"] = []; - - const result = await testing.waitForObservedMessage({ - token: "token", - initialOffset: 7, - timeoutMs: 5_000, - observedMessages, - observationScenarioId: "telegram-mentioned-message-reply", - observationScenarioTitle: "Telegram mentioned message gets a reply", - predicate: (message) => - testing.matchesTelegramScenarioReply({ - groupId: "-100123", - message, - sentMessageId: 55, - sutBotId: 88, - }), - }); - - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(result.message.messageId).toBe(99); - expect(result.nextOffset).toBe(11); - expect(observedMessages).toHaveLength(1); - expect(observedMessages[0]?.matchedScenario).toBe(true); - expect(observedMessages[0]?.messageId).toBe(99); - expect(observedMessages[0]?.scenarioId).toBe("telegram-mentioned-message-reply"); - }); - - it("prints Telegram scenario RTT in the Markdown report", () => { - expect( - testing.renderTelegramQaMarkdown({ - cleanupIssues: [], - credentialSource: "env", - groupId: "-100123", - redactMetadata: false, - startedAt: "2026-04-23T00:00:00.000Z", - finishedAt: "2026-04-23T00:00:10.000Z", - scenarios: [ - { - id: "telegram-canary", - title: "Telegram canary", - status: "pass", - details: "reply message 12 matched in 4321ms", - rttMs: 4321, - }, - ], - }), - ).toContain("- RTT: 4321ms"); - }); - - it("prints Telegram repeated RTT timing in the Markdown report", () => { - const report = testing.renderTelegramQaMarkdown({ - cleanupIssues: [], - credentialSource: "env", - groupId: "-100123", - redactMetadata: false, - startedAt: "2026-04-23T00:00:00.000Z", - finishedAt: "2026-04-23T00:00:10.000Z", - scenarios: [ - { - id: "telegram-mentioned-message-reply", - title: "Telegram mentioned message gets a reply", - status: "pass", - details: "reply matched; 3/4 RTT checks passed", - rttMs: 2000, - timing: { - avgMs: 2333, - p50Ms: 2000, - p95Ms: 4000, - maxMs: 4000, - samples: 4, - failedSamples: 1, - }, - }, - ], - }); - - expect(report).toContain("- Samples: 3/4"); - expect(report).toContain("- P50: 2000ms"); - expect(report).toContain("- P95: 4000ms"); - }); - - it("formats phase-specific canary diagnostics with context", () => { - const error = new Error( - "SUT bot did not send any group reply after the canary command within 30s.", - ); - error.name = "TelegramQaCanaryError"; - Object.assign(error, { - phase: "sut_reply_timeout", - context: { - driverMessageId: 55, - sutBotId: 88, - }, - }); - - const message = testing.canaryFailureMessage({ - error, - groupId: "-100123", - driverBotId: 42, - driverUsername: "driver_bot", - sutBotId: 88, - sutUsername: "sut_bot", - }); - expect(message).toContain("Phase: sut_reply_timeout"); - expect(message).toContain("- driverMessageId: 55"); - expect(message).not.toContain("- sutBotId: 88\n- sutBotId: 88"); - expect(message).toContain( - "Confirm the SUT bot is present in the target private group and can receive /help@BotUsername commands there.", - ); - }); - - it("redacts canary context details in public metadata mode", () => { - const error = new Error("timed out"); - error.name = "TelegramQaCanaryError"; - Object.assign(error, { - phase: "sut_reply_timeout", - context: { - driverMessageId: 55, - }, - }); - - const message = testing.canaryFailureMessage({ - error, - groupId: "-100123", - driverBotId: 42, - driverUsername: "driver_bot", - redactMetadata: true, - sutBotId: 88, - sutUsername: "sut_bot", - }); - - expect(message).toContain("- groupId: "); - expect(message).toContain("- driverBotId: "); - expect(message).toContain("- driverUsername: "); - expect(message).toContain("- sutBotId: "); - expect(message).toContain("- sutUsername: "); - expect(message).toContain("- driverMessageId: "); - expect(message).toContain("timed out"); - expect(message).not.toContain("-100123"); - expect(message).not.toContain("driver_bot"); - expect(message).not.toContain("sut_bot"); - expect(message).not.toContain("55"); - }); - - it("treats null canary context as a non-canary error", () => { - const error = new Error("boom"); - error.name = "TelegramQaCanaryError"; - Object.assign(error, { - phase: "sut_reply_timeout", - context: null, - }); - - const message = testing.canaryFailureMessage({ - error, - groupId: "-100123", - driverBotId: 42, - driverUsername: "driver_bot", - sutBotId: 88, - sutUsername: "sut_bot", - }); - - expect(message).toContain("Phase: unknown"); - expect(message).toContain("boom"); - }); - - it("keeps bounded telegram QA progress details on UTF-16 boundaries", () => { - const prefix = "a".repeat(236); - expect(testing.formatTelegramQaProgressDetails(`${prefix}😀after`)).toBe(`${prefix}...`); - expect(testing.formatTelegramQaProgressDetails("a".repeat(241))).toBe(`${"a".repeat(237)}...`); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts deleted file mode 100644 index ab34804c8988..000000000000 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ /dev/null @@ -1,2064 +0,0 @@ -// Qa Lab plugin module implements telegram live behavior. -import { randomUUID } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; -import type { TelegramBotMessage, TelegramBotUpdate } from "@openclaw/telegram/api.js"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { - parseStrictPositiveInteger, - resolveTimerTimeoutMs, -} from "openclaw/plugin-sdk/number-runtime"; -import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http"; -import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; -import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; -import { z } from "zod"; -import { createQaArtifactRunId } from "../../artifact-run-id.js"; -import { - QA_EVIDENCE_FILENAME, - buildLiveTransportEvidenceSummary, - type QaEvidenceTiming, -} from "../../evidence-summary.js"; -import { startQaGatewayChild, type QaGatewayChildCommand } from "../../gateway-child.js"; -import { - assertQaGatewayCredentialLeaseQuarantine, - shouldRetainQaGatewayCredentialLease, -} from "../../gateway-process-boundary.js"; -import { isTruthyOptIn } from "../../mantis-options.runtime.js"; -import { - parseQaProgressBooleanEnv as parseTelegramQaProgressBooleanEnv, - sanitizeQaProgressValue as sanitizeTelegramQaProgressValue, -} from "../../progress-format.js"; -import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; -import { - defaultQaModelForMode, - normalizeQaProviderMode, - type QaProviderMode, - type QaProviderModeInput, -} from "../../run-config.js"; -import { - acquireQaCredentialLease, - startQaCredentialLeaseHeartbeat, -} from "../shared/credential-lease.runtime.js"; -import { - appendQaLiveLaneIssue as appendLiveLaneIssue, - buildQaLiveLaneArtifactsError as buildLiveLaneArtifactsError, - redactQaLiveLaneIssues, -} from "../shared/live-artifacts.js"; -import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; -import { assertLiveScenarioReply as assertTelegramScenarioReply } from "../shared/live-scenario-reply.js"; -import type { LiveTransportCheckResult } from "../shared/live-transport-result.js"; -import { - normalizeLiveTransportRttOptions, - summarizeLiveTransportRttSamples, - type LiveTransportRttOptions, - type LiveTransportRttSample, -} from "../shared/live-transport-rtt.js"; -import { - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, -} from "../shared/live-transport-scenarios.js"; - -type TelegramQaRuntimeEnv = { - groupId: string; - driverToken: string; - sutToken: string; -}; - -type TelegramBotIdentity = { - id: number; - is_bot: boolean; - first_name: string; - username?: string; -}; - -type TelegramQaScenarioId = - | "telegram-other-bot-command-gating" - | "telegram-stream-final-single-message" - | "telegram-long-final-three-chunks" - | "telegram-long-final-reuses-preview" - | "telegram-mentioned-message-reply" - | "telegram-mention-gating"; - -type TelegramQaScenarioStep = { - allowAnySutReply?: boolean; - expectReply: boolean; - input: string; - expectedTextIncludes?: string[]; - expectedJoinedSutTextIncludes?: string[]; - expectedSutMessageCount?: number; - expectedSutMessageCountRange?: readonly [number, number]; - matchText?: string; - replyToLatestSutMessage?: boolean; - settleMs?: number; - timeoutMs?: number; -}; - -type TelegramQaScenarioRun = { - steps: TelegramQaScenarioStep[]; -}; - -type TelegramQaScenarioDefinition = LiveTransportScenarioDefinition & { - buildRun: (sutUsername: string) => TelegramQaScenarioRun; - buildRttRun?: (params: { rttIndex: number; sutUsername: string }) => TelegramQaScenarioRun; - defaultEnabled?: boolean; - defaultProviderModes?: readonly QaProviderMode[]; - evidenceCoverageIds?: readonly string[]; - regressionRefs?: readonly string[]; - rationale: string; -}; - -type TelegramObservedMessage = { - updateId: number; - messageId: number; - chatId: number; - senderId: number; - senderIsBot: boolean; - senderUsername?: string; - scenarioId?: string; - scenarioTitle?: string; - matchedScenario?: boolean; - text: string; - caption?: string; - replyToMessageId?: number; - timestamp: number; - inlineButtons: string[]; - mediaKinds: string[]; -}; - -const DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS = 30_000; -const TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS = 45_000; - -type TelegramQaScenarioResult = LiveTransportCheckResult; - -function telegramLiveTransportCoverageIds(scenario: TelegramQaScenarioDefinition) { - if (scenario.evidenceCoverageIds) { - return scenario.evidenceCoverageIds; - } - return scenario.standardId ? [`channels.telegram.${scenario.standardId}`] : []; -} - -type TelegramQaRttOptions = LiveTransportRttOptions; - -type TelegramQaRttResult = { - details: string; - driverOffset: number; - failed: number; - latestSutMessageId?: number; - passed: number; - timing: QaEvidenceTiming; -}; - -type TelegramQaCanaryPhase = "sut_reply_timeout" | "sut_reply_not_threaded" | "sut_reply_empty"; - -type TelegramQaRunResult = { - outputDir: string; - reportPath: string; - summaryPath: string; - gatewayDebugDirPath?: string; - scenarios: TelegramQaScenarioResult[]; -}; - -type TelegramChannelStatus = { - connected?: boolean; - lastConnectedAt?: number; - lastDisconnect?: unknown; - lastError?: string | null; - restartPending?: boolean; - running?: boolean; -}; - -class TelegramQaCanaryError extends Error { - phase: TelegramQaCanaryPhase; - context: Record; - - constructor( - phase: TelegramQaCanaryPhase, - message: string, - context: Record, - ) { - super(message); - this.name = "TelegramQaCanaryError"; - this.phase = phase; - this.context = context; - } -} - -function isTelegramQaCanaryError(error: unknown): error is TelegramQaCanaryError { - return ( - error instanceof TelegramQaCanaryError || - (typeof error === "object" && - error !== null && - typeof (error as { phase?: unknown }).phase === "string" && - typeof (error as { context?: unknown }).context === "object" && - (error as { context?: unknown }).context !== null) - ); -} - -type TelegramApiEnvelope = { - ok: boolean; - result?: T; - description?: string; -}; - -type TelegramReplyMarkup = { - inline_keyboard?: Array>; -}; - -type TelegramRichMessage = { - markdown?: string; - html?: string; - blocks?: unknown[]; -}; - -type TelegramMessage = Pick & - Partial> & { - audio?: unknown; - chat: { id: number }; - document?: unknown; - from?: Pick, "id" | "is_bot" | "username">; - photo?: unknown[]; - rich_message?: TelegramRichMessage; - reply_markup?: TelegramReplyMarkup; - reply_to_message?: { message_id?: number }; - sticker?: unknown; - video?: unknown; - voice?: unknown; - }; - -type TelegramUpdate = Pick & { - edited_message?: TelegramMessage; - message?: TelegramMessage; -}; - -type TelegramSendMessageResult = { - message_id: number; - chat: { - id: number; - }; -}; - -function telegramQaStepRun(step: TelegramQaScenarioStep): TelegramQaScenarioRun { - return { steps: [step] }; -} - -const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ - { - id: "telegram-other-bot-command-gating", - title: "Telegram command addressed to another bot is ignored", - rationale: "Bot-to-bot groups must not let commands addressed to another bot wake the SUT.", - timeoutMs: 8_000, - buildRun: () => - telegramQaStepRun({ - expectReply: false, - input: "/status@OpenClawQaOtherBot", - }), - }, - { - id: "telegram-mentioned-message-reply", - title: "Telegram mentioned message gets a reply", - evidenceCoverageIds: ["channels.telegram.mention-gating"], - rationale: "Bot-to-bot group mention routing must produce a threaded SUT reply.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `@${sutUsername} Telegram QA mention routing check. Reply with a short acknowledgement.`, - replyToLatestSutMessage: true, - }), - buildRttRun: ({ rttIndex, sutUsername }) => { - const marker = `QA-TELEGRAM-RTT-${rttIndex}-${randomUUID().slice(0, 8).toUpperCase()}`; - return telegramQaStepRun({ - expectReply: true, - input: `@${sutUsername} Telegram RTT check ${rttIndex}. Reply exactly: ${marker}`, - expectedTextIncludes: [marker], - matchText: marker, - replyToLatestSutMessage: true, - }); - }, - }, - { - id: "telegram-stream-final-single-message", - title: "Telegram streamed final stays one message", - defaultEnabled: false, - defaultProviderModes: ["mock-openai"], - rationale: "Opt-in regression guard for duplicate final replies from Telegram streaming paths.", - regressionRefs: ["openclaw/openclaw#39905"], - timeoutMs: 75_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - allowAnySutReply: true, - expectReply: true, - input: `@${sutUsername} Quiet streaming QA check. Reply exactly: QA-TELEGRAM-STREAM-SINGLE-OK`, - expectedTextIncludes: ["QA-TELEGRAM-STREAM-SINGLE-OK"], - expectedJoinedSutTextIncludes: ["QA-TELEGRAM-STREAM-SINGLE-OK"], - expectedSutMessageCount: 1, - settleMs: 4_000, - }), - }, - { - id: "telegram-long-final-reuses-preview", - title: "Telegram long final reuses the preview message", - defaultProviderModes: ["mock-openai"], - rationale: "Regression guard for long streamed finals leaving stale preview messages behind.", - regressionRefs: ["openclaw/openclaw#39905"], - timeoutMs: 60_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - allowAnySutReply: true, - expectReply: true, - input: `@${sutUsername} Telegram long final QA check. Use the scripted long final response.`, - expectedTextIncludes: ["TELEGRAM-LONG-FINAL-BEGIN"], - expectedJoinedSutTextIncludes: ["TELEGRAM-LONG-FINAL-BEGIN", "TELEGRAM-LONG-FINAL-END"], - expectedSutMessageCountRange: [1, 2], - replyToLatestSutMessage: true, - settleMs: 4_000, - }), - }, - { - id: "telegram-long-final-three-chunks", - title: "Telegram three-chunk final keeps only final chunks", - defaultEnabled: false, - rationale: "Opt-in stress probe for Telegram long final chunk accounting.", - regressionRefs: ["openclaw/openclaw#39905"], - timeoutMs: 60_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - allowAnySutReply: true, - expectReply: true, - input: `@${sutUsername} Telegram long final three chunk QA check. Use the scripted three chunk final response.`, - expectedTextIncludes: ["TELEGRAM-LONG-FINAL-3CHUNK-BEGIN"], - expectedJoinedSutTextIncludes: [ - "TELEGRAM-LONG-FINAL-3CHUNK-BEGIN", - "TELEGRAM-LONG-FINAL-3CHUNK-END", - ], - expectedSutMessageCount: 3, - replyToLatestSutMessage: true, - settleMs: 4_000, - }), - }, - { - id: "telegram-mention-gating", - standardId: "mention-gating", - title: "Telegram group message without mention does not trigger", - rationale: "Required group mention gate should suppress ordinary group chatter.", - timeoutMs: 8_000, - buildRun: () => { - const token = `TELEGRAM_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return telegramQaStepRun({ - expectReply: false, - input: `reply with only this exact marker: ${token}`, - matchText: token, - }); - }, - }, -]; - -const TELEGRAM_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ - alwaysOnStandardScenarioIds: ["canary"], - scenarios: TELEGRAM_QA_SCENARIOS, -}); - -const TELEGRAM_QA_ENV_KEYS = [ - "OPENCLAW_QA_TELEGRAM_GROUP_ID", - "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", - "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", -] as const; -const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; -const QA_SUITE_PROGRESS_ENV = "OPENCLAW_QA_SUITE_PROGRESS"; -const TELEGRAM_QA_PROGRESS_DETAIL_LIMIT = 240; -const TELEGRAM_QA_PROGRESS_PREFIX = "[qa-telegram-live]"; - -const telegramQaCredentialPayloadSchema = z.object({ - groupId: z.string().trim().min(1), - driverToken: z.string().trim().min(1), - sutToken: z.string().trim().min(1), -}); - -function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof TELEGRAM_QA_ENV_KEYS)[number]) { - const value = env[key]?.trim(); - if (!value) { - throw new Error(`Missing ${key}.`); - } - return value; -} - -function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) { - const override = parseTelegramQaProgressBooleanEnv(env[QA_SUITE_PROGRESS_ENV]); - if (override !== undefined) { - return override; - } - return parseTelegramQaProgressBooleanEnv(env.CI) === true; -} - -function parsePositiveTelegramQaEnvMs(env: NodeJS.ProcessEnv, name: string, fallbackMs: number) { - const raw = env[name]; - if (raw === undefined) { - return fallbackMs; - } - const parsed = parseStrictPositiveInteger(raw); - if (parsed === undefined) { - return fallbackMs; - } - return parsed; -} - -function resolveTelegramQaCanaryTimeoutMs(env: NodeJS.ProcessEnv = process.env) { - return parsePositiveTelegramQaEnvMs( - env, - "OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", - DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS, - ); -} - -function resolveTelegramQaScenarioTimeoutMs( - fallbackMs: number, - env: NodeJS.ProcessEnv = process.env, -) { - return parsePositiveTelegramQaEnvMs(env, "OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", fallbackMs); -} - -function resolveTelegramQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) { - const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS; - if (!raw) { - return TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS; - } - return parseStrictPositiveInteger(raw) ?? TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS; -} - -function normalizeTelegramQaRttOptions(params: { - count?: number; - checkIds?: readonly string[]; - maxFailures?: number; - timeoutMs?: number; -}): TelegramQaRttOptions | undefined { - const knownScenarioIds = new Set(TELEGRAM_QA_SCENARIOS.map((scenario) => scenario.id)); - return normalizeLiveTransportRttOptions({ - count: params.count, - defaultCheckIds: ["telegram-mentioned-message-reply"], - knownCheckIds: knownScenarioIds, - maxFailures: params.maxFailures, - rawCheckIds: params.checkIds, - timeoutMs: params.timeoutMs, - unknownCheckMessage: (checkId) => `unknown Telegram QA RTT check: ${checkId}`, - }); -} - -function assertTelegramQaRttCheckSupport(params: { - rttOptions?: TelegramQaRttOptions; - scenarios: TelegramQaScenarioDefinition[]; -}) { - if (!params.rttOptions) { - return; - } - const selectedScenarioIds = new Set(params.scenarios.map((scenario) => scenario.id)); - for (const scenarioId of params.rttOptions.checkIds) { - if (!selectedScenarioIds.has(scenarioId)) { - throw new Error(`Telegram QA RTT check ${scenarioId} is not selected.`); - } - } - for (const scenario of params.scenarios) { - if (params.rttOptions.checkIds.has(scenario.id) && !scenario.buildRttRun) { - throw new Error(`Telegram QA scenario ${scenario.id} does not support RTT measurement.`); - } - } -} - -function formatTelegramQaTimeoutSeconds(timeoutMs: number) { - return `${Math.round(timeoutMs / 1_000)}s`; -} - -function writeTelegramQaProgress(enabled: boolean, message: string) { - if (!enabled) { - return; - } - process.stderr.write(`${TELEGRAM_QA_PROGRESS_PREFIX} ${message}\n`); -} - -function formatTelegramQaProgressDetails(details: string): string { - const sanitized = sanitizeTelegramQaProgressValue(details); - if (sanitized.length <= TELEGRAM_QA_PROGRESS_DETAIL_LIMIT) { - return sanitized; - } - return `${truncateUtf16Safe(sanitized, TELEGRAM_QA_PROGRESS_DETAIL_LIMIT - 3).trimEnd()}...`; -} - -function resolveTelegramQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): TelegramQaRuntimeEnv { - const groupId = resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_GROUP_ID"); - if (!/^-?\d+$/u.test(groupId)) { - throw new Error("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id."); - } - return { - groupId, - driverToken: resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN"), - sutToken: resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"), - }; -} - -function parseTelegramQaCredentialPayload(payload: unknown): TelegramQaRuntimeEnv { - const parsed = telegramQaCredentialPayloadSchema.parse(payload); - if (!/^-?\d+$/u.test(parsed.groupId)) { - throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id."); - } - return { - groupId: parsed.groupId, - driverToken: parsed.driverToken, - sutToken: parsed.sutToken, - }; -} - -function flattenInlineButtons(replyMarkup?: TelegramReplyMarkup) { - return (replyMarkup?.inline_keyboard ?? []) - .flat() - .map((button) => button.text?.trim()) - .filter((text): text is string => Boolean(text)); -} - -function detectMediaKinds(message: TelegramMessage) { - const kinds: string[] = []; - if (Array.isArray(message.photo) && message.photo.length > 0) { - kinds.push("photo"); - } - if (message.document) { - kinds.push("document"); - } - if (message.audio) { - kinds.push("audio"); - } - if (message.video) { - kinds.push("video"); - } - if (message.voice) { - kinds.push("voice"); - } - if (message.sticker) { - kinds.push("sticker"); - } - return kinds; -} - -function flattenTelegramRichText(value: unknown): string { - if (typeof value === "string") { - return value; - } - if (Array.isArray(value)) { - return value.map((part) => flattenTelegramRichText(part)).join(""); - } - if (!isRecord(value)) { - return ""; - } - if ("text" in value) { - return flattenTelegramRichText(value.text); - } - if (typeof value.alternative_text === "string") { - return value.alternative_text; - } - if (typeof value.expression === "string") { - return value.expression; - } - return ""; -} - -function flattenTelegramRichBlock(value: unknown): string { - if (typeof value === "string" || Array.isArray(value)) { - return flattenTelegramRichText(value); - } - if (!isRecord(value)) { - return ""; - } - const parts: string[] = []; - if ("text" in value) { - parts.push(flattenTelegramRichText(value.text)); - } - if ("summary" in value) { - parts.push(flattenTelegramRichText(value.summary)); - } - if (typeof value.label === "string") { - parts.push(value.label); - } - if (typeof value.expression === "string") { - parts.push(value.expression); - } - if ("blocks" in value) { - parts.push(flattenTelegramRichBlocks(value.blocks)); - } - if ("items" in value) { - parts.push(flattenTelegramRichBlocks(value.items)); - } - if ("cells" in value) { - parts.push(flattenTelegramRichTableCells(value.cells)); - } - if ("caption" in value) { - parts.push(flattenTelegramRichBlock(value.caption)); - } - if ("credit" in value) { - parts.push(flattenTelegramRichText(value.credit)); - } - return parts.filter((part) => part.trim()).join("\n"); -} - -function flattenTelegramRichBlocks(value: unknown): string { - const blocks = Array.isArray(value) ? value : [value]; - return blocks - .map((block) => flattenTelegramRichBlock(block)) - .filter((part) => part.trim()) - .join("\n"); -} - -function flattenTelegramRichTableCells(value: unknown): string { - if (!Array.isArray(value)) { - return flattenTelegramRichBlock(value); - } - return value - .map((row) => { - const cells = Array.isArray(row) ? row : [row]; - return cells - .map((cell) => flattenTelegramRichBlock(cell)) - .filter((cell) => cell.trim()) - .join("\t"); - }) - .filter((row) => row.trim()) - .join("\n"); -} - -function selectTelegramRichMessageText(richMessage: TelegramRichMessage | undefined) { - return ( - richMessage?.markdown || richMessage?.html || flattenTelegramRichBlocks(richMessage?.blocks) - ); -} - -function selectTelegramObservedText(message: TelegramMessage) { - return ( - message.text || message.caption || selectTelegramRichMessageText(message.rich_message) || "" - ); -} - -function normalizeTelegramObservedMessage(update: TelegramUpdate): TelegramObservedMessage | null { - const message = update.message ?? update.edited_message; - if (!message?.from?.id) { - return null; - } - return { - updateId: update.update_id, - messageId: message.message_id, - chatId: message.chat.id, - senderId: message.from.id, - senderIsBot: message.from.is_bot, - senderUsername: message.from.username, - text: selectTelegramObservedText(message), - caption: message.caption, - replyToMessageId: message.reply_to_message?.message_id, - timestamp: message.date * 1000, - inlineButtons: flattenInlineButtons(message.reply_markup), - mediaKinds: detectMediaKinds(message), - }; -} - -function buildTelegramQaConfig( - baseCfg: OpenClawConfig, - params: { - groupId: string; - sutToken: string; - driverBotId: number; - sutAccountId: string; - }, -): OpenClawConfig { - const pluginAllow = uniqueStrings([...(baseCfg.plugins?.allow ?? []), "telegram"]); - const pluginEntries = { - ...baseCfg.plugins?.entries, - telegram: { enabled: true }, - }; - return { - ...baseCfg, - agents: { - ...baseCfg.agents, - defaults: { - ...baseCfg.agents?.defaults, - models: { - ...baseCfg.agents?.defaults?.models, - "openai/gpt-5.6-luna": { - ...baseCfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"], - agentRuntime: { id: "openclaw" }, - }, - }, - skipBootstrap: true, - }, - }, - plugins: { - ...baseCfg.plugins, - allow: pluginAllow, - entries: pluginEntries, - }, - messages: { - ...baseCfg.messages, - groupChat: { - ...baseCfg.messages?.groupChat, - visibleReplies: "automatic", - }, - }, - channels: { - ...baseCfg.channels, - telegram: { - enabled: true, - defaultAccount: params.sutAccountId, - accounts: { - [params.sutAccountId]: { - enabled: true, - botToken: params.sutToken, - dmPolicy: "disabled", - replyToMode: "first", - groups: { - [params.groupId]: { - groupPolicy: "allowlist", - allowFrom: [String(params.driverBotId)], - requireMention: true, - }, - }, - }, - }, - }, - }, - }; -} - -async function callTelegramApi( - token: string, - method: string, - body?: Record, - timeoutMs = 15_000, -): Promise { - const requestTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 15_000); - const { response, release } = await fetchWithSsrFGuard({ - url: `https://api.telegram.org/bot${token}/${method}`, - init: { - method: "POST", - headers: { - "content-type": "application/json", - }, - body: JSON.stringify(body ?? {}), - }, - timeoutMs: requestTimeoutMs, - policy: { hostnameAllowlist: ["api.telegram.org"] }, - auditContext: "qa-lab-telegram-live", - capture: false, - }); - try { - const payload = await readProviderJsonResponse>( - response, - `qa-lab-telegram-live.${method}`, - ); - if (!response.ok || !payload.ok || payload.result === undefined) { - throw new Error( - payload.description?.trim() || `${method} failed with status ${response.status}`, - ); - } - return payload.result; - } finally { - await release(); - } -} - -function isRecoverableTelegramQaPollError(error: unknown): boolean { - const message = formatErrorMessage(error).toLowerCase(); - return ( - message.includes("fetch failed") || - message.includes("aborted due to timeout") || - message.includes("operation was aborted") || - message.includes("request timed out") || - message.includes("aborterror") || - message.includes("econnreset") || - message.includes("etimedout") || - message.includes("socket hang up") || - message.includes("terminated") - ); -} - -async function getBotIdentity(token: string) { - return await callTelegramApi(token, "getMe"); -} - -async function flushTelegramUpdates(token: string) { - const startedAt = Date.now(); - let offset = 0; - while (Date.now() - startedAt < 15_000) { - const updates = await callTelegramApi( - token, - "getUpdates", - { - offset, - timeout: 0, - allowed_updates: ["message", "edited_message"], - }, - 15_000, - ); - if (updates.length === 0) { - return offset; - } - offset = (updates.at(-1)?.update_id ?? offset) + 1; - } - throw new Error("timed out after 15000ms draining Telegram updates"); -} - -async function sendGroupMessage( - token: string, - groupId: string, - text: string, - opts: { replyToMessageId?: number } = {}, -) { - return await callTelegramApi(token, "sendMessage", { - chat_id: groupId, - text, - disable_notification: true, - ...(opts.replyToMessageId !== undefined - ? { - reply_parameters: { - message_id: opts.replyToMessageId, - allow_sending_without_reply: true, - }, - } - : {}), - }); -} - -async function waitForTelegramPollRetryDelay(remainingMs: number) { - await new Promise((resolve) => { - setTimeout(resolve, Math.min(250, Math.max(100, remainingMs))); - }); -} - -async function waitForObservedMessage(params: { - token: string; - initialOffset: number; - timeoutMs: number; - predicate: (message: TelegramObservedMessage) => boolean; - observedMessages: TelegramObservedMessage[]; - observationScenarioId: string; - observationScenarioTitle: string; - expectedTextIncludes?: string[]; - validateMatchedMessage?: (message: TelegramObservedMessage) => void; -}) { - const startedAt = Date.now(); - let offset = params.initialOffset; - let lastPollingError: unknown; - let lastExpectedMismatch: Error | undefined; - while (Date.now() - startedAt < params.timeoutMs) { - const remainingMs = Math.max( - 1_000, - Math.min(10_000, params.timeoutMs - (Date.now() - startedAt)), - ); - const timeoutSeconds = Math.max(1, Math.min(10, Math.floor(remainingMs / 1000))); - let updates: TelegramUpdate[]; - try { - updates = await callTelegramApi( - params.token, - "getUpdates", - { - offset, - timeout: timeoutSeconds, - allowed_updates: ["message", "edited_message"], - }, - timeoutSeconds * 1000 + 5_000, - ); - lastPollingError = undefined; - } catch (error) { - if (!isRecoverableTelegramQaPollError(error)) { - throw error; - } - lastPollingError = error; - await waitForTelegramPollRetryDelay(params.timeoutMs - (Date.now() - startedAt)); - continue; - } - const batchObservedAtMs = Date.now(); - if (updates.length === 0) { - continue; - } - offset = (updates.at(-1)?.update_id ?? offset) + 1; - for (const update of updates) { - const normalized = normalizeTelegramObservedMessage(update); - if (!normalized) { - continue; - } - const matchedScenario = params.predicate(normalized); - const observedMessage: TelegramObservedMessage = { - ...normalized, - scenarioId: params.observationScenarioId, - scenarioTitle: params.observationScenarioTitle, - matchedScenario, - }; - params.observedMessages.push(observedMessage); - if (matchedScenario) { - try { - if (params.validateMatchedMessage) { - params.validateMatchedMessage(observedMessage); - } else { - assertTelegramScenarioReply({ - expectedTextIncludes: params.expectedTextIncludes, - message: observedMessage, - }); - } - } catch (error) { - lastExpectedMismatch = - error instanceof Error ? error : new Error(formatErrorMessage(error)); - continue; - } - return { message: observedMessage, nextOffset: offset, observedAtMs: batchObservedAtMs }; - } - } - } - if (lastExpectedMismatch) { - throw lastExpectedMismatch; - } - const timeoutMessage = `timed out after ${params.timeoutMs}ms waiting for Telegram message`; - if (lastPollingError) { - throw new Error( - `${timeoutMessage}; last polling error: ${formatErrorMessage(lastPollingError)}`, - ); - } - throw new Error(timeoutMessage); -} - -async function collectObservedMessages(params: { - token: string; - initialOffset: number; - settleMs: number; - predicate: (message: TelegramObservedMessage) => boolean; - observedMessages: TelegramObservedMessage[]; - observationScenarioId: string; - observationScenarioTitle: string; -}) { - const startedAt = Date.now(); - let offset = params.initialOffset; - while (Date.now() - startedAt < params.settleMs) { - const remainingMs = Math.max(1, params.settleMs - (Date.now() - startedAt)); - const timeoutSeconds = Math.max(1, Math.min(2, Math.ceil(remainingMs / 1000))); - let updates: TelegramUpdate[]; - try { - updates = await callTelegramApi( - params.token, - "getUpdates", - { - offset, - timeout: timeoutSeconds, - allowed_updates: ["message", "edited_message"], - }, - timeoutSeconds * 1000 + 5_000, - ); - } catch (error) { - if (!isRecoverableTelegramQaPollError(error)) { - throw error; - } - await waitForTelegramPollRetryDelay(params.settleMs - (Date.now() - startedAt)); - continue; - } - if (updates.length === 0) { - continue; - } - offset = (updates.at(-1)?.update_id ?? offset) + 1; - for (const update of updates) { - const normalized = normalizeTelegramObservedMessage(update); - if (!normalized) { - continue; - } - params.observedMessages.push({ - ...normalized, - scenarioId: params.observationScenarioId, - scenarioTitle: params.observationScenarioTitle, - matchedScenario: params.predicate(normalized), - }); - } - } - return offset; -} - -function assertTelegramScenarioMessageSet(params: { - expectedJoinedSutTextIncludes?: string[]; - expectedSutMessageCount?: number; - expectedSutMessageCountRange?: readonly [number, number]; - groupId: string; - observedMessages: TelegramObservedMessage[]; - scenarioId: string; - sutBotId: number; -}) { - if ( - params.expectedSutMessageCount === undefined && - params.expectedSutMessageCountRange === undefined && - (params.expectedJoinedSutTextIncludes ?? []).length === 0 - ) { - return; - } - const byMessageId = new Map(); - for (const message of params.observedMessages) { - if ( - message.scenarioId === params.scenarioId && - message.chatId === Number(params.groupId) && - message.senderId === params.sutBotId - ) { - byMessageId.set(message.messageId, message); - } - } - const messages = [...byMessageId.values()].toSorted((a, b) => a.messageId - b.messageId); - if ( - params.expectedSutMessageCount !== undefined && - messages.length !== params.expectedSutMessageCount - ) { - throw new Error( - `expected ${params.expectedSutMessageCount} SUT message(s), observed ${messages.length}: ${messages - .map((message) => message.messageId) - .join(", ")}`, - ); - } - if (params.expectedSutMessageCountRange !== undefined) { - const [min, max] = params.expectedSutMessageCountRange; - if (messages.length < min || messages.length > max) { - throw new Error( - `expected ${min}-${max} SUT message(s), observed ${messages.length}: ${messages - .map((message) => message.messageId) - .join(", ")}`, - ); - } - } - const joinedText = messages.map((message) => message.text).join(""); - for (const expected of params.expectedJoinedSutTextIncludes ?? []) { - if (!joinedText.includes(expected)) { - throw new Error(`joined SUT reply text missing expected text: ${expected}`); - } - } -} - -async function waitForTelegramChannelRunning( - gateway: Awaited>, - accountId: string, - options?: { - env?: NodeJS.ProcessEnv; - pollMs?: number; - timeoutMs?: number; - }, -) { - const startedAt = Date.now(); - const timeoutMs = options?.timeoutMs ?? resolveTelegramQaReadyTimeoutMs(options?.env); - const pollMs = options?.pollMs ?? 500; - let lastProbeError: string | undefined; - let lastStatus: TelegramChannelStatus | undefined; - while (Date.now() - startedAt < timeoutMs) { - try { - const payload = (await gateway.call( - "channels.status", - { probe: false, timeoutMs: 2_000 }, - { timeoutMs: 5_000 }, - )) as { - channelAccounts?: Record< - string, - Array<{ - accountId?: string; - connected?: boolean; - lastConnectedAt?: number; - lastDisconnect?: unknown; - lastError?: string | null; - running?: boolean; - restartPending?: boolean; - }> - >; - }; - const accounts = payload.channelAccounts?.telegram ?? []; - const match = accounts.find((entry) => entry.accountId === accountId); - lastProbeError = undefined; - lastStatus = match - ? { - connected: match.connected, - lastConnectedAt: match.lastConnectedAt, - lastDisconnect: match.lastDisconnect, - lastError: match.lastError, - restartPending: match.restartPending, - running: match.running, - } - : undefined; - if (match?.running && match.connected === true && match.restartPending !== true) { - return; - } - } catch (error) { - lastProbeError = formatErrorMessage(error); - // retry - } - await new Promise((resolve) => { - setTimeout(resolve, pollMs); - }); - } - const details = lastStatus - ? `; last status: ${JSON.stringify(lastStatus)}` - : lastProbeError - ? `; last probe error: ${lastProbeError}` - : ""; - throw new Error(`telegram account "${accountId}" did not become ready${details}`); -} - -function renderTelegramQaMarkdown(params: { - cleanupIssues: string[]; - credentialSource: "convex" | "env"; - redactMetadata: boolean; - groupId: string; - gatewayDebugDirPath?: string; - startedAt: string; - finishedAt: string; - scenarios: TelegramQaScenarioResult[]; -}) { - const lines = [ - "# Telegram QA Report", - "", - `- Credential source: \`${params.credentialSource}\``, - `- Group: \`${params.groupId}\``, - `- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``, - `- Started: ${params.startedAt}`, - `- Finished: ${params.finishedAt}`, - "", - "## Scenarios", - "", - ]; - for (const scenario of params.scenarios) { - lines.push(`### ${scenario.title}`); - lines.push(""); - lines.push(`- Status: ${scenario.status}`); - lines.push(`- Details: ${scenario.details}`); - if (scenario.rttMs !== undefined) { - lines.push(`- RTT: ${scenario.rttMs}ms`); - } - if (scenario.timing?.samples !== undefined) { - lines.push( - `- Samples: ${scenario.timing.samples - (scenario.timing.failedSamples ?? 0)}/${scenario.timing.samples}`, - ); - if (scenario.timing.avgMs !== undefined) { - lines.push(`- Avg: ${scenario.timing.avgMs}ms`); - } - if (scenario.timing.p50Ms !== undefined) { - lines.push(`- P50: ${scenario.timing.p50Ms}ms`); - } - if (scenario.timing.p95Ms !== undefined) { - lines.push(`- P95: ${scenario.timing.p95Ms}ms`); - } - if (scenario.timing.maxMs !== undefined) { - lines.push(`- Max: ${scenario.timing.maxMs}ms`); - } - } - lines.push(""); - } - if (params.gatewayDebugDirPath) { - lines.push("## Gateway Debug"); - lines.push(""); - lines.push(`- Preserved at: \`${params.gatewayDebugDirPath}\``); - lines.push(""); - } - if (params.cleanupIssues.length > 0) { - lines.push("## Cleanup"); - lines.push(""); - for (const issue of params.cleanupIssues) { - lines.push(`- ${issue}`); - } - lines.push(""); - } - return lines.join("\n"); -} - -function shouldRunTelegramScenarioByDefault( - scenario: TelegramQaScenarioDefinition, - providerMode: QaProviderMode, -) { - if (scenario.defaultEnabled === false) { - return false; - } - return !scenario.defaultProviderModes || scenario.defaultProviderModes.includes(providerMode); -} - -function findScenario( - ids?: string[], - providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE, -) { - const scenarios = - ids && ids.length > 0 - ? TELEGRAM_QA_SCENARIOS - : TELEGRAM_QA_SCENARIOS.filter((scenario) => - shouldRunTelegramScenarioByDefault(scenario, providerMode), - ); - return selectLiveTransportScenarios({ - ids, - laneLabel: "Telegram", - scenarios, - }); -} - -export function listTelegramQaScenarioCatalog( - providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE, -) { - return TELEGRAM_QA_SCENARIOS.map((scenario) => ({ - id: scenario.id, - title: scenario.title, - defaultEnabled: shouldRunTelegramScenarioByDefault(scenario, providerMode), - rationale: scenario.rationale, - regressionRefs: [...(scenario.regressionRefs ?? [])], - })); -} - -function matchesTelegramScenarioReply(params: { - groupId: string; - allowAnySutReply?: boolean; - matchText?: string; - message: TelegramObservedMessage; - sentMessageId: number; - sutBotId: number; -}) { - if ( - params.message.chatId !== Number(params.groupId) || - params.message.senderId !== params.sutBotId - ) { - return false; - } - if (params.message.replyToMessageId === params.sentMessageId) { - return true; - } - if (params.allowAnySutReply === true) { - return params.message.messageId > params.sentMessageId; - } - return Boolean( - params.matchText && - params.message.messageId > params.sentMessageId && - params.message.text.includes(params.matchText), - ); -} - -function assertTelegramCanaryPresenceReply(message: TelegramObservedMessage) { - if (!message.senderIsBot) { - throw new Error(`canary reply message ${message.messageId} was not sent by a bot`); - } - // Telegram rich-message updates can arrive to the driver bot with no text - // body. The release canary proves command delivery plus threaded SUT output; - // text assertions stay on explicit command/scenario checks. -} - -function isTelegramObservedMessageTimeoutError(error: unknown, timeoutMs: number) { - return formatErrorMessage(error).startsWith( - `timed out after ${timeoutMs}ms waiting for Telegram message`, - ); -} - -function resolveTelegramQaScenarioSteps(run: TelegramQaScenarioRun): TelegramQaScenarioStep[] { - if (run.steps.length === 0) { - throw new Error("Telegram QA scenario must include at least one step"); - } - return run.steps; -} - -async function runTelegramQaScenarioStep(params: { - driverOffset: number; - driverToken: string; - env: NodeJS.ProcessEnv; - groupId: string; - latestSutMessageId?: number; - observedMessages: TelegramObservedMessage[]; - replyTimeoutMs?: number; - scenario: TelegramQaScenarioDefinition; - step: TelegramQaScenarioStep; - sutBotId: number; -}) { - const fallbackTimeoutMs = params.step.timeoutMs ?? params.scenario.timeoutMs; - const stepTimeoutMs = params.step.expectReply - ? (params.replyTimeoutMs ?? resolveTelegramQaScenarioTimeoutMs(fallbackTimeoutMs, params.env)) - : fallbackTimeoutMs; - const requestStartedAtMs = Date.now(); - const sent = await sendGroupMessage( - params.driverToken, - params.groupId, - params.step.input, - params.step.replyToLatestSutMessage - ? { replyToMessageId: params.latestSutMessageId } - : undefined, - ); - try { - const matched = await waitForObservedMessage({ - token: params.driverToken, - initialOffset: params.driverOffset, - timeoutMs: stepTimeoutMs, - observedMessages: params.observedMessages, - observationScenarioId: params.scenario.id, - observationScenarioTitle: params.scenario.title, - expectedTextIncludes: params.step.expectReply ? params.step.expectedTextIncludes : undefined, - predicate: (message) => - matchesTelegramScenarioReply({ - allowAnySutReply: params.step.allowAnySutReply, - groupId: params.groupId, - matchText: params.step.matchText, - message, - sentMessageId: sent.message_id, - sutBotId: params.sutBotId, - }), - }); - if (!params.step.expectReply) { - throw new Error(`unexpected reply message ${matched.message.messageId} matched`); - } - return { - matched, - requestStartedAt: new Date(requestStartedAtMs).toISOString(), - requestStartedAtMs, - sentMessageId: sent.message_id, - }; - } catch (error) { - if (!params.step.expectReply && isTelegramObservedMessageTimeoutError(error, stepTimeoutMs)) { - return { - matched: undefined, - requestStartedAt: new Date(requestStartedAtMs).toISOString(), - requestStartedAtMs, - sentMessageId: sent.message_id, - }; - } - throw error; - } -} - -async function runTelegramQaRttChecks(params: { - driverOffset: number; - driverToken: string; - env: NodeJS.ProcessEnv; - groupId: string; - latestSutMessageId?: number; - observedMessages: TelegramObservedMessage[]; - rttOptions: TelegramQaRttOptions; - scenario: TelegramQaScenarioDefinition; - sutBotId: number; - sutUsername: string; -}): Promise { - if (!params.scenario.buildRttRun) { - throw new Error(`Telegram QA scenario ${params.scenario.id} does not support RTT measurement.`); - } - let driverOffset = params.driverOffset; - let latestSutMessageId = params.latestSutMessageId; - const samples: LiveTransportRttSample[] = []; - let failures = 0; - let passed = 0; - for (let index = 1; passed < params.rttOptions.count; index += 1) { - const run = params.scenario.buildRttRun({ - rttIndex: index, - sutUsername: params.sutUsername, - }); - const steps = resolveTelegramQaScenarioSteps(run); - const step = steps[0]; - if (steps.length !== 1 || !step) { - throw new Error(`Telegram QA RTT check ${params.scenario.id} must have one step.`); - } - try { - driverOffset = await flushTelegramUpdates(params.driverToken); - const stepResult = await runTelegramQaScenarioStep({ - driverOffset, - driverToken: params.driverToken, - env: params.env, - groupId: params.groupId, - latestSutMessageId, - observedMessages: params.observedMessages, - replyTimeoutMs: params.rttOptions.timeoutMs, - scenario: params.scenario, - step, - sutBotId: params.sutBotId, - }); - if (!stepResult.matched) { - throw new Error("RTT check did not expect a reply"); - } - driverOffset = stepResult.matched.nextOffset; - latestSutMessageId = stepResult.matched.message.messageId; - const rttMs = stepResult.matched.observedAtMs - stepResult.requestStartedAtMs; - samples.push({ - status: "pass", - rttMs, - }); - passed += 1; - } catch { - failures += 1; - samples.push({ - status: "fail", - }); - } - if (failures >= params.rttOptions.maxFailures) { - break; - } - } - - const summary = summarizeLiveTransportRttSamples(samples); - return { - details: `${summary.passed}/${samples.length} RTT checks passed`, - driverOffset, - failed: summary.failed, - latestSutMessageId, - passed: summary.passed, - timing: summary.timing, - }; -} - -function classifyCanaryReply(params: { - message: TelegramObservedMessage; - groupId: string; - sutBotId: number; - driverMessageId: number; -}) { - if ( - params.message.chatId !== Number(params.groupId) || - params.message.senderId !== params.sutBotId - ) { - return "ignore" as const; - } - return params.message.replyToMessageId === params.driverMessageId - ? ("match" as const) - : ("unthreaded" as const); -} - -async function runCanary(params: { - driverToken: string; - groupId: string; - sutUsername: string; - sutBotId: number; - timeoutMs: number; - observedMessages: TelegramObservedMessage[]; -}) { - const offset = await flushTelegramUpdates(params.driverToken); - const requestStartedAtMs = Date.now(); - const driverMessage = await sendGroupMessage( - params.driverToken, - params.groupId, - `/help@${params.sutUsername}`, - ); - const requestStartedAt = new Date(requestStartedAtMs).toISOString(); - let firstUnthreadedReply: - | Pick - | undefined; - let sutObserved: Awaited>; - try { - sutObserved = await waitForObservedMessage({ - token: params.driverToken, - initialOffset: offset, - timeoutMs: params.timeoutMs, - observedMessages: params.observedMessages, - observationScenarioId: "telegram-canary", - observationScenarioTitle: "Telegram canary", - validateMatchedMessage: assertTelegramCanaryPresenceReply, - predicate: (message) => { - const classification = classifyCanaryReply({ - message, - groupId: params.groupId, - sutBotId: params.sutBotId, - driverMessageId: driverMessage.message_id, - }); - if (classification === "ignore") { - return false; - } - if (classification === "unthreaded") { - firstUnthreadedReply ??= { - messageId: message.messageId, - replyToMessageId: message.replyToMessageId, - text: message.text, - }; - return false; - } - return classification === "match"; - }, - }); - } catch (error) { - if (firstUnthreadedReply) { - throw new TelegramQaCanaryError( - "sut_reply_not_threaded", - "SUT bot replied, but not as a reply to the canary driver message.", - { - groupId: params.groupId, - sutBotId: params.sutBotId, - driverMessageId: driverMessage.message_id, - sutMessageId: firstUnthreadedReply.messageId, - sutReplyToMessageId: firstUnthreadedReply.replyToMessageId, - }, - ); - } - throw new TelegramQaCanaryError( - "sut_reply_timeout", - `SUT bot did not send any group reply after the canary command within ${formatTelegramQaTimeoutSeconds(params.timeoutMs)}.`, - { - groupId: params.groupId, - sutBotId: params.sutBotId, - driverMessageId: driverMessage.message_id, - cause: formatErrorMessage(error), - }, - ); - } - return { - requestStartedAt, - responseObservedAt: new Date(sutObserved.observedAtMs).toISOString(), - rttMs: sutObserved.observedAtMs - requestStartedAtMs, - sentMessageId: driverMessage.message_id, - responseMessageId: sutObserved.message.messageId, - }; -} - -function canaryFailureMessage(params: { - error: unknown; - groupId: string; - driverBotId: number; - driverUsername?: string; - redactMetadata?: boolean; - sutBotId: number; - sutUsername: string; -}) { - const error = params.error; - const details = formatErrorMessage(error); - const phase = isTelegramQaCanaryError(error) ? error.phase : "unknown"; - const canonicalContext = new Set([ - "groupId", - "driverBotId", - "driverUsername", - "sutBotId", - "sutUsername", - ]); - const context = isTelegramQaCanaryError(error) - ? Object.entries(error.context) - .filter(([key, value]) => value !== undefined && value !== "" && !canonicalContext.has(key)) - .map(([key, value]) => - params.redactMetadata ? `- ${key}: ` : `- ${key}: ${String(value)}`, - ) - : []; - const remediation = (() => { - switch (phase) { - case "sut_reply_timeout": - return [ - "1. Enable Bot-to-Bot Communication Mode for both the driver and SUT bots in @BotFather.", - "2. Confirm the SUT bot is present in the target private group and can receive /help@BotUsername commands there.", - "3. Confirm the QA child gateway started the SUT Telegram account with the expected token.", - ]; - case "sut_reply_not_threaded": - return [ - "1. Check whether the SUT bot is replying in the group without threading to the driver message.", - "2. Confirm the Telegram native command path preserves reply-to behavior for group commands.", - "3. Inspect telegram-qa-report.md and gateway debug logs for the mismatched SUT message id and reply target.", - ]; - case "sut_reply_empty": - return [ - "1. Check whether the Telegram native command response path produced an empty or suppressed reply.", - "2. Confirm the SUT command completed successfully in gateway logs.", - "3. Inspect telegram-qa-report.md for the matched message ids and phase context.", - ]; - default: - return [ - "1. Enable Bot-to-Bot Communication Mode for both the driver and SUT bots in @BotFather.", - "2. Ensure the driver bot can observe bot traffic in the private group by making it admin or disabling privacy mode, then re-add it.", - "3. Ensure both bots are members of the same private group.", - "4. Confirm the SUT bot is allowed to receive /help@BotUsername commands in that group.", - ]; - } - })(); - return [ - "Telegram QA canary failed.", - `Phase: ${phase}`, - details, - "Context:", - `- groupId: ${params.redactMetadata ? "" : params.groupId}`, - `- driverBotId: ${params.redactMetadata ? "" : params.driverBotId}`, - `- driverUsername: ${params.redactMetadata ? "" : (params.driverUsername ?? "")}`, - `- sutBotId: ${params.redactMetadata ? "" : params.sutBotId}`, - `- sutUsername: ${params.redactMetadata ? "" : params.sutUsername}`, - ...context, - "Remediation:", - ...remediation, - ].join("\n"); -} - -export async function runTelegramQaLive(params: { - env?: NodeJS.ProcessEnv; - repoRoot?: string; - outputDir?: string; - sutOpenClawCommand?: QaGatewayChildCommand; - providerMode?: QaProviderModeInput; - primaryModel?: string; - alternateModel?: string; - fastMode?: boolean; - scenarioIds?: string[]; - rttCount?: number; - rttTimeoutMs?: number; - maxRttFailures?: number; - rttCheckIds?: string[]; - sutAccountId?: string; - credentialSource?: string; - credentialRole?: string; - redactPublicMetadata?: boolean; - progressEnabled?: boolean; - canaryTimeoutMs?: number; -}): Promise { - const env = params.env ?? process.env; - const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); - const outputDir = - params.outputDir ?? - path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${createQaArtifactRunId()}`); - await fs.mkdir(outputDir, { recursive: true }); - - const providerMode = normalizeQaProviderMode( - params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, - ); - const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); - const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); - const sutAccountId = params.sutAccountId?.trim() || "sut"; - const scenarios = findScenario(params.scenarioIds, providerMode); - const rttOptions = normalizeTelegramQaRttOptions({ - checkIds: params.rttCheckIds, - count: params.rttCount, - maxFailures: params.maxRttFailures, - timeoutMs: params.rttTimeoutMs, - }); - assertTelegramQaRttCheckSupport({ rttOptions, scenarios }); - const progressEnabled = params.progressEnabled ?? shouldLogTelegramQaLiveProgress(env); - writeTelegramQaProgress( - progressEnabled, - `run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"} rttChecks=${rttOptions?.count ?? 0}`, - ); - - const credentialLease = await acquireQaCredentialLease({ - env, - kind: "telegram", - source: params.credentialSource, - role: params.credentialRole, - resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(env), - parsePayload: parseTelegramQaCredentialPayload, - }); - try { - assertQaGatewayCredentialLeaseQuarantine(credentialLease, env); - } catch (error) { - await credentialLease.release(); - throw error; - } - const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); - const assertLeaseHealthy = () => { - leaseHeartbeat.throwIfFailed(); - }; - writeTelegramQaProgress( - progressEnabled, - `credentials ready: source=${credentialLease.source} role=${credentialLease.role ?? ""}`, - ); - - const runtimeEnv = credentialLease.payload; - const observedMessages: TelegramObservedMessage[] = []; - const redactPublicMetadata = - params.redactPublicMetadata ?? isTruthyOptIn(env[QA_REDACT_PUBLIC_METADATA_ENV]); - writeTelegramQaProgress( - progressEnabled, - `runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"}`, - ); - const startedAt = new Date().toISOString(); - const scenarioResults: TelegramQaScenarioResult[] = []; - const cleanupIssues: string[] = []; - const gatewayDebugDirPath = path.join(outputDir, "gateway-debug"); - let preservedGatewayDebugArtifacts = false; - let canaryFailure: string | null = null; - try { - const driverIdentity = await getBotIdentity(runtimeEnv.driverToken); - const sutIdentity = await getBotIdentity(runtimeEnv.sutToken); - const sutUsername = sutIdentity.username?.trim(); - const uniqueIds = new Set([driverIdentity.id, sutIdentity.id]); - if (uniqueIds.size !== 2) { - throw new Error("Telegram QA requires two distinct bots for driver and SUT."); - } - if (!sutUsername) { - throw new Error("Telegram QA requires the SUT bot to have a Telegram username."); - } - - await Promise.all([ - flushTelegramUpdates(runtimeEnv.driverToken), - flushTelegramUpdates(runtimeEnv.sutToken), - ]); - - const gatewayHarness = await startQaLiveLaneGateway({ - repoRoot, - command: params.sutOpenClawCommand, - transport: { - requiredPluginIds: [], - createGatewayConfig: () => ({}), - }, - transportBaseUrl: "http://127.0.0.1:0", - providerMode, - primaryModel, - alternateModel, - fastMode: params.fastMode, - controlUiEnabled: false, - mutateConfig: (cfg) => - buildTelegramQaConfig(cfg, { - groupId: runtimeEnv.groupId, - sutToken: runtimeEnv.sutToken, - driverBotId: driverIdentity.id, - sutAccountId, - }), - }); - try { - await waitForTelegramChannelRunning(gatewayHarness.gateway, sutAccountId, { env }); - assertLeaseHealthy(); - let latestSutMessageId: number | undefined; - try { - writeTelegramQaProgress(progressEnabled, "canary start"); - const canaryTiming = await runCanary({ - driverToken: runtimeEnv.driverToken, - groupId: runtimeEnv.groupId, - sutUsername, - sutBotId: sutIdentity.id, - timeoutMs: params.canaryTimeoutMs ?? resolveTelegramQaCanaryTimeoutMs(env), - observedMessages, - }); - latestSutMessageId = canaryTiming.responseMessageId; - scenarioResults.push({ - id: "telegram-canary", - coverageIds: ["channels.telegram.canary"], - title: "Telegram canary", - status: "pass", - details: redactPublicMetadata - ? `reply matched in ${canaryTiming.rttMs}ms` - : `reply message ${canaryTiming.responseMessageId} matched in ${canaryTiming.rttMs}ms`, - rttMs: canaryTiming.rttMs, - requestStartedAt: canaryTiming.requestStartedAt, - responseObservedAt: canaryTiming.responseObservedAt, - rttMeasurement: { - finalMatchedReplyRttMs: canaryTiming.rttMs, - requestStartedAt: canaryTiming.requestStartedAt, - responseObservedAt: canaryTiming.responseObservedAt, - source: "request-to-observed-message", - }, - sentMessageId: redactPublicMetadata ? undefined : canaryTiming.sentMessageId, - responseMessageId: redactPublicMetadata ? undefined : canaryTiming.responseMessageId, - }); - writeTelegramQaProgress(progressEnabled, "canary pass"); - } catch (error) { - canaryFailure = canaryFailureMessage({ - error, - groupId: runtimeEnv.groupId, - driverBotId: driverIdentity.id, - driverUsername: driverIdentity.username, - redactMetadata: redactPublicMetadata, - sutBotId: sutIdentity.id, - sutUsername, - }); - scenarioResults.push({ - id: "telegram-canary", - coverageIds: ["channels.telegram.canary"], - title: "Telegram canary", - status: "fail", - details: canaryFailure, - }); - writeTelegramQaProgress( - progressEnabled, - `canary fail: details=${formatTelegramQaProgressDetails(canaryFailure)}`, - ); - } - assertLeaseHealthy(); - if (!canaryFailure) { - let driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); - for (const [scenarioIndex, scenario] of scenarios.entries()) { - const scenarioIndexLabel = `${scenarioIndex + 1}/${scenarios.length}`; - const scenarioIdForLog = sanitizeTelegramQaProgressValue(scenario.id); - writeTelegramQaProgress( - progressEnabled, - `scenario start ${scenarioIndexLabel}: ${scenarioIdForLog}`, - ); - assertLeaseHealthy(); - const scenarioRun = scenario.buildRun(sutUsername); - try { - const scenarioSteps = resolveTelegramQaScenarioSteps(scenarioRun); - let firstRequestStartedAt: string | undefined; - let lastRequestStartedAtMs = 0; - let lastMatched: Awaited> | undefined; - let lastSentMessageId: number | undefined; - for (const step of scenarioSteps) { - driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); - const stepResult = await runTelegramQaScenarioStep({ - driverOffset, - driverToken: runtimeEnv.driverToken, - env, - groupId: runtimeEnv.groupId, - latestSutMessageId, - observedMessages, - scenario, - step, - sutBotId: sutIdentity.id, - }); - firstRequestStartedAt ??= stepResult.requestStartedAt; - lastRequestStartedAtMs = stepResult.requestStartedAtMs; - lastSentMessageId = stepResult.sentMessageId; - const matched = stepResult.matched; - if (!matched) { - continue; - } - driverOffset = matched.nextOffset; - if (step.settleMs !== undefined) { - driverOffset = await collectObservedMessages({ - token: runtimeEnv.driverToken, - initialOffset: driverOffset, - settleMs: step.settleMs, - observedMessages, - observationScenarioId: scenario.id, - observationScenarioTitle: scenario.title, - predicate: (message) => - matchesTelegramScenarioReply({ - allowAnySutReply: step.allowAnySutReply, - groupId: runtimeEnv.groupId, - matchText: step.matchText, - message, - sentMessageId: stepResult.sentMessageId, - sutBotId: sutIdentity.id, - }), - }); - } - assertTelegramScenarioReply({ - expectedTextIncludes: step.expectedTextIncludes, - message: matched.message, - }); - assertTelegramScenarioMessageSet({ - expectedJoinedSutTextIncludes: step.expectedJoinedSutTextIncludes, - expectedSutMessageCount: step.expectedSutMessageCount, - expectedSutMessageCountRange: step.expectedSutMessageCountRange, - groupId: runtimeEnv.groupId, - observedMessages, - scenarioId: scenario.id, - sutBotId: sutIdentity.id, - }); - latestSutMessageId = matched.message.messageId; - lastMatched = matched; - } - if (!lastMatched || !firstRequestStartedAt || lastSentMessageId === undefined) { - const result = { - id: scenario.id, - coverageIds: telegramLiveTransportCoverageIds(scenario), - title: scenario.title, - status: "pass", - details: "no reply", - } satisfies TelegramQaScenarioResult; - scenarioResults.push(result); - writeTelegramQaProgress( - progressEnabled, - `scenario pass ${scenarioIndexLabel}: ${scenarioIdForLog}`, - ); - continue; - } - const lastStep = scenarioSteps.at(-1); - const rttMs = lastMatched.observedAtMs - lastRequestStartedAtMs; - const suffix = - scenarioSteps.length === 1 - ? lastStep?.expectedSutMessageCount === undefined - ? lastStep?.expectedSutMessageCountRange === undefined - ? "" - : `; observed ${lastStep.expectedSutMessageCountRange[0]}-${lastStep.expectedSutMessageCountRange[1]} SUT message(s)` - : `; observed ${lastStep.expectedSutMessageCount} SUT message(s)` - : `; ${scenarioSteps.filter((step) => step.expectReply).length} command replies matched`; - let resultStatus: "pass" | "fail" = "pass"; - let details = redactPublicMetadata - ? `reply matched in ${rttMs}ms${suffix}` - : `reply message ${lastMatched.message.messageId} matched in ${rttMs}ms${suffix}`; - let resultRttMs: number | undefined = rttMs; - let timing: QaEvidenceTiming | undefined; - if (rttOptions?.checkIds.has(scenario.id)) { - const rttResult = await runTelegramQaRttChecks({ - driverOffset, - driverToken: runtimeEnv.driverToken, - env, - groupId: runtimeEnv.groupId, - latestSutMessageId, - observedMessages, - rttOptions, - scenario, - sutBotId: sutIdentity.id, - sutUsername, - }); - driverOffset = rttResult.driverOffset; - latestSutMessageId = rttResult.latestSutMessageId ?? latestSutMessageId; - timing = rttResult.timing; - resultRttMs = rttResult.timing.p50Ms; - details = `${details}; ${rttResult.details}`; - if (rttResult.passed < rttOptions.count) { - resultStatus = "fail"; - } - } - const result = { - id: scenario.id, - coverageIds: telegramLiveTransportCoverageIds(scenario), - title: scenario.title, - status: resultStatus, - details, - rttMs: resultRttMs, - timing, - requestStartedAt: firstRequestStartedAt, - responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(), - rttMeasurement: timing - ? undefined - : { - finalMatchedReplyRttMs: rttMs, - requestStartedAt: new Date(lastRequestStartedAtMs).toISOString(), - responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(), - source: "request-to-observed-message", - }, - sentMessageId: redactPublicMetadata ? undefined : lastSentMessageId, - responseMessageId: redactPublicMetadata ? undefined : lastMatched.message.messageId, - } satisfies TelegramQaScenarioResult; - scenarioResults.push(result); - writeTelegramQaProgress( - progressEnabled, - `scenario ${resultStatus} ${scenarioIndexLabel}: ${scenarioIdForLog}`, - ); - } catch (error) { - const result = { - id: scenario.id, - coverageIds: telegramLiveTransportCoverageIds(scenario), - title: scenario.title, - status: "fail", - details: formatErrorMessage(error), - } satisfies TelegramQaScenarioResult; - scenarioResults.push(result); - writeTelegramQaProgress( - progressEnabled, - `scenario fail ${scenarioIndexLabel}: ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`, - ); - } - assertLeaseHealthy(); - } - } - } finally { - try { - const shouldPreserveGatewayDebugArtifacts = scenarioResults.some( - (scenario) => scenario.status === "fail", - ); - await gatewayHarness.stop( - shouldPreserveGatewayDebugArtifacts ? { preserveToDir: gatewayDebugDirPath } : undefined, - ); - preservedGatewayDebugArtifacts = shouldPreserveGatewayDebugArtifacts; - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "live gateway cleanup", error); - } - } - } finally { - if (await shouldRetainQaGatewayCredentialLease(env)) { - try { - await credentialLease.heartbeat(); - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "credential lease quarantine heartbeat", error); - } - try { - await leaseHeartbeat.stop(); - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "credential lease heartbeat stop", error); - } - cleanupIssues.push( - "credential lease retained for two hours because isolated SUT quiescence was not proven", - ); - } else { - await leaseHeartbeat.stop(); - try { - await credentialLease.release(); - } catch (error) { - appendLiveLaneIssue(cleanupIssues, "credential lease release", error); - } - } - } - - const finishedAt = new Date().toISOString(); - const publishedCleanupIssues = redactPublicMetadata - ? redactQaLiveLaneIssues(cleanupIssues) - : cleanupIssues; - const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length; - const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length; - writeTelegramQaProgress( - progressEnabled, - `run complete: passed=${passedCount} failed=${failedCount} total=${scenarioResults.length}`, - ); - if (cleanupIssues.length > 0) { - writeTelegramQaProgress(progressEnabled, `cleanup issues: count=${cleanupIssues.length}`); - } - const reportPath = path.join(outputDir, "telegram-qa-report.md"); - const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME); - const evidence = buildLiveTransportEvidenceSummary({ - artifactPaths: [ - { kind: "summary", path: path.basename(summaryPath) }, - { kind: "report", path: path.basename(reportPath) }, - ], - env, - generatedAt: finishedAt, - primaryModel, - providerMode, - repoRoot, - checks: scenarioResults, - transportId: "telegram", - }); - await fs.writeFile( - reportPath, - `${renderTelegramQaMarkdown({ - cleanupIssues: publishedCleanupIssues, - credentialSource: credentialLease.source, - redactMetadata: redactPublicMetadata, - groupId: redactPublicMetadata ? "" : runtimeEnv.groupId, - gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, - startedAt, - finishedAt, - scenarios: scenarioResults, - })}\n`, - { encoding: "utf8", mode: 0o600 }, - ); - await fs.writeFile(summaryPath, `${JSON.stringify(evidence, null, 2)}\n`, { - encoding: "utf8", - mode: 0o600, - }); - const artifactPaths = { - report: reportPath, - summary: summaryPath, - ...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}), - }; - if (canaryFailure) { - throw new Error( - buildLiveLaneArtifactsError({ - heading: canaryFailure, - artifacts: artifactPaths, - }), - ); - } - if (cleanupIssues.length > 0) { - throw new Error( - buildLiveLaneArtifactsError({ - heading: "Telegram QA cleanup failed after artifacts were written.", - details: publishedCleanupIssues, - artifacts: artifactPaths, - }), - ); - } - - return { - outputDir, - reportPath, - summaryPath, - ...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}), - scenarios: scenarioResults, - }; -} - -const testing = { - TELEGRAM_QA_SCENARIOS, - TELEGRAM_QA_STANDARD_SCENARIO_IDS, - buildTelegramQaConfig, - canaryFailureMessage, - callTelegramApi, - assertTelegramCanaryPresenceReply, - assertTelegramScenarioMessageSet, - isRecoverableTelegramQaPollError, - assertTelegramScenarioReply, - classifyCanaryReply, - findScenario, - flushTelegramUpdates, - isTelegramObservedMessageTimeoutError, - listTelegramQaScenarioCatalog, - matchesTelegramScenarioReply, - normalizeTelegramObservedMessage, - parseTelegramQaProgressBooleanEnv, - parseTelegramQaCredentialPayload, - normalizeTelegramQaRttOptions, - resolveTelegramQaCanaryTimeoutMs, - resolveTelegramQaReadyTimeoutMs, - resolveTelegramQaScenarioTimeoutMs, - resolveTelegramQaRuntimeEnv, - sanitizeTelegramQaProgressValue, - shouldLogTelegramQaLiveProgress, - formatTelegramQaProgressDetails, - renderTelegramQaMarkdown, - waitForTelegramChannelRunning, - waitForObservedMessage, -}; -export { testing as __testing }; -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/qa-lab/src/suite-round-trip.test.ts b/extensions/qa-lab/src/suite-round-trip.test.ts new file mode 100644 index 000000000000..6ef37a871ef2 --- /dev/null +++ b/extensions/qa-lab/src/suite-round-trip.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from "vitest"; +import { runQaSuiteRoundTripProbe } from "./suite-round-trip.js"; + +describe("QA suite round-trip probe", () => { + it("collects requested samples and chains native replies", async () => { + const messages: Array<{ direction: "outbound"; id: string }> = []; + const sendInbound = vi.fn().mockResolvedValue({ id: "inbound" }); + const waitForOutbound = vi.fn().mockImplementation(async () => { + const reply = { id: `out-${messages.length + 1}` }; + messages.push({ direction: "outbound", id: reply.id }); + return reply; + }); + + const result = await runQaSuiteRoundTripProbe({ + probe: { + scenarioId: "channel-canary", + count: 2, + maxFailures: 2, + timeoutMs: 1_000, + markerPrefix: "QA-RTT", + input: { + conversation: { id: "room", kind: "group" }, + senderId: "driver", + }, + textPrefix: "Reply exactly: ", + chainReplies: true, + }, + transport: { + state: { + getSnapshot: () => ({ messages }), + }, + sendInbound, + waitForOutbound, + } as never, + }); + + expect(result.passed).toBe(2); + expect(result.failed).toBe(0); + expect(result.timing.samples).toBe(2); + expect(sendInbound.mock.calls[1]?.[0]).toMatchObject({ replyToId: "out-1" }); + }); + + it("stops when the failure budget is exhausted", async () => { + const result = await runQaSuiteRoundTripProbe({ + probe: { + scenarioId: "channel-canary", + count: 3, + maxFailures: 1, + timeoutMs: 10, + markerPrefix: "QA-RTT", + input: { + conversation: { id: "room", kind: "group" }, + senderId: "driver", + }, + textPrefix: "Reply exactly: ", + }, + transport: { + state: { getSnapshot: () => ({ messages: [] }) }, + sendInbound: vi.fn(), + waitForOutbound: vi.fn().mockRejectedValue(new Error("timeout")), + } as never, + }); + + expect(result).toMatchObject({ passed: 0, failed: 1 }); + }); +}); diff --git a/extensions/qa-lab/src/suite-round-trip.ts b/extensions/qa-lab/src/suite-round-trip.ts new file mode 100644 index 000000000000..c43629c045c6 --- /dev/null +++ b/extensions/qa-lab/src/suite-round-trip.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { + summarizeLiveTransportRttSamples, + type LiveTransportRttSample, +} from "./live-transports/shared/live-transport-rtt.js"; +import type { QaTransportAdapter } from "./qa-transport.js"; +import type { QaBusInboundMessageInput } from "./runtime-api.js"; + +export type QaSuiteRoundTripProbe = { + scenarioId: string; + count: number; + maxFailures: number; + timeoutMs: number; + markerPrefix: string; + input: Omit; + textPrefix: string; + chainReplies?: boolean; +}; + +export async function runQaSuiteRoundTripProbe(params: { + probe: QaSuiteRoundTripProbe; + transport: QaTransportAdapter; +}) { + const samples: LiveTransportRttSample[] = []; + let failures = 0; + let passed = 0; + let latestReplyId = params.transport.state + .getSnapshot() + .messages.findLast((message) => message.direction === "outbound")?.id; + + for (let index = 1; passed < params.probe.count; index += 1) { + const marker = `${params.probe.markerPrefix}-${index}-${randomUUID().slice(0, 8).toUpperCase()}`; + const outboundStartIndex = params.transport.state + .getSnapshot() + .messages.filter((message) => message.direction === "outbound").length; + const startedAt = Date.now(); + try { + await params.transport.sendInbound({ + ...params.probe.input, + text: `${params.probe.textPrefix}${marker}`, + ...(params.probe.chainReplies && latestReplyId ? { replyToId: latestReplyId } : {}), + }); + const reply = await params.transport.waitForOutbound({ + conversation: params.probe.input.conversation, + sinceIndex: outboundStartIndex, + textIncludes: marker, + timeoutMs: params.probe.timeoutMs, + }); + latestReplyId = reply.id; + samples.push({ status: "pass", rttMs: Math.max(1, Date.now() - startedAt) }); + passed += 1; + } catch { + samples.push({ status: "fail" }); + failures += 1; + } + if (failures >= params.probe.maxFailures) { + break; + } + } + + const summary = summarizeLiveTransportRttSamples(samples); + return { + ...summary, + details: `${summary.passed}/${samples.length} RTT checks passed`, + }; +} diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts index 6a8d09839bda..3f87e2300ba6 100644 --- a/extensions/qa-lab/src/suite-summary.ts +++ b/extensions/qa-lab/src/suite-summary.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { QaSuiteArtifactError } from "./errors.js"; -import type { QaEvidenceSummaryJson } from "./evidence-summary.js"; +import type { QaEvidenceSummaryJson, QaEvidenceTiming } from "./evidence-summary.js"; import type { QaProviderMode } from "./model-selection.js"; import type { RuntimeId, RuntimeParityResult } from "./runtime-parity.js"; import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js"; @@ -12,6 +12,7 @@ type QaSuiteSummaryScenario = { status: "pass" | "fail" | "skip" | "skipped"; steps: unknown[]; details?: string; + timing?: QaEvidenceTiming; runtimeParity?: RuntimeParityResult; }; diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 69f6e18f6228..061020d85554 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -26,6 +26,7 @@ import { import { buildQaSuiteEvidenceSummary, QA_EVIDENCE_FILENAME, + type QaEvidenceTiming, type QaEvidenceSummaryJson, } from "./evidence-summary.js"; import { @@ -88,6 +89,7 @@ import { shouldUseIsolatedQaSuiteScenarioWorkers, splitModelRef, } from "./suite-planning.js"; +import { runQaSuiteRoundTripProbe, type QaSuiteRoundTripProbe } from "./suite-round-trip.js"; import { createQaSuiteScenarioStepRunner, runQaSuiteScenarioDefinition, @@ -115,6 +117,7 @@ export type QaSuiteScenarioResult = { status: "pass" | "fail"; steps: QaReportCheck[]; details?: string; + timing?: QaEvidenceTiming; runtimeParity?: RuntimeParityResult; }; @@ -201,6 +204,7 @@ export type QaSuiteRunParams = { forcedRuntime?: RuntimeId; runtimePair?: [RuntimeId, RuntimeId]; captureRuntimeParityCell?: boolean; + roundTripProbe?: QaSuiteRoundTripProbe; // Unified suite partitions consume child evidence in memory; only the // parent should write the aggregate qa-evidence.json artifact. writeEvidenceFile?: boolean; @@ -545,6 +549,10 @@ function buildQaIsolatedScenarioWorkerParams(params: { transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs, workerStartStaggerMs: params.input?.workerStartStaggerMs, forcedRuntime: params.input?.forcedRuntime, + roundTripProbe: + params.input?.roundTripProbe?.scenarioId === params.scenario.id + ? params.input.roundTripProbe + : undefined, writeEvidenceFile: params.input?.writeEvidenceFile, }; } @@ -1276,6 +1284,17 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise scenario.id === params.roundTripProbe?.scenarioId) + ) { + throw new Error( + `QA round-trip probe scenario is not selected: ${params.roundTripProbe.scenarioId}`, + ); + } + if (params?.roundTripProbe && params.runtimePair) { + throw new Error("QA round-trip probes are not supported with runtime-pair runs."); + } const enabledPluginIds = [ ...new Set([ ...collectQaSuitePluginIds(selectedScenarios), @@ -1443,7 +1462,14 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise Promise> = [ - () => transportFactoryResult.cleanup(), + ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanup()] : []), () => disposeRegisteredAgentHarnesses(), ]; if (ownsLab) { @@ -1810,7 +1836,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise runScenarioDefinition(activeEnv, scenario); const scenarioRetryCount = scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined; - const result = + let result: QaSuiteScenarioResult = scenarioRetryCount === 0 ? await runSelectedScenario() : await runQaScenarioWithFlakeRetry(runSelectedScenario, () => @@ -1819,6 +1845,27 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise= params.roundTripProbe.count; + result = { + ...result, + status: probePassed ? "pass" : "fail", + details: [result.details, probeResult.details].filter(Boolean).join(" | "), + timing: probeResult.timing, + steps: [ + ...result.steps, + { + name: "Round-trip samples", + status: probePassed ? "pass" : "fail", + details: probeResult.details, + }, + ], + }; + } sampleGatewayProcessRss(`scenario:${scenario.id}:finish`); scenarios.push(result); writeQaSuiteProgress( diff --git a/qa/scenarios/channels/channel-canary.yaml b/qa/scenarios/channels/channel-canary.yaml index f382ef3860c2..b0f559949320 100644 --- a/qa/scenarios/channels/channel-canary.yaml +++ b/qa/scenarios/channels/channel-canary.yaml @@ -60,4 +60,7 @@ flow: expr: matchingOutbound.length === 1 message: expr: "`expected one canary reply, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: config.conversationId })}`" + - assert: + expr: "transport.id !== 'telegram' || Boolean(outbound.replyToId)" + message: Telegram canary reply was not threaded to the driver message detailsExpr: outbound.text diff --git a/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml new file mode 100644 index 000000000000..7b6517e2c020 --- /dev/null +++ b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml @@ -0,0 +1,56 @@ +title: Telegram long final reuses its preview + +scenario: + id: telegram-long-final-reuses-preview + surface: channel-framework + category: channel-framework.conversation-routing-and-delivery + coverage: + primary: [runtime.delivery] + regressionRefs: + - openclaw/openclaw#39905 + objective: Verify a long Telegram final replaces its preview instead of leaving stale text. + successCriteria: + - The joined final contains both scripted boundary markers. + - Telegram leaves one or two final chunks after streaming settles. + codeRefs: + - extensions/telegram/src/send.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run the scripted long final and verify preview reuse and chunk count. + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + +flow: + steps: + - name: reuses the long-final preview + actions: + - resetTransport: true + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-stream-room, kind: channel } + senderId: qa-stream-operator + senderName: QA Stream Operator + text: "@openclaw Telegram long final QA check. Use the scripted long final response." + - waitForOutbound: + conversation: { id: telegram-stream-room, kind: channel } + sinceIndex: { ref: outboundStartIndex } + textIncludes: TELEGRAM-LONG-FINAL-BEGIN + timeoutMs: 60000 + - call: sleep + args: [4000] + - set: finalMessages + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)" + - set: joinedFinal + value: + expr: "finalMessages.map((message) => message.text).join('')" + - assert: + expr: "finalMessages.length >= 1 && finalMessages.length <= 2 && joinedFinal.includes('TELEGRAM-LONG-FINAL-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-END')" + message: + expr: "`expected 1-2 complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`" + detailsExpr: "`${finalMessages.length} final chunk(s)`" diff --git a/qa/scenarios/channels/telegram-long-final-three-chunks.yaml b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml new file mode 100644 index 000000000000..db946b7eea22 --- /dev/null +++ b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml @@ -0,0 +1,56 @@ +title: Telegram long final keeps three final chunks + +scenario: + id: telegram-long-final-three-chunks + surface: channel-framework + category: channel-framework.conversation-routing-and-delivery + coverage: + primary: [runtime.delivery] + regressionRefs: + - openclaw/openclaw#39905 + objective: Verify a three-chunk Telegram final removes previews and keeps only final chunks. + successCriteria: + - The joined final contains both scripted three-chunk boundary markers. + - Exactly three final Telegram messages remain after streaming settles. + codeRefs: + - extensions/telegram/src/send.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run the scripted three-chunk final and verify final chunk accounting. + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + +flow: + steps: + - name: keeps exactly three final chunks + actions: + - resetTransport: true + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-stream-room, kind: channel } + senderId: qa-stream-operator + senderName: QA Stream Operator + text: "@openclaw Telegram long final three chunk QA check. Use the scripted three chunk final response." + - waitForOutbound: + conversation: { id: telegram-stream-room, kind: channel } + sinceIndex: { ref: outboundStartIndex } + textIncludes: TELEGRAM-LONG-FINAL-3CHUNK-BEGIN + timeoutMs: 60000 + - call: sleep + args: [4000] + - set: finalMessages + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)" + - set: joinedFinal + value: + expr: "finalMessages.map((message) => message.text).join('')" + - assert: + expr: "finalMessages.length === 3 && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-END')" + message: + expr: "`expected three complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`" + detailsExpr: "`${finalMessages.length} final chunks`" diff --git a/qa/scenarios/channels/telegram-other-bot-command-gating.yaml b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml new file mode 100644 index 000000000000..d437a4c90792 --- /dev/null +++ b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml @@ -0,0 +1,39 @@ +title: Telegram command addressed to another bot + +scenario: + id: telegram-other-bot-command-gating + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify a command addressed to another Telegram bot does not wake OpenClaw. + successCriteria: + - The driver sends a native status command addressed to another bot. + - The Telegram group remains quiet for the observation window. + codeRefs: + - extensions/telegram/src/bot-message-context.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Address a command to another bot and verify OpenClaw ignores it. + config: + requiredChannelDriver: live + +flow: + steps: + - name: ignores another bot's command + actions: + - resetTransport: true + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: /status@OpenClawQaOtherBot + - waitForNoOutbound: + quietMs: 8000 + sinceIndex: { ref: outboundStartIndex } + detailsExpr: "'command addressed to another bot stayed quiet'" diff --git a/qa/scenarios/channels/telegram-stream-final-single-message.yaml b/qa/scenarios/channels/telegram-stream-final-single-message.yaml new file mode 100644 index 000000000000..4eb3803f91cb --- /dev/null +++ b/qa/scenarios/channels/telegram-stream-final-single-message.yaml @@ -0,0 +1,55 @@ +title: Telegram streamed final stays one message + +scenario: + id: telegram-stream-final-single-message + surface: channel-framework + category: channel-framework.conversation-routing-and-delivery + coverage: + primary: [runtime.delivery] + regressionRefs: + - openclaw/openclaw#39905 + objective: Verify Telegram quiet streaming finalizes one preview without a duplicate reply. + successCriteria: + - The scripted streaming reply contains the exact marker. + - One final outbound Telegram message remains after streaming settles. + codeRefs: + - extensions/telegram/src/send.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run the scripted quiet-streaming reply and verify one final message. + config: + marker: QA-TELEGRAM-STREAM-SINGLE-OK + requiredChannelDriver: live + requiredProviderMode: mock-openai + +flow: + steps: + - name: finalizes one streaming message + actions: + - resetTransport: true + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-stream-room, kind: channel } + senderId: qa-stream-operator + senderName: QA Stream Operator + text: + expr: "`@openclaw Quiet streaming QA check. Reply exactly: ${config.marker}`" + - waitForOutbound: + conversation: { id: telegram-stream-room, kind: channel } + sinceIndex: { ref: outboundStartIndex } + textIncludes: { ref: config.marker } + timeoutMs: 75000 + - call: sleep + args: [4000] + - set: finalMessages + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)" + - assert: + expr: "finalMessages.length === 1 && finalMessages[0].text.includes(config.marker)" + message: + expr: "`expected one final Telegram message; saw ${finalMessages.length}: ${finalMessages.map((message) => message.text).join(' | ')}`" + detailsExpr: finalMessages[0].text diff --git a/scripts/e2e/npm-telegram-live-runner.ts b/scripts/e2e/npm-telegram-live-runner.ts index 8829037ef2b4..05b267b59253 100644 --- a/scripts/e2e/npm-telegram-live-runner.ts +++ b/scripts/e2e/npm-telegram-live-runner.ts @@ -6,6 +6,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import type { QaProviderMode } from "../../extensions/qa-lab/src/run-config.ts"; +import type { QaSuiteRoundTripProbe } from "../../extensions/qa-lab/src/suite-round-trip.ts"; function parseBoolean(value: string | undefined) { const normalized = value?.trim().toLowerCase(); @@ -53,24 +54,47 @@ function resolvePackageTelegramOutputDir(env: NodeJS.ProcessEnv, repoRoot: strin ); } -const DEFAULT_RTT_CHECK_ID = "telegram-mentioned-message-reply"; +const DEFAULT_RTT_CHECK_ID = "channel-canary"; function resolveRttOptions(env: NodeJS.ProcessEnv, selectedScenarioIds: readonly string[] = []) { const explicitCheckIds = splitCsv(env.OPENCLAW_NPM_TELEGRAM_RTT_CHECKS); + const checkIds = explicitCheckIds.length > 0 ? explicitCheckIds : [DEFAULT_RTT_CHECK_ID]; + const unknownCheckIds = checkIds.filter((checkId) => checkId !== DEFAULT_RTT_CHECK_ID); + if (unknownCheckIds.length > 0) { + throw new Error(`unknown Telegram QA RTT check: ${unknownCheckIds[0]}`); + } if ( explicitCheckIds.length === 0 && selectedScenarioIds.length > 0 && !selectedScenarioIds.includes(DEFAULT_RTT_CHECK_ID) ) { - return {}; + return undefined; } - const rttCount = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20; + const count = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20; return { - rttCount, - rttTimeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"), - maxRttFailures: - parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? rttCount, - rttCheckIds: explicitCheckIds, + scenarioId: DEFAULT_RTT_CHECK_ID, + count, + timeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS") ?? 30_000, + maxFailures: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? count, + }; +} + +function createRoundTripProbe( + options: ReturnType, +): QaSuiteRoundTripProbe | undefined { + if (!options) { + return undefined; + } + return { + ...options, + markerPrefix: "QA-TELEGRAM-RTT", + input: { + conversation: { id: "telegram-rtt-room", kind: "group" }, + senderId: "qa-rtt-driver", + senderName: "QA RTT Driver", + }, + textPrefix: "@openclaw Telegram RTT check. Reply exactly: ", + chainReplies: true, }; } @@ -117,8 +141,8 @@ async function resolveTrustedOpenClawCommand( } async function main() { - const { runTelegramQaLive } = - await import("../../extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts"); + const { runQaTelegramSuite } = + await import("../../extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts"); const rawSutOpenClawCommand = process.env.OPENCLAW_NPM_TELEGRAM_SUT_COMMAND?.trim(); if (!rawSutOpenClawCommand) { throw new Error("Missing OPENCLAW_NPM_TELEGRAM_SUT_COMMAND."); @@ -128,8 +152,8 @@ async function main() { const repoRoot = path.resolve(process.env.OPENCLAW_NPM_TELEGRAM_REPO_ROOT ?? process.cwd()); const outputDir = resolvePackageTelegramOutputDir(process.env, repoRoot); const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS); - const result = await runTelegramQaLive({ - env: process.env, + const result = await runQaTelegramSuite({ + allowFailures: true, repoRoot, outputDir, sutOpenClawCommand, @@ -138,11 +162,14 @@ async function main() { alternateModel: process.env.OPENCLAW_NPM_TELEGRAM_ALT_MODEL, fastMode: parseBoolean(process.env.OPENCLAW_NPM_TELEGRAM_FAST), scenarioIds, - ...resolveRttOptions(process.env, scenarioIds), + roundTripProbe: createRoundTripProbe(resolveRttOptions(process.env, scenarioIds)), sutAccountId: process.env.OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT, credentialSource: resolveCredentialSource(process.env), credentialRole: resolveCredentialRole(process.env), }); + if (!result) { + throw new Error("Package Telegram QA did not produce suite artifacts."); + } process.stdout.write(`Package Telegram QA report: ${result.reportPath}\n`); process.stdout.write(`Package Telegram QA summary: ${result.summaryPath}\n`); @@ -180,6 +207,7 @@ export const testing = { resolvePackageTelegramOutputDir, resolveCredentialRole, resolveCredentialSource, + createRoundTripProbe, resolveRttOptions, resolveTrustedOpenClawCommand, shouldFailPackageTelegramRun, diff --git a/scripts/mantis/build-telegram-evidence.mjs b/scripts/mantis/build-telegram-evidence.mjs index 05c70ff2cbce..17c9f31c121b 100644 --- a/scripts/mantis/build-telegram-evidence.mjs +++ b/scripts/mantis/build-telegram-evidence.mjs @@ -70,62 +70,6 @@ function evidenceCredentialSource(summary) { ); } -// Historical Telegram summary artifacts can still appear in old Mantis uploads. -// Current QA producers write qa-evidence.json for gate inputs. -function legacyTelegramSummaryToEvidenceSummary(summary) { - const scenarios = Array.isArray(summary.scenarios) ? summary.scenarios : []; - return { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: new Date().toISOString(), - entries: scenarios.map((scenario) => ({ - test: { - kind: "legacy-telegram-qa-scenario", - id: scenario?.id ?? "legacy-telegram-scenario", - title: scenario?.title ?? scenario?.id ?? "Legacy Telegram scenario", - }, - mapping: { - profile: "release", - coverage: [], - }, - execution: { - runner: "legacy-telegram-qa", - environment: { - ref: null, - os: "unknown", - nodeVersion: "unknown", - }, - provider: { - id: "unknown", - live: true, - model: { name: null, ref: null }, - auth: summary.credentials?.source ?? "unknown", - }, - channel: { - id: "telegram", - live: true, - driver: "native", - }, - packageSource: { kind: "unknown" }, - artifacts: [ - { - kind: "summary", - path: "telegram-qa-summary.json", - source: "legacy-telegram-qa", - }, - ], - }, - result: { - status: scenario?.status === "skip" ? "skipped" : (scenario?.status ?? "fail"), - ...(scenario?.status === "pass" - ? {} - : { failure: { reason: scenario?.details ?? "legacy scenario did not pass" } }), - ...(typeof scenario?.rttMs === "number" ? { timing: { rttMs: scenario.rttMs } } : {}), - }, - })), - }; -} - function renderScenarioList(summary) { const entries = evidenceEntries(summary); if (entries.length === 0) { @@ -420,7 +364,7 @@ export function buildTelegramEvidenceManifest({ kind: "report", lane: "run", label: "Telegram QA report", - path: "telegram-qa-report.md", + path: "qa-suite-report.md", targetPath: "report.md", }, ]; @@ -453,17 +397,11 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) { const outputDir = path.resolve(args.output_dir); mkdirSync(outputDir, { recursive: true }); const evidenceSummaryPath = path.join(outputDir, "qa-evidence.json"); - const legacySummaryPath = path.join(outputDir, "telegram-qa-summary.json"); - const usesCurrentEvidenceSummary = existsSync(evidenceSummaryPath); - const summaryPath = usesCurrentEvidenceSummary ? evidenceSummaryPath : legacySummaryPath; - const observedPath = path.join(outputDir, "telegram-qa-observed-messages.json"); - const reportPath = path.join(outputDir, "telegram-qa-report.md"); - if (!existsSync(summaryPath)) { + const reportPath = path.join(outputDir, "qa-suite-report.md"); + if (!existsSync(evidenceSummaryPath)) { throw new Error(`Missing Telegram QA evidence summary: ${evidenceSummaryPath}`); } - const summary = usesCurrentEvidenceSummary - ? readJson(evidenceSummaryPath) - : legacyTelegramSummaryToEvidenceSummary(readJson(legacySummaryPath)); + const summary = readJson(evidenceSummaryPath); const counts = evidenceCounts(summary); const pass = counts.failed === 0 && Number(counts.total ?? 0) > 0; if (!existsSync(reportPath)) { @@ -472,17 +410,15 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) { } writeFileSync(reportPath, "# Mantis Telegram Live QA\n\nTelegram QA report was unavailable.\n"); } - const hasLegacyObservedMessages = !usesCurrentEvidenceSummary && existsSync(observedPath); - const observedMessages = hasLegacyObservedMessages ? readJson(observedPath) : []; - const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages, summary }); + const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages: [], summary }); writeFileSync(path.join(outputDir, "telegram-live-transcript.html"), transcriptHtml, "utf8"); const manifest = buildTelegramEvidenceManifest({ candidateRef: args.candidate_ref, candidateSha: args.candidate_sha, - hasObservedMessages: hasLegacyObservedMessages, + hasObservedMessages: false, scenarioLabel: args.scenario_label, summary, - summaryArtifactPath: path.basename(summaryPath), + summaryArtifactPath: path.basename(evidenceSummaryPath), }); writeFileSync( path.join(outputDir, "mantis-evidence.json"), diff --git a/scripts/release-beta-smoke.ts b/scripts/release-beta-smoke.ts index fdae79f140eb..a7d33d089975 100644 --- a/scripts/release-beta-smoke.ts +++ b/scripts/release-beta-smoke.ts @@ -449,7 +449,7 @@ async function main(): Promise { if (telegramRunId) { await pollRun(options.repo, telegramRunId); const artifactDir = downloadTelegramArtifact(options.repo, telegramRunId); - const report = findFile(artifactDir, "telegram-qa-report.md"); + const report = findFile(artifactDir, "qa-suite-report.md"); if (report && existsSync(report)) { console.log(`\nTelegram report: ${report}\n`); console.log(readFileSync(report, "utf8")); diff --git a/test/scripts/mantis-build-telegram-evidence.test.ts b/test/scripts/mantis-build-telegram-evidence.test.ts index 6ea729356bfa..d1431fac8eed 100644 --- a/test/scripts/mantis-build-telegram-evidence.test.ts +++ b/test/scripts/mantis-build-telegram-evidence.test.ts @@ -80,35 +80,11 @@ function makeTelegramOutput({ includeReport = true, summary = {} } = {}) { ]), ); if (includeReport) { - writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n"); + writeFileSync(path.join(dir, "qa-suite-report.md"), "# Telegram QA\n\npass\n"); } return dir; } -function makeLegacyTelegramOutput() { - const dir = mkdtempSync(path.join(tmpdir(), "mantis-telegram-evidence-test-")); - tempDirs.push(dir); - mkdirSync(dir, { recursive: true }); - writeFileSync( - path.join(dir, "telegram-qa-summary.json"), - JSON.stringify({ - credentials: { source: "convex" }, - counts: { total: 1, passed: 1, failed: 0 }, - scenarios: [ - { - id: "telegram-status-command", - title: "Telegram status command reply", - status: "pass", - details: "Observed expected status response.", - }, - ], - }), - ); - writeFileSync(path.join(dir, "telegram-qa-observed-messages.json"), JSON.stringify([])); - writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n"); - return dir; -} - describe("scripts/mantis/build-telegram-evidence", () => { it("renders redacted Telegram observed messages as a transcript HTML page", () => { const html = renderTelegramEvidenceHtml({ @@ -202,21 +178,6 @@ describe("scripts/mantis/build-telegram-evidence", () => { ); }); - it("renders historical Telegram summaries when evidence summaries are absent", () => { - const dir = makeLegacyTelegramOutput(); - - const result = writeTelegramEvidence(["--output-dir", dir]); - - expect(readFileSync(result.transcriptPath, "utf8")).toContain("Telegram status command reply"); - expect(result.manifest.comparison.pass).toBe(true); - expect( - result.manifest.artifacts.find((artifact) => artifact.targetPath === "summary.json"), - ).toMatchObject({ path: "telegram-qa-summary.json" }); - expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).toContain( - "observed-messages.json", - ); - }); - it("does not fabricate a required report artifact for passing Telegram summaries", () => { const dir = makeTelegramOutput({ includeReport: false }); @@ -250,7 +211,7 @@ describe("scripts/mantis/build-telegram-evidence", () => { const result = writeTelegramEvidence(["--output-dir", dir]); expect(result.manifest.comparison.pass).toBe(false); - expect(readFileSync(path.join(dir, "telegram-qa-report.md"), "utf8")).toContain( + expect(readFileSync(path.join(dir, "qa-suite-report.md"), "utf8")).toContain( "Telegram QA report was unavailable", ); expect(loadEvidenceManifest(result.manifestPath).comparison.pass).toBe(false); diff --git a/test/scripts/npm-telegram-live.test.ts b/test/scripts/npm-telegram-live.test.ts index b4dea05aa646..189be02c6730 100644 --- a/test/scripts/npm-telegram-live.test.ts +++ b/test/scripts/npm-telegram-live.test.ts @@ -284,15 +284,15 @@ describe("package Telegram live Docker E2E", () => { it("defaults package Telegram RTT for the normal package live lane", () => { expect(testing.resolveRttOptions({})).toEqual({ - rttCount: 20, - rttTimeoutMs: undefined, - maxRttFailures: 20, - rttCheckIds: [], + scenarioId: "channel-canary", + count: 20, + timeoutMs: 30_000, + maxFailures: 20, }); }); it("does not force default RTT onto focused non-RTT scenario runs", () => { - expect(testing.resolveRttOptions({}, ["telegram-canary"])).toEqual({}); + expect(testing.resolveRttOptions({}, ["telegram-status-command"])).toBeUndefined(); }); it("maps repeated RTT env onto package Telegram live options", () => { @@ -301,16 +301,40 @@ describe("package Telegram live Docker E2E", () => { OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7", OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS: "45000", OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES: "2", - OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply", + OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "channel-canary", }), ).toEqual({ - rttCount: 7, - rttTimeoutMs: 45_000, - maxRttFailures: 2, - rttCheckIds: ["telegram-mentioned-message-reply"], + scenarioId: "channel-canary", + count: 7, + timeoutMs: 45_000, + maxFailures: 2, }); }); + it("builds a generic suite probe for the Telegram RTT lane", () => { + const probe = testing.createRoundTripProbe(testing.resolveRttOptions({})); + + expect(probe).toMatchObject({ + scenarioId: "channel-canary", + count: 20, + timeoutMs: 30_000, + markerPrefix: "QA-TELEGRAM-RTT", + textPrefix: "@openclaw Telegram RTT check. Reply exactly: ", + chainReplies: true, + input: { + conversation: { id: "telegram-rtt-room", kind: "group" }, + }, + }); + }); + + it("rejects retired RTT scenario ids", () => { + expect(() => + testing.resolveRttOptions({ + OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply", + }), + ).toThrow("unknown Telegram QA RTT check: telegram-mentioned-message-reply"); + }); + it("rejects invalid repeated RTT env", () => { expect(() => testing.resolveRttOptions({