From c22a21759b3c15dd368511780daba53f82d4429a Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Thu, 23 Apr 2026 14:07:07 +0530 Subject: [PATCH] test(qa): report telegram reply rtt --- CHANGELOG.md | 1 + docs/concepts/qa-e2e-automation.md | 2 + docs/help/testing.md | 2 +- .../telegram/telegram-live.runtime.test.ts | 22 +++++++++ .../telegram/telegram-live.runtime.ts | 47 +++++++++++++++++-- 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adffa3ed9b2..49c44042801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai ### Changes +- QA/Telegram: record per-scenario reply RTT in the live Telegram QA report and summary, starting with the canary response. - Providers/xAI: add image generation, text-to-speech, and speech-to-text support, including `grok-imagine-image` / `grok-imagine-image-pro`, reference-image edits, six live xAI voices, MP3/WAV/PCM/G.711 TTS formats, `grok-stt` audio transcription, and xAI realtime transcription for Voice Call streaming. (#68694) Thanks @KateWilkins. - Providers/STT: add Voice Call streaming transcription for Deepgram, ElevenLabs, and Mistral, alongside the existing OpenAI and xAI realtime STT paths; ElevenLabs also gains Scribe v2 batch audio transcription for inbound media. - TUI: add local embedded mode for running terminal chats without a Gateway while keeping plugin approval gates enforced. (#66767) Thanks @fuller-stack-dev. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index c6219b2f1ab..c499c01988f 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -82,6 +82,8 @@ observation works best when both bots have Bot-to-Bot Communication Mode enabled in `@BotFather`. The command exits non-zero when any scenario fails. Use `--allow-failures` when you want artifacts without a failing exit code. +The Telegram report and summary include per-reply RTT from the driver message +send request to the observed SUT reply, starting with the canary. Live transport lanes now share one smaller contract instead of each inventing their own scenario list shape: diff --git a/docs/help/testing.md b/docs/help/testing.md index 627e50ee06d..b69bb0fd074 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -133,7 +133,7 @@ runs the same lanes before release approval. want artifacts without a failing exit code. - Requires two distinct bots in the same private group, with the SUT bot exposing a Telegram username. - For stable bot-to-bot observation, enable Bot-to-Bot Communication Mode in `@BotFather` for both bots and ensure the driver bot can observe group bot traffic. - - Writes a Telegram QA report, summary, and observed-messages artifact under `.artifacts/qa-e2e/...`. + - Writes a Telegram QA report, summary, and observed-messages artifact under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply. Live transport lanes share one standard contract so new transports do not drift: diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index 3f3302b8186..3759bcbd007 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -594,6 +594,28 @@ describe("telegram live qa runtime", () => { ]); }); + it("prints Telegram scenario RTT in the Markdown report", () => { + expect( + __testing.renderTelegramQaMarkdown({ + cleanupIssues: [], + credentialSource: "env", + groupId: "-100123", + redactMetadata: false, + startedAt: "2026-04-23T00:00:00.000Z", + finishedAt: "2026-04-23T00:00:10.000Z", + scenarios: [ + { + id: "telegram-canary", + title: "Telegram canary", + status: "pass", + details: "reply message 12 matched in 4321ms", + rttMs: 4321, + }, + ], + }), + ).toContain("- RTT: 4321ms"); + }); + it("formats phase-specific canary diagnostics with context", () => { const error = new Error( "SUT bot did not send any group reply after the canary command within 30s.", diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts index 45a2b33c409..9c343c9bb66 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts @@ -100,6 +100,11 @@ type TelegramQaScenarioResult = { title: string; status: "pass" | "fail"; details: string; + rttMs?: number; + requestStartedAt?: string; + responseObservedAt?: string; + sentMessageId?: number; + responseMessageId?: number; }; type TelegramQaCanaryPhase = "sut_reply_timeout" | "sut_reply_not_threaded" | "sut_reply_empty"; @@ -608,7 +613,7 @@ async function waitForObservedMessage(params: { }; params.observedMessages.push(observedMessage); if (matchedScenario) { - return { message: observedMessage, nextOffset: offset }; + return { message: observedMessage, nextOffset: offset, observedAtMs: Date.now() }; } } } @@ -671,6 +676,9 @@ function renderTelegramQaMarkdown(params: { lines.push(""); lines.push(`- Status: ${scenario.status}`); lines.push(`- Details: ${scenario.details}`); + if (scenario.rttMs !== undefined) { + lines.push(`- RTT: ${scenario.rttMs}ms`); + } lines.push(""); } if (params.cleanupIssues.length > 0) { @@ -796,11 +804,13 @@ async function runCanary(params: { observedMessages: TelegramObservedMessage[]; }) { const offset = await flushTelegramUpdates(params.driverToken); + const requestStartedAtMs = Date.now(); const driverMessage = await sendGroupMessage( params.driverToken, params.groupId, `/help@${params.sutUsername}`, ); + const requestStartedAt = new Date(requestStartedAtMs).toISOString(); let firstUnthreadedReply: | Pick | undefined; @@ -871,6 +881,13 @@ async function runCanary(params: { }, ); } + return { + requestStartedAt, + responseObservedAt: new Date(sutObserved.observedAtMs).toISOString(), + rttMs: sutObserved.observedAtMs - requestStartedAtMs, + sentMessageId: driverMessage.message_id, + responseMessageId: sutObserved.message.messageId, + }; } function canaryFailureMessage(params: { @@ -1045,13 +1062,26 @@ export async function runTelegramQaLive(params: { assertLeaseHealthy(); try { writeTelegramQaProgress(progressEnabled, "canary start"); - await runCanary({ + const canaryTiming = await runCanary({ driverToken: runtimeEnv.driverToken, groupId: runtimeEnv.groupId, sutUsername, sutBotId: sutIdentity.id, observedMessages, }); + scenarioResults.push({ + id: "telegram-canary", + title: "Telegram canary", + status: "pass", + details: redactPublicMetadata + ? `reply matched in ${canaryTiming.rttMs}ms` + : `reply message ${canaryTiming.responseMessageId} matched in ${canaryTiming.rttMs}ms`, + rttMs: canaryTiming.rttMs, + requestStartedAt: canaryTiming.requestStartedAt, + responseObservedAt: canaryTiming.responseObservedAt, + sentMessageId: redactPublicMetadata ? undefined : canaryTiming.sentMessageId, + responseMessageId: redactPublicMetadata ? undefined : canaryTiming.responseMessageId, + }); writeTelegramQaProgress(progressEnabled, "canary pass"); } catch (error) { canaryFailure = canaryFailureMessage({ @@ -1087,11 +1117,13 @@ export async function runTelegramQaLive(params: { assertLeaseHealthy(); const scenarioRun = scenario.buildRun(sutUsername); try { + const requestStartedAtMs = Date.now(); const sent = await sendGroupMessage( runtimeEnv.driverToken, runtimeEnv.groupId, scenarioRun.input, ); + const requestStartedAt = new Date(requestStartedAtMs).toISOString(); const matched = await waitForObservedMessage({ token: runtimeEnv.driverToken, initialOffset: driverOffset, @@ -1116,13 +1148,19 @@ export async function runTelegramQaLive(params: { expectedTextIncludes: scenarioRun.expectedTextIncludes, message: matched.message, }); + const rttMs = matched.observedAtMs - requestStartedAtMs; const result = { id: scenario.id, title: scenario.title, status: "pass", details: redactPublicMetadata - ? "reply matched" - : `reply message ${matched.message.messageId} matched`, + ? `reply matched in ${rttMs}ms` + : `reply message ${matched.message.messageId} matched in ${rttMs}ms`, + rttMs, + requestStartedAt, + responseObservedAt: new Date(matched.observedAtMs).toISOString(), + sentMessageId: redactPublicMetadata ? undefined : sent.message_id, + responseMessageId: redactPublicMetadata ? undefined : matched.message.messageId, } satisfies TelegramQaScenarioResult; scenarioResults.push(result); writeTelegramQaProgress( @@ -1295,4 +1333,5 @@ export const __testing = { sanitizeTelegramQaProgressValue, shouldLogTelegramQaLiveProgress, formatTelegramQaProgressDetails, + renderTelegramQaMarkdown, };