diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 060fb944550..3a2c4f8f042 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -6,6 +6,7 @@ const { runQaSuiteFromRuntime, runQaCharacterEval, runQaMultipass, + runTelegramQaLive, startQaLabServer, writeQaDockerHarnessFiles, buildQaDockerHarnessImage, @@ -15,6 +16,7 @@ const { runQaSuiteFromRuntime: vi.fn(), runQaCharacterEval: vi.fn(), runQaMultipass: vi.fn(), + runTelegramQaLive: vi.fn(), startQaLabServer: vi.fn(), writeQaDockerHarnessFiles: vi.fn(), buildQaDockerHarnessImage: vi.fn(), @@ -37,6 +39,10 @@ vi.mock("./multipass.runtime.js", () => ({ runQaMultipass, })); +vi.mock("./telegram-live.runtime.js", () => ({ + runTelegramQaLive, +})); + vi.mock("./lab-server.js", () => ({ startQaLabServer, })); @@ -58,6 +64,7 @@ import { runQaCharacterEvalCommand, runQaManualLaneCommand, runQaSuiteCommand, + runQaTelegramCommand, } from "./cli.runtime.js"; describe("qa cli runtime", () => { @@ -69,6 +76,7 @@ describe("qa cli runtime", () => { runQaCharacterEval.mockReset(); runQaManualLane.mockReset(); runQaMultipass.mockReset(); + runTelegramQaLive.mockReset(); startQaLabServer.mockReset(); writeQaDockerHarnessFiles.mockReset(); buildQaDockerHarnessImage.mockReset(); @@ -98,6 +106,13 @@ describe("qa cli runtime", () => { vmName: "openclaw-qa-test", scenarioIds: ["channel-chat-baseline"], }); + runTelegramQaLive.mockResolvedValue({ + outputDir: "/tmp/telegram", + reportPath: "/tmp/telegram/report.md", + summaryPath: "/tmp/telegram/summary.json", + observedMessagesPath: "/tmp/telegram/observed.json", + scenarios: [], + }); startQaLabServer.mockResolvedValue({ baseUrl: "http://127.0.0.1:58000", runSelfCheck: vi.fn().mockResolvedValue({ @@ -146,6 +161,30 @@ describe("qa cli runtime", () => { }); }); + it("resolves telegram qa repo-root-relative paths before dispatching", async () => { + await runQaTelegramCommand({ + repoRoot: "/tmp/openclaw-repo", + outputDir: ".artifacts/qa/telegram", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + fastMode: true, + scenarioIds: ["telegram-help-command"], + sutAccountId: "sut-live", + }); + + expect(runTelegramQaLive).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"), + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + fastMode: true, + scenarioIds: ["telegram-help-command"], + sutAccountId: "sut-live", + }); + }); + it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 22693eded23..6a1921b24dc 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -15,6 +15,7 @@ import { type QaProviderModeInput, } from "./run-config.js"; import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js"; +import { runTelegramQaLive } from "./telegram-live.runtime.js"; type InterruptibleServer = { baseUrl: string; @@ -280,6 +281,32 @@ export async function runQaSuiteCommand(opts: { process.stdout.write(`QA suite summary: ${result.summaryPath}\n`); } +export async function runQaTelegramCommand(opts: { + repoRoot?: string; + outputDir?: string; + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + sutAccountId?: string; +}) { + const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); + const result = await runTelegramQaLive({ + repoRoot, + outputDir: opts.outputDir ? path.resolve(repoRoot, opts.outputDir) : undefined, + providerMode: opts.providerMode, + primaryModel: opts.primaryModel, + alternateModel: opts.alternateModel, + fastMode: opts.fastMode, + scenarioIds: opts.scenarioIds, + sutAccountId: opts.sutAccountId, + }); + process.stdout.write(`Telegram QA report: ${result.reportPath}\n`); + process.stdout.write(`Telegram QA summary: ${result.summaryPath}\n`); + process.stdout.write(`Telegram QA observed messages: ${result.observedMessagesPath}\n`); +} + export async function runQaCharacterEvalCommand(opts: { repoRoot?: string; outputDir?: string; diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index a3b3b0d0adc..b254e901646 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -35,6 +35,20 @@ async function runQaSuite(opts: { await runtime.runQaSuiteCommand(opts); } +async function runQaTelegram(opts: { + repoRoot?: string; + outputDir?: string; + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + sutAccountId?: string; +}) { + const runtime = await loadQaLabCliRuntime(); + await runtime.runQaTelegramCommand(opts); +} + async function runQaCharacterEval(opts: { repoRoot?: string; outputDir?: string; @@ -202,6 +216,53 @@ export function registerQaLabCli(program: Command) { }, ); + qa.command("telegram") + .description("Run the manual Telegram live QA lane against a private bot-to-bot group harness") + .option("--repo-root ", "Repository root to target when running from a neutral cwd") + .option("--output-dir ", "Telegram QA artifact directory") + .option( + "--provider-mode ", + "Provider mode: mock-openai or live-frontier (legacy live-openai still works)", + "mock-openai", + ) + .option("--model ", "Primary provider/model ref") + .option("--alt-model ", "Alternate provider/model ref") + .option( + "--scenario ", + "Run only the named Telegram QA scenario (repeatable)", + collectString, + [], + ) + .option("--fast", "Enable provider fast mode where supported", false) + .option( + "--sut-account ", + "Temporary Telegram account id inside the QA gateway config", + "sut", + ) + .action( + async (opts: { + repoRoot?: string; + outputDir?: string; + providerMode?: QaProviderModeInput; + model?: string; + altModel?: string; + scenario?: string[]; + fast?: boolean; + sutAccount?: string; + }) => { + await runQaTelegram({ + repoRoot: opts.repoRoot, + outputDir: opts.outputDir, + providerMode: opts.providerMode, + primaryModel: opts.model, + alternateModel: opts.altModel, + fastMode: opts.fast, + scenarioIds: opts.scenario, + sutAccountId: opts.sutAccount, + }); + }, + ); + qa.command("character-eval") .description("Run the character QA scenario across live models and write a judged report") .option("--repo-root ", "Repository root to target when running from a neutral cwd") diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index 5a6df46ba7f..a384e40bc32 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -6,6 +6,7 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; @@ -609,6 +610,7 @@ export async function startQaGatewayChild(params: { thinkingDefault?: QaThinkingLevel; claudeCliAuthMode?: QaCliBackendAuthMode; controlUiEnabled?: boolean; + mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig; }) { const tempRoot = await fs.mkdtemp( path.join(resolvePreferredOpenClawTmpDir(), "openclaw-qa-suite-"), @@ -654,7 +656,7 @@ export async function startQaGatewayChild(params: { providerConfigs: liveProviderConfigs, }) : undefined; - const cfg = buildQaGatewayConfig({ + const baseCfg = buildQaGatewayConfig({ bind: "loopback", gatewayPort, gatewayToken, @@ -675,6 +677,7 @@ export async function startQaGatewayChild(params: { thinkingDefault: params.thinkingDefault, controlUiEnabled: params.controlUiEnabled, }); + const cfg = params.mutateConfig ? params.mutateConfig(baseCfg) : baseCfg; await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8"); const allowedPluginIds = [...(cfg.plugins?.allow ?? []), "openai"].filter( (pluginId, index, array): pluginId is string => { diff --git a/extensions/qa-lab/src/telegram-live.runtime.test.ts b/extensions/qa-lab/src/telegram-live.runtime.test.ts new file mode 100644 index 00000000000..d8d78e70e6f --- /dev/null +++ b/extensions/qa-lab/src/telegram-live.runtime.test.ts @@ -0,0 +1,124 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; +import { describe, expect, it } from "vitest"; +import { __testing } from "./telegram-live.runtime.js"; + +describe("telegram live qa runtime", () => { + it("resolves required Telegram QA env vars", () => { + expect( + __testing.resolveTelegramQaRuntimeEnv({ + OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", + OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", + OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut", + }), + ).toEqual({ + groupId: "-100123", + driverToken: "driver", + sutToken: "sut", + }); + }); + + it("fails when a required Telegram QA env var is missing", () => { + expect(() => + __testing.resolveTelegramQaRuntimeEnv({ + OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123", + OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", + }), + ).toThrow("OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"); + }); + + it("fails when the Telegram group id is not numeric", () => { + expect(() => + __testing.resolveTelegramQaRuntimeEnv({ + OPENCLAW_QA_TELEGRAM_GROUP_ID: "qa-group", + OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver", + OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut", + }), + ).toThrow("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id."); + }); + + it("injects a temporary Telegram account into the QA gateway config", () => { + const baseCfg: OpenClawConfig = { + plugins: { + allow: ["memory-core", "qa-channel"], + entries: { + "memory-core": { enabled: true }, + "qa-channel": { enabled: true }, + }, + }, + channels: { + "qa-channel": { + enabled: true, + baseUrl: "http://127.0.0.1:43123", + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + allowFrom: ["*"], + }, + }, + }; + + const next = __testing.buildTelegramQaConfig(baseCfg, { + groupId: "-100123", + sutToken: "sut-token", + driverBotId: 42, + sutAccountId: "sut", + }); + + expect(next.plugins?.allow).toContain("telegram"); + expect(next.plugins?.entries?.telegram).toEqual({ enabled: true }); + expect(next.channels?.telegram).toEqual({ + enabled: true, + defaultAccount: "sut", + accounts: { + sut: { + enabled: true, + botToken: "sut-token", + dmPolicy: "disabled", + groups: { + "-100123": { + groupPolicy: "allowlist", + allowFrom: ["42"], + requireMention: true, + }, + }, + }, + }, + }); + }); + + it("normalizes observed Telegram messages", () => { + expect( + __testing.normalizeTelegramObservedMessage({ + update_id: 7, + message: { + message_id: 9, + date: 1_700_000_000, + text: "hello", + chat: { id: -100123 }, + from: { + id: 42, + is_bot: true, + username: "driver_bot", + }, + reply_to_message: { message_id: 8 }, + reply_markup: { + inline_keyboard: [[{ text: "Approve" }, { text: "Deny" }]], + }, + photo: [{}], + }, + }), + ).toEqual({ + updateId: 7, + messageId: 9, + chatId: -100123, + senderId: 42, + senderIsBot: true, + senderUsername: "driver_bot", + text: "hello", + caption: undefined, + replyToMessageId: 8, + timestamp: 1_700_000_000_000, + inlineButtons: ["Approve", "Deny"], + mediaKinds: ["photo"], + }); + }); +}); diff --git a/extensions/qa-lab/src/telegram-live.runtime.ts b/extensions/qa-lab/src/telegram-live.runtime.ts new file mode 100644 index 00000000000..35639569b06 --- /dev/null +++ b/extensions/qa-lab/src/telegram-live.runtime.ts @@ -0,0 +1,617 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { startQaGatewayChild } from "./gateway-child.js"; +import { + defaultQaModelForMode, + normalizeQaProviderMode, + type QaProviderModeInput, +} from "./run-config.js"; + +type TelegramQaRuntimeEnv = { + groupId: string; + driverToken: string; + sutToken: string; +}; + +type TelegramBotIdentity = { + id: number; + is_bot: boolean; + first_name: string; + username?: string; +}; + +type TelegramQaScenarioDefinition = { + id: "telegram-help-command"; + title: string; + timeoutMs: number; + buildInput: (sutUsername: string) => string; +}; + +type TelegramObservedMessage = { + updateId: number; + messageId: number; + chatId: number; + senderId: number; + senderIsBot: boolean; + senderUsername?: string; + text: string; + caption?: string; + replyToMessageId?: number; + timestamp: number; + inlineButtons: string[]; + mediaKinds: string[]; +}; + +type TelegramQaScenarioResult = { + id: string; + title: string; + status: "pass" | "fail"; + details: string; +}; + +export type TelegramQaRunResult = { + outputDir: string; + reportPath: string; + summaryPath: string; + observedMessagesPath: string; + scenarios: TelegramQaScenarioResult[]; +}; + +type TelegramQaSummary = { + groupId: string; + startedAt: string; + finishedAt: string; + counts: { + total: number; + passed: number; + failed: number; + }; + scenarios: TelegramQaScenarioResult[]; +}; + +type TelegramApiEnvelope = { + ok: boolean; + result?: T; + description?: string; +}; + +type TelegramReplyMarkup = { + inline_keyboard?: Array>; +}; + +type TelegramMessage = { + message_id: number; + date: number; + text?: string; + caption?: string; + reply_markup?: TelegramReplyMarkup; + reply_to_message?: { message_id?: number }; + from?: { + id?: number; + is_bot?: boolean; + username?: string; + }; + chat: { + id: number; + }; + photo?: unknown[]; + document?: unknown; + audio?: unknown; + video?: unknown; + voice?: unknown; + sticker?: unknown; +}; + +type TelegramUpdate = { + update_id: number; + message?: TelegramMessage; +}; + +type TelegramSendMessageResult = { + message_id: number; + chat: { + id: number; + }; +}; + +const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ + { + id: "telegram-help-command", + title: "Telegram help command reply", + timeoutMs: 45_000, + buildInput: (sutUsername) => `/help@${sutUsername}`, + }, +]; + +const TELEGRAM_QA_ENV_KEYS = [ + "OPENCLAW_QA_TELEGRAM_GROUP_ID", + "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN", + "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN", +] as const; + +function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof TELEGRAM_QA_ENV_KEYS)[number]) { + const value = env[key]?.trim(); + if (!value) { + throw new Error(`Missing ${key}.`); + } + return value; +} + +export function resolveTelegramQaRuntimeEnv( + env: NodeJS.ProcessEnv = process.env, +): TelegramQaRuntimeEnv { + const groupId = resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_GROUP_ID"); + if (!/^-?\d+$/u.test(groupId)) { + throw new Error("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id."); + } + return { + groupId, + driverToken: resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN"), + sutToken: resolveEnvValue(env, "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"), + }; +} + +function flattenInlineButtons(replyMarkup?: TelegramReplyMarkup) { + return (replyMarkup?.inline_keyboard ?? []) + .flat() + .map((button) => button.text?.trim()) + .filter((text): text is string => Boolean(text)); +} + +function detectMediaKinds(message: TelegramMessage) { + const kinds: string[] = []; + if (Array.isArray(message.photo) && message.photo.length > 0) { + kinds.push("photo"); + } + if (message.document) { + kinds.push("document"); + } + if (message.audio) { + kinds.push("audio"); + } + if (message.video) { + kinds.push("video"); + } + if (message.voice) { + kinds.push("voice"); + } + if (message.sticker) { + kinds.push("sticker"); + } + return kinds; +} + +export function normalizeTelegramObservedMessage( + update: TelegramUpdate, +): TelegramObservedMessage | null { + const message = update.message; + if (!message?.from?.id) { + return null; + } + return { + updateId: update.update_id, + messageId: message.message_id, + chatId: message.chat.id, + senderId: message.from.id, + senderIsBot: message.from.is_bot === true, + senderUsername: message.from.username, + text: message.text ?? message.caption ?? "", + caption: message.caption, + replyToMessageId: message.reply_to_message?.message_id, + timestamp: message.date * 1000, + inlineButtons: flattenInlineButtons(message.reply_markup), + mediaKinds: detectMediaKinds(message), + }; +} + +function buildTelegramQaConfig( + baseCfg: OpenClawConfig, + params: { + groupId: string; + sutToken: string; + driverBotId: number; + sutAccountId: string; + }, +): OpenClawConfig { + const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "telegram"])]; + const pluginEntries = { + ...baseCfg.plugins?.entries, + telegram: { enabled: true }, + }; + return { + ...baseCfg, + plugins: { + ...baseCfg.plugins, + allow: pluginAllow, + entries: pluginEntries, + }, + channels: { + ...baseCfg.channels, + telegram: { + enabled: true, + defaultAccount: params.sutAccountId, + accounts: { + [params.sutAccountId]: { + enabled: true, + botToken: params.sutToken, + dmPolicy: "disabled", + groups: { + [params.groupId]: { + groupPolicy: "allowlist", + allowFrom: [String(params.driverBotId)], + requireMention: true, + }, + }, + }, + }, + }, + }, + }; +} + +async function callTelegramApi( + token: string, + method: string, + body?: Record, +): Promise { + const response = await fetch(`https://api.telegram.org/bot${token}/${method}`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify(body ?? {}), + }); + const payload = (await response.json()) as TelegramApiEnvelope; + if (!response.ok || !payload.ok || payload.result === undefined) { + throw new Error( + payload.description?.trim() || `${method} failed with status ${response.status}`, + ); + } + return payload.result; +} + +async function getBotIdentity(token: string) { + return await callTelegramApi(token, "getMe"); +} + +async function flushTelegramUpdates(token: string) { + let offset = 0; + while (true) { + const updates = await callTelegramApi(token, "getUpdates", { + offset, + timeout: 0, + allowed_updates: ["message"], + }); + if (updates.length === 0) { + return offset; + } + offset = (updates.at(-1)?.update_id ?? offset) + 1; + } +} + +async function sendGroupMessage(token: string, groupId: string, text: string) { + return await callTelegramApi(token, "sendMessage", { + chat_id: groupId, + text, + disable_notification: true, + }); +} + +async function waitForObservedMessage(params: { + token: string; + initialOffset: number; + timeoutMs: number; + predicate: (message: TelegramObservedMessage) => boolean; + observedMessages: TelegramObservedMessage[]; +}) { + const startedAt = Date.now(); + let offset = params.initialOffset; + while (Date.now() - startedAt < params.timeoutMs) { + const remainingMs = Math.max( + 1_000, + Math.min(10_000, params.timeoutMs - (Date.now() - startedAt)), + ); + const timeoutSeconds = Math.max(1, Math.min(10, Math.floor(remainingMs / 1000))); + const updates = await callTelegramApi(params.token, "getUpdates", { + offset, + timeout: timeoutSeconds, + allowed_updates: ["message"], + }); + if (updates.length === 0) { + continue; + } + offset = (updates.at(-1)?.update_id ?? offset) + 1; + for (const update of updates) { + const normalized = normalizeTelegramObservedMessage(update); + if (!normalized) { + continue; + } + params.observedMessages.push(normalized); + if (params.predicate(normalized)) { + return { message: normalized, nextOffset: offset }; + } + } + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for Telegram message`); +} + +async function waitForTelegramChannelRunning( + gateway: Awaited>, + accountId: string, +) { + const startedAt = Date.now(); + while (Date.now() - startedAt < 45_000) { + try { + const payload = (await gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { + channelAccounts?: Record< + string, + Array<{ accountId?: string; running?: boolean; restartPending?: boolean }> + >; + }; + const accounts = payload.channelAccounts?.telegram ?? []; + const match = accounts.find((entry) => entry.accountId === accountId); + if (match?.running && match.restartPending !== true) { + return; + } + } catch { + // retry + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`telegram account "${accountId}" did not become ready`); +} + +function renderTelegramQaMarkdown(params: { + groupId: string; + startedAt: string; + finishedAt: string; + scenarios: TelegramQaScenarioResult[]; +}) { + const lines = [ + "# Telegram QA Report", + "", + `- Group: \`${params.groupId}\``, + `- Started: ${params.startedAt}`, + `- Finished: ${params.finishedAt}`, + "", + "## Scenarios", + "", + ]; + for (const scenario of params.scenarios) { + lines.push(`### ${scenario.title}`); + lines.push(""); + lines.push(`- Status: ${scenario.status}`); + lines.push(`- Details: ${scenario.details}`); + lines.push(""); + } + return lines.join("\n"); +} + +function findScenario(ids?: string[]) { + if (!ids || ids.length === 0) { + return [...TELEGRAM_QA_SCENARIOS]; + } + const selected = TELEGRAM_QA_SCENARIOS.filter((scenario) => ids.includes(scenario.id)); + if (selected.length === 0) { + throw new Error(`No Telegram QA scenarios matched: ${ids.join(", ")}`); + } + return selected; +} + +async function runCanary(params: { + driverToken: string; + groupId: string; + sutUsername: string; + driverBotId: number; + sutBotId: number; + observedMessages: TelegramObservedMessage[]; +}) { + let offset = await flushTelegramUpdates(params.driverToken); + const driverMessage = await sendGroupMessage( + params.driverToken, + params.groupId, + `/help@${params.sutUsername}`, + ); + const driverObserved = await waitForObservedMessage({ + token: params.driverToken, + initialOffset: offset, + timeoutMs: 20_000, + observedMessages: params.observedMessages, + predicate: (message) => + message.chatId === Number(params.groupId) && + message.senderId === params.driverBotId && + message.messageId === driverMessage.message_id, + }); + offset = driverObserved.nextOffset; + await waitForObservedMessage({ + token: params.driverToken, + initialOffset: offset, + timeoutMs: 30_000, + observedMessages: params.observedMessages, + predicate: (message) => + message.chatId === Number(params.groupId) && + message.senderId === params.sutBotId && + message.replyToMessageId === driverMessage.message_id && + message.text.trim().length > 0, + }); +} + +function canaryFailureMessage(error: unknown) { + const details = formatErrorMessage(error); + return [ + "Telegram QA canary failed.", + details, + "Remediation:", + "1. Enable Bot-to-Bot Communication Mode for both the driver and SUT bots in @BotFather.", + "2. Ensure the driver bot can observe bot traffic in the private group by making it admin or disabling privacy mode, then re-add it.", + "3. Ensure both bots are members of the same private group.", + "4. Confirm the SUT bot is allowed to receive /help@BotUsername commands in that group.", + ].join("\n"); +} + +export async function runTelegramQaLive(params: { + repoRoot?: string; + outputDir?: string; + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + sutAccountId?: string; +}): Promise { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const outputDir = + params.outputDir ?? + path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${Date.now().toString(36)}`); + await fs.mkdir(outputDir, { recursive: true }); + + const runtimeEnv = resolveTelegramQaRuntimeEnv(); + const providerMode = normalizeQaProviderMode(params.providerMode ?? "mock-openai"); + const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const sutAccountId = params.sutAccountId?.trim() || "sut"; + const scenarios = findScenario(params.scenarioIds); + const observedMessages: TelegramObservedMessage[] = []; + const startedAt = new Date().toISOString(); + + const driverIdentity = await getBotIdentity(runtimeEnv.driverToken); + const sutIdentity = await getBotIdentity(runtimeEnv.sutToken); + const sutUsername = sutIdentity.username?.trim(); + const uniqueIds = new Set([driverIdentity.id, sutIdentity.id]); + if (uniqueIds.size !== 2) { + throw new Error("Telegram QA requires two distinct bots for driver and SUT."); + } + if (!sutUsername) { + throw new Error("Telegram QA requires the SUT bot to have a Telegram username."); + } + + await Promise.all([ + flushTelegramUpdates(runtimeEnv.driverToken), + flushTelegramUpdates(runtimeEnv.sutToken), + ]); + + const gateway = await startQaGatewayChild({ + repoRoot, + qaBusBaseUrl: "http://127.0.0.1:43123", + providerMode, + primaryModel, + alternateModel, + fastMode: params.fastMode, + controlUiEnabled: false, + mutateConfig: (cfg) => + buildTelegramQaConfig(cfg, { + groupId: runtimeEnv.groupId, + sutToken: runtimeEnv.sutToken, + driverBotId: driverIdentity.id, + sutAccountId, + }), + }); + + const scenarioResults: TelegramQaScenarioResult[] = []; + try { + await waitForTelegramChannelRunning(gateway, sutAccountId); + try { + await runCanary({ + driverToken: runtimeEnv.driverToken, + groupId: runtimeEnv.groupId, + sutUsername, + driverBotId: driverIdentity.id, + sutBotId: sutIdentity.id, + observedMessages, + }); + } catch (error) { + throw new Error(canaryFailureMessage(error), { cause: error }); + } + + let driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); + for (const scenario of scenarios) { + try { + const sent = await sendGroupMessage( + runtimeEnv.driverToken, + runtimeEnv.groupId, + scenario.buildInput(sutUsername), + ); + const matched = await waitForObservedMessage({ + token: runtimeEnv.driverToken, + initialOffset: driverOffset, + timeoutMs: scenario.timeoutMs, + observedMessages, + predicate: (message) => + message.chatId === Number(runtimeEnv.groupId) && + message.senderId === sutIdentity.id && + message.replyToMessageId === sent.message_id && + message.text.trim().length > 0, + }); + driverOffset = matched.nextOffset; + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: `reply message ${matched.message.messageId} matched`, + }); + } catch (error) { + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "fail", + details: formatErrorMessage(error), + }); + } + } + } finally { + await gateway.stop(); + } + + const finishedAt = new Date().toISOString(); + const summary: TelegramQaSummary = { + groupId: runtimeEnv.groupId, + startedAt, + finishedAt, + counts: { + total: scenarioResults.length, + passed: scenarioResults.filter((entry) => entry.status === "pass").length, + failed: scenarioResults.filter((entry) => entry.status === "fail").length, + }, + scenarios: scenarioResults, + }; + const reportPath = path.join(outputDir, "telegram-qa-report.md"); + const summaryPath = path.join(outputDir, "telegram-qa-summary.json"); + const observedMessagesPath = path.join(outputDir, "telegram-qa-observed-messages.json"); + await fs.writeFile( + reportPath, + `${renderTelegramQaMarkdown({ + groupId: runtimeEnv.groupId, + startedAt, + finishedAt, + scenarios: scenarioResults, + })}\n`, + "utf8", + ); + await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + await fs.writeFile( + observedMessagesPath, + `${JSON.stringify(observedMessages, null, 2)}\n`, + "utf8", + ); + + return { + outputDir, + reportPath, + summaryPath, + observedMessagesPath, + scenarios: scenarioResults, + }; +} + +export const __testing = { + TELEGRAM_QA_SCENARIOS, + buildTelegramQaConfig, + normalizeTelegramObservedMessage, + resolveTelegramQaRuntimeEnv, +};