diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index ade18fa4cf7..3f3302b8186 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -66,6 +66,51 @@ describe("telegram live qa runtime", () => { ).toThrow("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id."); }); + it("parses Telegram live progress env booleans", () => { + expect(__testing.parseTelegramQaProgressBooleanEnv("true")).toBe(true); + expect(__testing.parseTelegramQaProgressBooleanEnv("on")).toBe(true); + expect(__testing.parseTelegramQaProgressBooleanEnv("false")).toBe(false); + expect(__testing.parseTelegramQaProgressBooleanEnv("off")).toBe(false); + expect(__testing.parseTelegramQaProgressBooleanEnv("maybe")).toBeUndefined(); + }); + + it("defaults Telegram live progress logging from CI when no override is set", () => { + expect(__testing.shouldLogTelegramQaLiveProgress({ CI: "true" })).toBe(true); + expect(__testing.shouldLogTelegramQaLiveProgress({ CI: "false" })).toBe(false); + }); + + it("applies OPENCLAW_QA_SUITE_PROGRESS override to Telegram live logging", () => { + expect( + __testing.shouldLogTelegramQaLiveProgress({ + CI: "false", + OPENCLAW_QA_SUITE_PROGRESS: "true", + }), + ).toBe(true); + expect( + __testing.shouldLogTelegramQaLiveProgress({ + CI: "true", + OPENCLAW_QA_SUITE_PROGRESS: "false", + }), + ).toBe(false); + expect( + __testing.shouldLogTelegramQaLiveProgress({ + CI: "true", + OPENCLAW_QA_SUITE_PROGRESS: "definitely", + }), + ).toBe(true); + }); + + it("sanitizes and truncates Telegram live progress details", () => { + expect(__testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe( + "scenario id value", + ); + expect(__testing.sanitizeTelegramQaProgressValue("\u0000\u0001")).toBe(""); + const details = __testing.formatTelegramQaProgressDetails(`header\n${"x".repeat(500)}`); + expect(details.startsWith("header ")).toBe(true); + expect(details.length).toBeLessThanOrEqual(240); + expect(details.endsWith("...")).toBe(true); + }); + it("parses Telegram pooled credential payloads", () => { expect( __testing.parseTelegramQaCredentialPayload({ diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts index 2f7b53b4240..6611b9582f4 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts @@ -298,6 +298,9 @@ const TELEGRAM_QA_ENV_KEYS = [ ] as const; const TELEGRAM_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT"; const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; +const QA_SUITE_PROGRESS_ENV = "OPENCLAW_QA_SUITE_PROGRESS"; +const TELEGRAM_QA_PROGRESS_DETAIL_LIMIT = 240; +const TELEGRAM_QA_PROGRESS_PREFIX = "[qa-telegram-live]"; const telegramQaCredentialPayloadSchema = z.object({ groupId: z.string().trim().min(1), @@ -318,6 +321,57 @@ function isTruthyOptIn(value: string | undefined) { return normalized === "1" || normalized === "true" || normalized === "yes"; } +function parseTelegramQaProgressBooleanEnv(value: string | undefined): boolean | undefined { + const normalized = value?.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + if (normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on") { + return true; + } + if (normalized === "0" || normalized === "false" || normalized === "no" || normalized === "off") { + return false; + } + return undefined; +} + +function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) { + const override = parseTelegramQaProgressBooleanEnv(env[QA_SUITE_PROGRESS_ENV]); + if (override !== undefined) { + return override; + } + return parseTelegramQaProgressBooleanEnv(env.CI) === true; +} + +function writeTelegramQaProgress(enabled: boolean, message: string) { + if (!enabled) { + return; + } + process.stderr.write(`${TELEGRAM_QA_PROGRESS_PREFIX} ${message}\n`); +} + +function sanitizeTelegramQaProgressValue(value: string): string { + let normalized = ""; + for (const char of value) { + const code = char.codePointAt(0); + if (code === undefined) { + continue; + } + const isControl = code <= 0x1f || (code >= 0x7f && code <= 0x9f); + normalized += isControl ? " " : char; + } + normalized = normalized.replace(/\s+/gu, " ").trim(); + return normalized.length > 0 ? normalized : ""; +} + +function formatTelegramQaProgressDetails(details: string): string { + const sanitized = sanitizeTelegramQaProgressValue(details); + if (sanitized.length <= TELEGRAM_QA_PROGRESS_DETAIL_LIMIT) { + return sanitized; + } + return `${sanitized.slice(0, TELEGRAM_QA_PROGRESS_DETAIL_LIMIT - 3).trimEnd()}...`; +} + export function resolveTelegramQaRuntimeEnv( env: NodeJS.ProcessEnv = process.env, ): TelegramQaRuntimeEnv { @@ -908,6 +962,19 @@ export async function runTelegramQaLive(params: { path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${Date.now().toString(36)}`); await fs.mkdir(outputDir, { recursive: true }); + const providerMode = normalizeQaProviderMode( + params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, + ); + const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const sutAccountId = params.sutAccountId?.trim() || "sut"; + const scenarios = findScenario(params.scenarioIds); + const progressEnabled = shouldLogTelegramQaLiveProgress(); + writeTelegramQaProgress( + progressEnabled, + `run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"}`, + ); + const credentialLease = await acquireQaCredentialLease({ kind: "telegram", source: params.credentialSource, @@ -919,18 +986,19 @@ export async function runTelegramQaLive(params: { const assertLeaseHealthy = () => { leaseHeartbeat.throwIfFailed(); }; + writeTelegramQaProgress( + progressEnabled, + `credentials ready: source=${credentialLease.source} role=${credentialLease.role ?? ""}`, + ); const runtimeEnv = credentialLease.payload; - const providerMode = normalizeQaProviderMode( - params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, - ); - const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); - const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); - const sutAccountId = params.sutAccountId?.trim() || "sut"; - const scenarios = findScenario(params.scenarioIds); const observedMessages: TelegramObservedMessage[] = []; const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); const includeObservedMessageContent = isTruthyOptIn(process.env[TELEGRAM_QA_CAPTURE_CONTENT_ENV]); + writeTelegramQaProgress( + progressEnabled, + `runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"} captureContent=${includeObservedMessageContent ? "on" : "off"}`, + ); const startedAt = new Date().toISOString(); const scenarioResults: TelegramQaScenarioResult[] = []; const cleanupIssues: string[] = []; @@ -976,6 +1044,7 @@ export async function runTelegramQaLive(params: { await waitForTelegramChannelRunning(gatewayHarness.gateway, sutAccountId); assertLeaseHealthy(); try { + writeTelegramQaProgress(progressEnabled, "canary start"); await runCanary({ driverToken: runtimeEnv.driverToken, groupId: runtimeEnv.groupId, @@ -983,6 +1052,7 @@ export async function runTelegramQaLive(params: { sutBotId: sutIdentity.id, observedMessages, }); + writeTelegramQaProgress(progressEnabled, "canary pass"); } catch (error) { canaryFailure = canaryFailureMessage({ error, @@ -999,11 +1069,21 @@ export async function runTelegramQaLive(params: { status: "fail", details: canaryFailure, }); + writeTelegramQaProgress( + progressEnabled, + `canary fail: details=${formatTelegramQaProgressDetails(canaryFailure)}`, + ); } assertLeaseHealthy(); if (!canaryFailure) { let driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); - for (const scenario of scenarios) { + for (const [scenarioIndex, scenario] of scenarios.entries()) { + const scenarioIndexLabel = `${scenarioIndex + 1}/${scenarios.length}`; + const scenarioIdForLog = sanitizeTelegramQaProgressValue(scenario.id); + writeTelegramQaProgress( + progressEnabled, + `scenario start (${scenarioIndexLabel}): ${scenarioIdForLog}`, + ); assertLeaseHealthy(); const scenarioRun = scenario.buildRun(sutUsername); try { @@ -1036,35 +1116,50 @@ export async function runTelegramQaLive(params: { expectedTextIncludes: scenarioRun.expectedTextIncludes, message: matched.message, }); - scenarioResults.push({ + const result = { id: scenario.id, title: scenario.title, status: "pass", details: redactPublicMetadata ? "reply matched" : `reply message ${matched.message.messageId} matched`, - }); + } satisfies TelegramQaScenarioResult; + scenarioResults.push(result); + writeTelegramQaProgress( + progressEnabled, + `scenario pass (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`, + ); } catch (error) { if (!scenarioRun.expectReply) { const details = formatErrorMessage(error); if ( details === `timed out after ${scenario.timeoutMs}ms waiting for Telegram message` ) { - scenarioResults.push({ + const result = { id: scenario.id, title: scenario.title, status: "pass", details: "no reply", - }); + } satisfies TelegramQaScenarioResult; + scenarioResults.push(result); + writeTelegramQaProgress( + progressEnabled, + `scenario pass (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`, + ); continue; } } - scenarioResults.push({ + const result = { id: scenario.id, title: scenario.title, status: "fail", details: formatErrorMessage(error), - }); + } satisfies TelegramQaScenarioResult; + scenarioResults.push(result); + writeTelegramQaProgress( + progressEnabled, + `scenario fail (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`, + ); } assertLeaseHealthy(); } @@ -1089,6 +1184,15 @@ export async function runTelegramQaLive(params: { const publishedCleanupIssues = redactPublicMetadata ? cleanupIssues.map(() => "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)") : cleanupIssues; + const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length; + const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length; + writeTelegramQaProgress( + progressEnabled, + `run complete: passed=${passedCount} failed=${failedCount} total=${scenarioResults.length}`, + ); + if (cleanupIssues.length > 0) { + writeTelegramQaProgress(progressEnabled, `cleanup issues: count=${cleanupIssues.length}`); + } const summary: TelegramQaSummary = { credentials: { source: credentialLease.source, @@ -1103,8 +1207,8 @@ export async function runTelegramQaLive(params: { cleanupIssues: publishedCleanupIssues, counts: { total: scenarioResults.length, - passed: scenarioResults.filter((entry) => entry.status === "pass").length, - failed: scenarioResults.filter((entry) => entry.status === "fail").length, + passed: passedCount, + failed: failedCount, }, scenarios: scenarioResults, }; @@ -1185,6 +1289,10 @@ export const __testing = { findScenario, matchesTelegramScenarioReply, normalizeTelegramObservedMessage, + parseTelegramQaProgressBooleanEnv, parseTelegramQaCredentialPayload, resolveTelegramQaRuntimeEnv, + sanitizeTelegramQaProgressValue, + shouldLogTelegramQaLiveProgress, + formatTelegramQaProgressDetails, };