From 64533ed7b1e757d061a4d9bade3444595d3e00e4 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 04:12:50 +0100 Subject: [PATCH] ci(release): allow slower qa live canaries --- .github/workflows/openclaw-release-checks.yml | 2 ++ .../telegram/telegram-live.runtime.test.ts | 14 ++++++++ .../telegram/telegram-live.runtime.ts | 33 +++++++++++++++++-- .../src/runners/contract/runtime.test.ts | 18 ++++++++++ .../qa-matrix/src/runners/contract/runtime.ts | 11 ++++++- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 968b0ca8fcc..dd517450e3f 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -669,6 +669,7 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS: "90000" OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000" run: | set -euo pipefail @@ -761,6 +762,7 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" + OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "90000" run: | set -euo pipefail diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index f4a9389aa44..73a72405671 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -100,6 +100,20 @@ describe("telegram live qa runtime", () => { ).toBe(true); }); + it("normalizes the Telegram QA canary timeout env", () => { + expect(__testing.resolveTelegramQaCanaryTimeoutMs({})).toBe(30_000); + expect( + __testing.resolveTelegramQaCanaryTimeoutMs({ + OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "90000", + }), + ).toBe(90_000); + expect( + __testing.resolveTelegramQaCanaryTimeoutMs({ + OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "nope", + }), + ).toBe(30_000); + }); + it("sanitizes and truncates Telegram live progress details", () => { expect(__testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe( "scenario id value", diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts index d444f486911..8ff5843ab6d 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts @@ -99,6 +99,8 @@ type TelegramObservedMessageArtifact = { mediaKinds: string[]; }; +const DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS = 30_000; + type TelegramQaScenarioResult = { id: string; title: string; @@ -350,6 +352,30 @@ function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) { return parseTelegramQaProgressBooleanEnv(env.CI) === true; } +function parsePositiveTelegramQaEnvMs(env: NodeJS.ProcessEnv, name: string, fallbackMs: number) { + const raw = env[name]; + if (raw === undefined) { + return fallbackMs; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 1) { + return fallbackMs; + } + return Math.floor(parsed); +} + +function resolveTelegramQaCanaryTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + return parsePositiveTelegramQaEnvMs( + env, + "OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", + DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS, + ); +} + +function formatTelegramQaTimeoutSeconds(timeoutMs: number) { + return `${Math.round(timeoutMs / 1_000)}s`; +} + function writeTelegramQaProgress(enabled: boolean, message: string) { if (!enabled) { return; @@ -862,6 +888,7 @@ async function runCanary(params: { groupId: string; sutUsername: string; sutBotId: number; + timeoutMs: number; observedMessages: TelegramObservedMessage[]; }) { const offset = await flushTelegramUpdates(params.driverToken); @@ -880,7 +907,7 @@ async function runCanary(params: { sutObserved = await waitForObservedMessage({ token: params.driverToken, initialOffset: offset, - timeoutMs: 30_000, + timeoutMs: params.timeoutMs, observedMessages: params.observedMessages, observationScenarioId: "telegram-canary", observationScenarioTitle: "Telegram canary", @@ -921,7 +948,7 @@ async function runCanary(params: { } throw new TelegramQaCanaryError( "sut_reply_timeout", - "SUT bot did not send any group reply after the canary command within 30s.", + `SUT bot did not send any group reply after the canary command within ${formatTelegramQaTimeoutSeconds(params.timeoutMs)}.`, { groupId: params.groupId, sutBotId: params.sutBotId, @@ -1208,6 +1235,7 @@ export async function runTelegramQaLive(params: { groupId: runtimeEnv.groupId, sutUsername, sutBotId: sutIdentity.id, + timeoutMs: resolveTelegramQaCanaryTimeoutMs(), observedMessages, }); scenarioResults.push({ @@ -1481,6 +1509,7 @@ export const __testing = { normalizeTelegramObservedMessage, parseTelegramQaProgressBooleanEnv, parseTelegramQaCredentialPayload, + resolveTelegramQaCanaryTimeoutMs, resolveTelegramQaRuntimeEnv, sanitizeTelegramQaProgressValue, shouldLogTelegramQaLiveProgress, diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 93763a769d9..8385edb25bd 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -106,6 +106,24 @@ describe("matrix live qa runtime", () => { } }); + it("normalizes the Matrix QA canary timeout env", () => { + const previous = process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS; + try { + delete process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS; + expect(liveTesting.resolveMatrixQaCanaryTimeoutMs()).toBe(45_000); + process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS = "90000"; + expect(liveTesting.resolveMatrixQaCanaryTimeoutMs()).toBe(90_000); + process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS = "nope"; + expect(liveTesting.resolveMatrixQaCanaryTimeoutMs()).toBe(45_000); + } finally { + if (previous === undefined) { + delete process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS; + } else { + process.env.OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS = previous; + } + } + }); + it("injects a temporary Matrix account into the QA gateway config", () => { const baseCfg: OpenClawConfig = { plugins: { diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index f042d9cfe5e..2d500124348 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -51,6 +51,7 @@ type MatrixQaGatewayChild = { const DEFAULT_MATRIX_QA_RUN_TIMEOUT_MS = 30 * 60_000; const DEFAULT_MATRIX_QA_CLEANUP_TIMEOUT_MS = 90_000; +const DEFAULT_MATRIX_QA_CANARY_TIMEOUT_MS = 45_000; type MatrixQaLiveLaneGatewayHarness = { gateway: MatrixQaGatewayChild; @@ -192,6 +193,13 @@ function createMatrixQaRunDeadline() { }; } +function resolveMatrixQaCanaryTimeoutMs() { + return parsePositiveMatrixQaEnvMs( + "OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS", + DEFAULT_MATRIX_QA_CANARY_TIMEOUT_MS, + ); +} + function remainingMatrixQaRunMs(deadline: { deadlineMs: number }) { return Math.max(1, deadline.deadlineMs - Date.now()); } @@ -720,7 +728,7 @@ export async function runMatrixQaLive(params: { syncState, syncStreams, sutUserId: provisioning.sut.userId, - timeoutMs: 45_000, + timeoutMs: resolveMatrixQaCanaryTimeoutMs(), }), ), ); @@ -1128,6 +1136,7 @@ export const __testing = { findMatrixQaScenarios, isMatrixAccountReady, patchMatrixQaGatewayConfig, + resolveMatrixQaCanaryTimeoutMs, resolveMatrixQaModels, shouldWriteMatrixQaProgress, summarizeMatrixQaConfigSnapshot,