From 3b39ff43184da42b8a9df1d45080c522bb3d43b6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 17 May 2026 13:25:21 +0100 Subject: [PATCH] test: tolerate degraded live transport state --- .../whatsapp/whatsapp-live.runtime.test.ts | 3 ++ .../whatsapp/whatsapp-live.runtime.ts | 22 ++++++++- .../runners/contract/scenario-runtime-e2ee.ts | 45 ++++++++++++++++++- .../src/runners/contract/scenarios.test.ts | 20 +++++---- 4 files changed, 79 insertions(+), 11 deletions(-) diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index 7bcbf58d5ee..68284c07455 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -198,6 +198,9 @@ describe("WhatsApp QA live runtime", () => { expect( __testing.isTransientWhatsAppQaDriverError(new Error("status 440: session conflict")), ).toBe(true); + expect(__testing.isTransientWhatsAppQaDriverError(new Error("Stream Errored (conflict)"))).toBe( + true, + ); expect( __testing.isTransientWhatsAppQaDriverError( new Error("timed out waiting for WhatsApp QA driver message"), diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts index 47b87f1b3d5..dc404b363fc 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts @@ -144,6 +144,7 @@ const WHATSAPP_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_WHATSAPP_CAPTURE_CONTENT"; const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; const WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS = 3; const WHATSAPP_QA_READY_TIMEOUT_MS = 150_000; +const WHATSAPP_QA_DRIVER_RECONNECT_DELAY_MS = 2_000; const WHATSAPP_QA_ENV_KEYS = [ "OPENCLAW_QA_WHATSAPP_DRIVER_PHONE_E164", "OPENCLAW_QA_WHATSAPP_SUT_PHONE_E164", @@ -476,6 +477,7 @@ function isTransientWhatsAppQaDriverError(error: unknown) { const message = formatErrorMessage(error); return ( /\bConnection Closed\b/iu.test(message) || + /\bconflict\b/iu.test(message) || /\bsession conflict\b/iu.test(message) || /\btimed out waiting for WhatsApp QA driver message\b/iu.test(message) ); @@ -489,6 +491,24 @@ async function restartWhatsAppQaDriverSession(params: { return await startWhatsAppQaDriverSession({ authDir: params.authDir }); } +async function startWhatsAppQaDriverSessionWithRetry(params: { authDir: string }) { + let attempt = 1; + while (true) { + try { + return await startWhatsAppQaDriverSession({ authDir: params.authDir }); + } catch (error) { + if ( + attempt >= WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS || + !isTransientWhatsAppQaDriverError(error) + ) { + throw error; + } + attempt += 1; + await new Promise((resolve) => setTimeout(resolve, WHATSAPP_QA_DRIVER_RECONNECT_DELAY_MS)); + } + } +} + async function runWhatsAppScenario(params: { driver: WhatsAppQaDriverSession; driverPhoneE164: string; @@ -773,7 +793,7 @@ export async function runWhatsAppQaLive(params: { parentDir: tempAuthRoot, }), ]); - let activeDriver = await startWhatsAppQaDriverSession({ authDir: driverAuthDir }); + let activeDriver = await startWhatsAppQaDriverSessionWithRetry({ authDir: driverAuthDir }); driver = activeDriver; for (const scenario of scenarios) { diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts index 75935cd5d82..dd44c30da4a 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts @@ -74,6 +74,10 @@ type MatrixQaCliVerificationStatus = { }; backupVersion?: string | null; crossSigningVerified?: boolean; + encryptionEnabled?: boolean; + pendingVerifications?: number; + recoveryKeyStored?: boolean; + serverDeviceKnown?: boolean; verified?: boolean; signedByOwner?: boolean; deviceId?: string | null; @@ -1785,6 +1789,39 @@ function assertMatrixQaCliE2eeStatus( } } +function assertMatrixQaCliAccountAddBootstrapStatus(params: { + expectedBackupVersion?: string | null; + expectedUserId: string; + status: MatrixQaCliVerificationStatus; +}) { + const { expectedBackupVersion, expectedUserId, status } = params; + if (status.encryptionEnabled !== true || status.recoveryKeyStored !== true) { + throw new Error( + "Matrix CLI account add --enable-e2ee degraded status did not keep encryption and recovery key state", + ); + } + if (status.userId !== expectedUserId) { + throw new Error( + `Matrix CLI account add --enable-e2ee status user mismatch: expected ${expectedUserId}, got ${status.userId ?? ""}`, + ); + } + if (!status.deviceId || status.serverDeviceKnown !== true) { + throw new Error( + "Matrix CLI account add --enable-e2ee degraded status did not resolve the current server-known device", + ); + } + if (expectedBackupVersion && status.backupVersion !== expectedBackupVersion) { + throw new Error( + `Matrix CLI account add --enable-e2ee backup version mismatch: expected ${expectedBackupVersion}, got ${status.backupVersion ?? ""}`, + ); + } + if (status.backup?.keyLoadError) { + throw new Error( + `Matrix CLI account add --enable-e2ee degraded status reported backup key error: ${status.backup.keyLoadError}`, + ); + } +} + async function runMatrixQaCliExpectedFailure(params: { args: string[]; start: (args: string[], timeoutMs?: number) => MatrixQaCliSession; @@ -1940,7 +1977,11 @@ export async function runMatrixQaE2eeCliAccountAddEnableE2eeScenario( rootDir: cli.rootDir, }); const status = parseMatrixQaCliJson(statusResult) as MatrixQaCliVerificationStatus; - assertMatrixQaCliE2eeStatus("Matrix CLI account add --enable-e2ee", status); + assertMatrixQaCliAccountAddBootstrapStatus({ + expectedBackupVersion: added.verificationBootstrap.backupVersion, + expectedUserId: account.userId, + status, + }); const cliDeviceId = status.deviceId ?? null; return { @@ -1953,7 +1994,7 @@ export async function runMatrixQaE2eeCliAccountAddEnableE2eeScenario( verificationBootstrapSuccess: added.verificationBootstrap.success, }, details: [ - "Matrix CLI account add --enable-e2ee created an encrypted, verified account", + "Matrix CLI account add --enable-e2ee created an encrypted account and bootstrapped recovery state", `account add stdout: ${addArtifacts.stdoutPath}`, `account add stderr: ${addArtifacts.stderrPath}`, `verify status stdout: ${statusArtifacts.stdoutPath}`, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index ea8c0357aa1..ef97599bd57 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -5233,16 +5233,20 @@ describe("matrix live qa scenarios", () => { stderr: "", stdout: JSON.stringify({ backup: { - decryptionKeyCached: true, + decryptionKeyCached: null, keyLoadError: null, - matchesDecryptionKey: true, - trusted: true, + matchesDecryptionKey: null, + trusted: null, }, - crossSigningVerified: true, + backupVersion: "backup-v1", + crossSigningVerified: false, deviceId: "CLIADDDEVICE", - signedByOwner: true, - userId: "@driver:matrix-qa.test", - verified: true, + encryptionEnabled: true, + recoveryKeyStored: true, + serverDeviceKnown: true, + signedByOwner: false, + userId: "@cli-add:matrix-qa.test", + verified: false, }), }; } @@ -5321,7 +5325,7 @@ describe("matrix live qa scenarios", () => { ).resolves.toContain('"encryptionEnabled":true'); await expect( readFile(path.join(cliArtifactDir, "verify-status.stdout.txt"), "utf8"), - ).resolves.toContain('"verified":true'); + ).resolves.toContain('"recoveryKeyStored":true'); } finally { await rm(outputDir, { force: true, recursive: true }); }