From 7bf4dfeff33e8e1661e0462d6cd4deead0bc0efa Mon Sep 17 00:00:00 2001
From: Peter Steinberger <steipete@gmail.com>
Date: Sun, 17 May 2026 12:08:32 +0100
Subject: [PATCH] test: harden live QA transport probes

---
 .../whatsapp/whatsapp-live.runtime.test.ts    |  5 ++
 .../whatsapp/whatsapp-live.runtime.ts         | 77 ++++++++++++-------
 .../runners/contract/scenario-runtime-room.ts | 14 +++-
 3 files changed, 67 insertions(+), 29 deletions(-)

diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
index 4e692175d84..2274c6b0df3 100644
--- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
+++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
@@ -192,4 +192,9 @@ describe("WhatsApp QA live runtime", () => {
       },
     ]);
   });
+
+  it("classifies WhatsApp driver connection closures as retryable", () => {
+    expect(__testing.isTransientWhatsAppQaDriverError(new Error("Connection Closed"))).toBe(true);
+    expect(__testing.isTransientWhatsAppQaDriverError(new Error("timed out waiting"))).toBe(false);
+  });
 });
diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts
index 8fa395e2d19..8d49870e58b 100644
--- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts
+++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts
@@ -470,6 +470,10 @@ function messageMatches(message: WhatsAppObservedMessage, matchText: string | Re
     : matchText.test(message.text);
 }
 
+function isTransientWhatsAppQaDriverError(error: unknown) {
+  return /\bConnection Closed\b/iu.test(formatErrorMessage(error));
+}
+
 async function runWhatsAppScenario(params: {
   driver: WhatsAppQaDriverSession;
   driverPhoneE164: string;
@@ -754,7 +758,7 @@ export async function runWhatsAppQaLive(params: {
         parentDir: tempAuthRoot,
       }),
     ]);
-    const activeDriver = await startWhatsAppQaDriverSession({ authDir: driverAuthDir });
+    let activeDriver = await startWhatsAppQaDriverSession({ authDir: driverAuthDir });
     driver = activeDriver;
 
     for (const scenario of scenarios) {
@@ -768,32 +772,50 @@ export async function runWhatsAppQaLive(params: {
         );
         continue;
       }
-      try {
-        const result = await runWhatsAppScenario({
-          driver: activeDriver,
-          driverPhoneE164: runtimeEnv.driverPhoneE164,
-          gatewayDebugDirPath,
-          observedMessages,
-          providerMode,
-          primaryModel,
-          alternateModel,
-          fastMode: params.fastMode,
-          groupJid: runtimeEnv.groupJid,
-          repoRoot,
-          scenario,
-          sutAccountId,
-          sutAuthDir,
-          sutPhoneE164: runtimeEnv.sutPhoneE164,
-        });
-        scenarioResults.push(result);
-      } catch (error) {
-        preservedGatewayDebugArtifacts = true;
-        scenarioResults.push({
-          id: scenario.id,
-          title: scenario.title,
-          status: "fail",
-          details: formatErrorMessage(error),
-        });
+      let retriedDriver = false;
+      while (true) {
+        try {
+          const result = await runWhatsAppScenario({
+            driver: activeDriver,
+            driverPhoneE164: runtimeEnv.driverPhoneE164,
+            gatewayDebugDirPath,
+            observedMessages,
+            providerMode,
+            primaryModel,
+            alternateModel,
+            fastMode: params.fastMode,
+            groupJid: runtimeEnv.groupJid,
+            repoRoot,
+            scenario,
+            sutAccountId,
+            sutAuthDir,
+            sutPhoneE164: runtimeEnv.sutPhoneE164,
+          });
+          scenarioResults.push(
+            retriedDriver
+              ? { ...result, details: `${result.details}; driver reconnected` }
+              : result,
+          );
+          break;
+        } catch (error) {
+          if (!retriedDriver && isTransientWhatsAppQaDriverError(error)) {
+            retriedDriver = true;
+            await activeDriver.close().catch(() => {});
+            activeDriver = await startWhatsAppQaDriverSession({ authDir: driverAuthDir });
+            driver = activeDriver;
+            continue;
+          }
+          preservedGatewayDebugArtifacts = true;
+          scenarioResults.push({
+            id: scenario.id,
+            title: scenario.title,
+            status: "fail",
+            details: formatErrorMessage(error),
+          });
+          break;
+        }
+      }
+      if (scenarioResults.at(-1)?.status === "fail") {
         break;
       }
     }
@@ -921,6 +943,7 @@ export const __testing = {
   buildWhatsAppQaConfig,
   createMissingGroupJidScenarioResult,
   findScenarios,
+  isTransientWhatsAppQaDriverError,
   parseWhatsAppQaCredentialPayload,
   resolveWhatsAppQaRuntimeEnv,
   resolveWhatsAppMetadataRedaction,
diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
index 0e259800414..719c21d15ea 100644
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
@@ -561,7 +561,7 @@ export async function runAllowlistHotReloadScenario(context: MatrixQaScenarioCon
 export async function runQuietStreamingPreviewScenario(context: MatrixQaScenarioContext) {
   return runMatrixStreamingPreviewScenario(context, {
     expectedPreviewKind: "notice",
-    finalText: `MATRIX_QA_QUIET_STREAM_${randomUUID().slice(0, 8).toUpperCase()} preview complete`,
+    finalText: buildMatrixStreamingPreviewFinalText("MATRIX_QA_QUIET_STREAM"),
     label: "quiet streaming",
     triggerBodyBuilder: buildMatrixQuietStreamingPrompt,
   });
@@ -570,12 +570,22 @@ export async function runQuietStreamingPreviewScenario(context: MatrixQaScenario
 export async function runPartialStreamingPreviewScenario(context: MatrixQaScenarioContext) {
   return runMatrixStreamingPreviewScenario(context, {
     expectedPreviewKind: "message",
-    finalText: `MATRIX_QA_PARTIAL_STREAM_${randomUUID().slice(0, 8).toUpperCase()} preview complete`,
+    finalText: buildMatrixStreamingPreviewFinalText("MATRIX_QA_PARTIAL_STREAM"),
     label: "partial streaming",
     triggerBodyBuilder: buildMatrixPartialStreamingPrompt,
   });
 }
 
+function buildMatrixStreamingPreviewFinalText(prefix: string) {
+  const token = `${prefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
+  return [
+    `${token} preview complete.`,
+    `${token} alpha segment confirms the draft stream started before final delivery.`,
+    `${token} beta segment keeps the exact final answer long enough for preview updates.`,
+    `${token} omega segment marks the finalized Matrix QA reply.`,
+  ].join(" ");
+}
+
 async function runMatrixStreamingPreviewScenario(
   context: MatrixQaScenarioContext,
   params: {