diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts index 8d49870e58b..30c7eaf1fc5 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts @@ -142,6 +142,7 @@ type WhatsAppCredentialHeartbeat = ReturnType {}); + return await startWhatsAppQaDriverSession({ authDir: params.authDir }); +} + async function runWhatsAppScenario(params: { driver: WhatsAppQaDriverSession; driverPhoneE164: string; @@ -772,7 +781,7 @@ export async function runWhatsAppQaLive(params: { ); continue; } - let retriedDriver = false; + let driverAttempt = 1; while (true) { try { const result = await runWhatsAppScenario({ @@ -792,17 +801,31 @@ export async function runWhatsAppQaLive(params: { sutPhoneE164: runtimeEnv.sutPhoneE164, }); scenarioResults.push( - retriedDriver - ? { ...result, details: `${result.details}; driver reconnected` } + driverAttempt > 1 + ? { + ...result, + details: `${result.details}; driver reconnected ${driverAttempt - 1}x`, + } : result, ); break; } catch (error) { - if (!retriedDriver && isTransientWhatsAppQaDriverError(error)) { - retriedDriver = true; - await activeDriver.close().catch(() => {}); - activeDriver = await startWhatsAppQaDriverSession({ authDir: driverAuthDir }); - driver = activeDriver; + if ( + driverAttempt < WHATSAPP_QA_TRANSIENT_DRIVER_ATTEMPTS && + isTransientWhatsAppQaDriverError(error) + ) { + driverAttempt += 1; + try { + activeDriver = await restartWhatsAppQaDriverSession({ + authDir: driverAuthDir, + current: activeDriver, + }); + driver = activeDriver; + } catch (restartError) { + if (!isTransientWhatsAppQaDriverError(restartError)) { + throw restartError; + } + } continue; } preservedGatewayDebugArtifacts = true; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index 719c21d15ea..6eafb05691e 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -613,6 +613,30 @@ async function runMatrixStreamingPreviewScenario( since: startSince, timeoutMs: context.timeoutMs, }); + if (preview.event.body === params.finalText) { + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: preview.since, + startSince, + }); + const finalReply = buildMatrixReplyArtifact(preview.event, params.finalText); + return { + artifacts: { + driverEventId, + previewEventId: undefined, + reply: finalReply, + token: params.finalText, + triggerBody, + }, + details: [ + `driver event: ${driverEventId}`, + `scenario: ${params.label}`, + "preview event: ; final delivered without draft replacement", + ...buildMatrixReplyDetails("final reply", finalReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; + } const finalized = await client.waitForRoomEvent({ observedEvents: context.observedEvents, predicate: (event) => diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 85412bc0d4e..cf1e3981ce5 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -2991,6 +2991,47 @@ describe("matrix live qa scenarios", () => { }); }); + it("accepts final-only partial streaming replies without a draft replacement", async () => { + const fallbackFinalText = "MATRIX_QA_PARTIAL_STREAM_PREVIEW_COMPLETE"; + const { sendTextMessage, waitForRoomEvent } = mockMatrixQaRoomClient({ + driverEventId: "$partial-stream-trigger", + events: [ + { + event: ({ sendTextMessage }) => + matrixQaMessageEvent({ + kind: "message", + eventId: "$partial-final-only", + body: readMatrixQaReplyDirective( + mockMessageBody(sendTextMessage, "sendTextMessage"), + fallbackFinalText, + ), + }), + since: "driver-sync-final", + }, + ], + }); + + const scenario = requireMatrixQaScenario("matrix-room-partial-streaming-preview"); + + const result = await runMatrixQaScenario(scenario, matrixQaScenarioContext()); + const artifacts = result.artifacts as { + driverEventId?: unknown; + previewEventId?: unknown; + reply?: { eventId?: unknown }; + }; + expect(artifacts.driverEventId).toBe("$partial-stream-trigger"); + expect(artifacts.previewEventId).toBeUndefined(); + expect(artifacts.reply?.eventId).toBe("$partial-final-only"); + expect(result.details).toContain("final delivered without draft replacement"); + expect(waitForRoomEvent).toHaveBeenCalledTimes(1); + + expectSentTextMessage(sendTextMessage, { + bodyIncludes: "Partial streaming QA check", + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }); + }); + it("captures Matrix tool progress inside the quiet preview before finalizing", async () => { const previewEventId = "$tool-progress-preview"; const { sendTextMessage } = mockMatrixQaRoomClient({