From 5d8dceb37f185a401880f885ad28c32077a5fc53 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Fri, 17 Apr 2026 19:16:58 -0400 Subject: [PATCH] QA Matrix: add catchup incremental scenario --- .../qa-matrix/src/runners/contract/runtime.ts | 17 +++ .../src/runners/contract/scenario-catalog.ts | 7 ++ .../contract/scenario-runtime-restart.ts | 77 +++++++++++++ .../contract/scenario-runtime-shared.ts | 1 + .../src/runners/contract/scenario-runtime.ts | 3 + .../src/runners/contract/scenario-types.ts | 6 ++ .../src/runners/contract/scenarios.test.ts | 102 ++++++++++++++++++ 7 files changed, 213 insertions(+) diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index da6da2db7c5..6deb35198a5 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -647,6 +647,23 @@ export async function runMatrixQaLive(params: { `gateway restart done ${scenario.id} ${formatMatrixQaDurationMs(measuredRestart.durationMs)}`, ); }, + restartGatewayWithQueuedMessage: async (queueMessage) => { + if (!gatewayHarness) { + throw new Error("Matrix restart catchup scenario requires a live gateway"); + } + writeMatrixQaProgress(`gateway restart+queue start ${scenario.id}`); + const measuredRestart = await measureMatrixQaStep(async () => { + await scenarioGateway.harness.gateway.restart(); + await sleep(250); + await queueMessage(); + await waitForMatrixChannelReady(scenarioGateway.harness.gateway, sutAccountId); + }); + gatewayRestartMs += measuredRestart.durationMs; + scenarioRestartGatewayMs += measuredRestart.durationMs; + writeMatrixQaProgress( + `gateway restart+queue done ${scenario.id} ${formatMatrixQaDurationMs(measuredRestart.durationMs)}`, + ); + }, roomId: provisioning.roomId, sutAccessToken: provisioning.sut.accessToken, sutDeviceId: provisioning.sut.deviceId, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index b352c2dd7bf..8e864032a29 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -40,6 +40,7 @@ export type MatrixQaScenarioId = | "matrix-reaction-redaction-observed" | "matrix-restart-resume" | "matrix-post-restart-room-continue" + | "matrix-initial-catchup-then-incremental" | "matrix-room-membership-loss" | "matrix-homeserver-restart-resume" | "matrix-mention-gating" @@ -424,6 +425,12 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ title: "Matrix restarted room continues after the first recovered reply", topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY, }, + { + id: "matrix-initial-catchup-then-incremental", + timeoutMs: 90_000, + title: "Matrix initial catchup is followed by incremental replies", + topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY, + }, { id: "matrix-room-membership-loss", timeoutMs: 75_000, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts index 1679b04e190..c2f92c3a3d9 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts @@ -5,6 +5,13 @@ import { } from "./scenario-catalog.js"; import { buildMatrixReplyDetails, + buildMatrixQaToken, + buildMentionPrompt, + buildMatrixReplyArtifact, + isMatrixQaExactMarkerReply, + assertTopLevelReplyArtifact, + advanceMatrixQaActorCursor, + primeMatrixQaDriverScenarioClient, runAssertedDriverTopLevelScenario, type MatrixQaScenarioContext, } from "./scenario-runtime-shared.js"; @@ -107,3 +114,73 @@ export async function runPostRestartRoomContinueScenario(context: MatrixQaScenar ].join("\n"), } satisfies MatrixQaScenarioExecution; } + +export async function runInitialCatchupThenIncrementalScenario(context: MatrixQaScenarioContext) { + if (!context.restartGatewayWithQueuedMessage) { + throw new Error( + "Matrix initial catchup scenario requires a queued-message gateway restart callback", + ); + } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const catchupToken = buildMatrixQaToken("MATRIX_QA_CATCHUP"); + const catchupBody = buildMentionPrompt(context.sutUserId, catchupToken); + let catchupDriverEventId = ""; + + await context.restartGatewayWithQueuedMessage(async () => { + catchupDriverEventId = await client.sendTextMessage({ + body: catchupBody, + mentionUserIds: [context.sutUserId], + roomId, + }); + }); + + const catchupMatched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token: catchupToken, + }) && event.relatesTo === undefined, + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: catchupMatched.since, + startSince, + }); + const catchupReply = buildMatrixReplyArtifact(catchupMatched.event, catchupToken); + assertTopLevelReplyArtifact("catchup reply", catchupReply); + + const incremental = await runAssertedDriverTopLevelScenario({ + context, + label: "incremental reply after catchup", + roomId, + tokenPrefix: "MATRIX_QA_INCREMENTAL", + }); + + return { + artifacts: { + catchupDriverEventId, + catchupReply, + catchupToken, + incrementalDriverEventId: incremental.driverEventId, + incrementalReply: incremental.reply, + incrementalToken: incremental.token, + restartSignal: "SIGUSR1", + roomId, + }, + details: [ + `room id: ${roomId}`, + "restart signal: SIGUSR1", + `catchup driver event: ${catchupDriverEventId}`, + ...buildMatrixReplyDetails("catchup reply", catchupReply), + `incremental driver event: ${incremental.driverEventId}`, + ...buildMatrixReplyDetails("incremental reply", incremental.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index 7847381c008..f6a216e5973 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -29,6 +29,7 @@ export type MatrixQaScenarioContext = { observerUserId: string; outputDir?: string; restartGateway?: () => Promise; + restartGatewayWithQueuedMessage?: (queueMessage: () => Promise) => Promise; roomId: string; interruptTransport?: () => Promise; sutAccessToken: string; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index 10bea7d9e36..91c869828dc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -41,6 +41,7 @@ import { } from "./scenario-runtime-reaction.js"; import { runHomeserverRestartResumeScenario, + runInitialCatchupThenIncrementalScenario, runPostRestartRoomContinueScenario, runRestartResumeScenario, } from "./scenario-runtime-restart.js"; @@ -229,6 +230,8 @@ export async function runMatrixQaScenario( return await runRestartResumeScenario(context); case "matrix-post-restart-room-continue": return await runPostRestartRoomContinueScenario(context); + case "matrix-initial-catchup-then-incremental": + return await runInitialCatchupThenIncrementalScenario(context); case "matrix-room-membership-loss": return await runMembershipLossScenario(context); case "matrix-homeserver-restart-resume": diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index d2b5958ee62..03c859ee918 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -32,6 +32,9 @@ export type MatrixQaScenarioArtifacts = { attachmentMsgtype?: string; actorUserId?: string; blocked?: MatrixQaScenarioArtifacts; + catchupDriverEventId?: string; + catchupReply?: MatrixQaReplyArtifact; + catchupToken?: string; driverEventId?: string; editEventId?: string; editedToken?: string; @@ -39,6 +42,9 @@ export type MatrixQaScenarioArtifacts = { firstDriverEventId?: string; firstReply?: MatrixQaReplyArtifact; firstToken?: string; + incrementalDriverEventId?: string; + incrementalReply?: MatrixQaReplyArtifact; + incrementalToken?: string; originalDriverEventId?: string; originalReply?: MatrixQaReplyArtifact; originalToken?: string; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index d14b31cb031..21ff8a7c89c 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -102,6 +102,7 @@ describe("matrix live qa scenarios", () => { "matrix-reaction-redaction-observed", "matrix-restart-resume", "matrix-post-restart-room-continue", + "matrix-initial-catchup-then-incremental", "matrix-room-membership-loss", "matrix-homeserver-restart-resume", "matrix-mention-gating", @@ -515,6 +516,107 @@ describe("matrix live qa scenarios", () => { }); }); + it("queues a Matrix trigger during restart before proving incremental sync continues", async () => { + const callOrder: string[] = []; + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockImplementation(async (params) => { + callOrder.push(`send:${String(params.body).includes("CATCHUP") ? "catchup" : "incremental"}`); + return String(params.body).includes("CATCHUP") ? "$catchup-trigger" : "$incremental-trigger"; + }); + const waitForRoomEvent = vi.fn().mockImplementation(async () => { + const sentBody = String(sendTextMessage.mock.calls.at(-1)?.[0]?.body ?? ""); + const token = sentBody.replace("@sut:matrix-qa.test reply with only this exact marker: ", ""); + callOrder.push(`wait:${token.includes("CATCHUP") ? "catchup" : "incremental"}`); + return { + event: { + kind: "message", + roomId: "!restart:matrix-qa.test", + eventId: token.includes("CATCHUP") ? "$catchup-reply" : "$incremental-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: token, + }, + since: token.includes("CATCHUP") + ? "driver-sync-after-catchup" + : "driver-sync-after-incremental", + }; + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-initial-catchup-then-incremental", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + restartGatewayWithQueuedMessage: async (queueMessage) => { + callOrder.push("restart"); + await queueMessage(); + callOrder.push("ready"); + }, + roomId: "!room:matrix-qa.test", + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "restart", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Restart room", + requireMention: true, + roomId: "!restart:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + catchupDriverEventId: "$catchup-trigger", + catchupReply: { + eventId: "$catchup-reply", + tokenMatched: true, + }, + incrementalDriverEventId: "$incremental-trigger", + incrementalReply: { + eventId: "$incremental-reply", + tokenMatched: true, + }, + }, + }); + + expect(callOrder).toEqual([ + "restart", + "send:catchup", + "ready", + "wait:catchup", + "send:incremental", + "wait:incremental", + ]); + }); + it("runs the DM scenario against the provisioned DM room without a mention", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$dm-trigger");