From 7fc5a18d8913c4e7570a3403e5060aabad3f091e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 15 Apr 2026 06:09:18 +0100 Subject: [PATCH] test(qa-matrix): isolate flaky beta scenarios --- .../src/runners/contract/scenario-catalog.ts | 36 +++++++++ .../runners/contract/scenario-runtime-room.ts | 78 +++++++++++++------ .../src/runners/contract/scenario-types.ts | 1 + .../src/runners/contract/scenarios.test.ts | 26 +++++-- .../qa-matrix/src/substrate/config.test.ts | 24 ++++++ extensions/qa-matrix/src/substrate/config.ts | 25 ++++++ 6 files changed, 162 insertions(+), 28 deletions(-) diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index 133a6deb1ff..b3af1667fef 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -39,9 +39,12 @@ export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition + event.roomId === context.roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.relatesTo === undefined && + typeof event.body === "string" && + event.body.trim().length > 0, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "observer", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + const reply = buildMatrixReplyArtifact(matched.event, token); return { artifacts: { actorUserId: context.observerUserId, - driverEventId: result.driverEventId, - reply: result.reply, - token: result.token, - triggerBody: result.body, + driverEventId, + reply, + token, + triggerBody: body, }, details: [ `trigger sender: ${context.observerUserId}`, - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), + `driver event: ${driverEventId}`, + ...buildMatrixReplyDetails("reply", reply), ].join("\n"), } satisfies MatrixQaScenarioExecution; } @@ -312,6 +336,7 @@ export async function runQuietStreamingPreviewScenario(context: MatrixQaScenario } export async function runBlockStreamingScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_BLOCK_ROOM_KEY); const { client, startSince } = await primeMatrixQaActorCursor({ accessToken: context.driverAccessToken, actorId: "driver", @@ -324,27 +349,28 @@ export async function runBlockStreamingScenario(context: MatrixQaScenarioContext const driverEventId = await client.sendTextMessage({ body: triggerBody, mentionUserIds: [context.sutUserId], - roomId: context.roomId, + roomId, }); const firstBlock = await client.waitForRoomEvent({ observedEvents: context.observedEvents, predicate: (event) => - event.roomId === context.roomId && + event.roomId === roomId && event.sender === context.sutUserId && isMatrixQaMessageLikeKind(event.kind) && - event.body === firstText, - roomId: context.roomId, + (event.body ?? "").includes(firstText) && + !(event.body ?? "").includes(secondText), + roomId, since: startSince, timeoutMs: context.timeoutMs, }); const secondBlock = await client.waitForRoomEvent({ observedEvents: context.observedEvents, predicate: (event) => - event.roomId === context.roomId && + event.roomId === roomId && event.sender === context.sutUserId && isMatrixQaMessageLikeKind(event.kind) && - event.body === secondText, - roomId: context.roomId, + (event.body ?? "").includes(secondText), + roomId, since: firstBlock.since, timeoutMs: context.timeoutMs, }); @@ -364,10 +390,12 @@ export async function runBlockStreamingScenario(context: MatrixQaScenarioContext blockEventIds: [firstBlock.event.eventId, secondBlock.event.eventId], driverEventId, reply: buildMatrixReplyArtifact(secondBlock.event, secondText), + roomId, token: secondText, triggerBody, }, details: [ + `room id: ${roomId}`, `driver event: ${driverEventId}`, `block one event: ${firstBlock.event.eventId}`, `block two event: ${secondBlock.event.eventId}`, @@ -592,12 +620,13 @@ export async function runHomeserverRestartResumeScenario(context: MatrixQaScenar if (!context.interruptTransport) { throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_HOMESERVER_ROOM_KEY); await context.interruptTransport(); const resumed = await runDriverTopLevelMentionScenario({ baseUrl: context.baseUrl, driverAccessToken: context.driverAccessToken, observedEvents: context.observedEvents, - roomId: context.roomId, + roomId, syncState: context.syncState, sutUserId: context.sutUserId, timeoutMs: context.timeoutMs, @@ -608,10 +637,12 @@ export async function runHomeserverRestartResumeScenario(context: MatrixQaScenar artifacts: { driverEventId: resumed.driverEventId, reply: resumed.reply, + roomId, token: resumed.token, transportInterruption: "homeserver-restart", }, details: [ + `room id: ${roomId}`, "transport interruption: homeserver-restart", `driver event: ${resumed.driverEventId}`, ...buildMatrixReplyDetails("reply", resumed.reply), @@ -623,12 +654,13 @@ export async function runRestartResumeScenario(context: MatrixQaScenarioContext) if (!context.restartGateway) { throw new Error("Matrix restart scenario requires a gateway restart callback"); } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY); await context.restartGateway(); const result = await runDriverTopLevelMentionScenario({ baseUrl: context.baseUrl, driverAccessToken: context.driverAccessToken, observedEvents: context.observedEvents, - roomId: context.roomId, + roomId, syncState: context.syncState, sutUserId: context.sutUserId, timeoutMs: context.timeoutMs, @@ -640,9 +672,11 @@ export async function runRestartResumeScenario(context: MatrixQaScenarioContext) driverEventId: result.driverEventId, reply: result.reply, restartSignal: "SIGUSR1", + roomId, token: result.token, }, details: [ + `room id: ${roomId}`, "restart signal: SIGUSR1", `post-restart driver event: ${result.driverEventId}`, ...buildMatrixReplyDetails("reply", result.reply), diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index 45081a73826..81454ab95d6 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -26,6 +26,7 @@ export type MatrixQaScenarioArtifacts = { recoveredDriverEventId?: string; recoveredReply?: MatrixQaReplyArtifact; roomKey?: string; + roomId?: string; restartSignal?: string; rootEventId?: string; threadDriverEventId?: string; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 75bb3c3c271..7293addd626 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -336,10 +336,7 @@ describe("matrix live qa scenarios", () => { eventId: "$sut-reply", sender: "@sut:matrix-qa.test", type: "m.room.message", - body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( - "@sut:matrix-qa.test reply with only this exact marker: ", - "", - ), + body: "observer sender accepted", }, since: "observer-sync-next", })); @@ -380,6 +377,9 @@ describe("matrix live qa scenarios", () => { artifacts: { actorUserId: "@observer:matrix-qa.test", driverEventId: "$observer-allow-trigger", + reply: { + tokenMatched: false, + }, }, }); @@ -707,7 +707,21 @@ describe("matrix live qa scenarios", () => { topology: { defaultRoomId: "!main:matrix-qa.test", defaultRoomKey: "main", - rooms: [], + rooms: [ + { + key: "block", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Block", + requireMention: true, + roomId: "!block:matrix-qa.test", + }, + ], }, }), ).resolves.toMatchObject({ @@ -720,7 +734,7 @@ describe("matrix live qa scenarios", () => { expect(sendTextMessage).toHaveBeenCalledWith({ body: expect.stringContaining("Matrix block streaming QA check"), mentionUserIds: ["@sut:matrix-qa.test"], - roomId: "!main:matrix-qa.test", + roomId: "!block:matrix-qa.test", }); expect(waitForRoomEvent).toHaveBeenNthCalledWith( 2, diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index fe7ca8296e8..dae92ecabde 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -86,6 +86,18 @@ describe("matrix qa config", () => { overrides: { autoJoin: "allowlist", autoJoinAllowlist: [" !dm:matrix-qa.test ", "#ops:matrix-qa.test"], + agentDefaults: { + blockStreamingChunk: { + breakPreference: "newline", + maxChars: 48, + minChars: 1, + }, + blockStreamingCoalesce: { + idleMs: 0, + maxChars: 48, + minChars: 1, + }, + }, blockStreaming: true, dm: { sessionScope: "per-room", @@ -108,6 +120,18 @@ describe("matrix qa config", () => { topology, }); + expect(next.agents?.defaults).toMatchObject({ + blockStreamingChunk: { + breakPreference: "newline", + maxChars: 48, + minChars: 1, + }, + blockStreamingCoalesce: { + idleMs: 0, + maxChars: 48, + minChars: 1, + }, + }); expect(next.channels?.matrix?.accounts?.sut).toMatchObject({ autoJoin: "allowlist", autoJoinAllowlist: ["!dm:matrix-qa.test", "#ops:matrix-qa.test"], diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index c712341b9e2..1dc7be5125f 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -9,6 +9,19 @@ export type MatrixQaAutoJoinMode = "allowlist" | "always" | "off"; export type MatrixQaStreamingMode = "off" | "partial" | "quiet"; export type MatrixQaActorRole = "driver" | "observer" | "sut"; +export type MatrixQaAgentDefaultsOverrides = { + blockStreamingChunk?: { + breakPreference?: "newline" | "paragraph" | "sentence"; + maxChars?: number; + minChars?: number; + }; + blockStreamingCoalesce?: { + idleMs?: number; + maxChars?: number; + minChars?: number; + }; +}; + export type MatrixQaGroupConfigOverrides = { enabled?: boolean; requireMention?: boolean; @@ -23,6 +36,7 @@ export type MatrixQaDmConfigOverrides = { }; export type MatrixQaConfigOverrides = { + agentDefaults?: MatrixQaAgentDefaultsOverrides; autoJoin?: MatrixQaAutoJoinMode; autoJoinAllowlist?: string[]; blockStreaming?: boolean; @@ -352,6 +366,17 @@ export function buildMatrixQaConfig( return { ...baseCfg, + ...(params.overrides?.agentDefaults + ? { + agents: { + ...baseCfg.agents, + defaults: { + ...baseCfg.agents?.defaults, + ...params.overrides.agentDefaults, + }, + }, + } + : {}), plugins: { ...baseCfg.plugins, allow: pluginAllow,