diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 58d941e6c22..84595c2ad4d 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -21,6 +21,7 @@ describe("matrix live qa runtime", () => { const next = liveTesting.buildMatrixQaConfig(baseCfg, { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", sutAccessToken: "syt_sut", sutAccountId: "sut", sutDeviceId: "DEVICE123", @@ -84,6 +85,7 @@ describe("matrix live qa runtime", () => { { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", sutAccessToken: "syt_sut", sutAccountId: "sut", sutUserId: "@sut:matrix-qa.test", @@ -156,6 +158,7 @@ describe("matrix live qa runtime", () => { config: { default: liveTesting.buildMatrixQaConfigSnapshot({ driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", sutUserId: "@sut:matrix-qa.test", topology: { defaultRoomId: "!room:matrix-qa.test", @@ -183,6 +186,7 @@ describe("matrix live qa runtime", () => { title: "Matrix threadReplies always keeps room replies threaded", config: liveTesting.buildMatrixQaConfigSnapshot({ driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", overrides: { threadReplies: "always", }, @@ -258,6 +262,7 @@ describe("matrix live qa runtime", () => { config: { default: liveTesting.buildMatrixQaConfigSnapshot({ driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", sutUserId: "@sut:matrix-qa.test", topology: { defaultRoomId: "!room:matrix-qa.test", diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index afd0faf050c..01604162938 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -61,6 +61,8 @@ type MatrixQaScenarioResult = { title: string; }; +type MatrixQaScenarioConfigEntry = MatrixQaSummary["config"]["scenarios"][number]; + type MatrixQaSummary = { checks: QaReportCheck[]; config: { @@ -121,6 +123,61 @@ function formatMatrixQaScenarioDetails(params: { details: string; configSummary? return [`effective config: ${params.configSummary}`, params.details].join("\n"); } +function buildMatrixQaScenarioConfigEntry(params: { + gatewayConfigParams: { + driverUserId: string; + homeserver: string; + observerUserId: string; + sutAccessToken: string; + sutAccountId: string; + sutDeviceId?: string; + sutUserId: string; + topology: MatrixQaProvisionResult["topology"]; + }; + scenario: (typeof MATRIX_QA_SCENARIOS)[number]; +}): { + entry: MatrixQaScenarioConfigEntry; + summary?: string; +} { + const snapshot = buildMatrixQaConfigSnapshot({ + ...params.gatewayConfigParams, + overrides: params.scenario.configOverrides, + }); + return { + entry: { + config: snapshot, + id: params.scenario.id, + title: params.scenario.title, + }, + summary: + params.scenario.configOverrides === undefined + ? undefined + : summarizeMatrixQaConfigSnapshot(snapshot), + }; +} + +function buildMatrixQaScenarioResult(params: { + artifacts?: MatrixQaScenarioArtifacts; + configSummary?: string; + details: string; + scenario: { + id: string; + title: string; + }; + status: "fail" | "pass"; +}): MatrixQaScenarioResult { + return { + artifacts: params.artifacts, + id: params.scenario.id, + title: params.scenario.title, + status: params.status, + details: formatMatrixQaScenarioDetails({ + details: params.details, + configSummary: params.configSummary, + }), + }; +} + export type MatrixQaRunResult = { observedEventsPath: string; outputDir: string; @@ -321,6 +378,7 @@ export async function runMatrixQaLive(params: { const gatewayConfigParams = { driverUserId: provisioning.driver.userId, homeserver: harness.baseUrl, + observerUserId: provisioning.observer.userId, sutAccessToken: provisioning.sut.accessToken, sutAccountId, sutDeviceId: provisioning.sut.deviceId, @@ -328,7 +386,7 @@ export async function runMatrixQaLive(params: { topology: provisioning.topology, }; const defaultConfigSnapshot = buildMatrixQaConfigSnapshot(gatewayConfigParams); - const scenarioConfigSnapshots: MatrixQaSummary["config"]["scenarios"] = []; + const scenarioConfigSnapshots: MatrixQaScenarioConfigEntry[] = []; try { const ensureGatewayHarness = async (overrides?: MatrixQaConfigOverrides) => { @@ -403,19 +461,12 @@ export async function runMatrixQaLive(params: { if (!canaryFailed) { for (const scenario of scenarios) { - const scenarioConfigSnapshot = buildMatrixQaConfigSnapshot({ - ...gatewayConfigParams, - overrides: scenario.configOverrides, - }); - const scenarioConfigSummary = - scenario.configOverrides === undefined - ? undefined - : summarizeMatrixQaConfigSnapshot(scenarioConfigSnapshot); - scenarioConfigSnapshots.push({ - config: scenarioConfigSnapshot, - id: scenario.id, - title: scenario.title, - }); + const { entry: scenarioConfigEntry, summary: scenarioConfigSummary } = + buildMatrixQaScenarioConfigEntry({ + gatewayConfigParams, + scenario, + }); + scenarioConfigSnapshots.push(scenarioConfigEntry); try { const scenarioGateway = await ensureGatewayHarness(scenario.configOverrides); const result = await runMatrixQaScenario(scenario, { @@ -446,26 +497,24 @@ export async function runMatrixQaLive(params: { timeoutMs: scenario.timeoutMs, topology: provisioning.topology, }); - scenarioResults.push({ - artifacts: result.artifacts, - id: scenario.id, - title: scenario.title, - status: "pass", - details: formatMatrixQaScenarioDetails({ + scenarioResults.push( + buildMatrixQaScenarioResult({ + artifacts: result.artifacts, + configSummary: scenarioConfigSummary, details: result.details, - configSummary: scenarioConfigSummary, + scenario, + status: "pass", }), - }); + ); } catch (error) { - scenarioResults.push({ - id: scenario.id, - title: scenario.title, - status: "fail", - details: formatMatrixQaScenarioDetails({ - details: formatErrorMessage(error), + scenarioResults.push( + buildMatrixQaScenarioResult({ configSummary: scenarioConfigSummary, + details: formatErrorMessage(error), + scenario, + status: "fail", }), - }); + ); } } } diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index ce0330b6412..133a6deb1ff 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -17,6 +17,8 @@ export type MatrixQaScenarioId = | "matrix-thread-isolation" | "matrix-top-level-reply-shape" | "matrix-room-thread-reply-override" + | "matrix-room-quiet-streaming-preview" + | "matrix-room-block-streaming" | "matrix-dm-reply-shape" | "matrix-dm-shared-session-notice" | "matrix-dm-thread-reply-override" @@ -29,6 +31,7 @@ export type MatrixQaScenarioId = | "matrix-room-membership-loss" | "matrix-homeserver-restart-resume" | "matrix-mention-gating" + | "matrix-observer-allowlist-override" | "matrix-allowlist-block"; export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { @@ -134,6 +137,23 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ threadReplies: "always", }, }, + { + id: "matrix-room-quiet-streaming-preview", + timeoutMs: 45_000, + title: "Matrix quiet streaming emits notice previews before finalizing", + configOverrides: { + streaming: "quiet", + }, + }, + { + id: "matrix-room-block-streaming", + timeoutMs: 45_000, + title: "Matrix block streaming preserves completed quiet preview blocks", + configOverrides: { + blockStreaming: true, + streaming: "quiet", + }, + }, { id: "matrix-dm-reply-shape", timeoutMs: 45_000, @@ -226,6 +246,14 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 8_000, title: "Matrix room message without mention does not trigger", }, + { + id: "matrix-observer-allowlist-override", + timeoutMs: 45_000, + title: "Matrix sender allowlist override lets observer messages trigger replies", + configOverrides: { + groupAllowRoles: ["driver", "observer"], + }, + }, { id: "matrix-allowlist-block", standardId: "allowlist-block", diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-dm.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-dm.ts new file mode 100644 index 00000000000..42bcd2e32eb --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-dm.ts @@ -0,0 +1,215 @@ +import { randomUUID } from "node:crypto"; +import { + MATRIX_QA_DRIVER_DM_ROOM_KEY, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import { + assertThreadReplyArtifact, + assertTopLevelReplyArtifact, + buildExactMarkerPrompt, + buildMatrixNoticeArtifact, + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + createMatrixQaScenarioClient, + NO_REPLY_WINDOW_MS, + advanceMatrixQaActorCursor, + runConfigurableTopLevelScenario, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +async function runDmSharedSessionFlow(params: { + context: MatrixQaScenarioContext; + expectNotice: boolean; +}) { + const firstRoomId = resolveMatrixQaScenarioRoomId(params.context, MATRIX_QA_DRIVER_DM_ROOM_KEY); + const secondRoomId = resolveMatrixQaScenarioRoomId( + params.context, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + ); + + const firstResult = await runConfigurableTopLevelScenario({ + accessToken: params.context.driverAccessToken, + actorId: "driver", + baseUrl: params.context.baseUrl, + observedEvents: params.context.observedEvents, + roomId: firstRoomId, + syncState: params.context.syncState, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: "MATRIX_QA_DM_PRIMARY", + withMention: false, + }); + assertTopLevelReplyArtifact("primary DM reply", firstResult.reply); + + const replyClient = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const noticeClient = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const [replySince, noticeSince] = await Promise.all([ + replyClient.primeRoom(), + noticeClient.primeRoom(), + ]); + if (!replySince || !noticeSince) { + throw new Error("Matrix DM session scenario could not prime room cursors"); + } + + const secondToken = `MATRIX_QA_DM_SECONDARY_${randomUUID().slice(0, 8).toUpperCase()}`; + const secondBody = buildExactMarkerPrompt(secondToken); + const secondDriverEventId = await replyClient.sendTextMessage({ + body: secondBody, + roomId: secondRoomId, + }); + + const [replyResult, noticeResult] = await Promise.all([ + replyClient.waitForRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === secondRoomId && + event.sender === params.context.sutUserId && + event.type === "m.room.message" && + event.kind === "message" && + (event.body ?? "").includes(secondToken), + roomId: secondRoomId, + since: replySince, + timeoutMs: params.context.timeoutMs, + }), + noticeClient.waitForOptionalRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === secondRoomId && + event.sender === params.context.sutUserId && + event.kind === "notice" && + typeof event.body === "string" && + event.body.includes("channels.matrix.dm.sessionScope"), + roomId: secondRoomId, + since: noticeSince, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs), + }), + ]); + + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: params.context.syncState, + nextSince: replyResult.since, + startSince: replySince, + }); + + const secondReply = buildMatrixReplyArtifact(replyResult.event, secondToken); + assertTopLevelReplyArtifact("secondary DM reply", secondReply); + const noticeArtifact = noticeResult.matched + ? buildMatrixNoticeArtifact(noticeResult.event) + : undefined; + + if (params.expectNotice && !noticeArtifact) { + throw new Error( + "Matrix shared DM session scenario did not emit the expected cross-room notice", + ); + } + if (!params.expectNotice && noticeArtifact) { + throw new Error( + "Matrix per-room DM session scenario unexpectedly emitted a shared-session notice", + ); + } + + return { + firstRoomId, + noticeArtifact, + secondBody, + secondDriverEventId, + secondReply, + secondRoomId, + secondToken, + }; +} + +export async function runDmThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY); + const result = await runConfigurableTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + replyPredicate: (event, params) => + event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, + roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_DM_THREAD", + withMention: false, + }); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.driverEventId, + label: "DM thread override reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, + token: result.token, + triggerBody: result.body, + }, + details: [ + `room key: ${MATRIX_QA_DRIVER_DM_ROOM_KEY}`, + `room id: ${roomId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runDmSharedSessionNoticeScenario(context: MatrixQaScenarioContext) { + const result = await runDmSharedSessionFlow({ + context, + expectNotice: true, + }); + return { + artifacts: { + driverEventId: result.secondDriverEventId, + noticeBodyPreview: result.noticeArtifact?.bodyPreview, + noticeEventId: result.noticeArtifact?.eventId, + reply: result.secondReply, + roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + token: result.secondToken, + triggerBody: result.secondBody, + }, + details: [ + `primary room id: ${result.firstRoomId}`, + `secondary room id: ${result.secondRoomId}`, + `secondary driver event: ${result.secondDriverEventId}`, + `notice event: ${result.noticeArtifact?.eventId ?? ""}`, + `notice preview: ${result.noticeArtifact?.bodyPreview ?? ""}`, + ...buildMatrixReplyDetails("secondary reply", result.secondReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runDmPerRoomSessionOverrideScenario(context: MatrixQaScenarioContext) { + const result = await runDmSharedSessionFlow({ + context, + expectNotice: false, + }); + return { + artifacts: { + driverEventId: result.secondDriverEventId, + reply: result.secondReply, + roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + token: result.secondToken, + triggerBody: result.secondBody, + }, + details: [ + `primary room id: ${result.firstRoomId}`, + `secondary room id: ${result.secondRoomId}`, + `secondary driver event: ${result.secondDriverEventId}`, + "shared-session notice: suppressed", + ...buildMatrixReplyDetails("secondary reply", result.secondReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts new file mode 100644 index 00000000000..0d2264592fb --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -0,0 +1,651 @@ +import { randomUUID } from "node:crypto"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { + MATRIX_QA_MEMBERSHIP_ROOM_KEY, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import { + assertThreadReplyArtifact, + assertTopLevelReplyArtifact, + advanceMatrixQaActorCursor, + buildMatrixBlockStreamingPrompt, + buildMatrixQuietStreamingPrompt, + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + buildMentionPrompt, + createMatrixQaScenarioClient, + isMatrixQaMessageLikeKind, + NO_REPLY_WINDOW_MS, + primeMatrixQaActorCursor, + runConfigurableTopLevelScenario, + runDriverTopLevelMentionScenario, + runNoReplyExpectedScenario, + runTopologyScopedTopLevelScenario, + runTopLevelMentionScenario, + waitForMembershipEvent, + type MatrixQaScenarioContext, + type MatrixQaSyncState, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaCanaryArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; + +async function runThreadScenario(params: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.driverAccessToken, + actorId: "driver", + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const rootBody = `thread root ${randomUUID().slice(0, 8)}`; + const rootEventId = await client.sendTextMessage({ + body: rootBody, + roomId: params.roomId, + }); + const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`; + const driverEventId = await client.sendTextMessage({ + body: buildMentionPrompt(params.sutUserId, token), + mentionUserIds: [params.sutUserId], + replyToEventId: rootEventId, + roomId: params.roomId, + threadRootEventId: rootEventId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message" && + (event.body ?? "").includes(token) && + event.relatesTo?.relType === "m.thread" && + event.relatesTo.eventId === rootEventId, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return { + driverEventId, + reply: buildMatrixReplyArtifact(matched.event, token), + rootEventId, + token, + }; +} + +export async function runMatrixQaCanary(params: { + baseUrl: string; + driverAccessToken: string; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; +}): Promise<{ + driverEventId: string; + reply: MatrixQaCanaryArtifact["reply"]; + token: string; +}> { + const canary = await runDriverTopLevelMentionScenario({ + baseUrl: params.baseUrl, + driverAccessToken: params.driverAccessToken, + observedEvents: params.observedEvents, + roomId: params.roomId, + syncState: params.syncState, + sutUserId: params.sutUserId, + timeoutMs: params.timeoutMs, + tokenPrefix: "MATRIX_QA_CANARY", + }); + assertTopLevelReplyArtifact("canary reply", canary.reply); + return canary; +} + +export async function runThreadFollowUpScenario(context: MatrixQaScenarioContext) { + const result = await runThreadScenario(context); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.rootEventId, + label: "thread reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + rootEventId: result.rootEventId, + token: result.token, + }, + details: [ + `root event: ${result.rootEventId}`, + `driver thread event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runThreadIsolationScenario(context: MatrixQaScenarioContext) { + const threadPhase = await runThreadScenario(context); + assertThreadReplyArtifact(threadPhase.reply, { + expectedRootEventId: threadPhase.rootEventId, + label: "thread isolation reply", + }); + const topLevelPhase = await runDriverTopLevelMentionScenario({ + baseUrl: context.baseUrl, + driverAccessToken: context.driverAccessToken, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_TOPLEVEL", + }); + assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply); + return { + artifacts: { + threadDriverEventId: threadPhase.driverEventId, + threadReply: threadPhase.reply, + threadRootEventId: threadPhase.rootEventId, + threadToken: threadPhase.token, + topLevelDriverEventId: topLevelPhase.driverEventId, + topLevelReply: topLevelPhase.reply, + topLevelToken: topLevelPhase.token, + }, + details: [ + `thread root event: ${threadPhase.rootEventId}`, + `thread driver event: ${threadPhase.driverEventId}`, + ...buildMatrixReplyDetails("thread reply", threadPhase.reply), + `top-level driver event: ${topLevelPhase.driverEventId}`, + ...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runTopLevelReplyShapeScenario(context: MatrixQaScenarioContext) { + const result = await runDriverTopLevelMentionScenario({ + baseUrl: context.baseUrl, + driverAccessToken: context.driverAccessToken, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_TOPLEVEL", + }); + assertTopLevelReplyArtifact("top-level reply", result.reply); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + token: result.token, + }, + details: [ + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runRoomThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { + const result = await runConfigurableTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + replyPredicate: (event, params) => + event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_ROOM_THREAD", + }); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.driverEventId, + label: "room thread override reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + token: result.token, + triggerBody: result.body, + }, + details: [ + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runObserverAllowlistOverrideScenario(context: MatrixQaScenarioContext) { + const result = await runTopLevelMentionScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_OBSERVER_ALLOWLIST", + }); + assertTopLevelReplyArtifact("observer allowlist override reply", result.reply); + return { + artifacts: { + actorUserId: context.observerUserId, + driverEventId: result.driverEventId, + reply: result.reply, + token: result.token, + triggerBody: result.body, + }, + details: [ + `trigger sender: ${context.observerUserId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runQuietStreamingPreviewScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const finalText = `MATRIX_QA_QUIET_STREAM_${randomUUID().slice(0, 8).toUpperCase()} preview complete`; + const triggerBody = buildMatrixQuietStreamingPrompt(context.sutUserId, finalText); + const driverEventId = await client.sendTextMessage({ + body: triggerBody, + mentionUserIds: [context.sutUserId], + roomId: context.roomId, + }); + const preview = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + event.kind === "notice", + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const finalized = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + isMatrixQaMessageLikeKind(event.kind) && + event.relatesTo?.relType === "m.replace" && + event.relatesTo.eventId === preview.event.eventId && + event.body === finalText, + roomId: context.roomId, + since: preview.since, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: finalized.since, + startSince, + }); + const finalReply = buildMatrixReplyArtifact(finalized.event, finalText); + return { + artifacts: { + driverEventId, + previewBodyPreview: preview.event.body?.slice(0, 200), + previewEventId: preview.event.eventId, + reply: finalReply, + token: finalText, + triggerBody, + }, + details: [ + `driver event: ${driverEventId}`, + `preview event: ${preview.event.eventId}`, + `preview kind: ${preview.event.kind}`, + `preview body: ${preview.event.body ?? ""}`, + `final reply relation: ${finalized.event.relatesTo?.relType ?? ""}`, + `final reply target: ${finalized.event.relatesTo?.eventId ?? ""}`, + ...buildMatrixReplyDetails("final reply", finalReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runBlockStreamingScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const firstText = `MATRIX_QA_BLOCK_ONE_${randomUUID().slice(0, 8).toUpperCase()}`; + const secondText = `MATRIX_QA_BLOCK_TWO_${randomUUID().slice(0, 8).toUpperCase()}`; + const triggerBody = buildMatrixBlockStreamingPrompt(context.sutUserId, firstText, secondText); + const driverEventId = await client.sendTextMessage({ + body: triggerBody, + mentionUserIds: [context.sutUserId], + roomId: context.roomId, + }); + const firstBlock = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + isMatrixQaMessageLikeKind(event.kind) && + event.body === firstText, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const secondBlock = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + isMatrixQaMessageLikeKind(event.kind) && + event.body === secondText, + roomId: context.roomId, + since: firstBlock.since, + timeoutMs: context.timeoutMs, + }); + if (firstBlock.event.eventId === secondBlock.event.eventId) { + throw new Error( + "Matrix block streaming scenario reused one event instead of preserving blocks", + ); + } + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: secondBlock.since, + startSince, + }); + return { + artifacts: { + blockEventIds: [firstBlock.event.eventId, secondBlock.event.eventId], + driverEventId, + reply: buildMatrixReplyArtifact(secondBlock.event, secondText), + token: secondText, + triggerBody, + }, + details: [ + `driver event: ${driverEventId}`, + `block one event: ${firstBlock.event.eventId}`, + `block two event: ${secondBlock.event.eventId}`, + `block one kind: ${firstBlock.event.kind}`, + `block two kind: ${secondBlock.event.kind}`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runRoomAutoJoinInviteScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const dynamicRoomId = await client.createPrivateRoom({ + inviteUserIds: [context.observerUserId, context.sutUserId], + name: `Matrix QA AutoJoin ${randomUUID().slice(0, 8)}`, + }); + const joinResult = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === dynamicRoomId && + event.type === "m.room.member" && + event.stateKey === context.sutUserId && + event.membership === "join", + roomId: dynamicRoomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const joinEvent = joinResult.event; + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: joinResult.since, + startSince, + }); + + const result = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: dynamicRoomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_AUTOJOIN", + }); + assertTopLevelReplyArtifact("auto-join room reply", result.reply); + + return { + artifacts: { + driverEventId: result.driverEventId, + joinedRoomId: dynamicRoomId, + membershipJoinEventId: joinEvent.eventId, + reply: result.reply, + token: result.token, + triggerBody: result.body, + }, + details: [ + `joined room id: ${dynamicRoomId}`, + `join event: ${joinEvent.eventId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runMembershipLossScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEMBERSHIP_ROOM_KEY); + const driverClient = createMatrixQaScenarioClient({ + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + }); + const sutClient = createMatrixQaScenarioClient({ + accessToken: context.sutAccessToken, + baseUrl: context.baseUrl, + }); + + await driverClient.kickUserFromRoom({ + reason: "matrix qa membership loss", + roomId, + userId: context.sutUserId, + }); + const leaveEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "leave", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const noReplyToken = `MATRIX_QA_MEMBERSHIP_LOSS_${randomUUID().slice(0, 8).toUpperCase()}`; + await runNoReplyExpectedScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + baseUrl: context.baseUrl, + body: buildMentionPrompt(context.sutUserId, noReplyToken), + mentionUserIds: [context.sutUserId], + observedEvents: context.observedEvents, + roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + token: noReplyToken, + }); + + await driverClient.inviteUserToRoom({ + roomId, + userId: context.sutUserId, + }); + await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "invite", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + await sutClient.joinRoom(roomId); + const joinEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "join", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const recovered = await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_MEMBERSHIP_ROOM_KEY, + tokenPrefix: "MATRIX_QA_MEMBERSHIP_RETURN", + }); + + return { + artifacts: { + ...recovered.artifacts, + membershipJoinEventId: joinEvent.eventId, + membershipLeaveEventId: leaveEvent.eventId, + recoveredDriverEventId: recovered.artifacts?.driverEventId, + recoveredReply: recovered.artifacts?.reply, + }, + details: [ + `room key: ${MATRIX_QA_MEMBERSHIP_ROOM_KEY}`, + `room id: ${roomId}`, + `leave event: ${leaveEvent.eventId}`, + `join event: ${joinEvent.eventId}`, + recovered.details, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { + const reactionTargetEventId = context.canary?.reply.eventId?.trim(); + if (!reactionTargetEventId) { + throw new Error("Matrix reaction scenario requires a canary reply event id"); + } + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const reactionEmoji = "👍"; + const reactionEventId = await client.sendReaction({ + emoji: reactionEmoji, + messageId: reactionTargetEventId, + roomId: context.roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.driverUserId && + event.type === "m.reaction" && + event.eventId === reactionEventId && + event.reaction?.eventId === reactionTargetEventId && + event.reaction?.key === reactionEmoji, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + return { + artifacts: { + reactionEmoji, + reactionEventId, + reactionTargetEventId, + }, + details: [ + `reaction event: ${reactionEventId}`, + `reaction target: ${reactionTargetEventId}`, + `reaction emoji: ${reactionEmoji}`, + `observed reaction key: ${matched.event.reaction?.key ?? ""}`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.interruptTransport) { + throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); + } + await context.interruptTransport(); + const resumed = await runDriverTopLevelMentionScenario({ + baseUrl: context.baseUrl, + driverAccessToken: context.driverAccessToken, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_HOMESERVER", + }); + assertTopLevelReplyArtifact("post-homeserver-restart reply", resumed.reply); + return { + artifacts: { + driverEventId: resumed.driverEventId, + reply: resumed.reply, + token: resumed.token, + transportInterruption: "homeserver-restart", + }, + details: [ + "transport interruption: homeserver-restart", + `driver event: ${resumed.driverEventId}`, + ...buildMatrixReplyDetails("reply", resumed.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.restartGateway) { + throw new Error("Matrix restart scenario requires a gateway restart callback"); + } + await context.restartGateway(); + const result = await runDriverTopLevelMentionScenario({ + baseUrl: context.baseUrl, + driverAccessToken: context.driverAccessToken, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_RESTART", + }); + assertTopLevelReplyArtifact("post-restart reply", result.reply); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + restartSignal: "SIGUSR1", + token: result.token, + }, + details: [ + "restart signal: SIGUSR1", + `post-restart driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts new file mode 100644 index 00000000000..ea24e5f857f --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -0,0 +1,427 @@ +import { randomUUID } from "node:crypto"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { type MatrixQaProvisionedTopology } from "../../substrate/topology.js"; +import { resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js"; +import type { + MatrixQaCanaryArtifact, + MatrixQaReplyArtifact, + MatrixQaScenarioExecution, +} from "./scenario-types.js"; + +export type MatrixQaActorId = "driver" | "observer"; + +export type MatrixQaSyncState = Partial>; + +export type MatrixQaScenarioContext = { + baseUrl: string; + canary?: MatrixQaCanaryArtifact; + driverAccessToken: string; + driverUserId: string; + observedEvents: MatrixQaObservedEvent[]; + observerAccessToken: string; + observerUserId: string; + restartGateway?: () => Promise; + roomId: string; + interruptTransport?: () => Promise; + sutAccessToken: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + topology: MatrixQaProvisionedTopology; +}; + +export const NO_REPLY_WINDOW_MS = 8_000; + +export function buildMentionPrompt(sutUserId: string, token: string) { + return `${sutUserId} reply with only this exact marker: ${token}`; +} + +export function buildExactMarkerPrompt(token: string) { + return `reply with only this exact marker: ${token}`; +} + +export function buildMatrixQuietStreamingPrompt(sutUserId: string, text: string) { + return `${sutUserId} Matrix quiet streaming QA check: reply exactly \`${text}\`.`; +} + +export function buildMatrixBlockStreamingPrompt( + sutUserId: string, + firstText: string, + secondText: string, +) { + return [ + sutUserId, + "Matrix block streaming QA check:", + "emit exactly two assistant message blocks in order.", + `First exact marker: \`${firstText}\`.`, + `Second exact marker: \`${secondText}\`.`, + ].join(" "); +} + +export function isMatrixQaMessageLikeKind(kind: MatrixQaObservedEvent["kind"]) { + return kind === "message" || kind === "notice"; +} + +export function buildMatrixReplyArtifact( + event: MatrixQaObservedEvent, + token?: string, +): MatrixQaReplyArtifact { + const replyBody = event.body?.trim(); + return { + bodyPreview: replyBody?.slice(0, 200), + eventId: event.eventId, + mentions: event.mentions, + relatesTo: event.relatesTo, + sender: event.sender, + ...(token ? { tokenMatched: replyBody === token } : {}), + }; +} + +export function buildMatrixNoticeArtifact(event: MatrixQaObservedEvent) { + return { + bodyPreview: event.body?.trim().slice(0, 200), + eventId: event.eventId, + sender: event.sender, + }; +} + +export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) { + return [ + `${label} event: ${artifact.eventId}`, + `${label} token matched: ${ + artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no" + }`, + `${label} rel_type: ${artifact.relatesTo?.relType ?? ""}`, + `${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? ""}`, + `${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`, + ]; +} + +export function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) { + if (!artifact.tokenMatched) { + throw new Error(`${label} did not contain the expected token`); + } + if (artifact.relatesTo !== undefined) { + throw new Error(`${label} unexpectedly included relation metadata`); + } +} + +export function assertThreadReplyArtifact( + artifact: MatrixQaReplyArtifact, + params: { + expectedRootEventId: string; + label: string; + }, +) { + if (!artifact.tokenMatched) { + throw new Error(`${params.label} did not contain the expected token`); + } + if (artifact.relatesTo?.relType !== "m.thread") { + throw new Error(`${params.label} did not use m.thread`); + } + if (artifact.relatesTo.eventId !== params.expectedRootEventId) { + throw new Error( + `${params.label} targeted ${artifact.relatesTo.eventId ?? ""} instead of ${params.expectedRootEventId}`, + ); + } + if (artifact.relatesTo.isFallingBack !== true) { + throw new Error(`${params.label} did not set is_falling_back`); + } + if (!artifact.relatesTo.inReplyToId) { + throw new Error(`${params.label} did not set m.in_reply_to`); + } +} + +export function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) { + return syncState[actorId]; +} + +export function writeMatrixQaSyncCursor( + syncState: MatrixQaSyncState, + actorId: MatrixQaActorId, + since?: string, +) { + if (since) { + syncState[actorId] = since; + } +} + +export async function primeMatrixQaActorCursor(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + syncState: MatrixQaSyncState; +}) { + const client = createMatrixQaClient({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + }); + const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId); + if (existingSince) { + return { client, startSince: existingSince }; + } + const startSince = await client.primeRoom(); + if (!startSince) { + throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`); + } + return { client, startSince }; +} + +export function advanceMatrixQaActorCursor(params: { + actorId: MatrixQaActorId; + syncState: MatrixQaSyncState; + nextSince?: string; + startSince: string; +}) { + writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); +} + +export function createMatrixQaScenarioClient(params: { accessToken: string; baseUrl: string }) { + return createMatrixQaClient({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + }); +} + +export async function runConfigurableTopLevelScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + observedEvents: MatrixQaObservedEvent[]; + replyPredicate?: ( + event: MatrixQaObservedEvent, + params: { driverEventId: string; token: string }, + ) => boolean; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + tokenPrefix: string; + withMention?: boolean; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`; + const body = + params.withMention === false + ? buildExactMarkerPrompt(token) + : buildMentionPrompt(params.sutUserId, token); + const driverEventId = await client.sendTextMessage({ + body, + ...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }), + roomId: params.roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message" && + (event.body ?? "").includes(token) && + (params.replyPredicate?.(event, { driverEventId, token }) ?? event.relatesTo === undefined), + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return { + body, + driverEventId, + reply: buildMatrixReplyArtifact(matched.event, token), + token, + }; +} + +export async function runTopLevelMentionScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + tokenPrefix: string; + withMention?: boolean; +}) { + return await runConfigurableTopLevelScenario(params); +} + +export async function runDriverTopLevelMentionScenario(params: { + baseUrl: string; + driverAccessToken: string; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + tokenPrefix: string; +}) { + return await runTopLevelMentionScenario({ + accessToken: params.driverAccessToken, + actorId: "driver", + baseUrl: params.baseUrl, + observedEvents: params.observedEvents, + roomId: params.roomId, + syncState: params.syncState, + sutUserId: params.sutUserId, + timeoutMs: params.timeoutMs, + tokenPrefix: params.tokenPrefix, + }); +} + +export async function waitForMembershipEvent(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + membership: "invite" | "join" | "leave"; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + stateKey: string; + syncState: MatrixQaSyncState; + timeoutMs: number; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.type === "m.room.member" && + event.stateKey === params.stateKey && + event.membership === params.membership, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return matched.event; +} + +export async function runTopologyScopedTopLevelScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + actorUserId: string; + context: MatrixQaScenarioContext; + roomKey: string; + tokenPrefix: string; + withMention?: boolean; +}) { + const roomId = resolveMatrixQaScenarioRoomId(params.context, params.roomKey); + const result = await runTopLevelMentionScenario({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.context.baseUrl, + observedEvents: params.context.observedEvents, + roomId, + syncState: params.context.syncState, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: params.tokenPrefix, + withMention: params.withMention, + }); + assertTopLevelReplyArtifact(`reply in ${params.roomKey}`, result.reply); + return { + artifacts: { + actorUserId: params.actorUserId, + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: params.roomKey, + token: result.token, + triggerBody: result.body, + }, + details: [ + `room key: ${params.roomKey}`, + `room id: ${roomId}`, + `driver event: ${result.driverEventId}`, + `trigger sender: ${params.actorUserId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runNoReplyExpectedScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + actorUserId: string; + baseUrl: string; + body: string; + mentionUserIds?: string[]; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + token: string; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const driverEventId = await client.sendTextMessage({ + body: params.body, + ...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}), + roomId: params.roomId, + }); + const result = await client.waitForOptionalRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message", + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + if (result.matched) { + const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token); + throw new Error( + [ + `unexpected SUT reply from ${params.sutUserId}`, + `trigger sender: ${params.actorUserId}`, + ...buildMatrixReplyDetails("unexpected reply", unexpectedReply), + ].join("\n"), + ); + } + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: result.since, + startSince, + }); + return { + artifacts: { + actorUserId: params.actorUserId, + driverEventId, + expectedNoReplyWindowMs: params.timeoutMs, + token: params.token, + triggerBody: params.body, + }, + details: [ + `trigger event: ${driverEventId}`, + `trigger sender: ${params.actorUserId}`, + `waited ${params.timeoutMs}ms with no SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index c593d954829..6fb35037454 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -1,1054 +1,70 @@ import { randomUUID } from "node:crypto"; -import { createMatrixQaClient } from "../../substrate/client.js"; -import type { MatrixQaObservedEvent } from "../../substrate/events.js"; -import { type MatrixQaProvisionedTopology } from "../../substrate/topology.js"; import { MATRIX_QA_DRIVER_DM_ROOM_KEY, - MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, - MATRIX_QA_MEMBERSHIP_ROOM_KEY, MATRIX_QA_SECONDARY_ROOM_KEY, - resolveMatrixQaScenarioRoomId, type MatrixQaScenarioDefinition, } from "./scenario-catalog.js"; -import type { - MatrixQaCanaryArtifact, - MatrixQaReplyArtifact, - MatrixQaScenarioExecution, -} from "./scenario-types.js"; +import { + runDmPerRoomSessionOverrideScenario, + runDmSharedSessionNoticeScenario, + runDmThreadReplyOverrideScenario, +} from "./scenario-runtime-dm.js"; +import { + runBlockStreamingScenario, + runHomeserverRestartResumeScenario, + runMatrixQaCanary, + runMembershipLossScenario, + runObserverAllowlistOverrideScenario, + runQuietStreamingPreviewScenario, + runReactionNotificationScenario, + runRestartResumeScenario, + runRoomAutoJoinInviteScenario, + runRoomThreadReplyOverrideScenario, + runThreadFollowUpScenario, + runThreadIsolationScenario, + runTopLevelReplyShapeScenario, +} from "./scenario-runtime-room.js"; +import { + buildExactMarkerPrompt, + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + buildMentionPrompt, + readMatrixQaSyncCursor, + runNoReplyExpectedScenario, + runTopologyScopedTopLevelScenario, + writeMatrixQaSyncCursor, + type MatrixQaScenarioContext, + type MatrixQaSyncState, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; -type MatrixQaActorId = "driver" | "observer"; - -export type MatrixQaSyncState = Partial>; - -export type MatrixQaScenarioContext = { - baseUrl: string; - canary?: MatrixQaCanaryArtifact; - driverAccessToken: string; - driverUserId: string; - observedEvents: MatrixQaObservedEvent[]; - observerAccessToken: string; - observerUserId: string; - restartGateway?: () => Promise; - roomId: string; - interruptTransport?: () => Promise; - sutAccessToken: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - topology: MatrixQaProvisionedTopology; +export { + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + buildMentionPrompt, + readMatrixQaSyncCursor, + runMatrixQaCanary, + writeMatrixQaSyncCursor, }; - -const NO_REPLY_WINDOW_MS = 8_000; - -export function buildMentionPrompt(sutUserId: string, token: string) { - return `${sutUserId} reply with only this exact marker: ${token}`; -} - -function buildExactMarkerPrompt(token: string) { - return `reply with only this exact marker: ${token}`; -} - -export function buildMatrixReplyArtifact( - event: MatrixQaObservedEvent, - token?: string, -): MatrixQaReplyArtifact { - const replyBody = event.body?.trim(); - return { - bodyPreview: replyBody?.slice(0, 200), - eventId: event.eventId, - mentions: event.mentions, - relatesTo: event.relatesTo, - sender: event.sender, - ...(token ? { tokenMatched: replyBody === token } : {}), - }; -} - -function buildMatrixNoticeArtifact(event: MatrixQaObservedEvent) { - return { - bodyPreview: event.body?.trim().slice(0, 200), - eventId: event.eventId, - sender: event.sender, - }; -} - -export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) { - return [ - `${label} event: ${artifact.eventId}`, - `${label} token matched: ${ - artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no" - }`, - `${label} rel_type: ${artifact.relatesTo?.relType ?? ""}`, - `${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? ""}`, - `${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`, - ]; -} - -function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) { - if (!artifact.tokenMatched) { - throw new Error(`${label} did not contain the expected token`); - } - if (artifact.relatesTo !== undefined) { - throw new Error(`${label} unexpectedly included relation metadata`); - } -} - -function assertThreadReplyArtifact( - artifact: MatrixQaReplyArtifact, - params: { - expectedRootEventId: string; - label: string; - }, -) { - if (!artifact.tokenMatched) { - throw new Error(`${params.label} did not contain the expected token`); - } - if (artifact.relatesTo?.relType !== "m.thread") { - throw new Error(`${params.label} did not use m.thread`); - } - if (artifact.relatesTo.eventId !== params.expectedRootEventId) { - throw new Error( - `${params.label} targeted ${artifact.relatesTo.eventId ?? ""} instead of ${params.expectedRootEventId}`, - ); - } - if (artifact.relatesTo.isFallingBack !== true) { - throw new Error(`${params.label} did not set is_falling_back`); - } - if (!artifact.relatesTo.inReplyToId) { - throw new Error(`${params.label} did not set m.in_reply_to`); - } -} - -export function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) { - return syncState[actorId]; -} - -export function writeMatrixQaSyncCursor( - syncState: MatrixQaSyncState, - actorId: MatrixQaActorId, - since?: string, -) { - if (since) { - syncState[actorId] = since; - } -} - -async function primeMatrixQaActorCursor(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - syncState: MatrixQaSyncState; -}) { - const client = createMatrixQaClient({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, - }); - const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId); - if (existingSince) { - return { client, startSince: existingSince }; - } - const startSince = await client.primeRoom(); - if (!startSince) { - throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`); - } - return { client, startSince }; -} - -function advanceMatrixQaActorCursor(params: { - actorId: MatrixQaActorId; - syncState: MatrixQaSyncState; - nextSince?: string; - startSince: string; -}) { - writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); -} - -function createMatrixQaScenarioClient(params: { accessToken: string; baseUrl: string }) { - return createMatrixQaClient({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, - }); -} - -async function runConfigurableTopLevelScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - observedEvents: MatrixQaObservedEvent[]; - replyPredicate?: ( - event: MatrixQaObservedEvent, - params: { driverEventId: string; token: string }, - ) => boolean; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - tokenPrefix: string; - withMention?: boolean; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`; - const body = - params.withMention === false - ? buildExactMarkerPrompt(token) - : buildMentionPrompt(params.sutUserId, token); - const driverEventId = await client.sendTextMessage({ - body, - ...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }), - roomId: params.roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && - (params.replyPredicate?.(event, { driverEventId, token }) ?? event.relatesTo === undefined), - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return { - body, - driverEventId, - reply: buildMatrixReplyArtifact(matched.event, token), - token, - }; -} - -async function runTopLevelMentionScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - tokenPrefix: string; - withMention?: boolean; -}) { - return await runConfigurableTopLevelScenario(params); -} - -async function waitForMembershipEvent(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - membership: "invite" | "join" | "leave"; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - stateKey: string; - syncState: MatrixQaSyncState; - timeoutMs: number; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.type === "m.room.member" && - event.stateKey === params.stateKey && - event.membership === params.membership, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return matched.event; -} - -async function runTopologyScopedTopLevelScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - actorUserId: string; - context: MatrixQaScenarioContext; - roomKey: string; - tokenPrefix: string; - withMention?: boolean; -}) { - const roomId = resolveMatrixQaScenarioRoomId(params.context, params.roomKey); - const result = await runTopLevelMentionScenario({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.context.baseUrl, - observedEvents: params.context.observedEvents, - roomId, - syncState: params.context.syncState, - sutUserId: params.context.sutUserId, - timeoutMs: params.context.timeoutMs, - tokenPrefix: params.tokenPrefix, - withMention: params.withMention, - }); - assertTopLevelReplyArtifact(`reply in ${params.roomKey}`, result.reply); - return { - artifacts: { - actorUserId: params.actorUserId, - driverEventId: result.driverEventId, - reply: result.reply, - roomKey: params.roomKey, - token: result.token, - triggerBody: result.body, - }, - details: [ - `room key: ${params.roomKey}`, - `room id: ${roomId}`, - `driver event: ${result.driverEventId}`, - `trigger sender: ${params.actorUserId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runThreadScenario(params: MatrixQaScenarioContext) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.driverAccessToken, - actorId: "driver", - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const rootBody = `thread root ${randomUUID().slice(0, 8)}`; - const rootEventId = await client.sendTextMessage({ - body: rootBody, - roomId: params.roomId, - }); - const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`; - const driverEventId = await client.sendTextMessage({ - body: buildMentionPrompt(params.sutUserId, token), - mentionUserIds: [params.sutUserId], - replyToEventId: rootEventId, - roomId: params.roomId, - threadRootEventId: rootEventId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && - event.relatesTo?.relType === "m.thread" && - event.relatesTo.eventId === rootEventId, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return { - driverEventId, - reply: buildMatrixReplyArtifact(matched.event, token), - rootEventId, - token, - }; -} - -async function runRoomThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { - const result = await runConfigurableTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - replyPredicate: (event, params) => - event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_ROOM_THREAD", - }); - assertThreadReplyArtifact(result.reply, { - expectedRootEventId: result.driverEventId, - label: "room thread override reply", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - token: result.token, - triggerBody: result.body, - }, - details: [ - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runDmThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY); - const result = await runConfigurableTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - replyPredicate: (event, params) => - event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, - roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_DM_THREAD", - withMention: false, - }); - assertThreadReplyArtifact(result.reply, { - expectedRootEventId: result.driverEventId, - label: "DM thread override reply", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, - token: result.token, - triggerBody: result.body, - }, - details: [ - `room key: ${MATRIX_QA_DRIVER_DM_ROOM_KEY}`, - `room id: ${roomId}`, - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runDmSharedSessionFlow(params: { - context: MatrixQaScenarioContext; - expectNotice: boolean; -}) { - const firstRoomId = resolveMatrixQaScenarioRoomId(params.context, MATRIX_QA_DRIVER_DM_ROOM_KEY); - const secondRoomId = resolveMatrixQaScenarioRoomId( - params.context, - MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, - ); - - const firstResult = await runConfigurableTopLevelScenario({ - accessToken: params.context.driverAccessToken, - actorId: "driver", - baseUrl: params.context.baseUrl, - observedEvents: params.context.observedEvents, - roomId: firstRoomId, - syncState: params.context.syncState, - sutUserId: params.context.sutUserId, - timeoutMs: params.context.timeoutMs, - tokenPrefix: "MATRIX_QA_DM_PRIMARY", - withMention: false, - }); - assertTopLevelReplyArtifact("primary DM reply", firstResult.reply); - - const replyClient = createMatrixQaScenarioClient({ - accessToken: params.context.driverAccessToken, - baseUrl: params.context.baseUrl, - }); - const noticeClient = createMatrixQaScenarioClient({ - accessToken: params.context.driverAccessToken, - baseUrl: params.context.baseUrl, - }); - const [replySince, noticeSince] = await Promise.all([ - replyClient.primeRoom(), - noticeClient.primeRoom(), - ]); - if (!replySince || !noticeSince) { - throw new Error("Matrix DM session scenario could not prime room cursors"); - } - - const secondToken = `MATRIX_QA_DM_SECONDARY_${randomUUID().slice(0, 8).toUpperCase()}`; - const secondBody = buildExactMarkerPrompt(secondToken); - const secondDriverEventId = await replyClient.sendTextMessage({ - body: secondBody, - roomId: secondRoomId, - }); - - const [replyResult, noticeResult] = await Promise.all([ - replyClient.waitForRoomEvent({ - observedEvents: params.context.observedEvents, - predicate: (event) => - event.roomId === secondRoomId && - event.sender === params.context.sutUserId && - event.type === "m.room.message" && - event.kind === "message" && - (event.body ?? "").includes(secondToken), - roomId: secondRoomId, - since: replySince, - timeoutMs: params.context.timeoutMs, - }), - noticeClient.waitForOptionalRoomEvent({ - observedEvents: params.context.observedEvents, - predicate: (event) => - event.roomId === secondRoomId && - event.sender === params.context.sutUserId && - event.kind === "notice" && - typeof event.body === "string" && - event.body.includes("channels.matrix.dm.sessionScope"), - roomId: secondRoomId, - since: noticeSince, - timeoutMs: Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs), - }), - ]); - - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: params.context.syncState, - nextSince: replyResult.since, - startSince: replySince, - }); - - const secondReply = buildMatrixReplyArtifact(replyResult.event, secondToken); - assertTopLevelReplyArtifact("secondary DM reply", secondReply); - const noticeArtifact = noticeResult.matched - ? buildMatrixNoticeArtifact(noticeResult.event) - : undefined; - - if (params.expectNotice && !noticeArtifact) { - throw new Error( - "Matrix shared DM session scenario did not emit the expected cross-room notice", - ); - } - if (!params.expectNotice && noticeArtifact) { - throw new Error( - "Matrix per-room DM session scenario unexpectedly emitted a shared-session notice", - ); - } - - return { - noticeArtifact, - secondBody, - secondDriverEventId, - secondReply, - secondRoomId, - secondToken, - }; -} - -async function runDmSharedSessionNoticeScenario(context: MatrixQaScenarioContext) { - const result = await runDmSharedSessionFlow({ - context, - expectNotice: true, - }); - return { - artifacts: { - driverEventId: result.secondDriverEventId, - noticeBodyPreview: result.noticeArtifact?.bodyPreview, - noticeEventId: result.noticeArtifact?.eventId, - reply: result.secondReply, - roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, - token: result.secondToken, - triggerBody: result.secondBody, - }, - details: [ - `primary room id: ${resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY)}`, - `secondary room id: ${result.secondRoomId}`, - `secondary driver event: ${result.secondDriverEventId}`, - `notice event: ${result.noticeArtifact?.eventId ?? ""}`, - `notice preview: ${result.noticeArtifact?.bodyPreview ?? ""}`, - ...buildMatrixReplyDetails("secondary reply", result.secondReply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runDmPerRoomSessionOverrideScenario(context: MatrixQaScenarioContext) { - const result = await runDmSharedSessionFlow({ - context, - expectNotice: false, - }); - return { - artifacts: { - driverEventId: result.secondDriverEventId, - reply: result.secondReply, - roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, - token: result.secondToken, - triggerBody: result.secondBody, - }, - details: [ - `primary room id: ${resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY)}`, - `secondary room id: ${result.secondRoomId}`, - `secondary driver event: ${result.secondDriverEventId}`, - "shared-session notice: suppressed", - ...buildMatrixReplyDetails("secondary reply", result.secondReply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runRoomAutoJoinInviteScenario(context: MatrixQaScenarioContext) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - syncState: context.syncState, - }); - const dynamicRoomId = await client.createPrivateRoom({ - inviteUserIds: [context.observerUserId, context.sutUserId], - name: `Matrix QA AutoJoin ${randomUUID().slice(0, 8)}`, - }); - const joinResult = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === dynamicRoomId && - event.type === "m.room.member" && - event.stateKey === context.sutUserId && - event.membership === "join", - roomId: dynamicRoomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); - const joinEvent = joinResult.event; - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: context.syncState, - nextSince: joinResult.since, - startSince, - }); - - const result = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: dynamicRoomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_AUTOJOIN", - }); - assertTopLevelReplyArtifact("auto-join room reply", result.reply); - - return { - artifacts: { - driverEventId: result.driverEventId, - joinedRoomId: dynamicRoomId, - membershipJoinEventId: joinEvent.eventId, - reply: result.reply, - token: result.token, - triggerBody: result.body, - }, - details: [ - `joined room id: ${dynamicRoomId}`, - `join event: ${joinEvent.eventId}`, - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runNoReplyExpectedScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - actorUserId: string; - baseUrl: string; - body: string; - mentionUserIds?: string[]; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - token: string; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const driverEventId = await client.sendTextMessage({ - body: params.body, - ...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}), - roomId: params.roomId, - }); - const result = await client.waitForOptionalRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message", - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - if (result.matched) { - const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token); - throw new Error( - [ - `unexpected SUT reply from ${params.sutUserId}`, - `trigger sender: ${params.actorUserId}`, - ...buildMatrixReplyDetails("unexpected reply", unexpectedReply), - ].join("\n"), - ); - } - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: result.since, - startSince, - }); - return { - artifacts: { - actorUserId: params.actorUserId, - driverEventId, - expectedNoReplyWindowMs: params.timeoutMs, - token: params.token, - triggerBody: params.body, - }, - details: [ - `trigger event: ${driverEventId}`, - `trigger sender: ${params.actorUserId}`, - `waited ${params.timeoutMs}ms with no SUT reply`, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runMembershipLossScenario(context: MatrixQaScenarioContext) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEMBERSHIP_ROOM_KEY); - const driverClient = createMatrixQaScenarioClient({ - accessToken: context.driverAccessToken, - baseUrl: context.baseUrl, - }); - const sutClient = createMatrixQaScenarioClient({ - accessToken: context.sutAccessToken, - baseUrl: context.baseUrl, - }); - - await driverClient.kickUserFromRoom({ - reason: "matrix qa membership loss", - roomId, - userId: context.sutUserId, - }); - const leaveEvent = await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "leave", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - - const noReplyToken = `MATRIX_QA_MEMBERSHIP_LOSS_${randomUUID().slice(0, 8).toUpperCase()}`; - await runNoReplyExpectedScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - baseUrl: context.baseUrl, - body: buildMentionPrompt(context.sutUserId, noReplyToken), - mentionUserIds: [context.sutUserId], - observedEvents: context.observedEvents, - roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), - token: noReplyToken, - }); - - await driverClient.inviteUserToRoom({ - roomId, - userId: context.sutUserId, - }); - await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "invite", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - await sutClient.joinRoom(roomId); - const joinEvent = await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "join", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - - const recovered = await runTopologyScopedTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - context, - roomKey: MATRIX_QA_MEMBERSHIP_ROOM_KEY, - tokenPrefix: "MATRIX_QA_MEMBERSHIP_RETURN", - }); - - return { - artifacts: { - ...recovered.artifacts, - membershipJoinEventId: joinEvent.eventId, - membershipLeaveEventId: leaveEvent.eventId, - recoveredDriverEventId: recovered.artifacts?.driverEventId, - recoveredReply: recovered.artifacts?.reply, - }, - details: [ - `room key: ${MATRIX_QA_MEMBERSHIP_ROOM_KEY}`, - `room id: ${roomId}`, - `leave event: ${leaveEvent.eventId}`, - `join event: ${joinEvent.eventId}`, - recovered.details, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { - const reactionTargetEventId = context.canary?.reply.eventId?.trim(); - if (!reactionTargetEventId) { - throw new Error("Matrix reaction scenario requires a canary reply event id"); - } - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - syncState: context.syncState, - }); - const reactionEmoji = "👍"; - const reactionEventId = await client.sendReaction({ - emoji: reactionEmoji, - messageId: reactionTargetEventId, - roomId: context.roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === context.roomId && - event.sender === context.driverUserId && - event.type === "m.reaction" && - event.eventId === reactionEventId && - event.reaction?.eventId === reactionTargetEventId && - event.reaction?.key === reactionEmoji, - roomId: context.roomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: context.syncState, - nextSince: matched.since, - startSince, - }); - return { - artifacts: { - reactionEmoji, - reactionEventId, - reactionTargetEventId, - }, - details: [ - `reaction event: ${reactionEventId}`, - `reaction target: ${reactionTargetEventId}`, - `reaction emoji: ${reactionEmoji}`, - `observed reaction key: ${matched.event.reaction?.key ?? ""}`, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.interruptTransport) { - throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); - } - await context.interruptTransport(); - const resumed = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_HOMESERVER", - }); - assertTopLevelReplyArtifact("post-homeserver-restart reply", resumed.reply); - return { - artifacts: { - driverEventId: resumed.driverEventId, - reply: resumed.reply, - token: resumed.token, - transportInterruption: "homeserver-restart", - }, - details: [ - "transport interruption: homeserver-restart", - `driver event: ${resumed.driverEventId}`, - ...buildMatrixReplyDetails("reply", resumed.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.restartGateway) { - throw new Error("Matrix restart scenario requires a gateway restart callback"); - } - await context.restartGateway(); - const result = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_RESTART", - }); - assertTopLevelReplyArtifact("post-restart reply", result.reply); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - restartSignal: "SIGUSR1", - token: result.token, - }, - details: [ - "restart signal: SIGUSR1", - `post-restart driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -export async function runMatrixQaCanary(params: { - baseUrl: string; - driverAccessToken: string; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; -}) { - const canary = await runTopLevelMentionScenario({ - accessToken: params.driverAccessToken, - actorId: "driver", - baseUrl: params.baseUrl, - observedEvents: params.observedEvents, - roomId: params.roomId, - syncState: params.syncState, - sutUserId: params.sutUserId, - timeoutMs: params.timeoutMs, - tokenPrefix: "MATRIX_QA_CANARY", - }); - assertTopLevelReplyArtifact("canary reply", canary.reply); - return canary; -} +export type { MatrixQaScenarioContext, MatrixQaSyncState }; export async function runMatrixQaScenario( scenario: MatrixQaScenarioDefinition, context: MatrixQaScenarioContext, ): Promise { switch (scenario.id) { - case "matrix-thread-follow-up": { - const result = await runThreadScenario(context); - assertThreadReplyArtifact(result.reply, { - expectedRootEventId: result.rootEventId, - label: "thread reply", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - rootEventId: result.rootEventId, - token: result.token, - }, - details: [ - `root event: ${result.rootEventId}`, - `driver thread event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - }; - } - case "matrix-thread-isolation": { - const threadPhase = await runThreadScenario(context); - assertThreadReplyArtifact(threadPhase.reply, { - expectedRootEventId: threadPhase.rootEventId, - label: "thread isolation reply", - }); - const topLevelPhase = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_TOPLEVEL", - }); - assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply); - return { - artifacts: { - threadDriverEventId: threadPhase.driverEventId, - threadReply: threadPhase.reply, - threadRootEventId: threadPhase.rootEventId, - threadToken: threadPhase.token, - topLevelDriverEventId: topLevelPhase.driverEventId, - topLevelReply: topLevelPhase.reply, - topLevelToken: topLevelPhase.token, - }, - details: [ - `thread root event: ${threadPhase.rootEventId}`, - `thread driver event: ${threadPhase.driverEventId}`, - ...buildMatrixReplyDetails("thread reply", threadPhase.reply), - `top-level driver event: ${topLevelPhase.driverEventId}`, - ...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply), - ].join("\n"), - }; - } - case "matrix-top-level-reply-shape": { - const result = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_TOPLEVEL", - }); - assertTopLevelReplyArtifact("top-level reply", result.reply); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - token: result.token, - }, - details: [ - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - }; - } + case "matrix-thread-follow-up": + return await runThreadFollowUpScenario(context); + case "matrix-thread-isolation": + return await runThreadIsolationScenario(context); + case "matrix-top-level-reply-shape": + return await runTopLevelReplyShapeScenario(context); case "matrix-room-thread-reply-override": return await runRoomThreadReplyOverrideScenario(context); + case "matrix-room-quiet-streaming-preview": + return await runQuietStreamingPreviewScenario(context); + case "matrix-room-block-streaming": + return await runBlockStreamingScenario(context); case "matrix-dm-reply-shape": return await runTopologyScopedTopLevelScenario({ accessToken: context.driverAccessToken, @@ -1110,6 +126,8 @@ export async function runMatrixQaScenario( token, }); } + case "matrix-observer-allowlist-override": + return await runObserverAllowlistOverrideScenario(context); case "matrix-allowlist-block": { const token = `MATRIX_QA_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`; return await runNoReplyExpectedScenario({ diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index dd22e1bd992..45081a73826 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -41,6 +41,9 @@ export type MatrixQaScenarioArtifacts = { membershipLeaveEventId?: string; noticeBodyPreview?: string; noticeEventId?: string; + previewBodyPreview?: string; + previewEventId?: string; + blockEventIds?: string[]; transportInterruption?: string; joinedRoomId?: string; }; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 872a2104fd5..75bb3c3c271 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -28,6 +28,8 @@ describe("matrix live qa scenarios", () => { "matrix-thread-isolation", "matrix-top-level-reply-shape", "matrix-room-thread-reply-override", + "matrix-room-quiet-streaming-preview", + "matrix-room-block-streaming", "matrix-dm-reply-shape", "matrix-dm-shared-session-notice", "matrix-dm-thread-reply-override", @@ -40,6 +42,7 @@ describe("matrix live qa scenarios", () => { "matrix-room-membership-loss", "matrix-homeserver-restart-resume", "matrix-mention-gating", + "matrix-observer-allowlist-override", "matrix-allowlist-block", ]); }); @@ -323,6 +326,74 @@ describe("matrix live qa scenarios", () => { }); }); + it("allows observer messages when the sender allowlist override includes them", async () => { + const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$observer-allow-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$sut-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), + }, + since: "observer-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-observer-allowlist-override", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!room:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + actorUserId: "@observer:matrix-qa.test", + driverEventId: "$observer-allow-trigger", + }, + }); + + expect(createMatrixQaClient).toHaveBeenCalledWith({ + accessToken: "observer-token", + baseUrl: "http://127.0.0.1:28008/", + }); + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("@sut:matrix-qa.test reply with only this exact marker:"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!room:matrix-qa.test", + }); + }); + it("runs the DM scenario against the provisioned DM room without a mention", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$dm-trigger"); @@ -479,6 +550,186 @@ describe("matrix live qa scenarios", () => { }); }); + it("captures quiet preview notices before the finalized Matrix reply", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$quiet-stream-trigger"); + const readFinalText = () => + /reply exactly `([^`]+)`/.exec(String(sendTextMessage.mock.calls[0]?.[0]?.body))?.[1] ?? + "MATRIX_QA_QUIET_STREAM_PREVIEW_COMPLETE"; + const waitForRoomEvent = vi + .fn() + .mockImplementationOnce(async () => ({ + event: { + kind: "notice", + roomId: "!main:matrix-qa.test", + eventId: "$quiet-preview", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }, + since: "driver-sync-preview", + })) + .mockImplementationOnce(async () => ({ + event: { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$quiet-final", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: readFinalText(), + relatesTo: { + relType: "m.replace", + eventId: "$quiet-preview", + }, + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-room-quiet-streaming-preview", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + driverEventId: "$quiet-stream-trigger", + previewEventId: "$quiet-preview", + reply: { + eventId: "$quiet-final", + }, + }, + }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("Matrix quiet streaming QA check"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + since: "driver-sync-start", + }), + ); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + predicate: expect.any(Function), + since: "driver-sync-preview", + }), + ); + }); + + it("preserves separate finalized block events when Matrix block streaming is enabled", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$block-stream-trigger"); + const readBlockText = (label: "First" | "Second") => + new RegExp(`${label} exact marker: \`([^\\\`]+)\``).exec( + String(sendTextMessage.mock.calls[0]?.[0]?.body), + )?.[1] ?? `MATRIX_QA_BLOCK_${label.toUpperCase()}_FIXED`; + const waitForRoomEvent = vi + .fn() + .mockImplementationOnce(async () => ({ + event: { + kind: "notice", + roomId: "!main:matrix-qa.test", + eventId: "$block-one", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: readBlockText("First"), + }, + since: "driver-sync-block-one", + })) + .mockImplementationOnce(async () => ({ + event: { + kind: "notice", + roomId: "!main:matrix-qa.test", + eventId: "$block-two", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: readBlockText("Second"), + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-room-block-streaming", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + blockEventIds: ["$block-one", "$block-two"], + driverEventId: "$block-stream-trigger", + }, + }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("Matrix block streaming QA check"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + since: "driver-sync-block-one", + }), + ); + }); + it("uses DM thread override scenarios against the provisioned DM room", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$dm-thread-trigger"); diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index 0350dea18c3..fe7ca8296e8 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -54,6 +54,7 @@ describe("matrix qa config", () => { const next = buildMatrixQaConfig({} as OpenClawConfig, { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", sutAccessToken: "sut-token", sutAccountId: "sut", sutUserId: "@sut:matrix-qa.test", @@ -81,6 +82,7 @@ describe("matrix qa config", () => { const next = buildMatrixQaConfig({} as OpenClawConfig, { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", overrides: { autoJoin: "allowlist", autoJoinAllowlist: [" !dm:matrix-qa.test ", "#ops:matrix-qa.test"], @@ -129,6 +131,7 @@ describe("matrix qa config", () => { it("builds an effective Matrix QA config snapshot for reporting", () => { const snapshot = buildMatrixQaConfigSnapshot({ driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", overrides: { autoJoin: "allowlist", autoJoinAllowlist: ["!ops:matrix-qa.test"], @@ -177,11 +180,26 @@ describe("matrix qa config", () => { expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("streaming=partial"); }); + it("resolves role-based Matrix sender allowlist overrides", () => { + const snapshot = buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", + overrides: { + groupAllowRoles: ["driver", "observer"], + }, + sutUserId: "@sut:matrix-qa.test", + topology, + }); + + expect(snapshot.groupAllowFrom).toEqual(["@driver:matrix-qa.test", "@observer:matrix-qa.test"]); + }); + it("rejects unknown room-key overrides", () => { expect(() => buildMatrixQaConfig({} as OpenClawConfig, { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", overrides: { groupsByKey: { ghost: { diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index 87d940c3aa0..c712341b9e2 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -7,6 +7,7 @@ export type MatrixQaDmPolicy = "allowlist" | "disabled" | "open" | "pairing"; export type MatrixQaGroupPolicy = "allowlist" | "disabled" | "open"; export type MatrixQaAutoJoinMode = "allowlist" | "always" | "off"; export type MatrixQaStreamingMode = "off" | "partial" | "quiet"; +export type MatrixQaActorRole = "driver" | "observer" | "sut"; export type MatrixQaGroupConfigOverrides = { enabled?: boolean; @@ -28,6 +29,7 @@ export type MatrixQaConfigOverrides = { dm?: MatrixQaDmConfigOverrides; encryption?: boolean; groupAllowFrom?: string[]; + groupAllowRoles?: MatrixQaActorRole[]; groupPolicy?: MatrixQaGroupPolicy; groupsByKey?: Record; replyToMode?: MatrixQaReplyToMode; @@ -62,6 +64,11 @@ export type MatrixQaConfigSnapshot = { threadReplies: MatrixQaThreadRepliesMode; }; +type MatrixQaChannelConfig = NonNullable["matrix"]; +type MatrixQaChannelAccountConfig = NonNullable< + NonNullable["accounts"] +>[string]; + type MatrixQaAccountDmConfig = | { enabled: false } | { @@ -172,6 +179,39 @@ function resolveMatrixQaAutoJoinAllowlist(params: { overrides?: MatrixQaConfigOv return normalizeMatrixQaAllowlist(params.overrides.autoJoinAllowlist); } +function resolveMatrixQaRoleAllowlist(params: { + roles?: MatrixQaActorRole[]; + driverUserId: string; + observerUserId: string; + sutUserId: string; +}) { + const roleToUserId = { + driver: params.driverUserId, + observer: params.observerUserId, + sut: params.sutUserId, + } satisfies Record; + return (params.roles ?? []).map((role) => roleToUserId[role]); +} + +function resolveMatrixQaGroupAllowFrom(params: { + driverUserId: string; + observerUserId: string; + overrides?: MatrixQaConfigOverrides; + sutUserId: string; +}) { + const explicitAllowFrom = params.overrides?.groupAllowFrom; + const roleAllowFrom = resolveMatrixQaRoleAllowlist({ + roles: params.overrides?.groupAllowRoles, + driverUserId: params.driverUserId, + observerUserId: params.observerUserId, + sutUserId: params.sutUserId, + }); + if (explicitAllowFrom !== undefined || params.overrides?.groupAllowRoles !== undefined) { + return normalizeMatrixQaAllowlist([...(explicitAllowFrom ?? []), ...roleAllowFrom]); + } + return [params.driverUserId]; +} + function formatMatrixQaBoolean(value: boolean) { return value ? "true" : "false"; } @@ -195,8 +235,60 @@ function buildMatrixQaAccountDmConfig(params: { }; } +function buildMatrixQaChannelAccountConfig(params: { + existingAccount?: MatrixQaChannelAccountConfig; + groups: Record; + homeserver: string; + overrides?: MatrixQaConfigOverrides; + snapshot: MatrixQaConfigSnapshot; + sutAccessToken: string; + sutDeviceId?: string; + sutUserId: string; +}): MatrixQaChannelAccountConfig { + const groupsConfig = Object.keys(params.groups).length > 0 ? { groups: params.groups } : {}; + const autoJoinConfig = + params.snapshot.autoJoin !== "off" ? { autoJoin: params.snapshot.autoJoin } : {}; + const autoJoinAllowlistConfig = + params.snapshot.autoJoin === "allowlist" && params.snapshot.autoJoinAllowlist.length > 0 + ? { autoJoinAllowlist: params.snapshot.autoJoinAllowlist } + : {}; + const blockStreamingConfig = + params.overrides?.blockStreaming !== undefined + ? { blockStreaming: params.snapshot.blockStreaming } + : {}; + const streamingConfig = + params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming } : {}; + + return { + ...params.existingAccount, + accessToken: params.sutAccessToken, + ...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}), + dm: buildMatrixQaAccountDmConfig({ + dmOverrides: params.overrides?.dm, + snapshot: params.snapshot, + }), + enabled: true, + encryption: params.snapshot.encryption, + groupAllowFrom: params.snapshot.groupAllowFrom, + groupPolicy: params.snapshot.groupPolicy, + ...groupsConfig, + homeserver: params.homeserver, + network: { + dangerouslyAllowPrivateNetwork: true, + }, + replyToMode: params.snapshot.replyToMode, + threadReplies: params.snapshot.threadReplies, + userId: params.sutUserId, + ...autoJoinConfig, + ...autoJoinAllowlistConfig, + ...blockStreamingConfig, + ...streamingConfig, + }; +} + export function buildMatrixQaConfigSnapshot(params: { driverUserId: string; + observerUserId: string; overrides?: MatrixQaConfigOverrides; sutUserId: string; topology: MatrixQaProvisionedTopology; @@ -207,9 +299,7 @@ export function buildMatrixQaConfigSnapshot(params: { blockStreaming: params.overrides?.blockStreaming ?? false, dm: resolveMatrixQaDmConfigSnapshot(params), encryption: params.overrides?.encryption ?? false, - groupAllowFrom: normalizeMatrixQaAllowlist( - params.overrides?.groupAllowFrom ?? [params.driverUserId], - ), + groupAllowFrom: resolveMatrixQaGroupAllowFrom(params), groupPolicy: params.overrides?.groupPolicy ?? "allowlist", groupsByKey: resolveMatrixQaGroupSnapshots({ overrides: params.overrides, @@ -241,6 +331,7 @@ export function buildMatrixQaConfig( params: { driverUserId: string; homeserver: string; + observerUserId: string; overrides?: MatrixQaConfigOverrides; sutAccessToken: string; sutAccountId: string; @@ -252,16 +343,12 @@ export function buildMatrixQaConfig( const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "matrix"])]; const snapshot = buildMatrixQaConfigSnapshot({ driverUserId: params.driverUserId, + observerUserId: params.observerUserId, overrides: params.overrides, sutUserId: params.sutUserId, topology: params.topology, }); const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey); - const dmOverrides = params.overrides?.dm; - const dm = buildMatrixQaAccountDmConfig({ - dmOverrides, - snapshot, - }); return { ...baseCfg, @@ -281,33 +368,16 @@ export function buildMatrixQaConfig( defaultAccount: params.sutAccountId, accounts: { ...baseCfg.channels?.matrix?.accounts, - [params.sutAccountId]: { - accessToken: params.sutAccessToken, - ...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}), - dm, - enabled: true, - encryption: snapshot.encryption, - groupAllowFrom: snapshot.groupAllowFrom, - groupPolicy: snapshot.groupPolicy, - ...(Object.keys(groups).length > 0 ? { groups } : {}), + [params.sutAccountId]: buildMatrixQaChannelAccountConfig({ + existingAccount: baseCfg.channels?.matrix?.accounts?.[params.sutAccountId], + groups, homeserver: params.homeserver, - network: { - dangerouslyAllowPrivateNetwork: true, - }, - replyToMode: snapshot.replyToMode, - threadReplies: snapshot.threadReplies, - userId: params.sutUserId, - ...(snapshot.autoJoin !== "off" ? { autoJoin: snapshot.autoJoin } : {}), - ...(snapshot.autoJoin === "allowlist" && snapshot.autoJoinAllowlist.length > 0 - ? { autoJoinAllowlist: snapshot.autoJoinAllowlist } - : {}), - ...(params.overrides?.blockStreaming !== undefined - ? { blockStreaming: snapshot.blockStreaming } - : {}), - ...(params.overrides?.streaming !== undefined - ? { streaming: params.overrides.streaming } - : {}), - }, + overrides: params.overrides, + snapshot, + sutAccessToken: params.sutAccessToken, + sutDeviceId: params.sutDeviceId, + sutUserId: params.sutUserId, + }), }, }, }, diff --git a/extensions/qa-matrix/src/substrate/events.test.ts b/extensions/qa-matrix/src/substrate/events.test.ts index 6b9b82dd635..a8e0cd346ab 100644 --- a/extensions/qa-matrix/src/substrate/events.test.ts +++ b/extensions/qa-matrix/src/substrate/events.test.ts @@ -67,6 +67,39 @@ describe("matrix observed event normalization", () => { ); }); + it("prefers m.new_content text for Matrix replacement events", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$replace", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + content: { + body: "* finalized", + msgtype: "m.text", + "m.new_content": { + body: "finalized", + msgtype: "m.text", + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$draft", + }, + }, + }), + ).toEqual( + expect.objectContaining({ + kind: "message", + eventId: "$replace", + body: "finalized", + msgtype: "m.text", + relatesTo: { + eventId: "$draft", + relType: "m.replace", + }, + }), + ); + }); + it("normalizes Matrix reaction events with target metadata", () => { expect( normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { diff --git a/extensions/qa-matrix/src/substrate/events.ts b/extensions/qa-matrix/src/substrate/events.ts index 0313057baed..c6bb1182094 100644 --- a/extensions/qa-matrix/src/substrate/events.ts +++ b/extensions/qa-matrix/src/substrate/events.ts @@ -49,6 +49,21 @@ function normalizeMentionUserIds(value: unknown) { : undefined; } +function resolveMatrixQaMessageContent( + content: Record, + relatesTo: Record | null, +) { + const newContentRaw = content["m.new_content"]; + const newContent = + typeof newContentRaw === "object" && newContentRaw !== null + ? (newContentRaw as Record) + : null; + if (relatesTo?.rel_type === "m.replace" && newContent) { + return newContent; + } + return content; +} + function resolveMatrixQaObservedEventKind(params: { msgtype?: string; type: string }) { if (params.type === "m.reaction") { return "reaction" as const; @@ -86,7 +101,10 @@ export function normalizeMatrixQaObservedEvent( typeof inReplyToRaw === "object" && inReplyToRaw !== null ? (inReplyToRaw as Record) : null; - const mentionsRaw = content["m.mentions"]; + const messageContent = resolveMatrixQaMessageContent(content, relatesTo); + const normalizedMsgtype = + typeof messageContent.msgtype === "string" ? messageContent.msgtype : msgtype; + const mentionsRaw = messageContent["m.mentions"] ?? content["m.mentions"]; const mentions = typeof mentionsRaw === "object" && mentionsRaw !== null ? (mentionsRaw as Record) @@ -100,7 +118,7 @@ export function normalizeMatrixQaObservedEvent( : undefined; return { - kind: resolveMatrixQaObservedEventKind({ msgtype, type }), + kind: resolveMatrixQaObservedEventKind({ msgtype: normalizedMsgtype, type }), roomId, eventId, sender: typeof event.sender === "string" ? event.sender : undefined, @@ -108,9 +126,10 @@ export function normalizeMatrixQaObservedEvent( type, originServerTs: typeof event.origin_server_ts === "number" ? Math.floor(event.origin_server_ts) : undefined, - body: typeof content.body === "string" ? content.body : undefined, - formattedBody: typeof content.formatted_body === "string" ? content.formatted_body : undefined, - msgtype, + body: typeof messageContent.body === "string" ? messageContent.body : undefined, + formattedBody: + typeof messageContent.formatted_body === "string" ? messageContent.formatted_body : undefined, + msgtype: normalizedMsgtype, membership: typeof content.membership === "string" ? content.membership : undefined, ...(relatesTo ? {