diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index d4a0952092a..58d941e6c22 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -144,6 +144,108 @@ describe("matrix live qa runtime", () => { }); }); + it("records default and per-scenario Matrix config snapshots in the summary", () => { + expect( + liveTesting.buildMatrixQaSummary({ + artifactPaths: { + observedEvents: "/tmp/observed.json", + report: "/tmp/report.md", + summary: "/tmp/summary.json", + }, + checks: [{ name: "Matrix harness ready", status: "pass" }], + config: { + default: liveTesting.buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA", + requireMention: true, + roomId: "!room:matrix-qa.test", + }, + ], + }, + }), + scenarios: [ + { + id: "matrix-room-thread-reply-override", + title: "Matrix threadReplies always keeps room replies threaded", + config: liveTesting.buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + overrides: { + threadReplies: "always", + }, + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA", + requireMention: true, + roomId: "!room:matrix-qa.test", + }, + ], + }, + }), + }, + ], + }, + finishedAt: "2026-04-10T10:05:00.000Z", + harness: { + baseUrl: "http://127.0.0.1:28008/", + composeFile: "/tmp/docker-compose.yml", + dmRoomIds: [], + image: "ghcr.io/matrix-construct/tuwunel:v1.5.1", + roomId: "!room:matrix-qa.test", + roomIds: ["!room:matrix-qa.test"], + serverName: "matrix-qa.test", + }, + observedEventCount: 0, + scenarios: [], + startedAt: "2026-04-10T10:00:00.000Z", + sutAccountId: "sut", + userIds: { + driver: "@driver:matrix-qa.test", + observer: "@observer:matrix-qa.test", + sut: "@sut:matrix-qa.test", + }, + }).config, + ).toMatchObject({ + default: { + replyToMode: "off", + threadReplies: "inbound", + }, + scenarios: [ + { + id: "matrix-room-thread-reply-override", + config: { + threadReplies: "always", + }, + }, + ], + }); + }); + it("preserves negative-scenario artifacts in the Matrix summary", () => { expect( liveTesting.buildMatrixQaSummary({ @@ -153,6 +255,18 @@ describe("matrix live qa runtime", () => { summary: "/tmp/summary.json", }, checks: [{ name: "Matrix harness ready", status: "pass" }], + config: { + default: liveTesting.buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + scenarios: [], + }, finishedAt: "2026-04-10T10:05:00.000Z", harness: { baseUrl: "http://127.0.0.1:28008/", diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index da9d560bec8..86517b09dba 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -14,7 +14,13 @@ import { } from "../../shared/live-lane-helpers.js"; import { buildMatrixQaObservedEventsArtifact } from "../../substrate/artifacts.js"; import { provisionMatrixQaRoom, type MatrixQaProvisionResult } from "../../substrate/client.js"; -import { buildMatrixQaConfig, type MatrixQaConfigOverrides } from "../../substrate/config.js"; +import { + buildMatrixQaConfig, + buildMatrixQaConfigSnapshot, + summarizeMatrixQaConfigSnapshot, + type MatrixQaConfigOverrides, + type MatrixQaConfigSnapshot, +} from "../../substrate/config.js"; import type { MatrixQaObservedEvent } from "../../substrate/events.js"; import { startMatrixQaHarness } from "../../substrate/harness.runtime.js"; import { resolveMatrixQaModels } from "./model-selection.js"; @@ -57,6 +63,14 @@ type MatrixQaScenarioResult = { type MatrixQaSummary = { checks: QaReportCheck[]; + config: { + default: MatrixQaConfigSnapshot; + scenarios: Array<{ + config: MatrixQaConfigSnapshot; + id: string; + title: string; + }>; + }; counts: { failed: number; passed: number; @@ -93,6 +107,20 @@ type MatrixQaArtifactPaths = { summary: string; }; +function countMatrixQaStatuses(entries: T[]) { + return { + failed: entries.filter((entry) => entry.status === "fail").length, + passed: entries.filter((entry) => entry.status === "pass").length, + }; +} + +function formatMatrixQaScenarioDetails(params: { details: string; configSummary?: string }) { + if (!params.configSummary) { + return params.details; + } + return [`effective config: ${params.configSummary}`, params.details].join("\n"); +} + export type MatrixQaRunResult = { observedEventsPath: string; outputDir: string; @@ -105,6 +133,7 @@ function buildMatrixQaSummary(params: { artifactPaths: MatrixQaArtifactPaths; canary?: MatrixQaCanaryArtifact; checks: QaReportCheck[]; + config: MatrixQaSummary["config"]; finishedAt: string; harness: MatrixQaSummary["harness"]; observedEventCount: number; @@ -113,16 +142,16 @@ function buildMatrixQaSummary(params: { sutAccountId: string; userIds: MatrixQaSummary["userIds"]; }): MatrixQaSummary { + const checkCounts = countMatrixQaStatuses(params.checks); + const scenarioCounts = countMatrixQaStatuses(params.scenarios); + return { checks: params.checks, + config: params.config, counts: { total: params.checks.length + params.scenarios.length, - passed: - params.checks.filter((check) => check.status === "pass").length + - params.scenarios.filter((scenario) => scenario.status === "pass").length, - failed: - params.checks.filter((check) => check.status === "fail").length + - params.scenarios.filter((scenario) => scenario.status === "fail").length, + passed: checkCounts.passed + scenarioCounts.passed, + failed: checkCounts.failed + scenarioCounts.failed, }, finishedAt: params.finishedAt, harness: params.harness, @@ -298,6 +327,8 @@ export async function runMatrixQaLive(params: { sutUserId: provisioning.sut.userId, topology: provisioning.topology, }; + const defaultConfigSnapshot = buildMatrixQaConfigSnapshot(gatewayConfigParams); + const scenarioConfigSnapshots: MatrixQaSummary["config"]["scenarios"] = []; try { const ensureGatewayHarness = async (overrides?: MatrixQaConfigOverrides) => { @@ -372,6 +403,19 @@ export async function runMatrixQaLive(params: { if (!canaryFailed) { for (const scenario of scenarios) { + const scenarioConfigSnapshot = buildMatrixQaConfigSnapshot({ + ...gatewayConfigParams, + overrides: scenario.configOverrides, + }); + const scenarioConfigSummary = + scenario.configOverrides === undefined + ? undefined + : summarizeMatrixQaConfigSnapshot(scenarioConfigSnapshot); + scenarioConfigSnapshots.push({ + config: scenarioConfigSnapshot, + id: scenario.id, + title: scenario.title, + }); try { const scenarioGateway = await ensureGatewayHarness(scenario.configOverrides); const result = await runMatrixQaScenario(scenario, { @@ -407,14 +451,20 @@ export async function runMatrixQaLive(params: { id: scenario.id, title: scenario.title, status: "pass", - details: result.details, + details: formatMatrixQaScenarioDetails({ + details: result.details, + configSummary: scenarioConfigSummary, + }), }); } catch (error) { scenarioResults.push({ id: scenario.id, title: scenario.title, status: "fail", - details: formatErrorMessage(error), + details: formatMatrixQaScenarioDetails({ + details: formatErrorMessage(error), + configSummary: scenarioConfigSummary, + }), }); } } @@ -464,6 +514,7 @@ export async function runMatrixQaLive(params: { notes: [ `roomId: ${provisioning.roomId}`, `roomIds: ${provisioning.topology.rooms.map((room) => room.roomId).join(", ")}`, + `default config: ${summarizeMatrixQaConfigSnapshot(defaultConfigSnapshot)}`, `driver: ${provisioning.driver.userId}`, `observer: ${provisioning.observer.userId}`, `sut: ${provisioning.sut.userId}`, @@ -475,6 +526,10 @@ export async function runMatrixQaLive(params: { artifactPaths, canary: canaryArtifact, checks, + config: { + default: defaultConfigSnapshot, + scenarios: scenarioConfigSnapshots, + }, finishedAt, harness: { baseUrl: harness.baseUrl, @@ -556,7 +611,9 @@ export const __testing = { buildMatrixQaSummary, MATRIX_QA_SCENARIOS, buildMatrixQaConfig, + buildMatrixQaConfigSnapshot, isMatrixAccountReady, resolveMatrixQaModels, + summarizeMatrixQaConfigSnapshot, waitForMatrixChannelReady, }; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts new file mode 100644 index 00000000000..ce0330b6412 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -0,0 +1,270 @@ +import { + collectLiveTransportStandardScenarioCoverage, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, +} from "../../shared/live-transport-scenarios.js"; +import { type MatrixQaConfigOverrides } from "../../substrate/config.js"; +import { + buildDefaultMatrixQaTopologySpec, + findMatrixQaProvisionedRoom, + mergeMatrixQaTopologySpecs, + type MatrixQaProvisionedTopology, + type MatrixQaTopologySpec, +} from "../../substrate/topology.js"; + +export type MatrixQaScenarioId = + | "matrix-thread-follow-up" + | "matrix-thread-isolation" + | "matrix-top-level-reply-shape" + | "matrix-room-thread-reply-override" + | "matrix-dm-reply-shape" + | "matrix-dm-shared-session-notice" + | "matrix-dm-thread-reply-override" + | "matrix-dm-per-room-session-override" + | "matrix-room-autojoin-invite" + | "matrix-secondary-room-reply" + | "matrix-secondary-room-open-trigger" + | "matrix-reaction-notification" + | "matrix-restart-resume" + | "matrix-room-membership-loss" + | "matrix-homeserver-restart-resume" + | "matrix-mention-gating" + | "matrix-allowlist-block"; + +export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { + configOverrides?: MatrixQaConfigOverrides; + topology?: MatrixQaTopologySpec; +}; + +export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm"; +export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared"; +export const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership"; +export const MATRIX_QA_SECONDARY_ROOM_KEY = "secondary"; + +function buildMatrixQaDmTopology( + rooms: Array<{ + key: string; + name: string; + }>, +): MatrixQaTopologySpec { + return { + defaultRoomKey: "main", + rooms: rooms.map((room) => ({ + key: room.key, + kind: "dm" as const, + members: ["driver", "sut"], + name: room.name, + })), + }; +} + +function buildMatrixQaSingleGroupTopology(params: { + key: string; + name: string; + requireMention: boolean; +}): MatrixQaTopologySpec { + return { + defaultRoomKey: "main", + rooms: [ + { + key: params.key, + kind: "group", + members: ["driver", "observer", "sut"], + name: params.name, + requireMention: params.requireMention, + }, + ], + }; +} + +const MATRIX_QA_DRIVER_DM_TOPOLOGY = buildMatrixQaDmTopology([ + { + key: MATRIX_QA_DRIVER_DM_ROOM_KEY, + name: "Matrix QA Driver/SUT DM", + }, +]); + +const MATRIX_QA_SHARED_DM_TOPOLOGY = buildMatrixQaDmTopology([ + { + key: MATRIX_QA_DRIVER_DM_ROOM_KEY, + name: "Matrix QA Driver/SUT DM", + }, + { + key: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + name: "Matrix QA Driver/SUT Shared DM", + }, +]); + +const MATRIX_QA_SECONDARY_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({ + key: MATRIX_QA_SECONDARY_ROOM_KEY, + name: "Matrix QA Secondary Room", + requireMention: true, +}); + +const MATRIX_QA_MEMBERSHIP_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({ + key: MATRIX_QA_MEMBERSHIP_ROOM_KEY, + name: "Matrix QA Membership Room", + requireMention: true, +}); + +export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ + { + id: "matrix-thread-follow-up", + standardId: "thread-follow-up", + timeoutMs: 60_000, + title: "Matrix thread follow-up reply", + }, + { + id: "matrix-thread-isolation", + standardId: "thread-isolation", + timeoutMs: 75_000, + title: "Matrix top-level reply stays out of prior thread", + }, + { + id: "matrix-top-level-reply-shape", + standardId: "top-level-reply-shape", + timeoutMs: 45_000, + title: "Matrix top-level reply keeps replyToMode off", + }, + { + id: "matrix-room-thread-reply-override", + timeoutMs: 45_000, + title: "Matrix threadReplies always keeps room replies threaded", + configOverrides: { + threadReplies: "always", + }, + }, + { + id: "matrix-dm-reply-shape", + timeoutMs: 45_000, + title: "Matrix DM reply stays top-level without a mention", + topology: MATRIX_QA_DRIVER_DM_TOPOLOGY, + }, + { + id: "matrix-dm-shared-session-notice", + timeoutMs: 45_000, + title: "Matrix shared DM sessions emit a cross-room notice", + topology: MATRIX_QA_SHARED_DM_TOPOLOGY, + }, + { + id: "matrix-dm-thread-reply-override", + timeoutMs: 45_000, + title: "Matrix DM thread override keeps DM replies threaded", + topology: MATRIX_QA_DRIVER_DM_TOPOLOGY, + configOverrides: { + dm: { + threadReplies: "always", + }, + threadReplies: "off", + }, + }, + { + id: "matrix-dm-per-room-session-override", + timeoutMs: 45_000, + title: "Matrix DM per-room session override suppresses cross-room notices", + topology: MATRIX_QA_SHARED_DM_TOPOLOGY, + configOverrides: { + dm: { + sessionScope: "per-room", + }, + }, + }, + { + id: "matrix-room-autojoin-invite", + timeoutMs: 60_000, + title: "Matrix invite auto-join accepts fresh group rooms", + configOverrides: { + autoJoin: "always", + groupPolicy: "open", + }, + }, + { + id: "matrix-secondary-room-reply", + timeoutMs: 45_000, + title: "Matrix secondary room reply stays scoped to that room", + topology: MATRIX_QA_SECONDARY_ROOM_TOPOLOGY, + }, + { + id: "matrix-secondary-room-open-trigger", + timeoutMs: 45_000, + title: "Matrix secondary room can opt out of mention gating", + topology: MATRIX_QA_SECONDARY_ROOM_TOPOLOGY, + configOverrides: { + groupsByKey: { + [MATRIX_QA_SECONDARY_ROOM_KEY]: { + requireMention: false, + }, + }, + }, + }, + { + id: "matrix-reaction-notification", + standardId: "reaction-observation", + timeoutMs: 45_000, + title: "Matrix reactions on bot replies are observed", + }, + { + id: "matrix-restart-resume", + standardId: "restart-resume", + timeoutMs: 60_000, + title: "Matrix lane resumes cleanly after gateway restart", + }, + { + id: "matrix-room-membership-loss", + timeoutMs: 75_000, + title: "Matrix room membership loss recovers after re-invite", + topology: MATRIX_QA_MEMBERSHIP_ROOM_TOPOLOGY, + }, + { + id: "matrix-homeserver-restart-resume", + timeoutMs: 75_000, + title: "Matrix lane resumes after homeserver restart", + }, + { + id: "matrix-mention-gating", + standardId: "mention-gating", + timeoutMs: 8_000, + title: "Matrix room message without mention does not trigger", + }, + { + id: "matrix-allowlist-block", + standardId: "allowlist-block", + timeoutMs: 8_000, + title: "Matrix allowlist blocks non-driver replies", + }, +]; + +export const MATRIX_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ + alwaysOnStandardScenarioIds: ["canary"], + scenarios: MATRIX_QA_SCENARIOS, +}); + +export function findMatrixQaScenarios(ids?: string[]) { + return selectLiveTransportScenarios({ + ids, + laneLabel: "Matrix", + scenarios: MATRIX_QA_SCENARIOS, + }); +} + +export function buildMatrixQaTopologyForScenarios(params: { + defaultRoomName: string; + scenarios: MatrixQaScenarioDefinition[]; +}): MatrixQaTopologySpec { + return mergeMatrixQaTopologySpecs([ + buildDefaultMatrixQaTopologySpec({ + defaultRoomName: params.defaultRoomName, + }), + ...params.scenarios.flatMap((scenario) => (scenario.topology ? [scenario.topology] : [])), + ]); +} + +export function resolveMatrixQaScenarioRoomId( + context: Pick<{ roomId: string; topology: MatrixQaProvisionedTopology }, "roomId" | "topology">, + roomKey?: string, +) { + if (!roomKey) { + return context.roomId; + } + return findMatrixQaProvisionedRoom(context.topology, roomKey).roomId; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts new file mode 100644 index 00000000000..83c8bdacb29 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -0,0 +1,1135 @@ +import { randomUUID } from "node:crypto"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { type MatrixQaProvisionedTopology } from "../../substrate/topology.js"; +import { + MATRIX_QA_DRIVER_DM_ROOM_KEY, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + MATRIX_QA_MEMBERSHIP_ROOM_KEY, + MATRIX_QA_SECONDARY_ROOM_KEY, + resolveMatrixQaScenarioRoomId, + type MatrixQaScenarioDefinition, +} from "./scenario-catalog.js"; +import type { + MatrixQaCanaryArtifact, + MatrixQaReplyArtifact, + MatrixQaScenarioExecution, +} from "./scenarios.js"; + +type MatrixQaActorId = "driver" | "observer"; + +export type MatrixQaSyncState = Partial>; + +export type MatrixQaScenarioContext = { + baseUrl: string; + canary?: MatrixQaCanaryArtifact; + driverAccessToken: string; + driverUserId: string; + observedEvents: MatrixQaObservedEvent[]; + observerAccessToken: string; + observerUserId: string; + restartGateway?: () => Promise; + roomId: string; + interruptTransport?: () => Promise; + sutAccessToken: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + topology: MatrixQaProvisionedTopology; +}; + +const NO_REPLY_WINDOW_MS = 8_000; + +export function buildMentionPrompt(sutUserId: string, token: string) { + return `${sutUserId} reply with only this exact marker: ${token}`; +} + +function buildExactMarkerPrompt(token: string) { + return `reply with only this exact marker: ${token}`; +} + +export function buildMatrixReplyArtifact( + event: MatrixQaObservedEvent, + token?: string, +): MatrixQaReplyArtifact { + const replyBody = event.body?.trim(); + return { + bodyPreview: replyBody?.slice(0, 200), + eventId: event.eventId, + mentions: event.mentions, + relatesTo: event.relatesTo, + sender: event.sender, + ...(token ? { tokenMatched: replyBody === token } : {}), + }; +} + +function buildMatrixNoticeArtifact(event: MatrixQaObservedEvent) { + return { + bodyPreview: event.body?.trim().slice(0, 200), + eventId: event.eventId, + sender: event.sender, + }; +} + +export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) { + return [ + `${label} event: ${artifact.eventId}`, + `${label} token matched: ${ + artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no" + }`, + `${label} rel_type: ${artifact.relatesTo?.relType ?? ""}`, + `${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? ""}`, + `${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`, + ]; +} + +function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) { + if (!artifact.tokenMatched) { + throw new Error(`${label} did not contain the expected token`); + } + if (artifact.relatesTo !== undefined) { + throw new Error(`${label} unexpectedly included relation metadata`); + } +} + +function assertThreadReplyArtifact( + artifact: MatrixQaReplyArtifact, + params: { + expectedRootEventId: string; + label: string; + }, +) { + if (!artifact.tokenMatched) { + throw new Error(`${params.label} did not contain the expected token`); + } + if (artifact.relatesTo?.relType !== "m.thread") { + throw new Error(`${params.label} did not use m.thread`); + } + if (artifact.relatesTo.eventId !== params.expectedRootEventId) { + throw new Error( + `${params.label} targeted ${artifact.relatesTo.eventId ?? ""} instead of ${params.expectedRootEventId}`, + ); + } + if (artifact.relatesTo.isFallingBack !== true) { + throw new Error(`${params.label} did not set is_falling_back`); + } + if (!artifact.relatesTo.inReplyToId) { + throw new Error(`${params.label} did not set m.in_reply_to`); + } +} + +export function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) { + return syncState[actorId]; +} + +export function writeMatrixQaSyncCursor( + syncState: MatrixQaSyncState, + actorId: MatrixQaActorId, + since?: string, +) { + if (since) { + syncState[actorId] = since; + } +} + +async function primeMatrixQaActorCursor(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + syncState: MatrixQaSyncState; +}) { + const client = createMatrixQaClient({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + }); + const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId); + if (existingSince) { + return { client, startSince: existingSince }; + } + const startSince = await client.primeRoom(); + if (!startSince) { + throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`); + } + return { client, startSince }; +} + +function advanceMatrixQaActorCursor(params: { + actorId: MatrixQaActorId; + syncState: MatrixQaSyncState; + nextSince?: string; + startSince: string; +}) { + writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); +} + +function createMatrixQaScenarioClient(params: { accessToken: string; baseUrl: string }) { + return createMatrixQaClient({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + }); +} + +async function runConfigurableTopLevelScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + observedEvents: MatrixQaObservedEvent[]; + replyPredicate?: ( + event: MatrixQaObservedEvent, + params: { driverEventId: string; token: string }, + ) => boolean; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + tokenPrefix: string; + withMention?: boolean; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`; + const body = + params.withMention === false + ? buildExactMarkerPrompt(token) + : buildMentionPrompt(params.sutUserId, token); + const driverEventId = await client.sendTextMessage({ + body, + ...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }), + roomId: params.roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message" && + (event.body ?? "").includes(token) && + (params.replyPredicate?.(event, { driverEventId, token }) ?? event.relatesTo === undefined), + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return { + body, + driverEventId, + reply: buildMatrixReplyArtifact(matched.event, token), + token, + }; +} + +async function runTopLevelMentionScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + tokenPrefix: string; + withMention?: boolean; +}) { + return await runConfigurableTopLevelScenario(params); +} + +async function waitForMembershipEvent(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + membership: "invite" | "join" | "leave"; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + stateKey: string; + syncState: MatrixQaSyncState; + timeoutMs: number; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.type === "m.room.member" && + event.stateKey === params.stateKey && + event.membership === params.membership, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return matched.event; +} + +async function runTopologyScopedTopLevelScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + actorUserId: string; + context: MatrixQaScenarioContext; + roomKey: string; + tokenPrefix: string; + withMention?: boolean; +}) { + const roomId = resolveMatrixQaScenarioRoomId(params.context, params.roomKey); + const result = await runTopLevelMentionScenario({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.context.baseUrl, + observedEvents: params.context.observedEvents, + roomId, + syncState: params.context.syncState, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: params.tokenPrefix, + withMention: params.withMention, + }); + assertTopLevelReplyArtifact(`reply in ${params.roomKey}`, result.reply); + return { + artifacts: { + actorUserId: params.actorUserId, + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: params.roomKey, + token: result.token, + triggerBody: result.body, + }, + details: [ + `room key: ${params.roomKey}`, + `room id: ${roomId}`, + `driver event: ${result.driverEventId}`, + `trigger sender: ${params.actorUserId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runThreadScenario(params: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.driverAccessToken, + actorId: "driver", + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const rootBody = `thread root ${randomUUID().slice(0, 8)}`; + const rootEventId = await client.sendTextMessage({ + body: rootBody, + roomId: params.roomId, + }); + const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`; + const driverEventId = await client.sendTextMessage({ + body: buildMentionPrompt(params.sutUserId, token), + mentionUserIds: [params.sutUserId], + replyToEventId: rootEventId, + roomId: params.roomId, + threadRootEventId: rootEventId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message" && + (event.body ?? "").includes(token) && + event.relatesTo?.relType === "m.thread" && + event.relatesTo.eventId === rootEventId, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return { + driverEventId, + reply: buildMatrixReplyArtifact(matched.event, token), + rootEventId, + token, + }; +} + +async function runRoomThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { + const result = await runConfigurableTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + replyPredicate: (event, params) => + event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_ROOM_THREAD", + }); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.driverEventId, + label: "room thread override reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + token: result.token, + triggerBody: result.body, + }, + details: [ + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runDmThreadReplyOverrideScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY); + const result = await runConfigurableTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + replyPredicate: (event, params) => + event.relatesTo?.relType === "m.thread" && event.relatesTo?.eventId === params.driverEventId, + roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_DM_THREAD", + withMention: false, + }); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.driverEventId, + label: "DM thread override reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, + token: result.token, + triggerBody: result.body, + }, + details: [ + `room key: ${MATRIX_QA_DRIVER_DM_ROOM_KEY}`, + `room id: ${roomId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runDmSharedSessionFlow(params: { + context: MatrixQaScenarioContext; + expectNotice: boolean; +}) { + const firstRoomId = resolveMatrixQaScenarioRoomId(params.context, MATRIX_QA_DRIVER_DM_ROOM_KEY); + const secondRoomId = resolveMatrixQaScenarioRoomId( + params.context, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + ); + + const firstResult = await runConfigurableTopLevelScenario({ + accessToken: params.context.driverAccessToken, + actorId: "driver", + baseUrl: params.context.baseUrl, + observedEvents: params.context.observedEvents, + roomId: firstRoomId, + syncState: params.context.syncState, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: "MATRIX_QA_DM_PRIMARY", + withMention: false, + }); + assertTopLevelReplyArtifact("primary DM reply", firstResult.reply); + + const replyClient = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const noticeClient = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const [replySince, noticeSince] = await Promise.all([ + replyClient.primeRoom(), + noticeClient.primeRoom(), + ]); + if (!replySince || !noticeSince) { + throw new Error("Matrix DM session scenario could not prime room cursors"); + } + + const secondToken = `MATRIX_QA_DM_SECONDARY_${randomUUID().slice(0, 8).toUpperCase()}`; + const secondBody = buildExactMarkerPrompt(secondToken); + const secondDriverEventId = await replyClient.sendTextMessage({ + body: secondBody, + roomId: secondRoomId, + }); + + const [replyResult, noticeResult] = await Promise.all([ + replyClient.waitForRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === secondRoomId && + event.sender === params.context.sutUserId && + event.type === "m.room.message" && + event.kind === "message" && + (event.body ?? "").includes(secondToken), + roomId: secondRoomId, + since: replySince, + timeoutMs: params.context.timeoutMs, + }), + noticeClient.waitForOptionalRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === secondRoomId && + event.sender === params.context.sutUserId && + event.kind === "notice" && + typeof event.body === "string" && + event.body.includes("channels.matrix.dm.sessionScope"), + roomId: secondRoomId, + since: noticeSince, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs), + }), + ]); + + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: params.context.syncState, + nextSince: replyResult.since, + startSince: replySince, + }); + + const secondReply = buildMatrixReplyArtifact(replyResult.event, secondToken); + assertTopLevelReplyArtifact("secondary DM reply", secondReply); + const noticeArtifact = noticeResult.matched + ? buildMatrixNoticeArtifact(noticeResult.event) + : undefined; + + if (params.expectNotice && !noticeArtifact) { + throw new Error( + "Matrix shared DM session scenario did not emit the expected cross-room notice", + ); + } + if (!params.expectNotice && noticeArtifact) { + throw new Error( + "Matrix per-room DM session scenario unexpectedly emitted a shared-session notice", + ); + } + + return { + noticeArtifact, + secondBody, + secondDriverEventId, + secondReply, + secondRoomId, + secondToken, + }; +} + +async function runDmSharedSessionNoticeScenario(context: MatrixQaScenarioContext) { + const result = await runDmSharedSessionFlow({ + context, + expectNotice: true, + }); + return { + artifacts: { + driverEventId: result.secondDriverEventId, + noticeBodyPreview: result.noticeArtifact?.bodyPreview, + noticeEventId: result.noticeArtifact?.eventId, + reply: result.secondReply, + roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + token: result.secondToken, + triggerBody: result.secondBody, + }, + details: [ + `primary room id: ${resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY)}`, + `secondary room id: ${result.secondRoomId}`, + `secondary driver event: ${result.secondDriverEventId}`, + `notice event: ${result.noticeArtifact?.eventId ?? ""}`, + `notice preview: ${result.noticeArtifact?.bodyPreview ?? ""}`, + ...buildMatrixReplyDetails("secondary reply", result.secondReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runDmPerRoomSessionOverrideScenario(context: MatrixQaScenarioContext) { + const result = await runDmSharedSessionFlow({ + context, + expectNotice: false, + }); + return { + artifacts: { + driverEventId: result.secondDriverEventId, + reply: result.secondReply, + roomKey: MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + token: result.secondToken, + triggerBody: result.secondBody, + }, + details: [ + `primary room id: ${resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY)}`, + `secondary room id: ${result.secondRoomId}`, + `secondary driver event: ${result.secondDriverEventId}`, + "shared-session notice: suppressed", + ...buildMatrixReplyDetails("secondary reply", result.secondReply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runRoomAutoJoinInviteScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const dynamicRoomId = await client.createPrivateRoom({ + inviteUserIds: [context.observerUserId, context.sutUserId], + name: `Matrix QA AutoJoin ${randomUUID().slice(0, 8)}`, + }); + const joinResult = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === dynamicRoomId && + event.type === "m.room.member" && + event.stateKey === context.sutUserId && + event.membership === "join", + roomId: dynamicRoomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const joinEvent = joinResult.event; + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: joinResult.since, + startSince, + }); + + const result = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: dynamicRoomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_AUTOJOIN", + }); + assertTopLevelReplyArtifact("auto-join room reply", result.reply); + + return { + artifacts: { + driverEventId: result.driverEventId, + joinedRoomId: dynamicRoomId, + membershipJoinEventId: joinEvent.eventId, + reply: result.reply, + token: result.token, + triggerBody: result.body, + }, + details: [ + `joined room id: ${dynamicRoomId}`, + `join event: ${joinEvent.eventId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runNoReplyExpectedScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + actorUserId: string; + baseUrl: string; + body: string; + mentionUserIds?: string[]; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; + token: string; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const driverEventId = await client.sendTextMessage({ + body: params.body, + ...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}), + roomId: params.roomId, + }); + const result = await client.waitForOptionalRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message", + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + if (result.matched) { + const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token); + throw new Error( + [ + `unexpected SUT reply from ${params.sutUserId}`, + `trigger sender: ${params.actorUserId}`, + ...buildMatrixReplyDetails("unexpected reply", unexpectedReply), + ].join("\n"), + ); + } + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: result.since, + startSince, + }); + return { + artifacts: { + actorUserId: params.actorUserId, + driverEventId, + expectedNoReplyWindowMs: params.timeoutMs, + token: params.token, + triggerBody: params.body, + }, + details: [ + `trigger event: ${driverEventId}`, + `trigger sender: ${params.actorUserId}`, + `waited ${params.timeoutMs}ms with no SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runMembershipLossScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEMBERSHIP_ROOM_KEY); + const driverClient = createMatrixQaScenarioClient({ + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + }); + const sutClient = createMatrixQaScenarioClient({ + accessToken: context.sutAccessToken, + baseUrl: context.baseUrl, + }); + + await driverClient.kickUserFromRoom({ + reason: "matrix qa membership loss", + roomId, + userId: context.sutUserId, + }); + const leaveEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "leave", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const noReplyToken = `MATRIX_QA_MEMBERSHIP_LOSS_${randomUUID().slice(0, 8).toUpperCase()}`; + await runNoReplyExpectedScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + baseUrl: context.baseUrl, + body: buildMentionPrompt(context.sutUserId, noReplyToken), + mentionUserIds: [context.sutUserId], + observedEvents: context.observedEvents, + roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + token: noReplyToken, + }); + + await driverClient.inviteUserToRoom({ + roomId, + userId: context.sutUserId, + }); + await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "invite", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + await sutClient.joinRoom(roomId); + const joinEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "join", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const recovered = await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_MEMBERSHIP_ROOM_KEY, + tokenPrefix: "MATRIX_QA_MEMBERSHIP_RETURN", + }); + + return { + artifacts: { + ...recovered.artifacts, + membershipJoinEventId: joinEvent.eventId, + membershipLeaveEventId: leaveEvent.eventId, + recoveredDriverEventId: recovered.artifacts?.driverEventId, + recoveredReply: recovered.artifacts?.reply, + }, + details: [ + `room key: ${MATRIX_QA_MEMBERSHIP_ROOM_KEY}`, + `room id: ${roomId}`, + `leave event: ${leaveEvent.eventId}`, + `join event: ${joinEvent.eventId}`, + recovered.details, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { + const reactionTargetEventId = context.canary?.reply.eventId?.trim(); + if (!reactionTargetEventId) { + throw new Error("Matrix reaction scenario requires a canary reply event id"); + } + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + syncState: context.syncState, + }); + const reactionEmoji = "👍"; + const reactionEventId = await client.sendReaction({ + emoji: reactionEmoji, + messageId: reactionTargetEventId, + roomId: context.roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.driverUserId && + event.type === "m.reaction" && + event.eventId === reactionEventId && + event.reaction?.eventId === reactionTargetEventId && + event.reaction?.key === reactionEmoji, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + return { + artifacts: { + reactionEmoji, + reactionEventId, + reactionTargetEventId, + }, + details: [ + `reaction event: ${reactionEventId}`, + `reaction target: ${reactionTargetEventId}`, + `reaction emoji: ${reactionEmoji}`, + `observed reaction key: ${matched.event.reaction?.key ?? ""}`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.interruptTransport) { + throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); + } + await context.interruptTransport(); + const resumed = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_HOMESERVER", + }); + assertTopLevelReplyArtifact("post-homeserver-restart reply", resumed.reply); + return { + artifacts: { + driverEventId: resumed.driverEventId, + reply: resumed.reply, + token: resumed.token, + transportInterruption: "homeserver-restart", + }, + details: [ + "transport interruption: homeserver-restart", + `driver event: ${resumed.driverEventId}`, + ...buildMatrixReplyDetails("reply", resumed.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +async function runRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.restartGateway) { + throw new Error("Matrix restart scenario requires a gateway restart callback"); + } + await context.restartGateway(); + const result = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_RESTART", + }); + assertTopLevelReplyArtifact("post-restart reply", result.reply); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + restartSignal: "SIGUSR1", + token: result.token, + }, + details: [ + "restart signal: SIGUSR1", + `post-restart driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runMatrixQaCanary(params: { + baseUrl: string; + driverAccessToken: string; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + syncState: MatrixQaSyncState; + sutUserId: string; + timeoutMs: number; +}) { + const canary = await runTopLevelMentionScenario({ + accessToken: params.driverAccessToken, + actorId: "driver", + baseUrl: params.baseUrl, + observedEvents: params.observedEvents, + roomId: params.roomId, + syncState: params.syncState, + sutUserId: params.sutUserId, + timeoutMs: params.timeoutMs, + tokenPrefix: "MATRIX_QA_CANARY", + }); + assertTopLevelReplyArtifact("canary reply", canary.reply); + return canary; +} + +export async function runMatrixQaScenario( + scenario: MatrixQaScenarioDefinition, + context: MatrixQaScenarioContext, +): Promise { + switch (scenario.id) { + case "matrix-thread-follow-up": { + const result = await runThreadScenario(context); + assertThreadReplyArtifact(result.reply, { + expectedRootEventId: result.rootEventId, + label: "thread reply", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + rootEventId: result.rootEventId, + token: result.token, + }, + details: [ + `root event: ${result.rootEventId}`, + `driver thread event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + }; + } + case "matrix-thread-isolation": { + const threadPhase = await runThreadScenario(context); + assertThreadReplyArtifact(threadPhase.reply, { + expectedRootEventId: threadPhase.rootEventId, + label: "thread isolation reply", + }); + const topLevelPhase = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_TOPLEVEL", + }); + assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply); + return { + artifacts: { + threadDriverEventId: threadPhase.driverEventId, + threadReply: threadPhase.reply, + threadRootEventId: threadPhase.rootEventId, + threadToken: threadPhase.token, + topLevelDriverEventId: topLevelPhase.driverEventId, + topLevelReply: topLevelPhase.reply, + topLevelToken: topLevelPhase.token, + }, + details: [ + `thread root event: ${threadPhase.rootEventId}`, + `thread driver event: ${threadPhase.driverEventId}`, + ...buildMatrixReplyDetails("thread reply", threadPhase.reply), + `top-level driver event: ${topLevelPhase.driverEventId}`, + ...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply), + ].join("\n"), + }; + } + case "matrix-top-level-reply-shape": { + const result = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_TOPLEVEL", + }); + assertTopLevelReplyArtifact("top-level reply", result.reply); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + token: result.token, + }, + details: [ + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + }; + } + case "matrix-room-thread-reply-override": + return await runRoomThreadReplyOverrideScenario(context); + case "matrix-dm-reply-shape": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, + tokenPrefix: "MATRIX_QA_DM", + withMention: false, + }); + case "matrix-dm-shared-session-notice": + return await runDmSharedSessionNoticeScenario(context); + case "matrix-dm-thread-reply-override": + return await runDmThreadReplyOverrideScenario(context); + case "matrix-dm-per-room-session-override": + return await runDmPerRoomSessionOverrideScenario(context); + case "matrix-room-autojoin-invite": + return await runRoomAutoJoinInviteScenario(context); + case "matrix-secondary-room-reply": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, + tokenPrefix: "MATRIX_QA_SECONDARY", + }); + case "matrix-secondary-room-open-trigger": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, + tokenPrefix: "MATRIX_QA_SECONDARY_OPEN", + withMention: false, + }); + case "matrix-reaction-notification": + return await runReactionNotificationScenario(context); + case "matrix-restart-resume": + return await runRestartResumeScenario(context); + case "matrix-room-membership-loss": + return await runMembershipLossScenario(context); + case "matrix-homeserver-restart-resume": + return await runHomeserverRestartResumeScenario(context); + case "matrix-mention-gating": { + const token = `MATRIX_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return await runNoReplyExpectedScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + baseUrl: context.baseUrl, + body: buildExactMarkerPrompt(token), + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + token, + }); + } + case "matrix-allowlist-block": { + const token = `MATRIX_QA_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`; + return await runNoReplyExpectedScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + actorUserId: context.observerUserId, + baseUrl: context.baseUrl, + body: buildMentionPrompt(context.sutUserId, token), + mentionUserIds: [context.sutUserId], + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + token, + }); + } + default: { + const exhaustiveScenarioId: never = scenario.id; + return exhaustiveScenarioId; + } + } +} diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 18c099aef0e..872a2104fd5 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -27,7 +27,12 @@ describe("matrix live qa scenarios", () => { "matrix-thread-follow-up", "matrix-thread-isolation", "matrix-top-level-reply-shape", + "matrix-room-thread-reply-override", "matrix-dm-reply-shape", + "matrix-dm-shared-session-notice", + "matrix-dm-thread-reply-override", + "matrix-dm-per-room-session-override", + "matrix-room-autojoin-invite", "matrix-secondary-room-reply", "matrix-secondary-room-open-trigger", "matrix-reaction-notification", @@ -405,6 +410,481 @@ describe("matrix live qa scenarios", () => { ); }); + it("uses room thread override scenarios against the main room", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$room-thread-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$sut-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), + relatesTo: { + relType: "m.thread", + eventId: "$room-thread-trigger", + inReplyToId: "$room-thread-trigger", + isFallingBack: true, + }, + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-room-thread-reply-override", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + driverEventId: "$room-thread-trigger", + reply: { + relatesTo: { + relType: "m.thread", + eventId: "$room-thread-trigger", + }, + }, + }, + }); + }); + + it("uses DM thread override scenarios against the provisioned DM room", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$dm-thread-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!dm:matrix-qa.test", + eventId: "$sut-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + relatesTo: { + relType: "m.thread", + eventId: "$dm-thread-trigger", + inReplyToId: "$dm-thread-trigger", + isFallingBack: true, + }, + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-dm-thread-reply-override", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + driverEventId: "$dm-thread-trigger", + reply: { + relatesTo: { + relType: "m.thread", + eventId: "$dm-thread-trigger", + }, + }, + }, + }); + }); + + it("surfaces the shared DM session notice in the secondary DM room", async () => { + const primePrimaryRoom = vi.fn().mockResolvedValue("driver-primary-sync-start"); + const sendPrimaryTextMessage = vi.fn().mockResolvedValue("$dm-primary-trigger"); + const waitPrimaryReply = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!dm:matrix-qa.test", + eventId: "$sut-primary-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendPrimaryTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + }, + since: "driver-primary-sync-next", + })); + const primeSecondaryReplyRoom = vi.fn().mockResolvedValue("driver-secondary-reply-sync-start"); + const sendSecondaryTextMessage = vi.fn().mockResolvedValue("$dm-secondary-trigger"); + const waitSecondaryReply = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!dm-shared:matrix-qa.test", + eventId: "$sut-secondary-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendSecondaryTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + }, + since: "driver-secondary-sync-next", + })); + const primeSecondaryNoticeRoom = vi + .fn() + .mockResolvedValue("driver-secondary-notice-sync-start"); + const waitSecondaryNotice = vi.fn().mockImplementation(async () => ({ + matched: true, + event: { + kind: "notice", + roomId: "!dm-shared:matrix-qa.test", + eventId: "$shared-notice", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: "This Matrix DM is sharing a session with another Matrix DM room. Set channels.matrix.dm.sessionScope to per-room to isolate each Matrix DM room.", + }, + since: "driver-secondary-notice-sync-next", + })); + + createMatrixQaClient + .mockReturnValueOnce({ + primeRoom: primePrimaryRoom, + sendTextMessage: sendPrimaryTextMessage, + waitForRoomEvent: waitPrimaryReply, + }) + .mockReturnValueOnce({ + primeRoom: primeSecondaryReplyRoom, + sendTextMessage: sendSecondaryTextMessage, + waitForRoomEvent: waitSecondaryReply, + }) + .mockReturnValueOnce({ + primeRoom: primeSecondaryNoticeRoom, + waitForOptionalRoomEvent: waitSecondaryNotice, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-dm-shared-session-notice", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "Shared DM", + requireMention: false, + roomId: "!dm-shared:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + noticeEventId: "$shared-notice", + roomKey: scenarioTesting.MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + }, + }); + + expect(sendPrimaryTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("reply with only this exact marker:"), + roomId: "!dm:matrix-qa.test", + }); + expect(sendSecondaryTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("reply with only this exact marker:"), + roomId: "!dm-shared:matrix-qa.test", + }); + expect(waitSecondaryNotice).toHaveBeenCalledWith( + expect.objectContaining({ + roomId: "!dm-shared:matrix-qa.test", + }), + ); + }); + + it("suppresses the shared DM notice when sessionScope is per-room", async () => { + const primePrimaryRoom = vi.fn().mockResolvedValue("driver-primary-sync-start"); + const sendPrimaryTextMessage = vi.fn().mockResolvedValue("$dm-primary-trigger"); + const waitPrimaryReply = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!dm:matrix-qa.test", + eventId: "$sut-primary-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendPrimaryTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + }, + since: "driver-primary-sync-next", + })); + const primeSecondaryReplyRoom = vi.fn().mockResolvedValue("driver-secondary-reply-sync-start"); + const sendSecondaryTextMessage = vi.fn().mockResolvedValue("$dm-secondary-trigger"); + const waitSecondaryReply = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!dm-shared:matrix-qa.test", + eventId: "$sut-secondary-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendSecondaryTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + }, + since: "driver-secondary-sync-next", + })); + const primeSecondaryNoticeRoom = vi + .fn() + .mockResolvedValue("driver-secondary-notice-sync-start"); + const waitSecondaryNotice = vi.fn().mockImplementation(async () => ({ + matched: false, + since: "driver-secondary-notice-sync-next", + })); + + createMatrixQaClient + .mockReturnValueOnce({ + primeRoom: primePrimaryRoom, + sendTextMessage: sendPrimaryTextMessage, + waitForRoomEvent: waitPrimaryReply, + }) + .mockReturnValueOnce({ + primeRoom: primeSecondaryReplyRoom, + sendTextMessage: sendSecondaryTextMessage, + waitForRoomEvent: waitSecondaryReply, + }) + .mockReturnValueOnce({ + primeRoom: primeSecondaryNoticeRoom, + waitForOptionalRoomEvent: waitSecondaryNotice, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-dm-per-room-session-override", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "Shared DM", + requireMention: false, + roomId: "!dm-shared:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + roomKey: scenarioTesting.MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + }, + }); + + expect(waitSecondaryNotice).toHaveBeenCalledTimes(1); + }); + + it("auto-joins a freshly invited Matrix group room before replying", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const createPrivateRoom = vi.fn().mockResolvedValue("!autojoin:matrix-qa.test"); + const sendTextMessage = vi.fn().mockResolvedValue("$autojoin-trigger"); + const waitForRoomEvent = vi + .fn() + .mockImplementationOnce(async () => ({ + event: { + kind: "membership", + roomId: "!autojoin:matrix-qa.test", + eventId: "$autojoin-join", + sender: "@sut:matrix-qa.test", + stateKey: "@sut:matrix-qa.test", + type: "m.room.member", + membership: "join", + }, + since: "driver-sync-join", + })) + .mockImplementationOnce(async () => ({ + event: { + kind: "message", + roomId: "!autojoin:matrix-qa.test", + eventId: "$sut-autojoin-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + createPrivateRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-room-autojoin-invite", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + joinedRoomId: "!autojoin:matrix-qa.test", + membershipJoinEventId: "$autojoin-join", + }, + }); + + expect(createPrivateRoom).toHaveBeenCalledWith({ + inviteUserIds: ["@observer:matrix-qa.test", "@sut:matrix-qa.test"], + name: expect.stringContaining("Matrix QA AutoJoin"), + }); + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("@sut:matrix-qa.test reply with only this exact marker:"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!autojoin:matrix-qa.test", + }); + }); + it("runs the secondary-room scenario against the provisioned secondary room", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$secondary-trigger"); diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.ts b/extensions/qa-matrix/src/runners/contract/scenarios.ts index 8a54ab5e15b..68fff045236 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.ts @@ -1,37 +1,41 @@ -import { randomUUID } from "node:crypto"; -import { - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, -} from "../../shared/live-transport-scenarios.js"; -import { createMatrixQaClient } from "../../substrate/client.js"; -import { type MatrixQaConfigOverrides } from "../../substrate/config.js"; import type { MatrixQaObservedEvent } from "../../substrate/events.js"; import { - buildDefaultMatrixQaTopologySpec, - findMatrixQaProvisionedRoom, - mergeMatrixQaTopologySpecs, - type MatrixQaProvisionedTopology, - type MatrixQaTopologySpec, -} from "../../substrate/topology.js"; + MATRIX_QA_DRIVER_DM_ROOM_KEY, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + MATRIX_QA_MEMBERSHIP_ROOM_KEY, + MATRIX_QA_SCENARIOS, + MATRIX_QA_SECONDARY_ROOM_KEY, + MATRIX_QA_STANDARD_SCENARIO_IDS, + buildMatrixQaTopologyForScenarios, + findMatrixQaScenarios, + resolveMatrixQaScenarioRoomId, + type MatrixQaScenarioDefinition, + type MatrixQaScenarioId, +} from "./scenario-catalog.js"; +import { + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + buildMentionPrompt, + readMatrixQaSyncCursor, + runMatrixQaCanary, + runMatrixQaScenario, + writeMatrixQaSyncCursor, + type MatrixQaScenarioContext, + type MatrixQaSyncState, +} from "./scenario-runtime.js"; -export type MatrixQaScenarioId = - | "matrix-thread-follow-up" - | "matrix-thread-isolation" - | "matrix-top-level-reply-shape" - | "matrix-dm-reply-shape" - | "matrix-secondary-room-reply" - | "matrix-secondary-room-open-trigger" - | "matrix-reaction-notification" - | "matrix-restart-resume" - | "matrix-room-membership-loss" - | "matrix-homeserver-restart-resume" - | "matrix-mention-gating" - | "matrix-allowlist-block"; - -export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { - configOverrides?: MatrixQaConfigOverrides; - topology?: MatrixQaTopologySpec; +export type { MatrixQaScenarioDefinition, MatrixQaScenarioId }; +export { + MATRIX_QA_SCENARIOS, + MATRIX_QA_STANDARD_SCENARIO_IDS, + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + buildMatrixQaTopologyForScenarios, + buildMentionPrompt, + findMatrixQaScenarios, + resolveMatrixQaScenarioRoomId, + runMatrixQaCanary, + runMatrixQaScenario, }; export type MatrixQaReplyArtifact = { @@ -73,7 +77,10 @@ export type MatrixQaScenarioArtifacts = { triggerBody?: string; membershipJoinEventId?: string; membershipLeaveEventId?: string; + noticeBodyPreview?: string; + noticeEventId?: string; transportInterruption?: string; + joinedRoomId?: string; }; export type MatrixQaScenarioExecution = { @@ -81,966 +88,11 @@ export type MatrixQaScenarioExecution = { details: string; }; -type MatrixQaActorId = "driver" | "observer"; - -type MatrixQaSyncState = Partial>; - -type MatrixQaScenarioContext = { - baseUrl: string; - canary?: MatrixQaCanaryArtifact; - driverAccessToken: string; - driverUserId: string; - observedEvents: MatrixQaObservedEvent[]; - observerAccessToken: string; - observerUserId: string; - restartGateway?: () => Promise; - roomId: string; - interruptTransport?: () => Promise; - sutAccessToken: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - topology: MatrixQaProvisionedTopology; -}; - -const NO_REPLY_WINDOW_MS = 8_000; -const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm"; -const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership"; -const MATRIX_QA_SECONDARY_ROOM_KEY = "secondary"; - -export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ - { - id: "matrix-thread-follow-up", - standardId: "thread-follow-up", - timeoutMs: 60_000, - title: "Matrix thread follow-up reply", - }, - { - id: "matrix-thread-isolation", - standardId: "thread-isolation", - timeoutMs: 75_000, - title: "Matrix top-level reply stays out of prior thread", - }, - { - id: "matrix-top-level-reply-shape", - standardId: "top-level-reply-shape", - timeoutMs: 45_000, - title: "Matrix top-level reply keeps replyToMode off", - }, - { - id: "matrix-dm-reply-shape", - timeoutMs: 45_000, - title: "Matrix DM reply stays top-level without a mention", - topology: { - defaultRoomKey: "main", - rooms: [ - { - key: MATRIX_QA_DRIVER_DM_ROOM_KEY, - kind: "dm", - members: ["driver", "sut"], - name: "Matrix QA Driver/SUT DM", - }, - ], - }, - }, - { - id: "matrix-secondary-room-reply", - timeoutMs: 45_000, - title: "Matrix secondary room reply stays scoped to that room", - topology: { - defaultRoomKey: "main", - rooms: [ - { - key: MATRIX_QA_SECONDARY_ROOM_KEY, - kind: "group", - members: ["driver", "observer", "sut"], - name: "Matrix QA Secondary Room", - requireMention: true, - }, - ], - }, - }, - { - id: "matrix-secondary-room-open-trigger", - timeoutMs: 45_000, - title: "Matrix secondary room can opt out of mention gating", - topology: { - defaultRoomKey: "main", - rooms: [ - { - key: MATRIX_QA_SECONDARY_ROOM_KEY, - kind: "group", - members: ["driver", "observer", "sut"], - name: "Matrix QA Secondary Room", - requireMention: true, - }, - ], - }, - configOverrides: { - groupsByKey: { - [MATRIX_QA_SECONDARY_ROOM_KEY]: { - requireMention: false, - }, - }, - }, - }, - { - id: "matrix-reaction-notification", - standardId: "reaction-observation", - timeoutMs: 45_000, - title: "Matrix reactions on bot replies are observed", - }, - { - id: "matrix-restart-resume", - standardId: "restart-resume", - timeoutMs: 60_000, - title: "Matrix lane resumes cleanly after gateway restart", - }, - { - id: "matrix-room-membership-loss", - timeoutMs: 75_000, - title: "Matrix room membership loss recovers after re-invite", - topology: { - defaultRoomKey: "main", - rooms: [ - { - key: MATRIX_QA_MEMBERSHIP_ROOM_KEY, - kind: "group", - members: ["driver", "observer", "sut"], - name: "Matrix QA Membership Room", - requireMention: true, - }, - ], - }, - }, - { - id: "matrix-homeserver-restart-resume", - timeoutMs: 75_000, - title: "Matrix lane resumes after homeserver restart", - }, - { - id: "matrix-mention-gating", - standardId: "mention-gating", - timeoutMs: NO_REPLY_WINDOW_MS, - title: "Matrix room message without mention does not trigger", - }, - { - id: "matrix-allowlist-block", - standardId: "allowlist-block", - timeoutMs: NO_REPLY_WINDOW_MS, - title: "Matrix allowlist blocks non-driver replies", - }, -]; - -export const MATRIX_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ - alwaysOnStandardScenarioIds: ["canary"], - scenarios: MATRIX_QA_SCENARIOS, -}); - -export function findMatrixQaScenarios(ids?: string[]) { - return selectLiveTransportScenarios({ - ids, - laneLabel: "Matrix", - scenarios: MATRIX_QA_SCENARIOS, - }); -} - -export function buildMatrixQaTopologyForScenarios(params: { - defaultRoomName: string; - scenarios: MatrixQaScenarioDefinition[]; -}): MatrixQaTopologySpec { - return mergeMatrixQaTopologySpecs([ - buildDefaultMatrixQaTopologySpec({ - defaultRoomName: params.defaultRoomName, - }), - ...params.scenarios.flatMap((scenario) => (scenario.topology ? [scenario.topology] : [])), - ]); -} - -export function resolveMatrixQaScenarioRoomId( - context: Pick, - roomKey?: string, -) { - if (!roomKey) { - return context.roomId; - } - return findMatrixQaProvisionedRoom(context.topology, roomKey).roomId; -} - -export function buildMentionPrompt(sutUserId: string, token: string) { - return `${sutUserId} reply with only this exact marker: ${token}`; -} - -function buildExactMarkerPrompt(token: string) { - return `reply with only this exact marker: ${token}`; -} - -function buildMatrixReplyArtifact( - event: MatrixQaObservedEvent, - token?: string, -): MatrixQaReplyArtifact { - const replyBody = event.body?.trim(); - return { - bodyPreview: replyBody?.slice(0, 200), - eventId: event.eventId, - mentions: event.mentions, - relatesTo: event.relatesTo, - sender: event.sender, - ...(token ? { tokenMatched: replyBody === token } : {}), - }; -} - -export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) { - return [ - `${label} event: ${artifact.eventId}`, - `${label} token matched: ${ - artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no" - }`, - `${label} rel_type: ${artifact.relatesTo?.relType ?? ""}`, - `${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? ""}`, - `${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`, - ]; -} - -function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) { - if (!artifact.tokenMatched) { - throw new Error(`${label} did not contain the expected token`); - } - if (artifact.relatesTo !== undefined) { - throw new Error(`${label} unexpectedly included relation metadata`); - } -} - -function assertThreadReplyArtifact( - artifact: MatrixQaReplyArtifact, - params: { - expectedRootEventId: string; - label: string; - }, -) { - if (!artifact.tokenMatched) { - throw new Error(`${params.label} did not contain the expected token`); - } - if (artifact.relatesTo?.relType !== "m.thread") { - throw new Error(`${params.label} did not use m.thread`); - } - if (artifact.relatesTo.eventId !== params.expectedRootEventId) { - throw new Error( - `${params.label} targeted ${artifact.relatesTo.eventId ?? ""} instead of ${params.expectedRootEventId}`, - ); - } - if (artifact.relatesTo.isFallingBack !== true) { - throw new Error(`${params.label} did not set is_falling_back`); - } - if (!artifact.relatesTo.inReplyToId) { - throw new Error(`${params.label} did not set m.in_reply_to`); - } -} - -function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) { - return syncState[actorId]; -} - -function writeMatrixQaSyncCursor( - syncState: MatrixQaSyncState, - actorId: MatrixQaActorId, - since?: string, -) { - if (since) { - syncState[actorId] = since; - } -} - -async function primeMatrixQaActorCursor(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - syncState: MatrixQaSyncState; -}) { - const client = createMatrixQaClient({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, - }); - const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId); - if (existingSince) { - return { client, startSince: existingSince }; - } - const startSince = await client.primeRoom(); - if (!startSince) { - throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`); - } - return { client, startSince }; -} - -function advanceMatrixQaActorCursor(params: { - actorId: MatrixQaActorId; - syncState: MatrixQaSyncState; - nextSince?: string; - startSince: string; -}) { - writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); -} - -function createMatrixQaScenarioClient(params: { accessToken: string; baseUrl: string }) { - return createMatrixQaClient({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, - }); -} - -async function runTopLevelMentionScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - tokenPrefix: string; - withMention?: boolean; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`; - const body = - params.withMention === false - ? buildExactMarkerPrompt(token) - : buildMentionPrompt(params.sutUserId, token); - const driverEventId = await client.sendTextMessage({ - body, - ...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }), - roomId: params.roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && - event.relatesTo === undefined, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return { - body, - driverEventId, - reply: buildMatrixReplyArtifact(matched.event, token), - since: matched.since, - token, - }; -} - -async function waitForMembershipEvent(params: { - accessToken: string; - actorId: MatrixQaActorId; - baseUrl: string; - membership: "invite" | "join" | "leave"; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - stateKey: string; - syncState: MatrixQaSyncState; - timeoutMs: number; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.type === "m.room.member" && - event.stateKey === params.stateKey && - event.membership === params.membership, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return matched.event; -} - -async function runTopologyScopedTopLevelScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - actorUserId: string; - context: MatrixQaScenarioContext; - roomKey: string; - tokenPrefix: string; - withMention?: boolean; -}) { - const roomId = resolveMatrixQaScenarioRoomId(params.context, params.roomKey); - const result = await runTopLevelMentionScenario({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.context.baseUrl, - observedEvents: params.context.observedEvents, - roomId, - syncState: params.context.syncState, - sutUserId: params.context.sutUserId, - timeoutMs: params.context.timeoutMs, - tokenPrefix: params.tokenPrefix, - withMention: params.withMention, - }); - assertTopLevelReplyArtifact(`reply in ${params.roomKey}`, result.reply); - return { - artifacts: { - actorUserId: params.actorUserId, - driverEventId: result.driverEventId, - reply: result.reply, - roomKey: params.roomKey, - token: result.token, - triggerBody: result.body, - }, - details: [ - `room key: ${params.roomKey}`, - `room id: ${roomId}`, - `driver event: ${result.driverEventId}`, - `trigger sender: ${params.actorUserId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runThreadScenario(params: MatrixQaScenarioContext) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.driverAccessToken, - actorId: "driver", - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const rootBody = `thread root ${randomUUID().slice(0, 8)}`; - const rootEventId = await client.sendTextMessage({ - body: rootBody, - roomId: params.roomId, - }); - const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`; - const driverEventId = await client.sendTextMessage({ - body: buildMentionPrompt(params.sutUserId, token), - mentionUserIds: [params.sutUserId], - replyToEventId: rootEventId, - roomId: params.roomId, - threadRootEventId: rootEventId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && - event.relatesTo?.relType === "m.thread" && - event.relatesTo.eventId === rootEventId, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: params.syncState, - nextSince: matched.since, - startSince, - }); - return { - driverEventId, - reply: buildMatrixReplyArtifact(matched.event, token), - rootEventId, - since: matched.since, - token, - }; -} - -async function runNoReplyExpectedScenario(params: { - accessToken: string; - actorId: MatrixQaActorId; - actorUserId: string; - baseUrl: string; - body: string; - mentionUserIds?: string[]; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; - token: string; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - syncState: params.syncState, - }); - const driverEventId = await client.sendTextMessage({ - body: params.body, - ...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}), - roomId: params.roomId, - }); - const result = await client.waitForOptionalRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message", - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - if (result.matched) { - const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token); - throw new Error( - [ - `unexpected SUT reply from ${params.sutUserId}`, - `trigger sender: ${params.actorUserId}`, - ...buildMatrixReplyDetails("unexpected reply", unexpectedReply), - ].join("\n"), - ); - } - advanceMatrixQaActorCursor({ - actorId: params.actorId, - syncState: params.syncState, - nextSince: result.since, - startSince, - }); - return { - artifacts: { - actorUserId: params.actorUserId, - driverEventId, - expectedNoReplyWindowMs: params.timeoutMs, - token: params.token, - triggerBody: params.body, - }, - details: [ - `trigger event: ${driverEventId}`, - `trigger sender: ${params.actorUserId}`, - `waited ${params.timeoutMs}ms with no SUT reply`, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runMembershipLossScenario(context: MatrixQaScenarioContext) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEMBERSHIP_ROOM_KEY); - const driverClient = createMatrixQaScenarioClient({ - accessToken: context.driverAccessToken, - baseUrl: context.baseUrl, - }); - const sutClient = createMatrixQaScenarioClient({ - accessToken: context.sutAccessToken, - baseUrl: context.baseUrl, - }); - - await driverClient.kickUserFromRoom({ - reason: "matrix qa membership loss", - roomId, - userId: context.sutUserId, - }); - const leaveEvent = await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "leave", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - - const noReplyToken = `MATRIX_QA_MEMBERSHIP_LOSS_${randomUUID().slice(0, 8).toUpperCase()}`; - await runNoReplyExpectedScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - baseUrl: context.baseUrl, - body: buildMentionPrompt(context.sutUserId, noReplyToken), - mentionUserIds: [context.sutUserId], - observedEvents: context.observedEvents, - roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), - token: noReplyToken, - }); - - await driverClient.inviteUserToRoom({ - roomId, - userId: context.sutUserId, - }); - await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "invite", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - await sutClient.joinRoom(roomId); - const joinEvent = await waitForMembershipEvent({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - membership: "join", - observedEvents: context.observedEvents, - roomId, - stateKey: context.sutUserId, - syncState: context.syncState, - timeoutMs: context.timeoutMs, - }); - - const recovered = await runTopologyScopedTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - context, - roomKey: MATRIX_QA_MEMBERSHIP_ROOM_KEY, - tokenPrefix: "MATRIX_QA_MEMBERSHIP_RETURN", - }); - - return { - artifacts: { - ...recovered.artifacts, - membershipJoinEventId: joinEvent.eventId, - membershipLeaveEventId: leaveEvent.eventId, - recoveredDriverEventId: recovered.artifacts?.driverEventId, - recoveredReply: recovered.artifacts?.reply, - }, - details: [ - `room key: ${MATRIX_QA_MEMBERSHIP_ROOM_KEY}`, - `room id: ${roomId}`, - `leave event: ${leaveEvent.eventId}`, - `join event: ${joinEvent.eventId}`, - recovered.details, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { - const reactionTargetEventId = context.canary?.reply.eventId?.trim(); - if (!reactionTargetEventId) { - throw new Error("Matrix reaction scenario requires a canary reply event id"); - } - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - syncState: context.syncState, - }); - const reactionEmoji = "👍"; - const reactionEventId = await client.sendReaction({ - emoji: reactionEmoji, - messageId: reactionTargetEventId, - roomId: context.roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === context.roomId && - event.sender === context.driverUserId && - event.type === "m.reaction" && - event.eventId === reactionEventId && - event.reaction?.eventId === reactionTargetEventId && - event.reaction?.key === reactionEmoji, - roomId: context.roomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: context.syncState, - nextSince: matched.since, - startSince, - }); - return { - artifacts: { - reactionEmoji, - reactionEventId, - reactionTargetEventId, - }, - details: [ - `reaction event: ${reactionEventId}`, - `reaction target: ${reactionTargetEventId}`, - `reaction emoji: ${reactionEmoji}`, - `observed reaction key: ${matched.event.reaction?.key ?? ""}`, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.interruptTransport) { - throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); - } - await context.interruptTransport(); - const resumed = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_HOMESERVER", - }); - assertTopLevelReplyArtifact("post-homeserver-restart reply", resumed.reply); - return { - artifacts: { - driverEventId: resumed.driverEventId, - reply: resumed.reply, - token: resumed.token, - transportInterruption: "homeserver-restart", - }, - details: [ - "transport interruption: homeserver-restart", - `driver event: ${resumed.driverEventId}`, - ...buildMatrixReplyDetails("reply", resumed.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -async function runRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.restartGateway) { - throw new Error("Matrix restart scenario requires a gateway restart callback"); - } - await context.restartGateway(); - const result = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_RESTART", - }); - assertTopLevelReplyArtifact("post-restart reply", result.reply); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - restartSignal: "SIGUSR1", - token: result.token, - }, - details: [ - "restart signal: SIGUSR1", - `post-restart driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -export async function runMatrixQaCanary(params: { - baseUrl: string; - driverAccessToken: string; - observedEvents: MatrixQaObservedEvent[]; - roomId: string; - syncState: MatrixQaSyncState; - sutUserId: string; - timeoutMs: number; -}) { - const canary = await runTopLevelMentionScenario({ - accessToken: params.driverAccessToken, - actorId: "driver", - baseUrl: params.baseUrl, - observedEvents: params.observedEvents, - roomId: params.roomId, - syncState: params.syncState, - sutUserId: params.sutUserId, - timeoutMs: params.timeoutMs, - tokenPrefix: "MATRIX_QA_CANARY", - }); - assertTopLevelReplyArtifact("canary reply", canary.reply); - return canary; -} - -export async function runMatrixQaScenario( - scenario: MatrixQaScenarioDefinition, - context: MatrixQaScenarioContext, -): Promise { - switch (scenario.id) { - case "matrix-thread-follow-up": { - const result = await runThreadScenario(context); - assertThreadReplyArtifact(result.reply, { - expectedRootEventId: result.rootEventId, - label: "thread reply", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - rootEventId: result.rootEventId, - token: result.token, - }, - details: [ - `root event: ${result.rootEventId}`, - `driver thread event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - }; - } - case "matrix-thread-isolation": { - const threadPhase = await runThreadScenario(context); - assertThreadReplyArtifact(threadPhase.reply, { - expectedRootEventId: threadPhase.rootEventId, - label: "thread isolation reply", - }); - const topLevelPhase = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_TOPLEVEL", - }); - assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply); - return { - artifacts: { - threadDriverEventId: threadPhase.driverEventId, - threadReply: threadPhase.reply, - threadRootEventId: threadPhase.rootEventId, - threadToken: threadPhase.token, - topLevelDriverEventId: topLevelPhase.driverEventId, - topLevelReply: topLevelPhase.reply, - topLevelToken: topLevelPhase.token, - }, - details: [ - `thread root event: ${threadPhase.rootEventId}`, - `thread driver event: ${threadPhase.driverEventId}`, - ...buildMatrixReplyDetails("thread reply", threadPhase.reply), - `top-level driver event: ${topLevelPhase.driverEventId}`, - ...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply), - ].join("\n"), - }; - } - case "matrix-top-level-reply-shape": { - const result = await runTopLevelMentionScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - tokenPrefix: "MATRIX_QA_TOPLEVEL", - }); - assertTopLevelReplyArtifact("top-level reply", result.reply); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - token: result.token, - }, - details: [ - `driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - }; - } - case "matrix-dm-reply-shape": - return await runTopologyScopedTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - context, - roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, - tokenPrefix: "MATRIX_QA_DM", - withMention: false, - }); - case "matrix-secondary-room-reply": - return await runTopologyScopedTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - context, - roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, - tokenPrefix: "MATRIX_QA_SECONDARY", - }); - case "matrix-secondary-room-open-trigger": - return await runTopologyScopedTopLevelScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - context, - roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, - tokenPrefix: "MATRIX_QA_SECONDARY_OPEN", - withMention: false, - }); - case "matrix-reaction-notification": - return await runReactionNotificationScenario(context); - case "matrix-restart-resume": - return await runRestartResumeScenario(context); - case "matrix-room-membership-loss": - return await runMembershipLossScenario(context); - case "matrix-homeserver-restart-resume": - return await runHomeserverRestartResumeScenario(context); - case "matrix-mention-gating": { - const token = `MATRIX_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return await runNoReplyExpectedScenario({ - accessToken: context.driverAccessToken, - actorId: "driver", - actorUserId: context.driverUserId, - baseUrl: context.baseUrl, - body: buildExactMarkerPrompt(token), - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - token, - }); - } - case "matrix-allowlist-block": { - const token = `MATRIX_QA_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`; - return await runNoReplyExpectedScenario({ - accessToken: context.observerAccessToken, - actorId: "observer", - actorUserId: context.observerUserId, - baseUrl: context.baseUrl, - body: buildMentionPrompt(context.sutUserId, token), - mentionUserIds: [context.sutUserId], - observedEvents: context.observedEvents, - roomId: context.roomId, - syncState: context.syncState, - sutUserId: context.sutUserId, - timeoutMs: context.timeoutMs, - token, - }); - } - default: { - const exhaustiveScenarioId: never = scenario.id; - return exhaustiveScenarioId; - } - } -} +export type { MatrixQaScenarioContext, MatrixQaSyncState }; export const __testing = { MATRIX_QA_DRIVER_DM_ROOM_KEY, + MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, MATRIX_QA_MEMBERSHIP_ROOM_KEY, MATRIX_QA_SECONDARY_ROOM_KEY, MATRIX_QA_STANDARD_SCENARIO_IDS, diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index 2feb2e833b5..0350dea18c3 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -1,6 +1,10 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import { describe, expect, it } from "vitest"; -import { buildMatrixQaConfig } from "./config.js"; +import { + buildMatrixQaConfig, + buildMatrixQaConfigSnapshot, + summarizeMatrixQaConfigSnapshot, +} from "./config.js"; import type { MatrixQaProvisionedTopology } from "./topology.js"; describe("matrix qa config", () => { @@ -78,6 +82,14 @@ describe("matrix qa config", () => { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", overrides: { + autoJoin: "allowlist", + autoJoinAllowlist: [" !dm:matrix-qa.test ", "#ops:matrix-qa.test"], + blockStreaming: true, + dm: { + sessionScope: "per-room", + threadReplies: "off", + }, + encryption: true, groupAllowFrom: ["@driver:matrix-qa.test", "@observer:matrix-qa.test"], groupsByKey: { secondary: { @@ -85,6 +97,7 @@ describe("matrix qa config", () => { }, }, replyToMode: "all", + streaming: "quiet", threadReplies: "always", }, sutAccessToken: "sut-token", @@ -94,16 +107,76 @@ describe("matrix qa config", () => { }); expect(next.channels?.matrix?.accounts?.sut).toMatchObject({ + autoJoin: "allowlist", + autoJoinAllowlist: ["!dm:matrix-qa.test", "#ops:matrix-qa.test"], + blockStreaming: true, + dm: { + sessionScope: "per-room", + threadReplies: "off", + }, + encryption: true, groupAllowFrom: ["@driver:matrix-qa.test", "@observer:matrix-qa.test"], groups: { "!main:matrix-qa.test": { enabled: true, requireMention: true }, "!secondary:matrix-qa.test": { enabled: true, requireMention: false }, }, replyToMode: "all", + streaming: "quiet", threadReplies: "always", }); }); + it("builds an effective Matrix QA config snapshot for reporting", () => { + const snapshot = buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + overrides: { + autoJoin: "allowlist", + autoJoinAllowlist: ["!ops:matrix-qa.test"], + blockStreaming: true, + dm: { + sessionScope: "per-room", + }, + groupPolicy: "open", + streaming: true, + }, + sutUserId: "@sut:matrix-qa.test", + topology, + }); + + expect(snapshot).toEqual({ + autoJoin: "allowlist", + autoJoinAllowlist: ["!ops:matrix-qa.test"], + blockStreaming: true, + dm: { + allowFrom: ["@driver:matrix-qa.test"], + enabled: true, + policy: "allowlist", + sessionScope: "per-room", + threadReplies: "inbound", + }, + encryption: false, + groupAllowFrom: ["@driver:matrix-qa.test"], + groupPolicy: "open", + groupsByKey: { + main: { + enabled: true, + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + secondary: { + enabled: true, + requireMention: true, + roomId: "!secondary:matrix-qa.test", + }, + }, + replyToMode: "off", + streaming: "partial", + threadReplies: "inbound", + }); + expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("autoJoin=allowlist"); + expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("streaming=partial"); + }); + it("rejects unknown room-key overrides", () => { expect(() => buildMatrixQaConfig({} as OpenClawConfig, { diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index d838f06a932..87d940c3aa0 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -1,10 +1,12 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import type { MatrixQaProvisionedTopology } from "./topology.js"; -type MatrixQaReplyToMode = "off" | "first" | "all" | "batched"; -type MatrixQaThreadRepliesMode = "off" | "inbound" | "always"; -type MatrixQaDmPolicy = "allowlist" | "disabled" | "open" | "pairing"; -type MatrixQaGroupPolicy = "allowlist" | "disabled" | "open"; +export type MatrixQaReplyToMode = "off" | "first" | "all" | "batched"; +export type MatrixQaThreadRepliesMode = "off" | "inbound" | "always"; +export type MatrixQaDmPolicy = "allowlist" | "disabled" | "open" | "pairing"; +export type MatrixQaGroupPolicy = "allowlist" | "disabled" | "open"; +export type MatrixQaAutoJoinMode = "allowlist" | "always" | "off"; +export type MatrixQaStreamingMode = "off" | "partial" | "quiet"; export type MatrixQaGroupConfigOverrides = { enabled?: boolean; @@ -20,7 +22,8 @@ export type MatrixQaDmConfigOverrides = { }; export type MatrixQaConfigOverrides = { - autoJoin?: "allowlist" | "always" | "off"; + autoJoin?: MatrixQaAutoJoinMode; + autoJoinAllowlist?: string[]; blockStreaming?: boolean; dm?: MatrixQaDmConfigOverrides; encryption?: boolean; @@ -32,7 +35,48 @@ export type MatrixQaConfigOverrides = { threadReplies?: MatrixQaThreadRepliesMode; }; -function resolveMatrixQaGroupEntries(params: { +export type MatrixQaConfigSnapshot = { + autoJoin: MatrixQaAutoJoinMode; + autoJoinAllowlist: string[]; + blockStreaming: boolean; + dm: { + allowFrom: string[]; + enabled: boolean; + policy: MatrixQaDmPolicy; + sessionScope: "per-room" | "per-user"; + threadReplies: MatrixQaThreadRepliesMode; + }; + encryption: boolean; + groupAllowFrom: string[]; + groupPolicy: MatrixQaGroupPolicy; + groupsByKey: Record< + string, + { + enabled: boolean; + requireMention: boolean; + roomId: string; + } + >; + replyToMode: MatrixQaReplyToMode; + streaming: MatrixQaStreamingMode; + threadReplies: MatrixQaThreadRepliesMode; +}; + +type MatrixQaAccountDmConfig = + | { enabled: false } + | { + allowFrom: string[]; + enabled: true; + policy: MatrixQaDmPolicy; + sessionScope?: "per-room" | "per-user"; + threadReplies?: MatrixQaThreadRepliesMode; + }; + +function normalizeMatrixQaAllowlist(entries?: string[]) { + return [...new Set((entries ?? []).map((entry) => entry.trim()).filter(Boolean))]; +} + +function resolveMatrixQaGroupSnapshots(params: { overrides?: MatrixQaConfigOverrides; topology: MatrixQaProvisionedTopology; }) { @@ -50,8 +94,9 @@ function resolveMatrixQaGroupEntries(params: { groupRooms.map((room) => { const override = groupsByKey[room.key]; return [ - room.roomId, + room.key, { + roomId: room.roomId, enabled: override?.enabled ?? true, requireMention: override?.requireMention ?? room.requireMention, }, @@ -60,6 +105,20 @@ function resolveMatrixQaGroupEntries(params: { ); } +function buildMatrixQaGroupEntries( + groupsByKey: MatrixQaConfigSnapshot["groupsByKey"], +): Record { + return Object.fromEntries( + Object.values(groupsByKey).map((group) => [ + group.roomId, + { + enabled: group.enabled, + requireMention: group.requireMention, + }, + ]), + ); +} + function resolveMatrixQaDmAllowFrom(params: { driverUserId: string; overrides?: MatrixQaConfigOverrides; @@ -67,19 +126,16 @@ function resolveMatrixQaDmAllowFrom(params: { topology: MatrixQaProvisionedTopology; }) { if (params.overrides?.dm?.allowFrom) { - return [...params.overrides.dm.allowFrom]; + return normalizeMatrixQaAllowlist(params.overrides.dm.allowFrom); } - const dmAllowFrom = [ - ...new Set( - params.topology.rooms - .filter((room) => room.kind === "dm") - .flatMap((room) => room.memberUserIds.filter((userId) => userId !== params.sutUserId)), - ), - ]; + const dmParticipantUserIds = params.topology.rooms + .filter((room) => room.kind === "dm") + .flatMap((room) => room.memberUserIds.filter((userId) => userId !== params.sutUserId)); + const dmAllowFrom = [...new Set(dmParticipantUserIds)]; return dmAllowFrom.length > 0 ? dmAllowFrom : [params.driverUserId]; } -function resolveMatrixQaDmConfig(params: { +function resolveMatrixQaDmConfigSnapshot(params: { driverUserId: string; overrides?: MatrixQaConfigOverrides; sutUserId: string; @@ -87,20 +143,99 @@ function resolveMatrixQaDmConfig(params: { }) { const hasDmRooms = params.topology.rooms.some((room) => room.kind === "dm"); const dmOverrides = params.overrides?.dm; + const enabled = hasDmRooms || dmOverrides?.enabled === true; + return { + allowFrom: enabled ? resolveMatrixQaDmAllowFrom(params) : [], + enabled, + policy: dmOverrides?.policy ?? "allowlist", + sessionScope: dmOverrides?.sessionScope ?? "per-user", + threadReplies: dmOverrides?.threadReplies ?? params.overrides?.threadReplies ?? "inbound", + }; +} - if (!hasDmRooms && dmOverrides?.enabled !== true) { +function resolveMatrixQaStreamingMode( + value: MatrixQaConfigOverrides["streaming"], +): MatrixQaStreamingMode { + if (value === true || value === "partial") { + return "partial"; + } + if (value === "quiet") { + return "quiet"; + } + return "off"; +} + +function resolveMatrixQaAutoJoinAllowlist(params: { overrides?: MatrixQaConfigOverrides }) { + if (params.overrides?.autoJoin !== "allowlist") { + return []; + } + return normalizeMatrixQaAllowlist(params.overrides.autoJoinAllowlist); +} + +function formatMatrixQaBoolean(value: boolean) { + return value ? "true" : "false"; +} + +function buildMatrixQaAccountDmConfig(params: { + dmOverrides?: MatrixQaConfigOverrides["dm"]; + snapshot: MatrixQaConfigSnapshot; +}): MatrixQaAccountDmConfig { + if (!params.snapshot.dm.enabled) { return { enabled: false }; } return { - allowFrom: resolveMatrixQaDmAllowFrom(params), - enabled: dmOverrides?.enabled ?? true, - policy: dmOverrides?.policy ?? "allowlist", - ...(dmOverrides?.sessionScope ? { sessionScope: dmOverrides.sessionScope } : {}), - ...(dmOverrides?.threadReplies ? { threadReplies: dmOverrides.threadReplies } : {}), + allowFrom: params.snapshot.dm.allowFrom, + enabled: true, + policy: params.snapshot.dm.policy, + ...(params.dmOverrides?.sessionScope ? { sessionScope: params.snapshot.dm.sessionScope } : {}), + ...(params.dmOverrides?.threadReplies + ? { threadReplies: params.snapshot.dm.threadReplies } + : {}), }; } +export function buildMatrixQaConfigSnapshot(params: { + driverUserId: string; + overrides?: MatrixQaConfigOverrides; + sutUserId: string; + topology: MatrixQaProvisionedTopology; +}): MatrixQaConfigSnapshot { + return { + autoJoin: params.overrides?.autoJoin ?? "off", + autoJoinAllowlist: resolveMatrixQaAutoJoinAllowlist(params), + blockStreaming: params.overrides?.blockStreaming ?? false, + dm: resolveMatrixQaDmConfigSnapshot(params), + encryption: params.overrides?.encryption ?? false, + groupAllowFrom: normalizeMatrixQaAllowlist( + params.overrides?.groupAllowFrom ?? [params.driverUserId], + ), + groupPolicy: params.overrides?.groupPolicy ?? "allowlist", + groupsByKey: resolveMatrixQaGroupSnapshots({ + overrides: params.overrides, + topology: params.topology, + }), + replyToMode: params.overrides?.replyToMode ?? "off", + streaming: resolveMatrixQaStreamingMode(params.overrides?.streaming), + threadReplies: params.overrides?.threadReplies ?? "inbound", + }; +} + +export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot) { + return [ + `replyToMode=${snapshot.replyToMode}`, + `threadReplies=${snapshot.threadReplies}`, + `dm.enabled=${formatMatrixQaBoolean(snapshot.dm.enabled)}`, + `dm.policy=${snapshot.dm.policy}`, + `dm.sessionScope=${snapshot.dm.sessionScope}`, + `dm.threadReplies=${snapshot.dm.threadReplies}`, + `streaming=${snapshot.streaming}`, + `blockStreaming=${formatMatrixQaBoolean(snapshot.blockStreaming)}`, + `autoJoin=${snapshot.autoJoin}`, + `encryption=${formatMatrixQaBoolean(snapshot.encryption)}`, + ].join(", "); +} + export function buildMatrixQaConfig( baseCfg: OpenClawConfig, params: { @@ -115,10 +250,18 @@ export function buildMatrixQaConfig( }, ): OpenClawConfig { const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "matrix"])]; - const groups = resolveMatrixQaGroupEntries({ + const snapshot = buildMatrixQaConfigSnapshot({ + driverUserId: params.driverUserId, overrides: params.overrides, + sutUserId: params.sutUserId, topology: params.topology, }); + const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey); + const dmOverrides = params.overrides?.dm; + const dm = buildMatrixQaAccountDmConfig({ + dmOverrides, + snapshot, + }); return { ...baseCfg, @@ -141,22 +284,25 @@ export function buildMatrixQaConfig( [params.sutAccountId]: { accessToken: params.sutAccessToken, ...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}), - dm: resolveMatrixQaDmConfig(params), + dm, enabled: true, - encryption: params.overrides?.encryption ?? false, - groupAllowFrom: params.overrides?.groupAllowFrom ?? [params.driverUserId], - groupPolicy: params.overrides?.groupPolicy ?? "allowlist", + encryption: snapshot.encryption, + groupAllowFrom: snapshot.groupAllowFrom, + groupPolicy: snapshot.groupPolicy, ...(Object.keys(groups).length > 0 ? { groups } : {}), homeserver: params.homeserver, network: { dangerouslyAllowPrivateNetwork: true, }, - replyToMode: params.overrides?.replyToMode ?? "off", - threadReplies: params.overrides?.threadReplies ?? "inbound", + replyToMode: snapshot.replyToMode, + threadReplies: snapshot.threadReplies, userId: params.sutUserId, - ...(params.overrides?.autoJoin ? { autoJoin: params.overrides.autoJoin } : {}), + ...(snapshot.autoJoin !== "off" ? { autoJoin: snapshot.autoJoin } : {}), + ...(snapshot.autoJoin === "allowlist" && snapshot.autoJoinAllowlist.length > 0 + ? { autoJoinAllowlist: snapshot.autoJoinAllowlist } + : {}), ...(params.overrides?.blockStreaming !== undefined - ? { blockStreaming: params.overrides.blockStreaming } + ? { blockStreaming: snapshot.blockStreaming } : {}), ...(params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming }