diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index f7e68d47809..08511811da7 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -21,11 +21,29 @@ describe("matrix live qa runtime", () => { const next = liveTesting.buildMatrixQaConfig(baseCfg, { driverUserId: "@driver:matrix-qa.test", homeserver: "http://127.0.0.1:28008/", - roomId: "!room:matrix-qa.test", sutAccessToken: "syt_sut", sutAccountId: "sut", sutDeviceId: "DEVICE123", sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA", + requireMention: true, + roomId: "!room:matrix-qa.test", + }, + ], + }, }); expect(next.plugins?.allow).toContain("matrix"); @@ -60,6 +78,72 @@ describe("matrix live qa runtime", () => { }); }); + it("derives Matrix DM + multi-room config from provisioned topology", () => { + const next = liveTesting.buildMatrixQaConfig( + {}, + { + driverUserId: "@driver:matrix-qa.test", + homeserver: "http://127.0.0.1:28008/", + sutAccessToken: "syt_sut", + sutAccountId: "sut", + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room-a:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA A", + requireMention: true, + roomId: "!room-a:matrix-qa.test", + }, + { + key: "secondary", + kind: "group", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "Matrix QA B", + requireMention: false, + roomId: "!room-b:matrix-qa.test", + }, + { + key: "sut-dm", + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "Matrix QA DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + ], + }, + }, + ); + + expect(next.channels?.matrix?.accounts?.sut?.dm).toEqual({ + allowFrom: ["@driver:matrix-qa.test"], + enabled: true, + policy: "allowlist", + }); + expect(next.channels?.matrix?.accounts?.sut?.groups).toEqual({ + "!room-a:matrix-qa.test": { + enabled: true, + requireMention: true, + }, + "!room-b:matrix-qa.test": { + enabled: true, + requireMention: false, + }, + }); + }); + it("redacts Matrix observed event content by default in artifacts", () => { expect( liveTesting.buildObservedEventsArtifact({ @@ -157,8 +241,10 @@ describe("matrix live qa runtime", () => { harness: { baseUrl: "http://127.0.0.1:28008/", composeFile: "/tmp/docker-compose.yml", + dmRoomIds: [], image: "ghcr.io/matrix-construct/tuwunel:v1.5.1", roomId: "!room:matrix-qa.test", + roomIds: ["!room:matrix-qa.test"], serverName: "matrix-qa.test", }, observedEventCount: 4, diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 824985a840f..14d41eb8664 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -17,10 +17,12 @@ import { type MatrixQaObservedEvent, type MatrixQaProvisionResult, } from "../../substrate/client.js"; +import { buildMatrixQaConfig, type MatrixQaConfigOverrides } from "../../substrate/config.js"; import { startMatrixQaHarness } from "../../substrate/harness.runtime.js"; import { resolveMatrixQaModels } from "./model-selection.js"; import { MATRIX_QA_SCENARIOS, + buildMatrixQaTopologyForScenarios, buildMatrixReplyDetails, findMatrixQaScenarios, runMatrixQaCanary, @@ -43,6 +45,10 @@ type MatrixQaLiveLaneGatewayHarness = { stop(): Promise; }; +function buildMatrixQaGatewayConfigKey(overrides?: MatrixQaConfigOverrides) { + return JSON.stringify(overrides ?? null); +} + type MatrixQaScenarioResult = { artifacts?: MatrixQaScenarioArtifacts; details: string; @@ -62,8 +68,10 @@ type MatrixQaSummary = { harness: { baseUrl: string; composeFile: string; + dmRoomIds: string[]; image: string; roomId: string; + roomIds: string[]; serverName: string; }; canary?: MatrixQaCanaryArtifact; @@ -132,63 +140,6 @@ function buildMatrixQaSummary(params: { }; } -function buildMatrixQaConfig( - baseCfg: OpenClawConfig, - params: { - driverUserId: string; - homeserver: string; - roomId: string; - sutAccessToken: string; - sutAccountId: string; - sutDeviceId?: string; - sutUserId: string; - }, -): OpenClawConfig { - const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "matrix"])]; - return { - ...baseCfg, - plugins: { - ...baseCfg.plugins, - allow: pluginAllow, - entries: { - ...baseCfg.plugins?.entries, - matrix: { enabled: true }, - }, - }, - channels: { - ...baseCfg.channels, - matrix: { - enabled: true, - defaultAccount: params.sutAccountId, - accounts: { - [params.sutAccountId]: { - accessToken: params.sutAccessToken, - ...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}), - dm: { enabled: false }, - enabled: true, - encryption: false, - groupAllowFrom: [params.driverUserId], - groupPolicy: "allowlist", - groups: { - [params.roomId]: { - enabled: true, - requireMention: true, - }, - }, - homeserver: params.homeserver, - network: { - dangerouslyAllowPrivateNetwork: true, - }, - replyToMode: "off", - threadReplies: "inbound", - userId: params.sutUserId, - }, - }, - }, - }, - }; -} - function buildObservedEventsArtifact(params: { includeContent: boolean; observedEvents: MatrixQaObservedEvent[]; @@ -312,11 +263,15 @@ export async function runMatrixQaLive(params: { }); const sutAccountId = params.sutAccountId?.trim() || "sut"; const scenarios = findMatrixQaScenarios(params.scenarioIds); + const runSuffix = randomUUID().slice(0, 8); + const topology = buildMatrixQaTopologyForScenarios({ + defaultRoomName: `OpenClaw Matrix QA ${runSuffix}`, + scenarios, + }); const observedEvents: MatrixQaObservedEvent[] = []; const includeObservedEventContent = process.env.OPENCLAW_QA_MATRIX_CAPTURE_CONTENT === "1"; const startedAtDate = new Date(); const startedAt = startedAtDate.toISOString(); - const runSuffix = randomUUID().slice(0, 8); const harness = await startMatrixQaHarness({ outputDir: path.join(outputDir, "matrix-harness"), @@ -331,6 +286,7 @@ export async function runMatrixQaLive(params: { registrationToken: harness.registrationToken, roomName: `OpenClaw Matrix QA ${runSuffix}`, sutLocalpart: `qa-sut-${runSuffix}`, + topology, }); } catch (error) { await harness.stop().catch(() => {}); @@ -347,6 +303,7 @@ export async function runMatrixQaLive(params: { `baseUrl: ${harness.baseUrl}`, `serverName: ${harness.serverName}`, `roomId: ${provisioning.roomId}`, + `roomCount: ${provisioning.topology.rooms.length}`, ].join("\n"), }, ]; @@ -354,34 +311,55 @@ export async function runMatrixQaLive(params: { const cleanupErrors: string[] = []; let canaryArtifact: MatrixQaCanaryArtifact | undefined; let gatewayHarness: MatrixQaLiveLaneGatewayHarness | null = null; + let gatewayHarnessKey: string | null = null; let canaryFailed = false; const syncState: { driver?: string; observer?: string } = {}; + const gatewayConfigParams = { + driverUserId: provisioning.driver.userId, + homeserver: harness.baseUrl, + sutAccessToken: provisioning.sut.accessToken, + sutAccountId, + sutDeviceId: provisioning.sut.deviceId, + sutUserId: provisioning.sut.userId, + topology: provisioning.topology, + }; try { - gatewayHarness = await startMatrixQaLiveLaneGateway({ - repoRoot, - transport: { - requiredPluginIds: [], - createGatewayConfig: () => ({}), - }, - transportBaseUrl: "http://127.0.0.1:43123", - providerMode, - primaryModel, - alternateModel, - fastMode: params.fastMode, - controlUiEnabled: false, - mutateConfig: (cfg) => - buildMatrixQaConfig(cfg, { - driverUserId: provisioning.driver.userId, - homeserver: harness.baseUrl, - roomId: provisioning.roomId, - sutAccessToken: provisioning.sut.accessToken, - sutAccountId, - sutDeviceId: provisioning.sut.deviceId, - sutUserId: provisioning.sut.userId, - }), - }); - await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId); + const ensureGatewayHarness = async (overrides?: MatrixQaConfigOverrides) => { + const nextKey = buildMatrixQaGatewayConfigKey(overrides); + if (gatewayHarness && gatewayHarnessKey === nextKey) { + return gatewayHarness; + } + if (gatewayHarness) { + await gatewayHarness.stop(); + gatewayHarness = null; + gatewayHarnessKey = null; + } + const started = await startMatrixQaLiveLaneGateway({ + repoRoot, + transport: { + requiredPluginIds: [], + createGatewayConfig: () => ({}), + }, + transportBaseUrl: "http://127.0.0.1:43123", + providerMode, + primaryModel, + alternateModel, + fastMode: params.fastMode, + controlUiEnabled: false, + mutateConfig: (cfg) => + buildMatrixQaConfig(cfg, { + ...gatewayConfigParams, + overrides, + }), + }); + await waitForMatrixChannelReady(started.gateway, sutAccountId); + gatewayHarness = started; + gatewayHarnessKey = nextKey; + return started; + }; + + gatewayHarness = await ensureGatewayHarness(); checks.push({ name: "Matrix channel ready", status: "pass", @@ -420,11 +398,18 @@ export async function runMatrixQaLive(params: { if (!canaryFailed) { for (const scenario of scenarios) { try { + const scenarioGateway = await ensureGatewayHarness(scenario.configOverrides); const result = await runMatrixQaScenario(scenario, { baseUrl: harness.baseUrl, canary: canaryArtifact, driverAccessToken: provisioning.driver.accessToken, driverUserId: provisioning.driver.userId, + interruptTransport: async () => { + await harness.restartService(); + await waitForMatrixChannelReady(scenarioGateway.gateway, sutAccountId, { + timeoutMs: 90_000, + }); + }, observedEvents, observerAccessToken: provisioning.observer.accessToken, observerUserId: provisioning.observer.userId, @@ -432,13 +417,15 @@ export async function runMatrixQaLive(params: { if (!gatewayHarness) { throw new Error("Matrix restart scenario requires a live gateway"); } - await gatewayHarness.gateway.restart(); - await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId); + await scenarioGateway.gateway.restart(); + await waitForMatrixChannelReady(scenarioGateway.gateway, sutAccountId); }, roomId: provisioning.roomId, + sutAccessToken: provisioning.sut.accessToken, syncState, sutUserId: provisioning.sut.userId, timeoutMs: scenario.timeoutMs, + topology: provisioning.topology, }); scenarioResults.push({ artifacts: result.artifacts, @@ -501,6 +488,7 @@ export async function runMatrixQaLive(params: { })), notes: [ `roomId: ${provisioning.roomId}`, + `roomIds: ${provisioning.topology.rooms.map((room) => room.roomId).join(", ")}`, `driver: ${provisioning.driver.userId}`, `observer: ${provisioning.observer.userId}`, `sut: ${provisioning.sut.userId}`, @@ -516,8 +504,12 @@ export async function runMatrixQaLive(params: { harness: { baseUrl: harness.baseUrl, composeFile: harness.composeFile, + dmRoomIds: provisioning.topology.rooms + .filter((room) => room.kind === "dm") + .map((room) => room.roomId), image: harness.image, roomId: provisioning.roomId, + roomIds: provisioning.topology.rooms.map((room) => room.roomId), serverName: harness.serverName, }, observedEventCount: observedEvents.length, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 41f670957bb..6df694b055d 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -27,8 +27,13 @@ describe("matrix live qa scenarios", () => { "matrix-thread-follow-up", "matrix-thread-isolation", "matrix-top-level-reply-shape", + "matrix-dm-reply-shape", + "matrix-secondary-room-reply", + "matrix-secondary-room-open-trigger", "matrix-reaction-notification", "matrix-restart-resume", + "matrix-room-membership-loss", + "matrix-homeserver-restart-resume", "matrix-mention-gating", "matrix-allowlist-block", ]); @@ -92,6 +97,160 @@ describe("matrix live qa scenarios", () => { ).toEqual([]); }); + it("merges default and scenario-requested Matrix topology once per run", () => { + expect( + scenarioTesting.buildMatrixQaTopologyForScenarios({ + defaultRoomName: "OpenClaw Matrix QA run", + scenarios: [ + MATRIX_QA_SCENARIOS[0], + { + id: "matrix-restart-resume", + standardId: "restart-resume", + timeoutMs: 60_000, + title: "Matrix restart resume", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: "driver-dm", + kind: "dm", + members: ["driver", "sut"], + name: "Driver/SUT DM", + }, + { + key: "ops", + kind: "group", + members: ["driver", "observer", "sut"], + name: "Ops room", + requireMention: false, + }, + ], + }, + }, + ], + }), + ).toEqual({ + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + members: ["driver", "observer", "sut"], + name: "OpenClaw Matrix QA run", + requireMention: true, + }, + { + key: "driver-dm", + kind: "dm", + members: ["driver", "sut"], + name: "Driver/SUT DM", + }, + { + key: "ops", + kind: "group", + members: ["driver", "observer", "sut"], + name: "Ops room", + requireMention: false, + }, + ], + }); + }); + + it("rejects conflicting Matrix topology room definitions", () => { + expect(() => + scenarioTesting.buildMatrixQaTopologyForScenarios({ + defaultRoomName: "OpenClaw Matrix QA run", + scenarios: [ + { + id: "matrix-thread-follow-up", + standardId: "thread-follow-up", + timeoutMs: 60_000, + title: "A", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: "ops", + kind: "group", + members: ["driver", "observer", "sut"], + name: "Ops room", + requireMention: true, + }, + ], + }, + }, + { + id: "matrix-thread-isolation", + standardId: "thread-isolation", + timeoutMs: 60_000, + title: "B", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: "ops", + kind: "group", + members: ["driver", "sut"], + name: "Ops room", + requireMention: true, + }, + ], + }, + }, + ], + }), + ).toThrow('Matrix QA topology room "ops" has conflicting definitions'); + }); + + it("resolves scenario room ids from provisioned topology keys", () => { + expect( + scenarioTesting.resolveMatrixQaScenarioRoomId( + { + roomId: "!main:matrix-qa.test", + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Main", + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + { + key: "driver-dm", + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "Driver DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + ], + }, + }, + "driver-dm", + ), + ).toBe("!dm:matrix-qa.test"); + expect( + scenarioTesting.resolveMatrixQaScenarioRoomId({ + roomId: "!main:matrix-qa.test", + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }), + ).toBe("!main:matrix-qa.test"); + }); + it("primes the observer sync cursor instead of reusing the driver's cursor", async () => { const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$observer-trigger"); @@ -128,8 +287,14 @@ describe("matrix live qa scenarios", () => { roomId: "!room:matrix-qa.test", restartGateway: undefined, syncState, + sutAccessToken: "sut-token", sutUserId: "@sut:matrix-qa.test", timeoutMs: 8_000, + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, }), ).resolves.toMatchObject({ artifacts: { @@ -150,4 +315,185 @@ describe("matrix live qa scenarios", () => { observer: "observer-sync-next", }); }); + + it("runs the DM scenario against the provisioned DM room without a mention", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$dm-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + roomId: "!dm:matrix-qa.test", + eventId: "$sut-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "reply with only this exact marker: ", + "", + ), + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find((entry) => entry.id === "matrix-dm-reply-shape"); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Main", + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + { + key: scenarioTesting.MATRIX_QA_DRIVER_DM_ROOM_KEY, + kind: "dm", + memberRoles: ["driver", "sut"], + memberUserIds: ["@driver:matrix-qa.test", "@sut:matrix-qa.test"], + name: "DM", + requireMention: false, + roomId: "!dm:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + actorUserId: "@driver:matrix-qa.test", + }, + }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("reply with only this exact marker:"), + roomId: "!dm:matrix-qa.test", + }); + expect(waitForRoomEvent).toHaveBeenCalledWith( + expect.objectContaining({ + roomId: "!dm:matrix-qa.test", + }), + ); + }); + + it("runs the secondary-room scenario against the provisioned secondary room", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$secondary-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + roomId: "!secondary:matrix-qa.test", + eventId: "$sut-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), + }, + since: "driver-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-secondary-room-reply", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Main", + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + { + key: scenarioTesting.MATRIX_QA_SECONDARY_ROOM_KEY, + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Secondary", + requireMention: true, + roomId: "!secondary:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + actorUserId: "@driver:matrix-qa.test", + }, + }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("@sut:matrix-qa.test"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!secondary:matrix-qa.test", + }); + expect(waitForRoomEvent).toHaveBeenCalledWith( + expect.objectContaining({ + roomId: "!secondary:matrix-qa.test", + }), + ); + }); }); diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.ts b/extensions/qa-matrix/src/runners/contract/scenarios.ts index 7c8638d845c..5c07c9c25f6 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.ts @@ -5,17 +5,33 @@ import { type LiveTransportScenarioDefinition, } from "../../shared/live-transport-scenarios.js"; import { createMatrixQaClient, type MatrixQaObservedEvent } from "../../substrate/client.js"; +import { type MatrixQaConfigOverrides } from "../../substrate/config.js"; +import { + buildDefaultMatrixQaTopologySpec, + findMatrixQaProvisionedRoom, + mergeMatrixQaTopologySpecs, + type MatrixQaProvisionedTopology, + type MatrixQaTopologySpec, +} from "../../substrate/topology.js"; export type MatrixQaScenarioId = | "matrix-thread-follow-up" | "matrix-thread-isolation" | "matrix-top-level-reply-shape" + | "matrix-dm-reply-shape" + | "matrix-secondary-room-reply" + | "matrix-secondary-room-open-trigger" | "matrix-reaction-notification" | "matrix-restart-resume" + | "matrix-room-membership-loss" + | "matrix-homeserver-restart-resume" | "matrix-mention-gating" | "matrix-allowlist-block"; -export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition; +export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { + configOverrides?: MatrixQaConfigOverrides; + topology?: MatrixQaTopologySpec; +}; export type MatrixQaReplyArtifact = { bodyPreview?: string; @@ -40,6 +56,9 @@ export type MatrixQaScenarioArtifacts = { reactionEventId?: string; reactionTargetEventId?: string; reply?: MatrixQaReplyArtifact; + recoveredDriverEventId?: string; + recoveredReply?: MatrixQaReplyArtifact; + roomKey?: string; restartSignal?: string; rootEventId?: string; threadDriverEventId?: string; @@ -51,6 +70,9 @@ export type MatrixQaScenarioArtifacts = { topLevelReply?: MatrixQaReplyArtifact; topLevelToken?: string; triggerBody?: string; + membershipJoinEventId?: string; + membershipLeaveEventId?: string; + transportInterruption?: string; }; export type MatrixQaScenarioExecution = { @@ -72,12 +94,18 @@ type MatrixQaScenarioContext = { observerUserId: string; restartGateway?: () => Promise; roomId: string; + interruptTransport?: () => Promise; + sutAccessToken: string; syncState: MatrixQaSyncState; sutUserId: string; timeoutMs: number; + topology: MatrixQaProvisionedTopology; }; const NO_REPLY_WINDOW_MS = 8_000; +const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm"; +const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership"; +const MATRIX_QA_SECONDARY_ROOM_KEY = "secondary"; export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ { @@ -98,6 +126,63 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 45_000, title: "Matrix top-level reply keeps replyToMode off", }, + { + id: "matrix-dm-reply-shape", + timeoutMs: 45_000, + title: "Matrix DM reply stays top-level without a mention", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: MATRIX_QA_DRIVER_DM_ROOM_KEY, + kind: "dm", + members: ["driver", "sut"], + name: "Matrix QA Driver/SUT DM", + }, + ], + }, + }, + { + id: "matrix-secondary-room-reply", + timeoutMs: 45_000, + title: "Matrix secondary room reply stays scoped to that room", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: MATRIX_QA_SECONDARY_ROOM_KEY, + kind: "group", + members: ["driver", "observer", "sut"], + name: "Matrix QA Secondary Room", + requireMention: true, + }, + ], + }, + }, + { + id: "matrix-secondary-room-open-trigger", + timeoutMs: 45_000, + title: "Matrix secondary room can opt out of mention gating", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: MATRIX_QA_SECONDARY_ROOM_KEY, + kind: "group", + members: ["driver", "observer", "sut"], + name: "Matrix QA Secondary Room", + requireMention: true, + }, + ], + }, + configOverrides: { + groupsByKey: { + [MATRIX_QA_SECONDARY_ROOM_KEY]: { + requireMention: false, + }, + }, + }, + }, { id: "matrix-reaction-notification", standardId: "reaction-observation", @@ -110,6 +195,28 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 60_000, title: "Matrix lane resumes cleanly after gateway restart", }, + { + id: "matrix-room-membership-loss", + timeoutMs: 75_000, + title: "Matrix room membership loss recovers after re-invite", + topology: { + defaultRoomKey: "main", + rooms: [ + { + key: MATRIX_QA_MEMBERSHIP_ROOM_KEY, + kind: "group", + members: ["driver", "observer", "sut"], + name: "Matrix QA Membership Room", + requireMention: true, + }, + ], + }, + }, + { + id: "matrix-homeserver-restart-resume", + timeoutMs: 75_000, + title: "Matrix lane resumes after homeserver restart", + }, { id: "matrix-mention-gating", standardId: "mention-gating", @@ -137,6 +244,28 @@ export function findMatrixQaScenarios(ids?: string[]) { }); } +export function buildMatrixQaTopologyForScenarios(params: { + defaultRoomName: string; + scenarios: MatrixQaScenarioDefinition[]; +}): MatrixQaTopologySpec { + return mergeMatrixQaTopologySpecs([ + buildDefaultMatrixQaTopologySpec({ + defaultRoomName: params.defaultRoomName, + }), + ...params.scenarios.flatMap((scenario) => (scenario.topology ? [scenario.topology] : [])), + ]); +} + +export function resolveMatrixQaScenarioRoomId( + context: Pick, + roomKey?: string, +) { + if (!roomKey) { + return context.roomId; + } + return findMatrixQaProvisionedRoom(context.topology, roomKey).roomId; +} + export function buildMentionPrompt(sutUserId: string, token: string) { return `${sutUserId} reply with only this exact marker: ${token}`; } @@ -251,6 +380,13 @@ function advanceMatrixQaActorCursor(params: { writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); } +function createMatrixQaScenarioClient(params: { accessToken: string; baseUrl: string }) { + return createMatrixQaClient({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + }); +} + async function runTopLevelMentionScenario(params: { accessToken: string; actorId: MatrixQaActorId; @@ -306,6 +442,85 @@ async function runTopLevelMentionScenario(params: { }; } +async function waitForMembershipEvent(params: { + accessToken: string; + actorId: MatrixQaActorId; + baseUrl: string; + membership: "invite" | "join" | "leave"; + observedEvents: MatrixQaObservedEvent[]; + roomId: string; + stateKey: string; + syncState: MatrixQaSyncState; + timeoutMs: number; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + syncState: params.syncState, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.type === "m.room.member" && + event.stateKey === params.stateKey && + event.membership === params.membership, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.syncState, + nextSince: matched.since, + startSince, + }); + return matched.event; +} + +async function runTopologyScopedTopLevelScenario(params: { + accessToken: string; + actorId: MatrixQaActorId; + actorUserId: string; + context: MatrixQaScenarioContext; + roomKey: string; + tokenPrefix: string; + withMention?: boolean; +}) { + const roomId = resolveMatrixQaScenarioRoomId(params.context, params.roomKey); + const result = await runTopLevelMentionScenario({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.context.baseUrl, + observedEvents: params.context.observedEvents, + roomId, + syncState: params.context.syncState, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: params.tokenPrefix, + withMention: params.withMention, + }); + assertTopLevelReplyArtifact(`reply in ${params.roomKey}`, result.reply); + return { + artifacts: { + actorUserId: params.actorUserId, + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: params.roomKey, + token: result.token, + triggerBody: result.body, + }, + details: [ + `room key: ${params.roomKey}`, + `room id: ${roomId}`, + `driver event: ${result.driverEventId}`, + `trigger sender: ${params.actorUserId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + async function runThreadScenario(params: MatrixQaScenarioContext) { const { client, startSince } = await primeMatrixQaActorCursor({ accessToken: params.driverAccessToken, @@ -421,6 +636,105 @@ async function runNoReplyExpectedScenario(params: { } satisfies MatrixQaScenarioExecution; } +async function runMembershipLossScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEMBERSHIP_ROOM_KEY); + const driverClient = createMatrixQaScenarioClient({ + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + }); + const sutClient = createMatrixQaScenarioClient({ + accessToken: context.sutAccessToken, + baseUrl: context.baseUrl, + }); + + await driverClient.kickUserFromRoom({ + reason: "matrix qa membership loss", + roomId, + userId: context.sutUserId, + }); + const leaveEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "leave", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const noReplyToken = `MATRIX_QA_MEMBERSHIP_LOSS_${randomUUID().slice(0, 8).toUpperCase()}`; + await runNoReplyExpectedScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + baseUrl: context.baseUrl, + body: buildMentionPrompt(context.sutUserId, noReplyToken), + mentionUserIds: [context.sutUserId], + observedEvents: context.observedEvents, + roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + token: noReplyToken, + }); + + await driverClient.inviteUserToRoom({ + roomId, + userId: context.sutUserId, + }); + await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "invite", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + await sutClient.joinRoom(roomId); + const joinEvent = await waitForMembershipEvent({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + membership: "join", + observedEvents: context.observedEvents, + roomId, + stateKey: context.sutUserId, + syncState: context.syncState, + timeoutMs: context.timeoutMs, + }); + + const recovered = await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_MEMBERSHIP_ROOM_KEY, + tokenPrefix: "MATRIX_QA_MEMBERSHIP_RETURN", + }); + + return { + artifacts: { + ...recovered.artifacts, + membershipJoinEventId: joinEvent.eventId, + membershipLeaveEventId: leaveEvent.eventId, + recoveredDriverEventId: recovered.artifacts?.driverEventId, + recoveredReply: recovered.artifacts?.reply, + }, + details: [ + `room key: ${MATRIX_QA_MEMBERSHIP_ROOM_KEY}`, + `room id: ${roomId}`, + `leave event: ${leaveEvent.eventId}`, + `join event: ${joinEvent.eventId}`, + recovered.details, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { const reactionTargetEventId = context.canary?.reply.eventId?.trim(); if (!reactionTargetEventId) { @@ -472,6 +786,38 @@ async function runReactionNotificationScenario(context: MatrixQaScenarioContext) } satisfies MatrixQaScenarioExecution; } +async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.interruptTransport) { + throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); + } + await context.interruptTransport(); + const resumed = await runTopLevelMentionScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + sutUserId: context.sutUserId, + timeoutMs: context.timeoutMs, + tokenPrefix: "MATRIX_QA_HOMESERVER", + }); + assertTopLevelReplyArtifact("post-homeserver-restart reply", resumed.reply); + return { + artifacts: { + driverEventId: resumed.driverEventId, + reply: resumed.reply, + token: resumed.token, + transportInterruption: "homeserver-restart", + }, + details: [ + "transport interruption: homeserver-restart", + `driver event: ${resumed.driverEventId}`, + ...buildMatrixReplyDetails("reply", resumed.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + async function runRestartResumeScenario(context: MatrixQaScenarioContext) { if (!context.restartGateway) { throw new Error("Matrix restart scenario requires a gateway restart callback"); @@ -615,10 +961,43 @@ export async function runMatrixQaScenario( ].join("\n"), }; } + case "matrix-dm-reply-shape": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_DRIVER_DM_ROOM_KEY, + tokenPrefix: "MATRIX_QA_DM", + withMention: false, + }); + case "matrix-secondary-room-reply": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, + tokenPrefix: "MATRIX_QA_SECONDARY", + }); + case "matrix-secondary-room-open-trigger": + return await runTopologyScopedTopLevelScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + context, + roomKey: MATRIX_QA_SECONDARY_ROOM_KEY, + tokenPrefix: "MATRIX_QA_SECONDARY_OPEN", + withMention: false, + }); case "matrix-reaction-notification": return await runReactionNotificationScenario(context); case "matrix-restart-resume": return await runRestartResumeScenario(context); + case "matrix-room-membership-loss": + return await runMembershipLossScenario(context); + case "matrix-homeserver-restart-resume": + return await runHomeserverRestartResumeScenario(context); case "matrix-mention-gating": { const token = `MATRIX_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; return await runNoReplyExpectedScenario({ @@ -660,11 +1039,16 @@ export async function runMatrixQaScenario( } export const __testing = { + MATRIX_QA_DRIVER_DM_ROOM_KEY, + MATRIX_QA_MEMBERSHIP_ROOM_KEY, + MATRIX_QA_SECONDARY_ROOM_KEY, MATRIX_QA_STANDARD_SCENARIO_IDS, + buildMatrixQaTopologyForScenarios, buildMatrixReplyDetails, buildMatrixReplyArtifact, buildMentionPrompt, findMatrixQaScenarios, readMatrixQaSyncCursor, + resolveMatrixQaScenarioRoomId, writeMatrixQaSyncCursor, };