diff --git a/extensions/qa-matrix/package.json b/extensions/qa-matrix/package.json index bc82a3f0722..4043ca01989 100644 --- a/extensions/qa-matrix/package.json +++ b/extensions/qa-matrix/package.json @@ -5,6 +5,7 @@ "description": "OpenClaw Matrix QA runner plugin", "type": "module", "devDependencies": { + "@openclaw/matrix": "workspace:*", "@openclaw/plugin-sdk": "workspace:*", "openclaw": "workspace:*" }, diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index fc950956777..da6da2db7c5 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -609,6 +609,8 @@ export async function runMatrixQaLive(params: { baseUrl: harness.baseUrl, canary: canaryArtifact, driverAccessToken: provisioning.driver.accessToken, + driverDeviceId: provisioning.driver.deviceId, + driverPassword: provisioning.driver.password, driverUserId: provisioning.driver.userId, interruptTransport: async () => { writeMatrixQaProgress(`transport interrupt start ${scenario.id}`); @@ -626,7 +628,10 @@ export async function runMatrixQaLive(params: { }, observedEvents, observerAccessToken: provisioning.observer.accessToken, + observerDeviceId: provisioning.observer.deviceId, + observerPassword: provisioning.observer.password, observerUserId: provisioning.observer.userId, + outputDir, restartGateway: async () => { if (!gatewayHarness) { throw new Error("Matrix restart scenario requires a live gateway"); @@ -644,6 +649,8 @@ export async function runMatrixQaLive(params: { }, roomId: provisioning.roomId, sutAccessToken: provisioning.sut.accessToken, + sutDeviceId: provisioning.sut.deviceId, + sutPassword: provisioning.sut.password, syncState, syncStreams, sutUserId: provisioning.sut.userId, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index 202e96d24f1..658c6c05bcc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -23,6 +23,9 @@ export type MatrixQaScenarioId = | "matrix-room-block-streaming" | "matrix-room-image-understanding-attachment" | "matrix-room-generated-image-delivery" + | "matrix-media-type-coverage" + | "matrix-attachment-only-ignored" + | "matrix-unsupported-media-safe" | "matrix-dm-reply-shape" | "matrix-dm-shared-session-notice" | "matrix-dm-thread-reply-override" @@ -33,12 +36,31 @@ export type MatrixQaScenarioId = | "matrix-reaction-notification" | "matrix-reaction-threaded" | "matrix-reaction-not-a-reply" + | "matrix-reaction-redaction-observed" | "matrix-restart-resume" + | "matrix-post-restart-room-continue" | "matrix-room-membership-loss" | "matrix-homeserver-restart-resume" | "matrix-mention-gating" + | "matrix-mention-metadata-spoof-block" | "matrix-observer-allowlist-override" - | "matrix-allowlist-block"; + | "matrix-allowlist-block" + | "matrix-multi-actor-ordering" + | "matrix-inbound-edit-ignored" + | "matrix-inbound-edit-no-duplicate-trigger" + | "matrix-e2ee-basic-reply" + | "matrix-e2ee-thread-follow-up" + | "matrix-e2ee-bootstrap-success" + | "matrix-e2ee-recovery-key-lifecycle" + | "matrix-e2ee-device-sas-verification" + | "matrix-e2ee-qr-verification" + | "matrix-e2ee-stale-device-hygiene" + | "matrix-e2ee-dm-sas-verification" + | "matrix-e2ee-restart-resume" + | "matrix-e2ee-verification-notice-no-trigger" + | "matrix-e2ee-artifact-redaction" + | "matrix-e2ee-media-image" + | "matrix-e2ee-key-bootstrap-failure"; export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { configOverrides?: MatrixQaConfigOverrides; @@ -48,6 +70,8 @@ export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition Buffer; + expectedAttachmentKind: "audio" | "file" | "image" | "video"; + expectedMsgtype: "m.audio" | "m.file" | "m.image" | "m.video"; + fileName: string; + kind: "audio" | "file" | "image" | "video"; + label: string; + tokenPrefix: string; +}; + +const MATRIX_QA_IMAGE_COLOR_GROUPS = [["red"], ["blue"]] as const; +const MATRIX_QA_ONE_PIXEL_JPEG_BASE64 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEBUQEBAVFRUVFRUVFRUVFRUVFRUVFRUXFhUVFRUYHSggGBolHRUVITEhJSkrLi4uFx8zODMsNygtLisBCgoKDg0OGhAQGi0fHyUtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAAEAAQMBIgACEQEDEQH/xAAXAAEBAQEAAAAAAAAAAAAAAAAAAQID/8QAFhEBAQEAAAAAAAAAAAAAAAAAAAER/9oADAMBAAIQAxAAAAH2AP/EABgQAQEAAwAAAAAAAAAAAAAAAAEAEQIS/9oACAEBAAEFAk1o7//EABYRAQEBAAAAAAAAAAAAAAAAAAABEf/aAAgBAwEBPwGn/8QAFhEBAQEAAAAAAAAAAAAAAAAAABEB/9oACAECAQE/AYf/xAAaEAACAgMAAAAAAAAAAAAAAAABEQAhMUFh/9oACAEBAAY/AjK9cY2f/8QAGhABAQACAwAAAAAAAAAAAAAAAAERITFBUf/aAAgBAQABPyGQk7W5jVYkA//Z"; + +export function createMatrixQaSplitColorImagePng() { + const width = 16; + const height = 16; + const rgba = Buffer.alloc(width * height * 4); + for (let y = 0; y < height; y += 1) { + const isTopHalf = y < height / 2; + for (let x = 0; x < width; x += 1) { + const [red, green, blue] = isTopHalf ? [255, 0, 0] : [0, 0, 255]; + fillPixel(rgba, x, y, width, red, green, blue); + } + } + return encodePngRgba(rgba, width, height); +} + +function createMatrixQaOnePixelJpeg() { + return Buffer.from(MATRIX_QA_ONE_PIXEL_JPEG_BASE64, "base64"); +} + +function createMatrixQaPdfFixture() { + return Buffer.from( + [ + "%PDF-1.4", + "1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj", + "2 0 obj << /Type /Pages /Count 0 >> endobj", + "trailer << /Root 1 0 R >>", + "%%EOF", + ].join("\n"), + "utf8", + ); +} + +function createMatrixQaEpubFixture() { + return Buffer.from("PK\u0003\u0004mimetypeapplication/epub+zip\n", "utf8"); +} + +function createMatrixQaWavFixture() { + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(1, 22); + header.writeUInt32LE(8_000, 24); + header.writeUInt32LE(16_000, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(0, 40); + return header; +} + +function createMatrixQaMp4Fixture() { + return Buffer.from([ + 0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, 0x00, 0x00, 0x02, 0x00, + 0x69, 0x73, 0x6f, 0x6d, 0x6d, 0x70, 0x34, 0x31, + ]); +} + +export const MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES: MatrixQaMediaTypeCoverageCase[] = [ + { + contentType: "image/jpeg", + createBuffer: createMatrixQaOnePixelJpeg, + expectedAttachmentKind: "image", + expectedMsgtype: "m.image", + fileName: "matrix-qa-one-pixel.jpg", + kind: "image", + label: "jpeg image", + tokenPrefix: "MATRIX_QA_MEDIA_JPEG", + }, + { + contentType: "application/pdf", + createBuffer: createMatrixQaPdfFixture, + expectedAttachmentKind: "file", + expectedMsgtype: "m.file", + fileName: "matrix-qa-document.pdf", + kind: "file", + label: "pdf file", + tokenPrefix: "MATRIX_QA_MEDIA_PDF", + }, + { + contentType: "application/epub+zip", + createBuffer: createMatrixQaEpubFixture, + expectedAttachmentKind: "file", + expectedMsgtype: "m.file", + fileName: "matrix-qa-book.epub", + kind: "file", + label: "epub file", + tokenPrefix: "MATRIX_QA_MEDIA_EPUB", + }, + { + contentType: "audio/wav", + createBuffer: createMatrixQaWavFixture, + expectedAttachmentKind: "audio", + expectedMsgtype: "m.audio", + fileName: "matrix-qa-audio.wav", + kind: "audio", + label: "wav audio", + tokenPrefix: "MATRIX_QA_MEDIA_AUDIO", + }, + { + contentType: "video/mp4", + createBuffer: createMatrixQaMp4Fixture, + expectedAttachmentKind: "video", + expectedMsgtype: "m.video", + fileName: "matrix-qa-video.mp4", + kind: "video", + label: "mp4 video", + tokenPrefix: "MATRIX_QA_MEDIA_VIDEO", + }, +]; + +export function buildMatrixQaImageUnderstandingPrompt(sutUserId: string) { + return `${sutUserId} Image understanding check: describe the top and bottom colors in the attached image in one short sentence.`; +} + +export function buildMatrixQaImageGenerationPrompt(sutUserId: string) { + return `${sutUserId} Image generation check: generate a QA lighthouse image and summarize it in one short sentence.`; +} + +export function hasMatrixQaExpectedColorReply(body: string | undefined) { + const normalizedBody = body?.toLowerCase() ?? ""; + return MATRIX_QA_IMAGE_COLOR_GROUPS.every((group) => + group.some((color) => normalizedBody.includes(color)), + ); +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts new file mode 100644 index 00000000000..d139053d31f --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts @@ -0,0 +1,1228 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { MatrixVerificationSummary } from "@openclaw/matrix/test-api.js"; +import { createMatrixQaClient } from "../../substrate/client.js"; +import { + createMatrixQaE2eeScenarioClient, + runMatrixQaE2eeBootstrap, +} from "../../substrate/e2ee-client.js"; +import type { MatrixQaE2eeScenarioClient } from "../../substrate/e2ee-client.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { + startMatrixQaFaultProxy, + type MatrixQaFaultProxyHit, + type MatrixQaFaultProxyRule, +} from "../../substrate/fault-proxy.js"; +import { + MATRIX_QA_E2EE_ROOM_KEY, + MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import { + buildMatrixQaImageUnderstandingPrompt, + createMatrixQaSplitColorImagePng, + hasMatrixQaExpectedColorReply, + MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, +} from "./scenario-media-fixtures.js"; +import { + assertThreadReplyArtifact, + assertTopLevelReplyArtifact, + buildMatrixQaToken, + buildMatrixReplyDetails, + buildMentionPrompt, + doesMatrixQaReplyBodyMatchToken, + isMatrixQaExactMarkerReply, + NO_REPLY_WINDOW_MS, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaReplyArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; + +type MatrixQaE2eeScenarioId = + | "matrix-e2ee-artifact-redaction" + | "matrix-e2ee-basic-reply" + | "matrix-e2ee-bootstrap-success" + | "matrix-e2ee-device-sas-verification" + | "matrix-e2ee-dm-sas-verification" + | "matrix-e2ee-key-bootstrap-failure" + | "matrix-e2ee-media-image" + | "matrix-e2ee-qr-verification" + | "matrix-e2ee-recovery-key-lifecycle" + | "matrix-e2ee-restart-resume" + | "matrix-e2ee-stale-device-hygiene" + | "matrix-e2ee-thread-follow-up" + | "matrix-e2ee-verification-notice-no-trigger"; + +const MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT = "/_matrix/client/v3/room_keys/version"; +const MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID = "room-key-backup-version-unavailable"; + +type MatrixQaE2eeBootstrapResult = Awaited>; + +function requireMatrixQaE2eeOutputDir(context: MatrixQaScenarioContext) { + if (!context.outputDir) { + throw new Error("Matrix E2EE QA scenarios require an output directory"); + } + return context.outputDir; +} + +function requireMatrixQaPassword(context: MatrixQaScenarioContext, actor: "driver" | "observer") { + const password = actor === "driver" ? context.driverPassword : context.observerPassword; + if (!password) { + throw new Error(`Matrix E2EE ${actor} password is required for this scenario`); + } + return password; +} + +function assertMatrixQaBootstrapSucceeded(label: string, result: MatrixQaE2eeBootstrapResult) { + if (!result.success) { + throw new Error(`${label} bootstrap failed: ${result.error ?? "unknown error"}`); + } + if (!result.verification.verified || !result.verification.signedByOwner) { + throw new Error(`${label} bootstrap did not leave the device verified by its owner`); + } + if (!result.crossSigning.published) { + throw new Error(`${label} bootstrap did not publish cross-signing keys`); + } + if (!result.verification.recoveryKeyStored) { + throw new Error(`${label} bootstrap did not store a recovery key`); + } + if (!result.verification.backupVersion) { + throw new Error(`${label} bootstrap did not create a room-key backup`); + } +} + +function isMatrixQaRepairableBackupBootstrapError(error: string | undefined) { + const normalized = error?.toLowerCase() ?? ""; + return ( + normalized.includes("room key backup is not usable") || + normalized.includes("m.megolm_backup.v1") || + normalized.includes("backup decryption key could not be loaded") + ); +} + +async function assertMatrixQaPeerDeviceTrusted(params: { + client: MatrixQaE2eeScenarioClient; + deviceId: string; + label: string; + userId: string; +}) { + const status = await params.client.getDeviceVerificationStatus(params.userId, params.deviceId); + if (!status.verified) { + throw new Error( + `${params.label} did not trust ${params.userId}/${params.deviceId} after verification`, + ); + } + return status; +} + +async function ensureMatrixQaE2eeOwnDeviceVerified(params: { + client: MatrixQaE2eeScenarioClient; + label: string; +}) { + let bootstrap = await params.client.bootstrapOwnDeviceVerification({ + forceResetCrossSigning: true, + }); + if (!bootstrap.success && isMatrixQaRepairableBackupBootstrapError(bootstrap.error)) { + const reset = await params.client.resetRoomKeyBackup(); + if (reset.success) { + bootstrap = await params.client.bootstrapOwnDeviceVerification({ + forceResetCrossSigning: true, + }); + } + } + assertMatrixQaBootstrapSucceeded(params.label, bootstrap); + return { + bootstrap, + recoveryKey: await params.client.getRecoveryKey(), + verification: bootstrap.verification, + }; +} + +async function waitForMatrixQaNonEmptyRoomKeyRestore(params: { + client: MatrixQaE2eeScenarioClient; + recoveryKey: string; + timeoutMs: number; +}) { + const startedAt = Date.now(); + let last: Awaited> | null = null; + while (Date.now() - startedAt < params.timeoutMs) { + const restored = await params.client.restoreRoomKeyBackup({ + recoveryKey: params.recoveryKey, + }); + last = restored; + if (!restored.success) { + throw new Error( + `Matrix E2EE room-key backup restore failed: ${restored.error ?? "unknown error"}`, + ); + } + if (restored.total > 0 && restored.imported > 0) { + return restored; + } + await sleep(500); + } + throw new Error( + `Matrix E2EE room-key backup restore did not import any keys before timeout (last imported/total: ${last?.imported ?? 0}/${last?.total ?? 0})`, + ); +} + +async function waitForMatrixQaVerificationSummary(params: { + client: MatrixQaE2eeScenarioClient; + label: string; + predicate: (summary: MatrixVerificationSummary) => boolean; + timeoutMs: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.timeoutMs) { + const summaries = await params.client.listVerifications(); + const found = summaries.find(params.predicate); + if (found) { + return found; + } + await sleep(Math.min(250, Math.max(25, params.timeoutMs - (Date.now() - startedAt)))); + } + throw new Error(`timed out waiting for Matrix verification summary: ${params.label}`); +} + +function sameMatrixQaVerificationTransaction( + left: MatrixVerificationSummary, + right: MatrixVerificationSummary, +) { + return Boolean(left.transactionId && left.transactionId === right.transactionId); +} + +function formatMatrixQaSasEmoji(summary: MatrixVerificationSummary) { + return summary.sas?.emoji?.map(([emoji, label]) => `${emoji} ${label}`) ?? []; +} + +function assertMatrixQaSasEmojiMatches(params: { + initiator: MatrixVerificationSummary; + recipient: MatrixVerificationSummary; +}) { + const initiatorEmoji = formatMatrixQaSasEmoji(params.initiator); + const recipientEmoji = formatMatrixQaSasEmoji(params.recipient); + if (initiatorEmoji.length === 0 || recipientEmoji.length === 0) { + throw new Error("Matrix SAS verification did not expose emoji data on both devices"); + } + if (JSON.stringify(initiatorEmoji) !== JSON.stringify(recipientEmoji)) { + throw new Error("Matrix SAS emoji did not match between verification devices"); + } + return initiatorEmoji; +} + +function isMatrixQaOwnerVerificationOnlyRecoveryError(error: string | undefined) { + return error?.toLowerCase().includes("device is still not verified by its owner") === true; +} + +function hasMatrixQaUsableRecoveryBackup( + result: Awaited>, +) { + return ( + Boolean(result.backup.serverVersion) && + result.backup.decryptionKeyCached !== false && + result.backup.keyLoadError === null + ); +} + +function isMatrixQaE2eeNoticeTriggeredSutReply(params: { + event: MatrixQaObservedEvent; + noticeEventId: string; + noticeSentAt: number; + roomId: string; + sutUserId: string; + token: string; +}) { + if ( + params.event.roomId !== params.roomId || + params.event.sender !== params.sutUserId || + params.event.type !== "m.room.message" + ) { + return false; + } + if (params.event.body?.includes(params.token)) { + return true; + } + if ( + params.event.relatesTo?.eventId === params.noticeEventId || + params.event.relatesTo?.inReplyToId === params.noticeEventId + ) { + return true; + } + return ( + typeof params.event.originServerTs === "number" && + params.event.originServerTs >= params.noticeSentAt + ); +} + +async function createMatrixQaE2eeDriverClient( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, +) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + deviceId: context.driverDeviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: context.driverPassword, + scenarioId, + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); +} + +async function createMatrixQaE2eeObserverClient( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, +) { + return await createMatrixQaE2eeScenarioClient({ + accessToken: context.observerAccessToken, + actorId: "observer", + baseUrl: context.baseUrl, + deviceId: context.observerDeviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: context.observerPassword, + scenarioId, + timeoutMs: context.timeoutMs, + userId: context.observerUserId, + }); +} + +async function withMatrixQaE2eeDriverAndObserver( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + run: (clients: { + driver: MatrixQaE2eeScenarioClient; + observer: MatrixQaE2eeScenarioClient; + }) => Promise, +) { + const driver = await createMatrixQaE2eeDriverClient(context, scenarioId); + const observer = await createMatrixQaE2eeObserverClient(context, scenarioId); + try { + return await run({ driver, observer }); + } finally { + await Promise.all([driver.stop(), observer.stop()]); + } +} + +async function completeMatrixQaSasVerification(params: { + initiator: MatrixQaE2eeScenarioClient; + recipient: MatrixQaE2eeScenarioClient; + recipientUserId: string; + request: { + deviceId?: string; + roomId?: string; + userId: string; + }; + timeoutMs: number; +}) { + const initiated = await params.initiator.requestVerification(params.request); + const recipientRequested = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient request", + predicate: (summary) => + !summary.initiatedByMe && + (sameMatrixQaVerificationTransaction(summary, initiated) || + (summary.otherUserId !== params.recipientUserId && summary.pending)), + timeoutMs: params.timeoutMs, + }); + if (recipientRequested.canAccept) { + await params.recipient.acceptVerification(recipientRequested.id); + } + await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator ready", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", + timeoutMs: params.timeoutMs, + }); + await params.initiator.startVerification(initiated.id, "sas"); + const initiatorSas = await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator SAS", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasSas, + timeoutMs: params.timeoutMs, + }); + const recipientSas = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient SAS", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiatorSas) && summary.hasSas, + timeoutMs: params.timeoutMs, + }); + const sasEmoji = assertMatrixQaSasEmojiMatches({ + initiator: initiatorSas, + recipient: recipientSas, + }); + await params.initiator.confirmVerificationSas(initiatorSas.id); + await params.recipient.confirmVerificationSas(recipientSas.id); + const completedInitiator = await waitForMatrixQaVerificationSummary({ + client: params.initiator, + label: "initiator complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, + timeoutMs: params.timeoutMs, + }); + const completedRecipient = await waitForMatrixQaVerificationSummary({ + client: params.recipient, + label: "recipient complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, completedInitiator) && summary.completed, + timeoutMs: params.timeoutMs, + }); + return { + completedInitiator, + completedRecipient, + sasEmoji, + }; +} + +function buildMatrixE2eeReplyArtifact( + event: MatrixQaObservedEvent, + token: string, +): MatrixQaReplyArtifact { + return { + eventId: event.eventId, + mentions: event.mentions, + relatesTo: event.relatesTo, + sender: event.sender, + tokenMatched: doesMatrixQaReplyBodyMatchToken(event, token), + }; +} + +function buildRoomKeyBackupUnavailableFaultRule(accessToken: string): MatrixQaFaultProxyRule { + return { + id: MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID, + match: (request) => + request.method === "GET" && + request.path === MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT && + request.bearerToken === accessToken, + response: () => ({ + body: { + errcode: "M_NOT_FOUND", + error: "No current key backup", + }, + status: 404, + }), + }; +} + +async function runMatrixQaFaultedE2eeBootstrap(context: MatrixQaScenarioContext): Promise<{ + faultHits: MatrixQaFaultProxyHit[]; + result: MatrixQaE2eeBootstrapResult; +}> { + const proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: context.baseUrl, + rules: [buildRoomKeyBackupUnavailableFaultRule(context.driverAccessToken)], + }); + try { + const result = await runMatrixQaE2eeBootstrap({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: proxy.baseUrl, + deviceId: context.driverDeviceId, + outputDir: requireMatrixQaE2eeOutputDir(context), + ...(context.driverPassword ? { password: context.driverPassword } : {}), + scenarioId: "matrix-e2ee-key-bootstrap-failure", + timeoutMs: context.timeoutMs, + userId: context.driverUserId, + }); + return { + faultHits: proxy.hits(), + result, + }; + } finally { + await proxy.stop(); + } +} + +function assertMatrixQaExpectedBootstrapFailure(params: { + faultHits: MatrixQaFaultProxyHit[]; + result: MatrixQaE2eeBootstrapResult; +}) { + if (params.faultHits.length === 0) { + throw new Error("Matrix E2EE bootstrap fault proxy was not exercised"); + } + if (params.result.success) { + throw new Error( + "Matrix E2EE bootstrap unexpectedly succeeded while room-key backup was faulted", + ); + } + const bootstrapError = params.result.error ?? ""; + if (!bootstrapError.toLowerCase().includes("room key backup")) { + throw new Error(`Matrix E2EE bootstrap failed for an unexpected reason: ${bootstrapError}`); + } + return bootstrapError; +} + +async function withMatrixQaE2eeDriver( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, + run: (client: MatrixQaE2eeScenarioClient) => Promise, +) { + const client = await createMatrixQaE2eeDriverClient(context, scenarioId); + try { + return await run(client); + } finally { + await client.stop(); + } +} + +async function runMatrixQaE2eeTopLevelScenario( + context: MatrixQaScenarioContext, + params: { + scenarioId: MatrixQaE2eeScenarioId; + tokenPrefix: string; + }, +) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + return await withMatrixQaE2eeDriver(context, params.scenarioId, async (client) => { + const startSince = await client.prime(); + const token = buildMatrixQaToken(params.tokenPrefix); + const body = buildMentionPrompt(context.sutUserId, token); + const driverEventId = await client.sendTextMessage({ + body, + mentionUserIds: [context.sutUserId], + roomId, + }); + const matched = await client.waitForRoomEvent({ + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token, + }) && event.relatesTo === undefined, + roomId, + timeoutMs: context.timeoutMs, + }); + const reply = buildMatrixE2eeReplyArtifact(matched.event, token); + assertTopLevelReplyArtifact("E2EE reply", reply); + return { + driverEventId, + reply, + roomId, + since: matched.since ?? startSince, + token, + }; + }); +} + +export async function runMatrixQaE2eeBasicReplyScenario( + context: MatrixQaScenarioContext, +): Promise { + const result = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-basic-reply", + tokenPrefix: "MATRIX_QA_E2EE_BASIC", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId: result.roomId, + }, + details: [ + `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room id: ${result.roomId}`, + `driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("E2EE reply", result.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeThreadFollowUpScenario( + context: MatrixQaScenarioContext, +): Promise { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const result = await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-thread-follow-up", + async (client) => { + await client.prime(); + const rootEventId = await client.sendTextMessage({ + body: `E2EE thread root ${randomUUID().slice(0, 8)}`, + roomId, + }); + const token = buildMatrixQaToken("MATRIX_QA_E2EE_THREAD"); + const driverEventId = await client.sendTextMessage({ + body: buildMentionPrompt(context.sutUserId, token), + mentionUserIds: [context.sutUserId], + replyToEventId: rootEventId, + roomId, + threadRootEventId: rootEventId, + }); + const matched = await client.waitForRoomEvent({ + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token, + }) && + event.relatesTo?.relType === "m.thread" && + event.relatesTo.eventId === rootEventId, + roomId, + timeoutMs: context.timeoutMs, + }); + const reply = buildMatrixE2eeReplyArtifact(matched.event, token); + assertThreadReplyArtifact(reply, { + expectedRootEventId: rootEventId, + label: "E2EE threaded reply", + }); + return { + driverEventId, + reply, + rootEventId, + token, + }; + }, + ); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + rootEventId: result.rootEventId, + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId, + }, + details: [ + `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room id: ${roomId}`, + `thread root event: ${result.rootEventId}`, + `mention trigger event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("E2EE threaded reply", result.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeBootstrapSuccessScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver(context, "matrix-e2ee-bootstrap-success", async (client) => { + const result = await client.bootstrapOwnDeviceVerification({ + forceResetCrossSigning: true, + }); + assertMatrixQaBootstrapSucceeded("driver", result); + return { + artifacts: { + backupCreatedVersion: result.verification.backupVersion, + bootstrapActor: "driver", + bootstrapSuccess: true, + currentDeviceId: result.verification.deviceId, + recoveryKeyId: result.verification.recoveryKeyId, + recoveryKeyStored: result.verification.recoveryKeyStored, + }, + details: [ + "driver bootstrap succeeded through real Matrix crypto bootstrap", + `device verified: ${result.verification.verified ? "yes" : "no"}`, + `signed by owner: ${result.verification.signedByOwner ? "yes" : "no"}`, + `cross-signing published: ${result.crossSigning.published ? "yes" : "no"}`, + `room-key backup version: ${result.verification.backupVersion ?? ""}`, + `recovery key id: ${result.verification.recoveryKeyId ?? ""}`, + ].join("\n"), + }; + }); +} + +export async function runMatrixQaE2eeRecoveryKeyLifecycleScenario( + context: MatrixQaScenarioContext, +): Promise { + const driverPassword = requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-recovery-key-lifecycle", + async (client) => { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ + client, + label: "driver", + }); + const recoveryKey = ready.recoveryKey; + const encodedRecoveryKey = recoveryKey?.encodedPrivateKey?.trim(); + if (!encodedRecoveryKey) { + throw new Error("Matrix E2EE bootstrap did not expose an encoded recovery key"); + } + const seededEventId = await client.sendTextMessage({ + body: `E2EE recovery-key restore seed ${randomUUID().slice(0, 8)}`, + roomId, + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const recoveryDevice = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Recovery Restore Device", + password: driverPassword, + userId: context.driverUserId, + }); + if (!recoveryDevice.deviceId) { + throw new Error("Matrix E2EE recovery login did not return a secondary device id"); + } + const recoveryClient = await createMatrixQaE2eeScenarioClient({ + accessToken: recoveryDevice.accessToken, + actorId: `driver-recovery-${randomUUID().slice(0, 8)}`, + baseUrl: context.baseUrl, + deviceId: recoveryDevice.deviceId, + observedEvents: context.observedEvents, + outputDir: requireMatrixQaE2eeOutputDir(context), + password: recoveryDevice.password, + scenarioId: "matrix-e2ee-recovery-key-lifecycle", + timeoutMs: context.timeoutMs, + userId: recoveryDevice.userId, + }); + let cleanupRecoveryDevice = true; + try { + const recoveryVerification = await recoveryClient.verifyWithRecoveryKey(encodedRecoveryKey); + const recoveryKeyUsable = + recoveryVerification.success || + isMatrixQaOwnerVerificationOnlyRecoveryError(recoveryVerification.error) || + hasMatrixQaUsableRecoveryBackup(recoveryVerification); + if (!recoveryVerification.success && !recoveryKeyUsable) { + throw new Error( + `Matrix E2EE recovery device verification failed: ${recoveryVerification.error ?? "unknown error"}`, + ); + } + const restored = await waitForMatrixQaNonEmptyRoomKeyRestore({ + client: recoveryClient, + recoveryKey: encodedRecoveryKey, + timeoutMs: context.timeoutMs, + }); + const reset = await recoveryClient.resetRoomKeyBackup(); + if (!reset.success) { + throw new Error( + `Matrix E2EE room-key backup reset failed: ${reset.error ?? "unknown error"}`, + ); + } + await recoveryClient.stop(); + await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); + cleanupRecoveryDevice = false; + return { + artifacts: { + backupCreatedVersion: reset.createdVersion, + backupReset: reset.success, + backupRestored: restored.success, + bootstrapActor: "driver", + bootstrapSuccess: ready.bootstrap?.success ?? true, + recoveryDeviceId: recoveryDevice.deviceId, + recoveryKeyId: recoveryKey?.keyId ?? null, + recoveryKeyUsable, + recoveryKeyStored: true, + recoveryVerified: recoveryVerification.success, + restoreImported: restored.imported, + restoreTotal: restored.total, + seededEventId, + }, + details: [ + "driver recovery lifecycle completed through real Matrix recovery APIs", + `bootstrap backup version: ${ready.verification.backupVersion ?? ""}`, + `seeded encrypted event: ${seededEventId}`, + `recovery device: ${recoveryDevice.deviceId}`, + `recovery key usable: ${recoveryKeyUsable ? "yes" : "no"}`, + `recovery device verified: ${recoveryVerification.success ? "yes" : "no"}`, + `restore imported/total: ${restored.imported}/${restored.total}`, + `restore loaded from secret storage: ${restored.loadedFromSecretStorage ? "yes" : "no"}`, + `reset previous version: ${reset.previousVersion ?? ""}`, + `reset created version: ${reset.createdVersion ?? ""}`, + ].join("\n"), + }; + } finally { + if (cleanupRecoveryDevice) { + await recoveryClient.stop().catch(() => undefined); + await client.deleteOwnDevices([recoveryDevice.deviceId]).catch(() => undefined); + } + } + }, + ); +} + +export async function runMatrixQaE2eeDeviceSasVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + if (!context.observerDeviceId) { + throw new Error("Matrix E2EE observer device id is required for device SAS verification"); + } + if (!context.driverDeviceId) { + throw new Error("Matrix E2EE driver device id is required for device SAS verification"); + } + const observerDeviceId = context.observerDeviceId; + const driverDeviceId = context.driverDeviceId; + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-device-sas-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const result = await completeMatrixQaSasVerification({ + initiator: driver, + recipient: observer, + recipientUserId: context.observerUserId, + request: { + deviceId: observerDeviceId, + userId: context.observerUserId, + }, + timeoutMs: context.timeoutMs, + }); + const driverTrust = await assertMatrixQaPeerDeviceTrusted({ + client: driver, + deviceId: observerDeviceId, + label: "driver", + userId: context.observerUserId, + }); + const observerTrust = await assertMatrixQaPeerDeviceTrusted({ + client: observer, + deviceId: driverDeviceId, + label: "observer", + userId: context.driverUserId, + }); + return { + artifacts: { + completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], + currentDeviceId: driverDeviceId, + driverTrustsObserverDevice: driverTrust.verified, + observerTrustsDriverDevice: observerTrust.verified, + sasEmoji: result.sasEmoji, + secondaryDeviceId: observerDeviceId, + }, + details: [ + "driver-to-observer device verification completed with real SAS", + `initiator transaction: ${result.completedInitiator.transactionId ?? ""}`, + `recipient transaction: ${result.completedRecipient.transactionId ?? ""}`, + `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, + `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, + `emoji: ${result.sasEmoji.join(", ")}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeQrVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + if (!context.observerDeviceId) { + throw new Error("Matrix E2EE observer device id is required for QR verification"); + } + if (!context.driverDeviceId) { + throw new Error("Matrix E2EE driver device id is required for QR verification"); + } + const observerDeviceId = context.observerDeviceId; + const driverDeviceId = context.driverDeviceId; + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-qr-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const initiated = await driver.requestVerification({ + deviceId: observerDeviceId, + userId: context.observerUserId, + }); + const incoming = await waitForMatrixQaVerificationSummary({ + client: observer, + label: "QR recipient request", + predicate: (summary) => + !summary.initiatedByMe && sameMatrixQaVerificationTransaction(summary, initiated), + timeoutMs: context.timeoutMs, + }); + if (incoming.canAccept) { + await observer.acceptVerification(incoming.id); + } + await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR request ready", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.phaseName === "ready", + timeoutMs: context.timeoutMs, + }); + const qr = await driver.generateVerificationQr(initiated.id); + await observer.scanVerificationQr(incoming.id, qr.qrDataBase64); + const reciprocate = await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR reciprocate", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.hasReciprocateQr, + timeoutMs: context.timeoutMs, + }); + await driver.confirmVerificationReciprocateQr(reciprocate.id); + const qrByteCount = Buffer.from(qr.qrDataBase64, "base64").byteLength; + const completedDriver = await waitForMatrixQaVerificationSummary({ + client: driver, + label: "QR driver complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, initiated) && summary.completed, + timeoutMs: context.timeoutMs, + }); + const completedObserver = await waitForMatrixQaVerificationSummary({ + client: observer, + label: "QR observer complete", + predicate: (summary) => + sameMatrixQaVerificationTransaction(summary, completedDriver) && summary.completed, + timeoutMs: context.timeoutMs, + }); + const driverTrust = await driver.getDeviceVerificationStatus( + context.observerUserId, + observerDeviceId, + ); + const observerTrust = await observer.getDeviceVerificationStatus( + context.driverUserId, + driverDeviceId, + ); + return { + artifacts: { + completedVerificationIds: [completedDriver.id, completedObserver.id], + driverTrustsObserverDevice: driverTrust.verified, + identityVerificationCompleted: true, + observerTrustsDriverDevice: observerTrust.verified, + qrBytes: qrByteCount, + secondaryDeviceId: observerDeviceId, + }, + details: [ + "driver-to-observer QR verification completed through real QR scan", + `transaction: ${completedDriver.transactionId ?? ""}`, + `driver trusts observer device: ${driverTrust.verified ? "yes" : "no"}`, + `observer trusts driver device: ${observerTrust.verified ? "yes" : "no"}`, + `qr bytes: ${qrByteCount}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeStaleDeviceHygieneScenario( + context: MatrixQaScenarioContext, +): Promise { + const driverPassword = requireMatrixQaPassword(context, "driver"); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-stale-device-hygiene", + async (client) => { + await ensureMatrixQaE2eeOwnDeviceVerified({ + client, + label: "driver", + }); + const loginClient = createMatrixQaClient({ + baseUrl: context.baseUrl, + }); + const secondary = await loginClient.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Stale Device", + password: driverPassword, + userId: context.driverUserId, + }); + if (!secondary.deviceId) { + throw new Error("Matrix stale-device login did not return a secondary device id"); + } + const before = await client.listOwnDevices(); + if (!before.some((device) => device.deviceId === secondary.deviceId)) { + throw new Error("Matrix stale-device list did not include the secondary login"); + } + const deleted = await client.deleteOwnDevices([secondary.deviceId]); + const remainingDeviceIds = deleted.remainingDevices.map((device) => device.deviceId); + if (remainingDeviceIds.includes(secondary.deviceId)) { + throw new Error( + "Matrix stale-device deletion left the secondary device in the device list", + ); + } + if ( + deleted.currentDeviceId && + !deleted.remainingDevices.some((device) => device.deviceId === deleted.currentDeviceId) + ) { + throw new Error("Matrix stale-device deletion removed the current device"); + } + return { + artifacts: { + currentDeviceId: deleted.currentDeviceId, + deletedDeviceIds: deleted.deletedDeviceIds, + remainingDeviceIds, + secondaryDeviceId: secondary.deviceId, + }, + details: [ + "driver secondary device was created, observed, and removed through real device APIs", + `current device: ${deleted.currentDeviceId ?? ""}`, + `deleted device: ${secondary.deviceId}`, + `remaining devices: ${remainingDeviceIds.join(", ")}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeDmSasVerificationScenario( + context: MatrixQaScenarioContext, +): Promise { + requireMatrixQaPassword(context, "driver"); + requireMatrixQaPassword(context, "observer"); + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY); + return await withMatrixQaE2eeDriverAndObserver( + context, + "matrix-e2ee-dm-sas-verification", + async ({ driver, observer }) => { + await Promise.all([ + ensureMatrixQaE2eeOwnDeviceVerified({ + client: driver, + label: "driver", + }), + ensureMatrixQaE2eeOwnDeviceVerified({ + client: observer, + label: "observer", + }), + ]); + const result = await completeMatrixQaSasVerification({ + initiator: driver, + recipient: observer, + recipientUserId: context.observerUserId, + request: { + roomId, + userId: context.observerUserId, + }, + timeoutMs: context.timeoutMs, + }); + if ( + result.completedInitiator.roomId !== roomId || + result.completedRecipient.roomId !== roomId + ) { + throw new Error("Matrix E2EE DM verification completed outside the expected DM room"); + } + return { + artifacts: { + completedVerificationIds: [result.completedInitiator.id, result.completedRecipient.id], + roomKey: MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, + sasEmoji: result.sasEmoji, + verificationRoomId: roomId, + }, + details: [ + "driver/observer encrypted DM verification completed with SAS in the expected room", + `verification DM room: ${roomId}`, + `transaction: ${result.completedInitiator.transactionId ?? ""}`, + `emoji: ${result.sasEmoji.join(", ")}`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeRestartResumeScenario( + context: MatrixQaScenarioContext, +): Promise { + if (!context.restartGateway) { + throw new Error("Matrix E2EE restart scenario requires gateway restart support"); + } + const first = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-restart-resume", + tokenPrefix: "MATRIX_QA_E2EE_BEFORE_RESTART", + }); + await context.restartGateway(); + const recovered = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-restart-resume", + tokenPrefix: "MATRIX_QA_E2EE_AFTER_RESTART", + }); + return { + artifacts: { + firstDriverEventId: first.driverEventId, + firstReply: first.reply, + recoveredDriverEventId: recovered.driverEventId, + recoveredReply: recovered.reply, + restartSignal: "gateway-restart", + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId: recovered.roomId, + }, + details: [ + `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room id: ${recovered.roomId}`, + `pre-restart event: ${first.driverEventId}`, + ...buildMatrixReplyDetails("pre-restart reply", first.reply), + `post-restart event: ${recovered.driverEventId}`, + ...buildMatrixReplyDetails("post-restart reply", recovered.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario( + context: MatrixQaScenarioContext, +): Promise { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + return await withMatrixQaE2eeDriver( + context, + "matrix-e2ee-verification-notice-no-trigger", + async (client) => { + await client.prime(); + const token = buildMatrixQaToken("MATRIX_QA_E2EE_VERIFY_NOTICE"); + const body = `Matrix verification started with ${context.driverUserId}; ${buildMentionPrompt( + context.sutUserId, + token, + )}`; + const noticeSentAt = Date.now(); + const noticeEventId = await client.sendNoticeMessage({ + body, + mentionUserIds: [context.sutUserId], + roomId, + }); + const result = await client.waitForOptionalRoomEvent({ + predicate: (event) => + isMatrixQaE2eeNoticeTriggeredSutReply({ + event, + noticeEventId, + noticeSentAt, + roomId, + sutUserId: context.sutUserId, + token, + }), + roomId, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + }); + if (result.matched) { + throw new Error(`unexpected E2EE verification-notice reply: ${result.event.eventId}`); + } + return { + artifacts: { + expectedNoReplyWindowMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + noticeEventId, + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId, + }, + details: [ + `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room id: ${roomId}`, + `verification notice event: ${noticeEventId}`, + `waited ${Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs)}ms with no SUT reply`, + ].join("\n"), + }; + }, + ); +} + +export async function runMatrixQaE2eeArtifactRedactionScenario( + context: MatrixQaScenarioContext, +): Promise { + const result = await runMatrixQaE2eeTopLevelScenario(context, { + scenarioId: "matrix-e2ee-artifact-redaction", + tokenPrefix: "MATRIX_QA_E2EE_REDACT", + }); + const leaked = context.observedEvents.some( + (event) => + event.roomId === result.roomId && + (event.body?.includes(result.token) || event.formattedBody?.includes(result.token)), + ); + if (!leaked) { + throw new Error("Matrix E2EE redaction scenario did not observe decrypted content in memory"); + } + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId: result.roomId, + }, + details: [ + "decrypted E2EE payload reached in-memory assertions only", + "observed-event artifacts redact body/formatted_body unless OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1", + `encrypted room id: ${result.roomId}`, + ...buildMatrixReplyDetails("E2EE reply", result.reply), + ].join("\n"), + }; +} + +export async function runMatrixQaE2eeMediaImageScenario( + context: MatrixQaScenarioContext, +): Promise { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + return await withMatrixQaE2eeDriver(context, "matrix-e2ee-media-image", async (client) => { + const startSince = await client.prime(); + const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); + const driverEventId = await client.sendImageMessage({ + body: triggerBody, + buffer: createMatrixQaSplitColorImagePng(), + contentType: "image/png", + fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + mentionUserIds: [context.sutUserId], + roomId, + }); + const attachmentEvent = await client.waitForRoomEvent({ + predicate: (event) => + event.roomId === roomId && + event.eventId === driverEventId && + event.sender === context.driverUserId && + event.attachment?.kind === "image" && + event.attachment.caption === triggerBody, + roomId, + timeoutMs: context.timeoutMs, + }); + const matched = await client.waitForRoomEvent({ + predicate: (event) => + event.roomId === roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.relatesTo === undefined && + hasMatrixQaExpectedColorReply(event.body), + roomId, + timeoutMs: context.timeoutMs, + }); + const reply: MatrixQaReplyArtifact = { + eventId: matched.event.eventId, + mentions: matched.event.mentions, + relatesTo: matched.event.relatesTo, + sender: matched.event.sender, + }; + return { + artifacts: { + attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + driverEventId, + reply, + roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomId, + }, + details: [ + `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room id: ${roomId}`, + `driver encrypted image event: ${driverEventId}`, + `driver encrypted image filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, + `driver encrypted image since: ${attachmentEvent.since ?? startSince ?? ""}`, + ...buildMatrixReplyDetails("E2EE image reply", reply), + ].join("\n"), + }; + }); +} + +export async function runMatrixQaE2eeKeyBootstrapFailureScenario( + context: MatrixQaScenarioContext, +): Promise { + const { faultHits, result } = await runMatrixQaFaultedE2eeBootstrap(context); + const bootstrapError = assertMatrixQaExpectedBootstrapFailure({ faultHits, result }); + + return { + artifacts: { + bootstrapActor: "driver", + bootstrapErrorPreview: bootstrapError.slice(0, 240), + bootstrapSuccess: result.success, + faultedEndpoint: MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT, + faultHitCount: faultHits.length, + ...(faultHits[0]?.ruleId ? { faultRuleId: faultHits[0].ruleId } : {}), + }, + details: [ + "Matrix E2EE bootstrap failure surfaced through real SDK bootstrap.", + `faulted endpoint: GET ${MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT}`, + `fault hits: ${faultHits.length}`, + `bootstrap success: ${result.success ? "yes" : "no"}`, + `bootstrap error: ${bootstrapError || ""}`, + ].join("\n"), + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-edit.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-edit.ts new file mode 100644 index 00000000000..998539ca274 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-edit.ts @@ -0,0 +1,90 @@ +import { randomUUID } from "node:crypto"; +import { + assertNoSutReplyWindow, + buildExactMarkerPrompt, + buildMatrixReplyDetails, + buildMentionPrompt, + primeMatrixQaDriverScenarioClient, + runAssertedDriverTopLevelScenario, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runInboundEditIgnoredScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const ignoredToken = `MATRIX_QA_EDIT_IGNORED_SOURCE_${randomUUID().slice(0, 8).toUpperCase()}`; + const editedToken = `MATRIX_QA_EDIT_IGNORED_${randomUUID().slice(0, 8).toUpperCase()}`; + const rootEventId = await client.sendTextMessage({ + body: buildExactMarkerPrompt(ignoredToken), + roomId: context.roomId, + }); + const editEventId = await client.sendReplacementMessage({ + body: buildMentionPrompt(context.sutUserId, editedToken), + mentionUserIds: [context.sutUserId], + roomId: context.roomId, + targetEventId: rootEventId, + }); + const { noReplyWindowMs } = await assertNoSutReplyWindow({ + actorId: "driver", + client, + context, + roomId: context.roomId, + since: startSince, + startSince, + unexpectedMessage: "unexpected SUT reply after Matrix edit-to-mention event", + }); + return { + artifacts: { + editEventId, + editedToken, + expectedNoReplyWindowMs: noReplyWindowMs, + rootEventId, + }, + details: [ + `root event: ${rootEventId}`, + `edit event: ${editEventId}`, + `waited ${noReplyWindowMs}ms with no SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runInboundEditNoDuplicateTriggerScenario(context: MatrixQaScenarioContext) { + const first = await runAssertedDriverTopLevelScenario({ + context, + label: "pre-edit reply", + tokenPrefix: "MATRIX_QA_EDIT_ORIGINAL", + }); + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const editedToken = `MATRIX_QA_EDIT_DUPLICATE_${randomUUID().slice(0, 8).toUpperCase()}`; + const editEventId = await client.sendReplacementMessage({ + body: buildMentionPrompt(context.sutUserId, editedToken), + mentionUserIds: [context.sutUserId], + roomId: context.roomId, + targetEventId: first.driverEventId, + }); + const { noReplyWindowMs } = await assertNoSutReplyWindow({ + actorId: "driver", + client, + context, + roomId: context.roomId, + since: startSince, + startSince, + unexpectedMessage: "unexpected duplicate SUT reply after Matrix edit", + }); + return { + artifacts: { + editEventId, + editedToken, + expectedNoReplyWindowMs: noReplyWindowMs, + originalDriverEventId: first.driverEventId, + originalReply: first.reply, + originalToken: first.token, + }, + details: [ + `original driver event: ${first.driverEventId}`, + ...buildMatrixReplyDetails("original reply", first.reply), + `edit event: ${editEventId}`, + `waited ${noReplyWindowMs}ms with no duplicate SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-media.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-media.ts new file mode 100644 index 00000000000..1c9f1107cd9 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-media.ts @@ -0,0 +1,368 @@ +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { MATRIX_QA_MEDIA_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js"; +import { + buildMatrixQaImageGenerationPrompt, + buildMatrixQaImageUnderstandingPrompt, + createMatrixQaSplitColorImagePng, + hasMatrixQaExpectedColorReply, + MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES, +} from "./scenario-media-fixtures.js"; +import { + advanceMatrixQaActorCursor, + assertNoSutReplyWindow, + buildMatrixQaToken, + buildMatrixReplyArtifact, + buildMatrixReplyDetails, + isMatrixQaExactMarkerReply, + isMatrixQaMessageLikeKind, + primeMatrixQaActorCursor, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +function requireMatrixQaImageAttachment(event: MatrixQaObservedEvent, scenarioLabel: string) { + if (event.msgtype !== "m.image" || event.attachment?.kind !== "image") { + throw new Error( + `${scenarioLabel} expected an m.image attachment but saw ${event.msgtype ?? ""}`, + ); + } + return event.attachment; +} + +function buildMatrixQaAttachmentDetailLines(params: { + attachmentEvent: MatrixQaObservedEvent; + label: string; +}) { + return [ + `${params.label} event: ${params.attachmentEvent.eventId}`, + `${params.label} msgtype: ${params.attachmentEvent.msgtype ?? ""}`, + `${params.label} attachment kind: ${params.attachmentEvent.attachment?.kind ?? ""}`, + `${params.label} attachment filename: ${params.attachmentEvent.attachment?.filename ?? ""}`, + `${params.label} body preview: ${params.attachmentEvent.body?.slice(0, 200) ?? ""}`, + ]; +} + +async function primeMatrixQaDriverMediaClient(context: MatrixQaScenarioContext) { + return await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + syncState: context.syncState, + syncStreams: context.syncStreams, + }); +} + +function buildMatrixQaMediaTypeCoveragePrompt(params: { + label: string; + sutUserId: string; + token: string; +}) { + return `${params.sutUserId} Matrix media type coverage (${params.label}): ignore the attachment content and reply with only this exact marker: ${params.token}`; +} + +export async function runImageUnderstandingAttachmentScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverMediaClient(context); + const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); + const driverEventId = await client.sendMediaMessage({ + body: triggerBody, + buffer: createMatrixQaSplitColorImagePng(), + contentType: "image/png", + fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + kind: "image", + mentionUserIds: [context.sutUserId], + roomId, + }); + const attachmentEvent = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === roomId && + event.eventId === driverEventId && + event.sender === context.driverUserId && + event.attachment?.kind === "image" && + event.attachment.caption === triggerBody, + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.relatesTo === undefined && + isMatrixQaMessageLikeKind(event.kind) && + hasMatrixQaExpectedColorReply(event.body), + roomId, + since: attachmentEvent.since, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + const reply = buildMatrixReplyArtifact(matched.event); + return { + artifacts: { + attachmentCaptionPreview: attachmentEvent.event.attachment?.caption?.slice(0, 200), + attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + driverEventId, + reply, + roomId, + triggerBody, + }, + details: [ + `room id: ${roomId}`, + `driver attachment event: ${driverEventId}`, + `sent attachment filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, + `sent attachment caption: ${attachmentEvent.event.attachment?.caption ?? ""}`, + ...buildMatrixReplyDetails("reply", reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runMediaTypeCoverageScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverMediaClient(context); + const attachments: NonNullable["attachments"] = []; + const replies: NonNullable["replies"] = []; + const details = [`room id: ${roomId}`]; + let since = startSince; + + for (const mediaCase of MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES) { + const token = buildMatrixQaToken(mediaCase.tokenPrefix); + const triggerBody = buildMatrixQaMediaTypeCoveragePrompt({ + label: mediaCase.label, + sutUserId: context.sutUserId, + token, + }); + const driverEventId = await client.sendMediaMessage({ + body: triggerBody, + buffer: mediaCase.createBuffer(), + contentType: mediaCase.contentType, + fileName: mediaCase.fileName, + kind: mediaCase.kind, + mentionUserIds: [context.sutUserId], + roomId, + }); + const attachmentEvent = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === roomId && + event.eventId === driverEventId && + event.sender === context.driverUserId && + event.msgtype === mediaCase.expectedMsgtype && + event.attachment?.kind === mediaCase.expectedAttachmentKind && + event.attachment.filename === mediaCase.fileName && + event.attachment.caption === triggerBody, + roomId, + since, + timeoutMs: context.timeoutMs, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token, + }) && event.relatesTo === undefined, + roomId, + since: attachmentEvent.since, + timeoutMs: context.timeoutMs, + }); + since = matched.since ?? since; + const reply = buildMatrixReplyArtifact(matched.event, token); + attachments.push({ + eventId: driverEventId, + filename: mediaCase.fileName, + kind: attachmentEvent.event.attachment?.kind, + label: mediaCase.label, + msgtype: attachmentEvent.event.msgtype, + }); + replies.push({ + eventId: reply.eventId, + label: mediaCase.label, + token, + tokenMatched: reply.tokenMatched, + }); + details.push( + `${mediaCase.label} event: ${driverEventId}`, + `${mediaCase.label} msgtype: ${attachmentEvent.event.msgtype ?? ""}`, + `${mediaCase.label} attachment kind: ${attachmentEvent.event.attachment?.kind ?? ""}`, + `${mediaCase.label} attachment filename: ${attachmentEvent.event.attachment?.filename ?? ""}`, + ...buildMatrixReplyDetails(`${mediaCase.label} reply`, reply), + ); + } + + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: since, + startSince, + }); + return { + artifacts: { + attachments, + replies, + roomId, + }, + details: details.join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runAttachmentOnlyIgnoredScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverMediaClient(context); + const driverEventId = await client.sendMediaMessage({ + buffer: createMatrixQaSplitColorImagePng(), + contentType: "image/png", + fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + kind: "image", + roomId, + }); + const attachmentEvent = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === roomId && + event.eventId === driverEventId && + event.sender === context.driverUserId && + event.attachment?.kind === "image" && + event.attachment.caption === undefined, + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const { noReplyWindowMs } = await assertNoSutReplyWindow({ + actorId: "driver", + client, + context, + roomId, + since: attachmentEvent.since, + startSince, + unexpectedMessage: "unexpected SUT reply to attachment-only group media", + }); + return { + artifacts: { + attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, + driverEventId, + expectedNoReplyWindowMs: noReplyWindowMs, + roomId, + }, + details: [ + `room id: ${roomId}`, + `driver attachment event: ${driverEventId}`, + `waited ${noReplyWindowMs}ms with no SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runUnsupportedMediaSafeScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverMediaClient(context); + const token = buildMatrixQaToken("MATRIX_QA_UNSUPPORTED_MEDIA"); + const triggerBody = `${context.sutUserId} Unsupported media QA check: ignore the attached text file and reply with only this exact marker: ${token}`; + const driverEventId = await client.sendMediaMessage({ + body: triggerBody, + buffer: Buffer.from("unsupported Matrix QA attachment body\n", "utf8"), + contentType: "text/plain", + fileName: "unsupported-matrix-qa.txt", + kind: "file", + mentionUserIds: [context.sutUserId], + roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId, + sutUserId: context.sutUserId, + token, + }) && event.relatesTo === undefined, + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + const reply = buildMatrixReplyArtifact(matched.event, token); + return { + artifacts: { + attachmentFilename: "unsupported-matrix-qa.txt", + attachmentKind: "file", + driverEventId, + reply, + roomId, + token, + triggerBody, + }, + details: [ + `room id: ${roomId}`, + `driver file event: ${driverEventId}`, + ...buildMatrixReplyDetails("reply", reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenarioContext) { + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); + const { client, startSince } = await primeMatrixQaDriverMediaClient(context); + const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId); + const driverEventId = await client.sendTextMessage({ + body: triggerBody, + mentionUserIds: [context.sutUserId], + roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.relatesTo === undefined && + event.msgtype === "m.image" && + event.attachment?.kind === "image", + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: matched.since, + startSince, + }); + const attachment = requireMatrixQaImageAttachment( + matched.event, + "Matrix generated image delivery scenario", + ); + return { + artifacts: { + attachmentBodyPreview: matched.event.body?.slice(0, 200), + attachmentEventId: matched.event.eventId, + attachmentFilename: attachment.filename, + attachmentKind: attachment.kind, + attachmentMsgtype: matched.event.msgtype, + driverEventId, + roomId, + triggerBody, + }, + details: [ + `room id: ${roomId}`, + `driver event: ${driverEventId}`, + ...buildMatrixQaAttachmentDetailLines({ + attachmentEvent: matched.event, + label: "generated image", + }), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-reaction.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-reaction.ts new file mode 100644 index 00000000000..c661296b844 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-reaction.ts @@ -0,0 +1,239 @@ +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { + advanceMatrixQaActorCursor, + assertNoSutReplyWindow, + createMatrixQaDriverScenarioClient, + primeMatrixQaActorCursor, + type MatrixQaActorId, + type MatrixQaScenarioContext, + type MatrixQaSyncState, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export function buildMatrixQaReactionDetailLines(params: { + actorUserId?: string; + observedReactionKey?: string; + reactionEmoji: string; + reactionEventId: string; + reactionTargetEventId: string; +}) { + return [ + `reaction event: ${params.reactionEventId}`, + `reaction target: ${params.reactionTargetEventId}`, + `reaction emoji: ${params.reactionEmoji}`, + ...(params.actorUserId ? [`reaction sender: ${params.actorUserId}`] : []), + ...(params.observedReactionKey ? [`observed reaction key: ${params.observedReactionKey}`] : []), + ]; +} + +function requireMatrixQaReactionTargetEventId( + reactionTargetEventId: string | undefined, + scenarioLabel: string, +) { + const normalizedReactionTargetEventId = reactionTargetEventId?.trim(); + if (!normalizedReactionTargetEventId) { + throw new Error(`${scenarioLabel} requires a canary reply event id`); + } + return normalizedReactionTargetEventId; +} + +export async function observeReactionScenario(params: { + actorId: MatrixQaActorId; + actorUserId: string; + accessToken: string; + baseUrl: string; + observedEvents: MatrixQaObservedEvent[]; + reactionEmoji?: string; + reactionTargetEventId: string; + roomId: string; + syncState: MatrixQaSyncState; + syncStreams?: MatrixQaScenarioContext["syncStreams"]; + timeoutMs: number; +}) { + const { client, startSince } = await primeMatrixQaActorCursor({ + accessToken: params.accessToken, + actorId: params.actorId, + baseUrl: params.baseUrl, + observedEvents: params.observedEvents, + syncState: params.syncState, + syncStreams: params.syncStreams, + }); + const reactionEmoji = params.reactionEmoji ?? "👍"; + const reactionEventId = await client.sendReaction({ + emoji: reactionEmoji, + messageId: params.reactionTargetEventId, + roomId: params.roomId, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.actorUserId && + event.type === "m.reaction" && + event.eventId === reactionEventId && + event.reaction?.eventId === params.reactionTargetEventId && + event.reaction?.key === reactionEmoji, + roomId: params.roomId, + since: startSince, + timeoutMs: params.timeoutMs, + }); + return { + actorId: params.actorId, + actorUserId: params.actorUserId, + event: matched.event, + reactionEmoji, + reactionEventId, + reactionTargetEventId: params.reactionTargetEventId, + since: matched.since, + startSince, + }; +} + +export function buildMatrixQaReactionArtifacts(params: { + actorUserId?: string; + expectedNoReplyWindowMs?: number; + reaction: Awaited>; +}) { + return { + ...(params.actorUserId ? { actorUserId: params.actorUserId } : {}), + ...(params.expectedNoReplyWindowMs === undefined + ? {} + : { expectedNoReplyWindowMs: params.expectedNoReplyWindowMs }), + reactionEmoji: params.reaction.reactionEmoji, + reactionEventId: params.reaction.reactionEventId, + reactionTargetEventId: params.reaction.reactionTargetEventId, + }; +} + +export async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { + const reactionTargetEventId = requireMatrixQaReactionTargetEventId( + context.canary?.reply.eventId, + "Matrix reaction scenario", + ); + const result = await observeReactionScenario({ + actorId: "driver", + actorUserId: context.driverUserId, + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + reactionTargetEventId, + roomId: context.roomId, + syncState: context.syncState, + syncStreams: context.syncStreams, + timeoutMs: context.timeoutMs, + }); + return { + artifacts: buildMatrixQaReactionArtifacts({ reaction: result }), + details: buildMatrixQaReactionDetailLines({ + actorUserId: result.actorUserId, + observedReactionKey: result.event.reaction?.key, + reactionEmoji: result.reactionEmoji, + reactionEventId: result.reactionEventId, + reactionTargetEventId: result.reactionTargetEventId, + }).join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runReactionNotAReplyScenario(context: MatrixQaScenarioContext) { + const reactionTargetEventId = requireMatrixQaReactionTargetEventId( + context.canary?.reply.eventId, + "Matrix reaction no-reply scenario", + ); + const reaction = await observeReactionScenario({ + actorId: "driver", + actorUserId: context.driverUserId, + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + reactionTargetEventId, + roomId: context.roomId, + syncState: context.syncState, + syncStreams: context.syncStreams, + timeoutMs: context.timeoutMs, + }); + const client = createMatrixQaDriverScenarioClient(context); + const { noReplyWindowMs } = await assertNoSutReplyWindow({ + actorId: reaction.actorId, + client, + context, + roomId: context.roomId, + since: reaction.since, + startSince: reaction.startSince, + unexpectedLines: [ + `reaction target: ${reaction.reactionTargetEventId}`, + `reaction event: ${reaction.reactionEventId}`, + ], + unexpectedMessage: `unexpected SUT reply after reaction from ${context.driverUserId}`, + }); + return { + artifacts: buildMatrixQaReactionArtifacts({ + actorUserId: context.driverUserId, + expectedNoReplyWindowMs: noReplyWindowMs, + reaction, + }), + details: [ + ...buildMatrixQaReactionDetailLines({ + reactionEmoji: reaction.reactionEmoji, + reactionEventId: reaction.reactionEventId, + reactionTargetEventId: reaction.reactionTargetEventId, + }), + `waited ${noReplyWindowMs}ms with no SUT reply`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runReactionRedactionObservedScenario(context: MatrixQaScenarioContext) { + const reactionTargetEventId = requireMatrixQaReactionTargetEventId( + context.canary?.reply.eventId, + "Matrix reaction redaction scenario", + ); + const reaction = await observeReactionScenario({ + actorId: "driver", + actorUserId: context.driverUserId, + accessToken: context.driverAccessToken, + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + reactionTargetEventId, + roomId: context.roomId, + syncState: context.syncState, + syncStreams: context.syncStreams, + timeoutMs: context.timeoutMs, + }); + const client = createMatrixQaDriverScenarioClient(context); + const redactionEventId = await client.redactEvent({ + eventId: reaction.reactionEventId, + reason: "matrix qa reaction removal", + roomId: context.roomId, + }); + const redaction = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.eventId === redactionEventId && + event.sender === context.driverUserId && + event.kind === "redaction", + roomId: context.roomId, + since: reaction.since, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: reaction.actorId, + syncState: context.syncState, + nextSince: redaction.since, + startSince: reaction.startSince, + }); + return { + artifacts: { + ...buildMatrixQaReactionArtifacts({ reaction }), + redactionEventId, + }, + details: [ + ...buildMatrixQaReactionDetailLines({ + reactionEmoji: reaction.reactionEmoji, + reactionEventId: reaction.reactionEventId, + reactionTargetEventId: reaction.reactionTargetEventId, + }), + `redaction event: ${redactionEventId}`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts new file mode 100644 index 00000000000..1679b04e190 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts @@ -0,0 +1,109 @@ +import { + MATRIX_QA_HOMESERVER_ROOM_KEY, + MATRIX_QA_RESTART_ROOM_KEY, + resolveMatrixQaScenarioRoomId, +} from "./scenario-catalog.js"; +import { + buildMatrixReplyDetails, + runAssertedDriverTopLevelScenario, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +export async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.interruptTransport) { + throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); + } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_HOMESERVER_ROOM_KEY); + await context.interruptTransport(); + const resumed = await runAssertedDriverTopLevelScenario({ + context, + label: "post-homeserver-restart reply", + roomId, + tokenPrefix: "MATRIX_QA_HOMESERVER", + }); + return { + artifacts: { + driverEventId: resumed.driverEventId, + reply: resumed.reply, + roomId, + token: resumed.token, + transportInterruption: "homeserver-restart", + }, + details: [ + `room id: ${roomId}`, + "transport interruption: homeserver-restart", + `driver event: ${resumed.driverEventId}`, + ...buildMatrixReplyDetails("reply", resumed.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runRestartResumeScenario(context: MatrixQaScenarioContext) { + if (!context.restartGateway) { + throw new Error("Matrix restart scenario requires a gateway restart callback"); + } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY); + await context.restartGateway(); + const result = await runAssertedDriverTopLevelScenario({ + context, + label: "post-restart reply", + roomId, + tokenPrefix: "MATRIX_QA_RESTART", + }); + return { + artifacts: { + driverEventId: result.driverEventId, + reply: result.reply, + restartSignal: "SIGUSR1", + roomId, + token: result.token, + }, + details: [ + `room id: ${roomId}`, + "restart signal: SIGUSR1", + `post-restart driver event: ${result.driverEventId}`, + ...buildMatrixReplyDetails("reply", result.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runPostRestartRoomContinueScenario(context: MatrixQaScenarioContext) { + if (!context.restartGateway) { + throw new Error("Matrix post-restart continuity scenario requires a gateway restart callback"); + } + const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY); + await context.restartGateway(); + const first = await runAssertedDriverTopLevelScenario({ + context, + label: "first post-restart reply", + roomId, + tokenPrefix: "MATRIX_QA_RESTART_FIRST", + }); + const second = await runAssertedDriverTopLevelScenario({ + context, + label: "second post-restart reply", + roomId, + tokenPrefix: "MATRIX_QA_RESTART_SECOND", + }); + return { + artifacts: { + firstDriverEventId: first.driverEventId, + firstReply: first.reply, + firstToken: first.token, + restartSignal: "SIGUSR1", + roomId, + secondDriverEventId: second.driverEventId, + secondReply: second.reply, + secondToken: second.token, + }, + details: [ + `room id: ${roomId}`, + "restart signal: SIGUSR1", + `first post-restart driver event: ${first.driverEventId}`, + ...buildMatrixReplyDetails("first reply", first.reply), + `second post-restart driver event: ${second.driverEventId}`, + ...buildMatrixReplyDetails("second reply", second.reply), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index ca71b82eb7a..4f373c912d0 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -1,95 +1,44 @@ import { randomUUID } from "node:crypto"; -import { encodePngRgba, fillPixel } from "openclaw/plugin-sdk/media-runtime"; import type { MatrixQaObservedEvent } from "../../substrate/events.js"; import { MATRIX_QA_BLOCK_ROOM_KEY, - MATRIX_QA_HOMESERVER_ROOM_KEY, - MATRIX_QA_MEDIA_ROOM_KEY, MATRIX_QA_MEMBERSHIP_ROOM_KEY, - MATRIX_QA_RESTART_ROOM_KEY, resolveMatrixQaScenarioRoomId, } from "./scenario-catalog.js"; +import { + buildMatrixQaReactionArtifacts, + buildMatrixQaReactionDetailLines, + observeReactionScenario, +} from "./scenario-runtime-reaction.js"; import { assertThreadReplyArtifact, assertTopLevelReplyArtifact, advanceMatrixQaActorCursor, buildMatrixBlockStreamingPrompt, buildMatrixQuietStreamingPrompt, + buildMatrixQaToken, buildMatrixReplyArtifact, buildMatrixReplyDetails, buildMentionPrompt, + createMatrixQaDriverScenarioClient, createMatrixQaScenarioClient, + isMatrixQaExactMarkerReply, isMatrixQaMessageLikeKind, NO_REPLY_WINDOW_MS, primeMatrixQaActorCursor, + primeMatrixQaDriverScenarioClient, + runAssertedDriverTopLevelScenario, runConfigurableTopLevelScenario, runDriverTopLevelMentionScenario, runNoReplyExpectedScenario, runTopologyScopedTopLevelScenario, waitForMembershipEvent, - type MatrixQaActorId, type MatrixQaScenarioContext, type MatrixQaSyncState, } from "./scenario-runtime-shared.js"; import type { MatrixQaCanaryArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; type MatrixQaThreadScenarioResult = Awaited>; -const MATRIX_QA_IMAGE_ATTACHMENT_FILENAME = "red-top-blue-bottom.png"; -const MATRIX_QA_IMAGE_COLOR_GROUPS = [["red"], ["blue"]] as const; - -function createMatrixQaSplitColorImagePng() { - const width = 16; - const height = 16; - const rgba = Buffer.alloc(width * height * 4); - for (let y = 0; y < height; y += 1) { - const isTopHalf = y < height / 2; - for (let x = 0; x < width; x += 1) { - if (isTopHalf) { - fillPixel(rgba, x, y, width, 255, 0, 0); - continue; - } - fillPixel(rgba, x, y, width, 0, 0, 255); - } - } - return encodePngRgba(rgba, width, height); -} - -function buildMatrixQaImageUnderstandingPrompt(sutUserId: string) { - return `${sutUserId} Image understanding check: describe the top and bottom colors in the attached image in one short sentence.`; -} - -function buildMatrixQaImageGenerationPrompt(sutUserId: string) { - return `${sutUserId} Image generation check: generate a QA lighthouse image and summarize it in one short sentence.`; -} - -function hasMatrixQaExpectedColorReply(body: string | undefined) { - const normalizedBody = body?.toLowerCase() ?? ""; - return MATRIX_QA_IMAGE_COLOR_GROUPS.every((group) => - group.some((color) => normalizedBody.includes(color)), - ); -} - -function requireMatrixQaImageAttachment(event: MatrixQaObservedEvent, scenarioLabel: string) { - if (event.msgtype !== "m.image" || event.attachment?.kind !== "image") { - throw new Error( - `${scenarioLabel} expected an m.image attachment but saw ${event.msgtype ?? ""}`, - ); - } - return event.attachment; -} - -function buildMatrixQaAttachmentDetailLines(params: { - attachmentEvent: MatrixQaObservedEvent; - label: string; -}) { - return [ - `${params.label} event: ${params.attachmentEvent.eventId}`, - `${params.label} msgtype: ${params.attachmentEvent.msgtype ?? ""}`, - `${params.label} attachment kind: ${params.attachmentEvent.attachment?.kind ?? ""}`, - `${params.label} attachment filename: ${params.attachmentEvent.attachment?.filename ?? ""}`, - `${params.label} body preview: ${params.attachmentEvent.body?.slice(0, 200) ?? ""}`, - ]; -} function assertMatrixQaInReplyTarget(params: { actualEventId?: string; @@ -139,49 +88,6 @@ function buildMatrixQaThreadDetailLines(params: { ]; } -async function primeMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) { - return await primeMatrixQaActorCursor({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - syncState: context.syncState, - syncStreams: context.syncStreams, - }); -} - -function createMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) { - return createMatrixQaScenarioClient({ - accessToken: context.driverAccessToken, - actorId: "driver", - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - syncState: context.syncState, - syncStreams: context.syncStreams, - }); -} - -async function runAssertedDriverTopLevelScenario(params: { - context: MatrixQaScenarioContext; - label: string; - roomId?: string; - tokenPrefix: string; -}) { - const result = await runDriverTopLevelMentionScenario({ - baseUrl: params.context.baseUrl, - driverAccessToken: params.context.driverAccessToken, - observedEvents: params.context.observedEvents, - roomId: params.roomId ?? params.context.roomId, - syncState: params.context.syncState, - syncStreams: params.context.syncStreams, - sutUserId: params.context.sutUserId, - timeoutMs: params.context.timeoutMs, - tokenPrefix: params.tokenPrefix, - }); - assertTopLevelReplyArtifact(params.label, result.reply); - return result; -} - async function runThreadScenario( params: MatrixQaScenarioContext, options?: { @@ -205,7 +111,7 @@ async function runThreadScenario( }) : undefined; const triggerEventId = nestedDriverEventId ?? rootEventId; - const token = `${options?.tokenPrefix ?? "MATRIX_QA_THREAD"}_${randomUUID().slice(0, 8).toUpperCase()}`; + const token = buildMatrixQaToken(options?.tokenPrefix ?? "MATRIX_QA_THREAD"); const driverEventId = await client.sendTextMessage({ body: buildMentionPrompt(params.sutUserId, token), mentionUserIds: [params.sutUserId], @@ -216,10 +122,11 @@ async function runThreadScenario( const matched = await client.waitForRoomEvent({ observedEvents: params.observedEvents, predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && + isMatrixQaExactMarkerReply(event, { + roomId: params.roomId, + sutUserId: params.sutUserId, + token, + }) && event.relatesTo?.relType === "m.thread" && event.relatesTo.eventId === rootEventId, roomId: params.roomId, @@ -435,7 +342,7 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen syncState: context.syncState, syncStreams: context.syncStreams, }); - const token = `MATRIX_QA_OBSERVER_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`; + const token = buildMatrixQaToken("MATRIX_QA_OBSERVER_ALLOWLIST"); const body = buildMentionPrompt(context.sutUserId, token); const driverEventId = await client.sendTextMessage({ body, @@ -445,12 +352,11 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen const matched = await client.waitForRoomEvent({ observedEvents: context.observedEvents, predicate: (event) => - event.roomId === context.roomId && - event.sender === context.sutUserId && - event.type === "m.room.message" && - event.relatesTo === undefined && - typeof event.body === "string" && - event.body.trim().length > 0, + isMatrixQaExactMarkerReply(event, { + roomId: context.roomId, + sutUserId: context.sutUserId, + token, + }) && event.relatesTo === undefined, roomId: context.roomId, since: startSince, timeoutMs: context.timeoutMs, @@ -462,6 +368,7 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen startSince, }); const reply = buildMatrixReplyArtifact(matched.event, token); + assertTopLevelReplyArtifact("observer allowlist reply", reply); return { artifacts: { actorUserId: context.observerUserId, @@ -603,110 +510,6 @@ export async function runBlockStreamingScenario(context: MatrixQaScenarioContext } satisfies MatrixQaScenarioExecution; } -export async function runImageUnderstandingAttachmentScenario(context: MatrixQaScenarioContext) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); - const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); - const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); - const driverEventId = await client.sendMediaMessage({ - body: triggerBody, - buffer: createMatrixQaSplitColorImagePng(), - contentType: "image/png", - fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, - kind: "image", - mentionUserIds: [context.sutUserId], - roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === roomId && - event.sender === context.sutUserId && - event.type === "m.room.message" && - event.relatesTo === undefined && - isMatrixQaMessageLikeKind(event.kind) && - hasMatrixQaExpectedColorReply(event.body), - roomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: context.syncState, - nextSince: matched.since, - startSince, - }); - const reply = buildMatrixReplyArtifact(matched.event); - return { - artifacts: { - attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, - driverEventId, - reply, - roomId, - triggerBody, - }, - details: [ - `room id: ${roomId}`, - `driver attachment event: ${driverEventId}`, - `sent attachment filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, - ...buildMatrixReplyDetails("reply", reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenarioContext) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); - const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); - const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId); - const driverEventId = await client.sendTextMessage({ - body: triggerBody, - mentionUserIds: [context.sutUserId], - roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === roomId && - event.sender === context.sutUserId && - event.type === "m.room.message" && - event.relatesTo === undefined && - event.msgtype === "m.image" && - event.attachment?.kind === "image", - roomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); - advanceMatrixQaActorCursor({ - actorId: "driver", - syncState: context.syncState, - nextSince: matched.since, - startSince, - }); - const attachment = requireMatrixQaImageAttachment( - matched.event, - "Matrix generated image delivery scenario", - ); - return { - artifacts: { - attachmentBodyPreview: matched.event.body?.slice(0, 200), - attachmentEventId: matched.event.eventId, - attachmentFilename: attachment.filename, - attachmentKind: attachment.kind, - attachmentMsgtype: matched.event.msgtype, - driverEventId, - roomId, - triggerBody, - }, - details: [ - `room id: ${roomId}`, - `driver event: ${driverEventId}`, - ...buildMatrixQaAttachmentDetailLines({ - attachmentEvent: matched.event, - label: "generated image", - }), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - export async function runRoomAutoJoinInviteScenario(context: MatrixQaScenarioContext) { const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); const dynamicRoomId = await client.createPrivateRoom({ @@ -857,130 +660,6 @@ export async function runMembershipLossScenario(context: MatrixQaScenarioContext } satisfies MatrixQaScenarioExecution; } -export async function runReactionNotificationScenario(context: MatrixQaScenarioContext) { - const reactionTargetEventId = requireMatrixQaReactionTargetEventId( - context.canary?.reply.eventId, - "Matrix reaction scenario", - ); - const result = await observeReactionScenario({ - actorId: "driver", - actorUserId: context.driverUserId, - accessToken: context.driverAccessToken, - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - reactionTargetEventId, - roomId: context.roomId, - syncState: context.syncState, - syncStreams: context.syncStreams, - timeoutMs: context.timeoutMs, - }); - return { - artifacts: buildMatrixQaReactionArtifacts({ reaction: result }), - details: buildMatrixQaReactionDetailLines({ - actorUserId: result.actorUserId, - observedReactionKey: result.event.reaction?.key, - reactionEmoji: result.reactionEmoji, - reactionEventId: result.reactionEventId, - reactionTargetEventId: result.reactionTargetEventId, - }).join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -function buildMatrixQaReactionDetailLines(params: { - actorUserId?: string; - observedReactionKey?: string; - reactionEmoji: string; - reactionEventId: string; - reactionTargetEventId: string; -}) { - return [ - `reaction event: ${params.reactionEventId}`, - `reaction target: ${params.reactionTargetEventId}`, - `reaction emoji: ${params.reactionEmoji}`, - ...(params.actorUserId ? [`reaction sender: ${params.actorUserId}`] : []), - ...(params.observedReactionKey ? [`observed reaction key: ${params.observedReactionKey}`] : []), - ]; -} - -function requireMatrixQaReactionTargetEventId( - reactionTargetEventId: string | undefined, - scenarioLabel: string, -) { - const normalizedReactionTargetEventId = reactionTargetEventId?.trim(); - if (!normalizedReactionTargetEventId) { - throw new Error(`${scenarioLabel} requires a canary reply event id`); - } - return normalizedReactionTargetEventId; -} - -async function observeReactionScenario(params: { - actorId: MatrixQaActorId; - actorUserId: string; - accessToken: string; - baseUrl: string; - observedEvents: MatrixQaObservedEvent[]; - reactionEmoji?: string; - reactionTargetEventId: string; - roomId: string; - syncState: MatrixQaSyncState; - syncStreams?: MatrixQaScenarioContext["syncStreams"]; - timeoutMs: number; -}) { - const { client, startSince } = await primeMatrixQaActorCursor({ - accessToken: params.accessToken, - actorId: params.actorId, - baseUrl: params.baseUrl, - observedEvents: params.observedEvents, - syncState: params.syncState, - syncStreams: params.syncStreams, - }); - const reactionEmoji = params.reactionEmoji ?? "👍"; - const reactionEventId = await client.sendReaction({ - emoji: reactionEmoji, - messageId: params.reactionTargetEventId, - roomId: params.roomId, - }); - const matched = await client.waitForRoomEvent({ - observedEvents: params.observedEvents, - predicate: (event) => - event.roomId === params.roomId && - event.sender === params.actorUserId && - event.type === "m.reaction" && - event.eventId === reactionEventId && - event.reaction?.eventId === params.reactionTargetEventId && - event.reaction?.key === reactionEmoji, - roomId: params.roomId, - since: startSince, - timeoutMs: params.timeoutMs, - }); - return { - actorId: params.actorId, - actorUserId: params.actorUserId, - event: matched.event, - reactionEmoji, - reactionEventId, - reactionTargetEventId: params.reactionTargetEventId, - since: matched.since, - startSince, - }; -} - -function buildMatrixQaReactionArtifacts(params: { - actorUserId?: string; - expectedNoReplyWindowMs?: number; - reaction: Awaited>; -}) { - return { - ...(params.actorUserId ? { actorUserId: params.actorUserId } : {}), - ...(params.expectedNoReplyWindowMs === undefined - ? {} - : { expectedNoReplyWindowMs: params.expectedNoReplyWindowMs }), - reactionEmoji: params.reaction.reactionEmoji, - reactionEventId: params.reaction.reactionEventId, - reactionTargetEventId: params.reaction.reactionTargetEventId, - }; -} - export async function runReactionThreadedScenario(context: MatrixQaScenarioContext) { const thread = await runThreadScenario(context, { createNestedReply: true, @@ -1031,124 +710,3 @@ export async function runReactionThreadedScenario(context: MatrixQaScenarioConte ].join("\n"), } satisfies MatrixQaScenarioExecution; } - -export async function runReactionNotAReplyScenario(context: MatrixQaScenarioContext) { - const reactionTargetEventId = requireMatrixQaReactionTargetEventId( - context.canary?.reply.eventId, - "Matrix reaction no-reply scenario", - ); - const reaction = await observeReactionScenario({ - actorId: "driver", - actorUserId: context.driverUserId, - accessToken: context.driverAccessToken, - baseUrl: context.baseUrl, - observedEvents: context.observedEvents, - reactionTargetEventId, - roomId: context.roomId, - syncState: context.syncState, - syncStreams: context.syncStreams, - timeoutMs: context.timeoutMs, - }); - const client = createMatrixQaDriverScenarioClient(context); - const noReplyWindowMs = Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs); - const noReplyResult = await client.waitForOptionalRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === context.roomId && - event.sender === context.sutUserId && - event.type === "m.room.message", - roomId: context.roomId, - since: reaction.since, - timeoutMs: noReplyWindowMs, - }); - if (noReplyResult.matched) { - const unexpectedReply = buildMatrixReplyArtifact(noReplyResult.event); - throw new Error( - [ - `unexpected SUT reply after reaction from ${context.driverUserId}`, - `reaction target: ${reaction.reactionTargetEventId}`, - `reaction event: ${reaction.reactionEventId}`, - ...buildMatrixReplyDetails("unexpected reply", unexpectedReply), - ].join("\n"), - ); - } - advanceMatrixQaActorCursor({ - actorId: reaction.actorId, - syncState: context.syncState, - nextSince: noReplyResult.since, - startSince: reaction.startSince, - }); - return { - artifacts: buildMatrixQaReactionArtifacts({ - actorUserId: context.driverUserId, - expectedNoReplyWindowMs: noReplyWindowMs, - reaction, - }), - details: [ - ...buildMatrixQaReactionDetailLines({ - reactionEmoji: reaction.reactionEmoji, - reactionEventId: reaction.reactionEventId, - reactionTargetEventId: reaction.reactionTargetEventId, - }), - `waited ${noReplyWindowMs}ms with no SUT reply`, - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -export async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.interruptTransport) { - throw new Error("Matrix homeserver restart scenario requires a transport interruption hook"); - } - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_HOMESERVER_ROOM_KEY); - await context.interruptTransport(); - const resumed = await runAssertedDriverTopLevelScenario({ - context, - label: "post-homeserver-restart reply", - roomId, - tokenPrefix: "MATRIX_QA_HOMESERVER", - }); - return { - artifacts: { - driverEventId: resumed.driverEventId, - reply: resumed.reply, - roomId, - token: resumed.token, - transportInterruption: "homeserver-restart", - }, - details: [ - `room id: ${roomId}`, - "transport interruption: homeserver-restart", - `driver event: ${resumed.driverEventId}`, - ...buildMatrixReplyDetails("reply", resumed.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} - -export async function runRestartResumeScenario(context: MatrixQaScenarioContext) { - if (!context.restartGateway) { - throw new Error("Matrix restart scenario requires a gateway restart callback"); - } - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY); - await context.restartGateway(); - const result = await runAssertedDriverTopLevelScenario({ - context, - label: "post-restart reply", - roomId, - tokenPrefix: "MATRIX_QA_RESTART", - }); - return { - artifacts: { - driverEventId: result.driverEventId, - reply: result.reply, - restartSignal: "SIGUSR1", - roomId, - token: result.token, - }, - details: [ - `room id: ${roomId}`, - "restart signal: SIGUSR1", - `post-restart driver event: ${result.driverEventId}`, - ...buildMatrixReplyDetails("reply", result.reply), - ].join("\n"), - } satisfies MatrixQaScenarioExecution; -} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index 5ebae807006..f9e51854bfd 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -19,14 +19,21 @@ export type MatrixQaScenarioContext = { baseUrl: string; canary?: MatrixQaCanaryArtifact; driverAccessToken: string; + driverDeviceId?: string; + driverPassword?: string; driverUserId: string; observedEvents: MatrixQaObservedEvent[]; observerAccessToken: string; + observerDeviceId?: string; + observerPassword?: string; observerUserId: string; + outputDir?: string; restartGateway?: () => Promise; roomId: string; interruptTransport?: () => Promise; sutAccessToken: string; + sutDeviceId?: string; + sutPassword?: string; syncState: MatrixQaSyncState; syncStreams?: MatrixQaSyncStreams; sutUserId: string; @@ -44,6 +51,10 @@ export function buildExactMarkerPrompt(token: string) { return `reply with only this exact marker: ${token}`; } +export function buildMatrixQaToken(prefix: string) { + return `${prefix}_${randomUUID().slice(0, 8).toUpperCase()}`; +} + export function buildMatrixQuietStreamingPrompt(sutUserId: string, text: string) { return `${sutUserId} Matrix quiet streaming QA check: reply exactly \`${text}\`.`; } @@ -66,6 +77,27 @@ export function isMatrixQaMessageLikeKind(kind: MatrixQaObservedEvent["kind"]) { return kind === "message" || kind === "notice"; } +export function doesMatrixQaReplyBodyMatchToken(event: MatrixQaObservedEvent, token: string) { + return event.body?.trim() === token; +} + +export function isMatrixQaExactMarkerReply( + event: MatrixQaObservedEvent, + params: { + roomId: string; + sutUserId: string; + token: string; + }, +) { + return ( + event.roomId === params.roomId && + event.sender === params.sutUserId && + event.type === "m.room.message" && + isMatrixQaMessageLikeKind(event.kind) && + doesMatrixQaReplyBodyMatchToken(event, params.token) + ); +} + export function buildMatrixReplyArtifact( event: MatrixQaObservedEvent, token?: string, @@ -77,7 +109,7 @@ export function buildMatrixReplyArtifact( mentions: event.mentions, relatesTo: event.relatesTo, sender: event.sender, - ...(token ? { tokenMatched: replyBody === token } : {}), + ...(token ? { tokenMatched: doesMatrixQaReplyBodyMatchToken(event, token) } : {}), }; } @@ -200,6 +232,17 @@ export function createMatrixQaScenarioClient(params: { }); } +export function createMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) { + return createMatrixQaScenarioClient({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + syncState: context.syncState, + syncStreams: context.syncStreams, + }); +} + export async function primeMatrixQaActorCursor(params: { accessToken: string; actorId: MatrixQaActorId; @@ -227,6 +270,17 @@ export async function primeMatrixQaActorCursor(params: { return { client, startSince }; } +export async function primeMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) { + return await primeMatrixQaActorCursor({ + accessToken: context.driverAccessToken, + actorId: "driver", + baseUrl: context.baseUrl, + observedEvents: context.observedEvents, + syncState: context.syncState, + syncStreams: context.syncStreams, + }); +} + export function advanceMatrixQaActorCursor(params: { actorId: MatrixQaActorId; syncState: MatrixQaSyncState; @@ -236,6 +290,50 @@ export function advanceMatrixQaActorCursor(params: { writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); } +type MatrixQaScenarioClient = ReturnType; + +export async function assertNoSutReplyWindow(params: { + actorId: MatrixQaActorId; + client: MatrixQaScenarioClient; + context: MatrixQaScenarioContext; + roomId: string; + since?: string; + startSince: string; + unexpectedLines?: string[]; + unexpectedMessage: string; +}) { + const noReplyWindowMs = Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs); + const result = await params.client.waitForOptionalRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.context.sutUserId && + event.type === "m.room.message", + roomId: params.roomId, + since: params.since, + timeoutMs: noReplyWindowMs, + }); + if (result.matched) { + throw new Error( + [ + params.unexpectedMessage, + ...(params.unexpectedLines ?? []), + ...buildMatrixReplyDetails("unexpected reply", buildMatrixReplyArtifact(result.event)), + ].join("\n"), + ); + } + advanceMatrixQaActorCursor({ + actorId: params.actorId, + syncState: params.context.syncState, + nextSince: result.since, + startSince: params.startSince, + }); + return { + noReplyWindowMs, + since: result.since, + }; +} + export async function runConfigurableTopLevelScenario(params: { accessToken: string; actorId: MatrixQaActorId; @@ -261,7 +359,7 @@ export async function runConfigurableTopLevelScenario(params: { syncState: params.syncState, syncStreams: params.syncStreams, }); - const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`; + const token = buildMatrixQaToken(params.tokenPrefix); const body = params.withMention === false ? buildExactMarkerPrompt(token) @@ -274,10 +372,11 @@ export async function runConfigurableTopLevelScenario(params: { const matched = await client.waitForRoomEvent({ observedEvents: params.observedEvents, predicate: (event) => - event.roomId === params.roomId && - event.sender === params.sutUserId && - event.type === "m.room.message" && - (event.body ?? "").includes(token) && + isMatrixQaExactMarkerReply(event, { + roomId: params.roomId, + sutUserId: params.sutUserId, + token, + }) && (params.replyPredicate?.(event, { driverEventId, token }) ?? event.relatesTo === undefined), roomId: params.roomId, since: startSince, @@ -338,6 +437,27 @@ export async function runDriverTopLevelMentionScenario(params: { }); } +export async function runAssertedDriverTopLevelScenario(params: { + context: MatrixQaScenarioContext; + label: string; + roomId?: string; + tokenPrefix: string; +}) { + const result = await runDriverTopLevelMentionScenario({ + baseUrl: params.context.baseUrl, + driverAccessToken: params.context.driverAccessToken, + observedEvents: params.context.observedEvents, + roomId: params.roomId ?? params.context.roomId, + syncState: params.context.syncState, + syncStreams: params.context.syncStreams, + sutUserId: params.context.sutUserId, + timeoutMs: params.context.timeoutMs, + tokenPrefix: params.tokenPrefix, + }); + assertTopLevelReplyArtifact(params.label, result.reply); + return result; +} + export async function waitForMembershipEvent(params: { accessToken: string; actorId: MatrixQaActorId; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index a420f775ce5..a9cad400cd3 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { MATRIX_QA_DRIVER_DM_ROOM_KEY, MATRIX_QA_SECONDARY_ROOM_KEY, @@ -10,18 +9,48 @@ import { runDmThreadReplyOverrideScenario, } from "./scenario-runtime-dm.js"; import { - runBlockStreamingScenario, + runMatrixQaE2eeArtifactRedactionScenario, + runMatrixQaE2eeBasicReplyScenario, + runMatrixQaE2eeBootstrapSuccessScenario, + runMatrixQaE2eeDeviceSasVerificationScenario, + runMatrixQaE2eeDmSasVerificationScenario, + runMatrixQaE2eeKeyBootstrapFailureScenario, + runMatrixQaE2eeMediaImageScenario, + runMatrixQaE2eeQrVerificationScenario, + runMatrixQaE2eeRecoveryKeyLifecycleScenario, + runMatrixQaE2eeRestartResumeScenario, + runMatrixQaE2eeStaleDeviceHygieneScenario, + runMatrixQaE2eeThreadFollowUpScenario, + runMatrixQaE2eeVerificationNoticeNoTriggerScenario, +} from "./scenario-runtime-e2ee.js"; +import { + runInboundEditIgnoredScenario, + runInboundEditNoDuplicateTriggerScenario, +} from "./scenario-runtime-edit.js"; +import { + runAttachmentOnlyIgnoredScenario, runGeneratedImageDeliveryScenario, - runHomeserverRestartResumeScenario, runImageUnderstandingAttachmentScenario, + runMediaTypeCoverageScenario, + runUnsupportedMediaSafeScenario, +} from "./scenario-runtime-media.js"; +import { + runReactionNotAReplyScenario, + runReactionNotificationScenario, + runReactionRedactionObservedScenario, +} from "./scenario-runtime-reaction.js"; +import { + runHomeserverRestartResumeScenario, + runPostRestartRoomContinueScenario, + runRestartResumeScenario, +} from "./scenario-runtime-restart.js"; +import { + runBlockStreamingScenario, runMatrixQaCanary, runMembershipLossScenario, runObserverAllowlistOverrideScenario, runQuietStreamingPreviewScenario, - runReactionNotAReplyScenario, - runReactionNotificationScenario, runReactionThreadedScenario, - runRestartResumeScenario, runRoomAutoJoinInviteScenario, runRoomThreadReplyOverrideScenario, runThreadFollowUpScenario, @@ -32,9 +61,11 @@ import { } from "./scenario-runtime-room.js"; import { buildExactMarkerPrompt, + buildMatrixQaToken, buildMatrixReplyArtifact, buildMatrixReplyDetails, buildMentionPrompt, + NO_REPLY_WINDOW_MS, readMatrixQaSyncCursor, runNoReplyExpectedScenario, runTopologyScopedTopLevelScenario, @@ -71,10 +102,6 @@ async function runDriverTopologyScopedScenario(params: { }); } -function buildMatrixQaToken(prefix: string) { - return `${prefix}_${randomUUID().slice(0, 8).toUpperCase()}`; -} - async function runNoReplyScenario(params: { accessToken: string; actorId: "driver" | "observer"; @@ -82,8 +109,10 @@ async function runNoReplyScenario(params: { body: string; context: MatrixQaScenarioContext; mentionUserIds?: string[]; + timeoutMs?: number; token: string; }) { + const timeoutMs = params.timeoutMs ?? params.context.timeoutMs; return await runNoReplyExpectedScenario({ accessToken: params.accessToken, actorId: params.actorId, @@ -95,11 +124,37 @@ async function runNoReplyScenario(params: { roomId: params.context.roomId, syncState: params.context.syncState, sutUserId: params.context.sutUserId, - timeoutMs: params.context.timeoutMs, + timeoutMs, token: params.token, }); } +async function runMultiActorOrderingScenario(context: MatrixQaScenarioContext) { + const blockedToken = buildMatrixQaToken("MATRIX_QA_MULTI_BLOCKED"); + const blocked = await runNoReplyScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + actorUserId: context.observerUserId, + body: buildMentionPrompt(context.sutUserId, blockedToken), + mentionUserIds: [context.sutUserId], + context, + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + token: blockedToken, + }); + const accepted = await runDriverTopologyScopedScenario({ + context, + roomKey: context.topology.defaultRoomKey, + tokenPrefix: "MATRIX_QA_MULTI_DRIVER", + }); + return { + artifacts: { + accepted: accepted.artifacts ?? {}, + blocked: blocked.artifacts ?? {}, + }, + details: [blocked.details, accepted.details].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + export async function runMatrixQaScenario( scenario: MatrixQaScenarioDefinition, context: MatrixQaScenarioContext, @@ -125,6 +180,12 @@ export async function runMatrixQaScenario( return await runImageUnderstandingAttachmentScenario(context); case "matrix-room-generated-image-delivery": return await runGeneratedImageDeliveryScenario(context); + case "matrix-media-type-coverage": + return await runMediaTypeCoverageScenario(context); + case "matrix-attachment-only-ignored": + return await runAttachmentOnlyIgnoredScenario(context); + case "matrix-unsupported-media-safe": + return await runUnsupportedMediaSafeScenario(context); case "matrix-dm-reply-shape": return await runDriverTopologyScopedScenario({ context, @@ -159,8 +220,12 @@ export async function runMatrixQaScenario( return await runReactionThreadedScenario(context); case "matrix-reaction-not-a-reply": return await runReactionNotAReplyScenario(context); + case "matrix-reaction-redaction-observed": + return await runReactionRedactionObservedScenario(context); case "matrix-restart-resume": return await runRestartResumeScenario(context); + case "matrix-post-restart-room-continue": + return await runPostRestartRoomContinueScenario(context); case "matrix-room-membership-loss": return await runMembershipLossScenario(context); case "matrix-homeserver-restart-resume": @@ -176,6 +241,18 @@ export async function runMatrixQaScenario( token, }); } + case "matrix-mention-metadata-spoof-block": { + const token = buildMatrixQaToken("MATRIX_QA_METADATA_SPOOF"); + return await runNoReplyScenario({ + accessToken: context.driverAccessToken, + actorId: "driver", + actorUserId: context.driverUserId, + body: buildExactMarkerPrompt(token), + mentionUserIds: [context.sutUserId], + context, + token, + }); + } case "matrix-observer-allowlist-override": return await runObserverAllowlistOverrideScenario(context); case "matrix-allowlist-block": { @@ -190,6 +267,38 @@ export async function runMatrixQaScenario( token, }); } + case "matrix-multi-actor-ordering": + return await runMultiActorOrderingScenario(context); + case "matrix-inbound-edit-ignored": + return await runInboundEditIgnoredScenario(context); + case "matrix-inbound-edit-no-duplicate-trigger": + return await runInboundEditNoDuplicateTriggerScenario(context); + case "matrix-e2ee-basic-reply": + return await runMatrixQaE2eeBasicReplyScenario(context); + case "matrix-e2ee-thread-follow-up": + return await runMatrixQaE2eeThreadFollowUpScenario(context); + case "matrix-e2ee-bootstrap-success": + return await runMatrixQaE2eeBootstrapSuccessScenario(context); + case "matrix-e2ee-recovery-key-lifecycle": + return await runMatrixQaE2eeRecoveryKeyLifecycleScenario(context); + case "matrix-e2ee-device-sas-verification": + return await runMatrixQaE2eeDeviceSasVerificationScenario(context); + case "matrix-e2ee-qr-verification": + return await runMatrixQaE2eeQrVerificationScenario(context); + case "matrix-e2ee-stale-device-hygiene": + return await runMatrixQaE2eeStaleDeviceHygieneScenario(context); + case "matrix-e2ee-dm-sas-verification": + return await runMatrixQaE2eeDmSasVerificationScenario(context); + case "matrix-e2ee-restart-resume": + return await runMatrixQaE2eeRestartResumeScenario(context); + case "matrix-e2ee-verification-notice-no-trigger": + return await runMatrixQaE2eeVerificationNoticeNoTriggerScenario(context); + case "matrix-e2ee-artifact-redaction": + return await runMatrixQaE2eeArtifactRedactionScenario(context); + case "matrix-e2ee-media-image": + return await runMatrixQaE2eeMediaImageScenario(context); + case "matrix-e2ee-key-bootstrap-failure": + return await runMatrixQaE2eeKeyBootstrapFailureScenario(context); default: { const exhaustiveScenarioId: never = scenario.id; return exhaustiveScenarioId; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index a03b30a41f0..84c56d0c895 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -16,24 +16,52 @@ export type MatrixQaCanaryArtifact = { }; export type MatrixQaScenarioArtifacts = { + accepted?: MatrixQaScenarioArtifacts; + attachments?: Array<{ + eventId: string; + filename?: string; + kind?: string; + label: string; + msgtype?: string; + }>; + attachmentCaptionPreview?: string; attachmentBodyPreview?: string; attachmentEventId?: string; attachmentFilename?: string; attachmentKind?: string; attachmentMsgtype?: string; actorUserId?: string; + blocked?: MatrixQaScenarioArtifacts; driverEventId?: string; + editEventId?: string; + editedToken?: string; expectedNoReplyWindowMs?: number; + firstDriverEventId?: string; + firstReply?: MatrixQaReplyArtifact; + firstToken?: string; + originalDriverEventId?: string; + originalReply?: MatrixQaReplyArtifact; + originalToken?: string; reactionEmoji?: string; reactionEventId?: string; reactionTargetEventId?: string; + redactionEventId?: string; reply?: MatrixQaReplyArtifact; + replies?: Array<{ + eventId: string; + label: string; + token: string; + tokenMatched?: boolean; + }>; recoveredDriverEventId?: string; recoveredReply?: MatrixQaReplyArtifact; + rootEventId?: string; roomKey?: string; roomId?: string; restartSignal?: string; - rootEventId?: string; + secondDriverEventId?: string; + secondReply?: MatrixQaReplyArtifact; + secondToken?: string; threadDriverEventId?: string; threadReply?: MatrixQaReplyArtifact; threadRootEventId?: string; @@ -50,7 +78,26 @@ export type MatrixQaScenarioArtifacts = { previewBodyPreview?: string; previewEventId?: string; blockEventIds?: string[]; + bootstrapActor?: "driver" | "observer" | "sut"; + bootstrapErrorPreview?: string; + bootstrapSuccess?: boolean; + backupCreatedVersion?: string | null; + backupRestored?: boolean; + backupReset?: boolean; + completedVerificationIds?: string[]; + currentDeviceId?: string | null; + deletedDeviceIds?: string[]; + faultedEndpoint?: string; + faultHitCount?: number; + faultRuleId?: string; + qrBytes?: number; + recoveryKeyId?: string | null; + recoveryKeyStored?: boolean; + remainingDeviceIds?: string[]; + sasEmoji?: string[]; + secondaryDeviceId?: string; transportInterruption?: string; + verificationRoomId?: string; joinedRoomId?: string; }; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 9ab774a0648..b37f37a60f9 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -2,15 +2,29 @@ import { describe, expect, it, beforeEach, vi } from "vitest"; const { createMatrixQaClient } = vi.hoisted(() => ({ createMatrixQaClient: vi.fn(), })); +const { createMatrixQaE2eeScenarioClient, runMatrixQaE2eeBootstrap, startMatrixQaFaultProxy } = + vi.hoisted(() => ({ + createMatrixQaE2eeScenarioClient: vi.fn(), + runMatrixQaE2eeBootstrap: vi.fn(), + startMatrixQaFaultProxy: vi.fn(), + })); vi.mock("../../substrate/client.js", () => ({ createMatrixQaClient, })); +vi.mock("../../substrate/e2ee-client.js", () => ({ + createMatrixQaE2eeScenarioClient, + runMatrixQaE2eeBootstrap, +})); +vi.mock("../../substrate/fault-proxy.js", () => ({ + startMatrixQaFaultProxy, +})); import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, findMissingLiveTransportStandardScenarios, } from "../../shared/live-transport-scenarios.js"; +import { MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES } from "./scenario-media-fixtures.js"; import { __testing as scenarioTesting, MATRIX_QA_SCENARIOS, @@ -20,6 +34,9 @@ import { describe("matrix live qa scenarios", () => { beforeEach(() => { createMatrixQaClient.mockReset(); + createMatrixQaE2eeScenarioClient.mockReset(); + runMatrixQaE2eeBootstrap.mockReset(); + startMatrixQaFaultProxy.mockReset(); }); it("ships the Matrix live QA scenario set by default", () => { @@ -34,6 +51,9 @@ describe("matrix live qa scenarios", () => { "matrix-room-block-streaming", "matrix-room-image-understanding-attachment", "matrix-room-generated-image-delivery", + "matrix-media-type-coverage", + "matrix-attachment-only-ignored", + "matrix-unsupported-media-safe", "matrix-dm-reply-shape", "matrix-dm-shared-session-notice", "matrix-dm-thread-reply-override", @@ -44,12 +64,31 @@ describe("matrix live qa scenarios", () => { "matrix-reaction-notification", "matrix-reaction-threaded", "matrix-reaction-not-a-reply", + "matrix-reaction-redaction-observed", "matrix-restart-resume", + "matrix-post-restart-room-continue", "matrix-room-membership-loss", "matrix-homeserver-restart-resume", "matrix-mention-gating", + "matrix-mention-metadata-spoof-block", "matrix-observer-allowlist-override", "matrix-allowlist-block", + "matrix-multi-actor-ordering", + "matrix-inbound-edit-ignored", + "matrix-inbound-edit-no-duplicate-trigger", + "matrix-e2ee-basic-reply", + "matrix-e2ee-thread-follow-up", + "matrix-e2ee-bootstrap-success", + "matrix-e2ee-recovery-key-lifecycle", + "matrix-e2ee-device-sas-verification", + "matrix-e2ee-qr-verification", + "matrix-e2ee-stale-device-hygiene", + "matrix-e2ee-dm-sas-verification", + "matrix-e2ee-restart-resume", + "matrix-e2ee-verification-notice-no-trigger", + "matrix-e2ee-artifact-redaction", + "matrix-e2ee-media-image", + "matrix-e2ee-key-bootstrap-failure", ]); }); @@ -149,6 +188,7 @@ describe("matrix live qa scenarios", () => { defaultRoomKey: "main", rooms: [ { + encrypted: false, key: "main", kind: "group", members: ["driver", "observer", "sut"], @@ -342,7 +382,10 @@ describe("matrix live qa scenarios", () => { eventId: "$sut-reply", sender: "@sut:matrix-qa.test", type: "m.room.message", - body: "observer sender accepted", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), }, since: "observer-sync-next", })); @@ -384,7 +427,7 @@ describe("matrix live qa scenarios", () => { actorUserId: "@observer:matrix-qa.test", driverEventId: "$observer-allow-trigger", reply: { - tokenMatched: false, + tokenMatched: true, }, }, }); @@ -753,17 +796,35 @@ describe("matrix live qa scenarios", () => { it("sends a real Matrix image attachment for image-understanding prompts", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendMediaMessage = vi.fn().mockResolvedValue("$image-understanding-trigger"); - const waitForRoomEvent = vi.fn().mockResolvedValue({ - event: { - kind: "message", - roomId: "!media:matrix-qa.test", - eventId: "$sut-image-reply", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - body: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.", - }, - since: "driver-sync-next", - }); + const waitForRoomEvent = vi + .fn() + .mockImplementationOnce(async () => ({ + event: { + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: "$image-understanding-trigger", + sender: "@driver:matrix-qa.test", + type: "m.room.message", + attachment: { + kind: "image", + filename: "red-top-blue-bottom.png", + caption: + "@sut:matrix-qa.test Image understanding check: describe the top and bottom colors in the attached image in one short sentence.", + }, + }, + since: "driver-sync-attachment", + })) + .mockImplementationOnce(async () => ({ + event: { + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: "$sut-image-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.", + }, + since: "driver-sync-next", + })); createMatrixQaClient.mockReturnValue({ primeRoom, @@ -823,6 +884,7 @@ describe("matrix live qa scenarios", () => { expect(sendMediaMessage).toHaveBeenCalledWith( expect.objectContaining({ + body: expect.stringContaining("Image understanding check"), contentType: "image/png", fileName: "red-top-blue-bottom.png", kind: "image", @@ -830,6 +892,12 @@ describe("matrix live qa scenarios", () => { roomId: "!media:matrix-qa.test", }), ); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + since: "driver-sync-attachment", + }), + ); }); it("waits for a real Matrix image attachment after image generation", async () => { @@ -915,6 +983,148 @@ describe("matrix live qa scenarios", () => { }); }); + it("covers every Matrix media msgtype with caption-triggered replies", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const mediaCases = MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES.map((mediaCase) => ({ + ...mediaCase, + eventId: `$media-${mediaCase.fileName}`, + })); + const sendMediaMessage = vi.fn().mockImplementation(async (opts: { fileName: string }) => { + const mediaCase = mediaCases.find((entry) => entry.fileName === opts.fileName); + return mediaCase?.eventId ?? "$unknown-media"; + }); + const waitForRoomEvent = vi.fn().mockImplementation(async () => { + const callIndex = waitForRoomEvent.mock.calls.length - 1; + const mediaCaseIndex = Math.floor(callIndex / 2); + const mediaCase = mediaCases[mediaCaseIndex]; + const sendOpts = sendMediaMessage.mock.calls[mediaCaseIndex]?.[0]; + if (callIndex % 2 === 0) { + return { + event: { + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: mediaCase.eventId, + sender: "@driver:matrix-qa.test", + type: "m.room.message", + msgtype: mediaCase.expectedMsgtype, + attachment: { + kind: mediaCase.expectedAttachmentKind, + filename: mediaCase.fileName, + caption: sendOpts?.body, + }, + }, + since: `driver-sync-attachment-${callIndex}`, + }; + } + const token = String(sendOpts?.body).match(/MATRIX_QA_MEDIA_[A-Z]+_[A-Z0-9]+/)?.[0] ?? ""; + return { + event: { + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: `$reply-${mediaCase.fileName}`, + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: token, + }, + since: `driver-sync-reply-${callIndex}`, + }; + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendMediaMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find((entry) => entry.id === "matrix-media-type-coverage"); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_MEDIA_ROOM_KEY, + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Media", + requireMention: true, + roomId: "!media:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + attachments: mediaCases.map((mediaCase) => ({ + eventId: mediaCase.eventId, + filename: mediaCase.fileName, + kind: mediaCase.expectedAttachmentKind, + msgtype: mediaCase.expectedMsgtype, + })), + roomId: "!media:matrix-qa.test", + }, + }); + + expect(sendMediaMessage).toHaveBeenCalledTimes(mediaCases.length); + for (const [index, mediaCase] of MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES.entries()) { + expect(sendMediaMessage).toHaveBeenNthCalledWith( + index + 1, + expect.objectContaining({ + contentType: mediaCase.contentType, + fileName: mediaCase.fileName, + kind: mediaCase.kind, + mentionUserIds: ["@sut:matrix-qa.test"], + }), + ); + } + const firstReplyWait = waitForRoomEvent.mock.calls[1]?.[0]; + const firstToken = + String(sendMediaMessage.mock.calls[0]?.[0]?.body).match( + /MATRIX_QA_MEDIA_[A-Z]+_[A-Z0-9]+/, + )?.[0] ?? ""; + expect( + firstReplyWait.predicate({ + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: "$verbose-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: `Sure, ${firstToken}`, + }), + ).toBe(false); + expect( + firstReplyWait.predicate({ + kind: "message", + roomId: "!media:matrix-qa.test", + eventId: "$exact-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: ` ${firstToken}\n`, + }), + ).toBe(true); + }); + it("uses DM thread override scenarios against the provisioned DM room", async () => { const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$dm-thread-trigger"); @@ -1414,4 +1624,367 @@ describe("matrix live qa scenarios", () => { }), ); }); + + it("ignores stale E2EE replies when checking a verification notice", async () => { + let noticeToken = ""; + const sendNoticeMessage = vi.fn().mockImplementation(async ({ body }) => { + noticeToken = body.match(/MATRIX_QA_E2EE_VERIFY_NOTICE_[A-Z0-9]+/)?.[0] ?? ""; + return "$verification-notice"; + }); + const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => { + expect( + params.predicate({ + body: "MATRIX_QA_E2EE_AFTER_RESTART_STALE", + eventId: "$stale-reply", + originServerTs: Date.now() - 60_000, + roomId: "!e2ee:matrix-qa.test", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }), + ).toBe(false); + expect( + params.predicate({ + body: noticeToken, + eventId: "$token-reply", + roomId: "!e2ee:matrix-qa.test", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }), + ).toBe(true); + expect( + params.predicate({ + eventId: "$related-reply", + relatesTo: { + inReplyToId: "$verification-notice", + }, + roomId: "!e2ee:matrix-qa.test", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }), + ).toBe(true); + expect( + params.predicate({ + eventId: "$new-unrelated-reply", + originServerTs: Date.now() + 1_000, + roomId: "!e2ee:matrix-qa.test", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }), + ).toBe(true); + return { + matched: false, + since: "e2ee:next", + }; + }); + + createMatrixQaE2eeScenarioClient.mockResolvedValue({ + prime: vi.fn().mockResolvedValue("e2ee:start"), + sendNoticeMessage, + stop: vi.fn().mockResolvedValue(undefined), + waitForOptionalRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-e2ee-verification-notice-no-trigger", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverDeviceId: "DRIVERDEVICE", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + outputDir: "/tmp/matrix-qa", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "E2EE", + requireMention: true, + roomId: "!e2ee:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + noticeEventId: "$verification-notice", + roomId: "!e2ee:matrix-qa.test", + }, + }); + + expect(noticeToken).toMatch(/^MATRIX_QA_E2EE_VERIFY_NOTICE_[A-Z0-9]+$/); + expect(waitForOptionalRoomEvent).toHaveBeenCalledWith( + expect.objectContaining({ + roomId: "!e2ee:matrix-qa.test", + }), + ); + }); + + it("applies a recovery key before restoring backed up room keys", async () => { + const verifyWithRecoveryKey = vi.fn().mockResolvedValue({ + backup: { + keyLoadError: null, + serverVersion: "backup-v1", + trusted: true, + }, + error: + "Matrix device is still not verified by its owner after applying the recovery key. Ensure cross-signing is available and the device is signed.", + success: false, + }); + const restoreRoomKeyBackup = vi.fn().mockResolvedValue({ + imported: 1, + loadedFromSecretStorage: true, + success: true, + total: 1, + }); + const resetRoomKeyBackup = vi.fn().mockResolvedValue({ + createdVersion: "backup-v2", + deletedVersion: "backup-v1", + previousVersion: "backup-v1", + success: true, + }); + const driverStop = vi.fn().mockResolvedValue(undefined); + const recoveryStop = vi.fn().mockResolvedValue(undefined); + createMatrixQaClient.mockReturnValue({ + loginWithPassword: vi.fn().mockResolvedValue({ + accessToken: "recovery-token", + deviceId: "RECOVERYDEVICE", + password: "driver-password", + userId: "@driver:matrix-qa.test", + }), + }); + createMatrixQaE2eeScenarioClient + .mockResolvedValueOnce({ + bootstrapOwnDeviceVerification: vi.fn().mockResolvedValue({ + crossSigning: { + published: true, + }, + success: true, + verification: { + backupVersion: "backup-v1", + recoveryKeyStored: true, + signedByOwner: true, + verified: true, + }, + }), + deleteOwnDevices: vi.fn().mockResolvedValue(undefined), + getRecoveryKey: vi.fn().mockResolvedValue({ + encodedPrivateKey: "encoded-recovery-key", + keyId: "SSSS", + }), + sendTextMessage: vi.fn().mockResolvedValue("$seeded-event"), + stop: driverStop, + }) + .mockResolvedValueOnce({ + resetRoomKeyBackup, + restoreRoomKeyBackup, + stop: recoveryStop, + verifyWithRecoveryKey, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-e2ee-recovery-key-lifecycle", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverDeviceId: "DRIVERDEVICE", + driverPassword: "driver-password", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + outputDir: "/tmp/matrix-qa", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + encrypted: true, + key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "E2EE", + requireMention: true, + roomId: "!e2ee:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + backupRestored: true, + recoveryDeviceId: "RECOVERYDEVICE", + recoveryKeyUsable: true, + recoveryVerified: false, + restoreImported: 1, + restoreTotal: 1, + }, + }); + + expect(verifyWithRecoveryKey).toHaveBeenCalledWith("encoded-recovery-key"); + expect(verifyWithRecoveryKey.mock.invocationCallOrder[0]).toBeLessThan( + restoreRoomKeyBackup.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER, + ); + }); + + it("runs Matrix E2EE bootstrap failure through a real faulted homeserver endpoint", async () => { + const stop = vi.fn().mockResolvedValue(undefined); + const hits = vi.fn().mockReturnValue([ + { + method: "GET", + path: "/_matrix/client/v3/room_keys/version", + ruleId: "room-key-backup-version-unavailable", + }, + ]); + startMatrixQaFaultProxy.mockResolvedValue({ + baseUrl: "http://127.0.0.1:39876", + hits, + stop, + }); + runMatrixQaE2eeBootstrap.mockResolvedValue({ + crossSigning: { + masterKeyPublished: true, + published: true, + selfSigningKeyPublished: true, + userId: "@driver:matrix-qa.test", + userSigningKeyPublished: true, + }, + cryptoBootstrap: null, + error: "Matrix room key backup is still missing after bootstrap", + pendingVerifications: 0, + success: false, + verification: { + backup: { + activeVersion: null, + enabled: false, + keyCached: false, + trusted: false, + }, + deviceId: "DRIVERDEVICE", + userId: "@driver:matrix-qa.test", + verified: true, + }, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-e2ee-key-bootstrap-failure", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverDeviceId: "DRIVERDEVICE", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + outputDir: "/tmp/matrix-qa", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "E2EE", + requireMention: true, + roomId: "!e2ee:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + bootstrapActor: "driver", + bootstrapSuccess: false, + faultedEndpoint: "/_matrix/client/v3/room_keys/version", + faultHitCount: 1, + faultRuleId: "room-key-backup-version-unavailable", + }, + }); + + const proxyArgs = startMatrixQaFaultProxy.mock.calls[0]?.[0]; + expect(proxyArgs).toBeDefined(); + if (!proxyArgs) { + throw new Error("expected Matrix QA fault proxy to start"); + } + const [faultRule] = proxyArgs.rules; + expect(faultRule).toBeDefined(); + if (!faultRule) { + throw new Error("expected Matrix QA fault proxy rule"); + } + expect(proxyArgs.targetBaseUrl).toBe("http://127.0.0.1:28008/"); + expect( + faultRule.match({ + bearerToken: "driver-token", + headers: {}, + method: "GET", + path: "/_matrix/client/v3/room_keys/version", + search: "", + }), + ).toBe(true); + expect(runMatrixQaE2eeBootstrap).toHaveBeenCalledWith({ + accessToken: "driver-token", + actorId: "driver", + baseUrl: "http://127.0.0.1:39876", + deviceId: "DRIVERDEVICE", + outputDir: "/tmp/matrix-qa", + scenarioId: "matrix-e2ee-key-bootstrap-failure", + timeoutMs: 8_000, + userId: "@driver:matrix-qa.test", + }); + expect(stop).toHaveBeenCalledTimes(1); + }); }); diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.ts b/extensions/qa-matrix/src/runners/contract/scenarios.ts index 6a07c1a467e..661257fa77e 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.ts @@ -1,6 +1,7 @@ import { MATRIX_QA_DRIVER_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + MATRIX_QA_E2EE_ROOM_KEY, MATRIX_QA_MEDIA_ROOM_KEY, MATRIX_QA_MEMBERSHIP_ROOM_KEY, MATRIX_QA_SCENARIOS, @@ -55,6 +56,7 @@ export type { MatrixQaScenarioContext, MatrixQaSyncState }; export const __testing = { MATRIX_QA_DRIVER_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, + MATRIX_QA_E2EE_ROOM_KEY, MATRIX_QA_MEDIA_ROOM_KEY, MATRIX_QA_MEMBERSHIP_ROOM_KEY, MATRIX_QA_SECONDARY_ROOM_KEY, diff --git a/extensions/qa-matrix/src/substrate/client.test.ts b/extensions/qa-matrix/src/substrate/client.test.ts index 55d49de2eab..bf8cdd6b7a8 100644 --- a/extensions/qa-matrix/src/substrate/client.test.ts +++ b/extensions/qa-matrix/src/substrate/client.test.ts @@ -63,6 +63,33 @@ describe("matrix driver client", () => { }); }); + it("builds Matrix replacement messages with replacement-local mention metadata", () => { + expect( + __testing.buildMatrixQaReplacementMessageContent({ + body: "@sut:matrix-qa.test updated prompt", + mentionUserIds: ["@sut:matrix-qa.test"], + targetEventId: " $msg-1 ", + }), + ).toEqual({ + body: "* @sut:matrix-qa.test updated prompt", + msgtype: "m.text", + "m.new_content": { + body: "@sut:matrix-qa.test updated prompt", + msgtype: "m.text", + format: "org.matrix.custom.html", + formatted_body: + '@sut:matrix-qa.test updated prompt', + "m.mentions": { + user_ids: ["@sut:matrix-qa.test"], + }, + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$msg-1", + }, + }); + }); + it("advances Matrix registration through token then dummy auth stages", () => { const firstStage = __testing.resolveNextRegistrationAuth({ registrationToken: "reg-token", @@ -105,6 +132,57 @@ describe("matrix driver client", () => { ).toThrow("Matrix registration requires unsupported auth stages:"); }); + it("logs in with Matrix password auth to create a secondary QA device", async () => { + const requests: Array<{ body: Record; url: string }> = []; + const fetchImpl: typeof fetch = async (input, init) => { + requests.push({ + body: parseJsonRequestBody(init), + url: resolveRequestUrl(input), + }); + return new Response( + JSON.stringify({ + access_token: "secondary-token", + device_id: "SECONDARYDEVICE", + user_id: "@qa-driver:matrix-qa.test", + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }; + + const client = createMatrixQaClient({ + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + }); + + await expect( + client.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Stale Device", + password: "driver-password", + userId: "@qa-driver:matrix-qa.test", + }), + ).resolves.toMatchObject({ + accessToken: "secondary-token", + deviceId: "SECONDARYDEVICE", + password: "driver-password", + userId: "@qa-driver:matrix-qa.test", + }); + + expect(requests).toEqual([ + { + url: "http://127.0.0.1:28008/_matrix/client/v3/login", + body: { + type: "m.login.password", + identifier: { + type: "m.id.user", + user: "@qa-driver:matrix-qa.test", + }, + initial_device_display_name: "OpenClaw Matrix QA Stale Device", + password: "driver-password", + }, + }, + ]); + }); + it("issues Matrix room membership control requests for QA topology changes", async () => { const requests: Array<{ body: Record; url: string }> = []; const fetchImpl: typeof fetch = async (input, init) => { @@ -189,6 +267,61 @@ describe("matrix driver client", () => { ).resolves.toBe("$reaction-1"); }); + it("sends Matrix replacements and redactions through protocol endpoints", async () => { + const requests: Array<{ body: Record; url: string }> = []; + const fetchImpl: typeof fetch = async (input, init) => { + requests.push({ + body: parseJsonRequestBody(init), + url: resolveRequestUrl(input), + }); + const eventId = requests.length === 1 ? "$replacement-1" : "$redaction-1"; + return new Response(JSON.stringify({ event_id: eventId }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + const client = createMatrixQaClient({ + accessToken: "token", + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + }); + + await expect( + client.sendReplacementMessage({ + body: "@sut:matrix-qa.test updated prompt", + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!room:matrix-qa.test", + targetEventId: "$msg-1", + }), + ).resolves.toBe("$replacement-1"); + await expect( + client.redactEvent({ + eventId: "$reaction-1", + reason: "qa cleanup", + roomId: "!room:matrix-qa.test", + }), + ).resolves.toBe("$redaction-1"); + + expect(requests[0]?.url).toContain( + "/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.room.message/", + ); + expect(requests[0]?.body).toMatchObject({ + "m.relates_to": { + rel_type: "m.replace", + event_id: "$msg-1", + }, + }); + expect(requests[1]).toEqual({ + url: expect.stringContaining( + "/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/redact/%24reaction-1/", + ), + body: { + reason: "qa cleanup", + }, + }); + }); + it("uploads Matrix media before sending the room event", async () => { const requests: Array<{ body: RequestInit["body"]; @@ -262,6 +395,38 @@ describe("matrix driver client", () => { }); }); + it("adds Matrix room encryption state when provisioning encrypted QA rooms", async () => { + const createRoomBodies: Array> = []; + const fetchImpl: typeof fetch = async (input, init) => { + createRoomBodies.push(parseJsonRequestBody(init)); + expect(resolveRequestUrl(input)).toBe("http://127.0.0.1:28008/_matrix/client/v3/createRoom"); + return new Response(JSON.stringify({ room_id: "!encrypted:matrix-qa.test" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + const client = createMatrixQaClient({ + accessToken: "token", + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + }); + + await expect( + client.createPrivateRoom({ + encrypted: true, + inviteUserIds: ["@sut:matrix-qa.test"], + name: "Encrypted QA Room", + }), + ).resolves.toBe("!encrypted:matrix-qa.test"); + + expect(createRoomBodies[0]?.initial_state).toContainEqual({ + type: "m.room.encryption", + state_key: "", + content: { algorithm: "m.megolm.v1.aes-sha2" }, + }); + }); + it("provisions a three-member room so Matrix QA runs in a group context", async () => { const createRoomBodies: Array> = []; const fetchImpl: typeof fetch = async (input, init) => { diff --git a/extensions/qa-matrix/src/substrate/client.ts b/extensions/qa-matrix/src/substrate/client.ts index 92f30aa27d4..e3f8380f5fd 100644 --- a/extensions/qa-matrix/src/substrate/client.ts +++ b/extensions/qa-matrix/src/substrate/client.ts @@ -30,6 +30,8 @@ type MatrixQaRegisterResponse = { user_id?: string; }; +type MatrixQaLoginResponse = MatrixQaRegisterResponse; + type MatrixQaRoomCreateResponse = { room_id?: string; }; @@ -38,17 +40,23 @@ type MatrixQaSendMessageContent = { body: string; format?: "org.matrix.custom.html"; formatted_body?: string; + "m.new_content"?: MatrixQaSendMessageContent; "m.mentions"?: { user_ids?: string[]; }; - "m.relates_to"?: { - rel_type: "m.thread"; - event_id: string; - is_falling_back: true; - "m.in_reply_to": { - event_id: string; - }; - }; + "m.relates_to"?: + | { + rel_type: "m.thread"; + event_id: string; + is_falling_back: true; + "m.in_reply_to": { + event_id: string; + }; + } + | { + rel_type: "m.replace"; + event_id: string; + }; msgtype: "m.text"; }; @@ -72,6 +80,12 @@ type MatrixQaSendReactionContent = { }; }; +type MatrixQaRoomInitialState = Array<{ + content: Record; + state_key: string; + type: string; +}>; + type MatrixQaUiaaResponse = { completed?: string[]; flows?: Array<{ stages?: string[] }>; @@ -107,6 +121,19 @@ function buildMatrixThreadRelation(threadRootEventId: string, replyToEventId?: s }; } +function buildMatrixReplacementRelation(targetEventId: string) { + const normalizedTargetEventId = targetEventId.trim(); + if (!normalizedTargetEventId) { + throw new Error("Matrix replacement requires a target event id"); + } + return { + "m.relates_to": { + rel_type: "m.replace" as const, + event_id: normalizedTargetEventId, + }, + }; +} + function buildMatrixReactionRelation( messageId: string, emoji: string, @@ -128,6 +155,24 @@ function buildMatrixReactionRelation( }; } +function buildMatrixQaRoomInitialState(encrypted?: boolean): MatrixQaRoomInitialState { + const initialState: MatrixQaRoomInitialState = [ + { + type: "m.room.history_visibility", + state_key: "", + content: { history_visibility: "joined" }, + }, + ]; + if (encrypted === true) { + initialState.push({ + type: "m.room.encryption", + state_key: "", + content: { algorithm: "m.megolm.v1.aes-sha2" }, + }); + } + return initialState; +} + function escapeMatrixHtml(value: string): string { return value.replace(/[&<>"']/g, (char) => { switch (char) { @@ -153,7 +198,7 @@ function buildMatrixMentionLink(userId: string) { return `${label}`; } -function buildMatrixQaMessageContent(params: { +export function buildMatrixQaMessageContent(params: { body: string; mentionUserIds?: string[]; replyToEventId?: string; @@ -201,6 +246,23 @@ function buildMatrixQaMessageContent(params: { }; } +function buildMatrixQaReplacementMessageContent(params: { + body: string; + mentionUserIds?: string[]; + targetEventId: string; +}): MatrixQaSendMessageContent { + const newContent = buildMatrixQaMessageContent({ + body: params.body, + mentionUserIds: params.mentionUserIds, + }); + return { + body: `* ${params.body}`, + msgtype: "m.text", + "m.new_content": newContent, + ...buildMatrixReplacementRelation(params.targetEventId), + }; +} + function resolveMatrixQaMediaMsgtype(params: { contentType?: string; kind?: "audio" | "file" | "image" | "video"; @@ -361,6 +423,14 @@ function buildRegisteredAccount(params: { } satisfies MatrixQaRegisteredAccount; } +function resolveMatrixQaLoginUser(params: { localpart?: string; userId?: string }) { + const user = params.userId?.trim() || params.localpart?.trim(); + if (!user) { + throw new Error("Matrix password login requires a localpart or userId."); + } + return user; +} + export function createMatrixQaClient(params: { accessToken?: string; baseUrl: string; @@ -369,21 +439,35 @@ export function createMatrixQaClient(params: { }) { const fetchImpl = params.fetchImpl ?? fetch; const syncObserver = params.syncObserver; + const sendEvent = async (opts: { body: unknown; endpoint: string; errorLabel: string }) => { + const result = await requestMatrixJson<{ event_id?: string }>({ + accessToken: params.accessToken, + baseUrl: params.baseUrl, + body: opts.body, + endpoint: opts.endpoint, + fetchImpl, + method: "PUT", + }); + const eventId = result.body.event_id?.trim(); + if (!eventId) { + throw new Error(`Matrix ${opts.errorLabel} did not return event_id.`); + } + return eventId; + }; return { - async createPrivateRoom(opts: { inviteUserIds: string[]; isDirect?: boolean; name: string }) { + async createPrivateRoom(opts: { + encrypted?: boolean; + inviteUserIds: string[]; + isDirect?: boolean; + name: string; + }) { const result = await requestMatrixJson({ accessToken: params.accessToken, baseUrl: params.baseUrl, body: { creation_content: { "m.federate": false }, - initial_state: [ - { - type: "m.room.history_visibility", - state_key: "", - content: { history_visibility: "joined" }, - }, - ], + initial_state: buildMatrixQaRoomInitialState(opts.encrypted), invite: opts.inviteUserIds, is_direct: opts.isDirect === true, name: opts.name, @@ -451,6 +535,34 @@ export function createMatrixQaClient(params: { `Matrix registration for ${opts.localpart} did not complete after 4 attempts.`, ); }, + async loginWithPassword(opts: { + deviceName: string; + localpart?: string; + password: string; + userId?: string; + }) { + const result = await requestMatrixJson({ + baseUrl: params.baseUrl, + body: { + type: "m.login.password", + identifier: { + type: "m.id.user", + user: resolveMatrixQaLoginUser(opts), + }, + initial_device_display_name: opts.deviceName, + password: opts.password, + }, + endpoint: "/_matrix/client/v3/login", + fetchImpl, + method: "POST", + timeoutMs: 30_000, + }); + return buildRegisteredAccount({ + localpart: opts.localpart ?? opts.userId ?? "", + password: opts.password, + response: result.body, + }); + }, async sendTextMessage(opts: { body: string; mentionUserIds?: string[]; @@ -459,19 +571,24 @@ export function createMatrixQaClient(params: { threadRootEventId?: string; }) { const txnId = randomUUID(); - const result = await requestMatrixJson<{ event_id?: string }>({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, + return await sendEvent({ body: buildMatrixQaMessageContent(opts), endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`, - fetchImpl, - method: "PUT", + errorLabel: "sendMessage", + }); + }, + async sendReplacementMessage(opts: { + body: string; + mentionUserIds?: string[]; + roomId: string; + targetEventId: string; + }) { + const txnId = randomUUID(); + return await sendEvent({ + body: buildMatrixQaReplacementMessageContent(opts), + endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`, + errorLabel: "sendReplacementMessage", }); - const eventId = result.body.event_id?.trim(); - if (!eventId) { - throw new Error("Matrix sendMessage did not return event_id."); - } - return eventId; }, async sendMediaMessage(opts: { body?: string; @@ -493,9 +610,7 @@ export function createMatrixQaClient(params: { fileName: opts.fileName, }); const txnId = randomUUID(); - const result = await requestMatrixJson<{ event_id?: string }>({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, + return await sendEvent({ body: buildMatrixQaMediaMessageContent({ body: opts.body, contentType: opts.contentType, @@ -508,30 +623,25 @@ export function createMatrixQaClient(params: { url: contentUri, }), endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`, - fetchImpl, - method: "PUT", + errorLabel: "sendMediaMessage", + }); + }, + async redactEvent(opts: { eventId: string; reason?: string; roomId: string }) { + const txnId = randomUUID(); + const reason = opts.reason?.trim(); + return await sendEvent({ + body: reason ? { reason } : {}, + endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/redact/${encodeURIComponent(opts.eventId)}/${encodeURIComponent(txnId)}`, + errorLabel: "redactEvent", }); - const eventId = result.body.event_id?.trim(); - if (!eventId) { - throw new Error("Matrix sendMediaMessage did not return event_id."); - } - return eventId; }, async sendReaction(opts: { emoji: string; messageId: string; roomId: string }) { const txnId = randomUUID(); - const result = await requestMatrixJson<{ event_id?: string }>({ - accessToken: params.accessToken, - baseUrl: params.baseUrl, + return await sendEvent({ body: buildMatrixReactionRelation(opts.messageId, opts.emoji), endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.reaction/${encodeURIComponent(txnId)}`, - fetchImpl, - method: "PUT", + errorLabel: "sendReaction", }); - const eventId = result.body.event_id?.trim(); - if (!eventId) { - throw new Error("Matrix sendReaction did not return event_id."); - } - return eventId; }, async joinRoom(roomId: string) { const result = await requestMatrixJson<{ room_id?: string }>({ @@ -684,6 +794,7 @@ async function provisionMatrixQaTopology(params: { fetchImpl: params.fetchImpl, }); const roomId = await creatorClient.createPrivateRoom({ + encrypted: room.encrypted === true, inviteUserIds: invitees.map((entry) => entry.account.userId), isDirect: room.kind === "dm", name: room.name, @@ -699,6 +810,7 @@ async function provisionMatrixQaTopology(params: { ), ); rooms.push({ + encrypted: room.encrypted === true, key: room.key, kind: room.kind, memberRoles: members.map((entry) => entry.role), @@ -793,7 +905,9 @@ export async function provisionMatrixQaRoom(params: { export const __testing = { buildMatrixQaMessageContent, + buildMatrixQaReplacementMessageContent, buildMatrixReactionRelation, + buildMatrixReplacementRelation, buildMatrixThreadRelation, createMatrixQaRoomObserver, resolveNextRegistrationAuth, diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index 24838ff2185..8a523118ef8 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -47,6 +47,7 @@ export type MatrixQaConfigOverrides = { groupPolicy?: MatrixQaGroupPolicy; groupsByKey?: Record; replyToMode?: MatrixQaReplyToMode; + startupVerification?: "if-unverified" | "off"; streaming?: "off" | "partial" | "quiet" | boolean; threadReplies?: MatrixQaThreadRepliesMode; }; @@ -74,6 +75,7 @@ export type MatrixQaConfigSnapshot = { } >; replyToMode: MatrixQaReplyToMode; + startupVerification?: "if-unverified" | "off"; streaming: MatrixQaStreamingMode; threadReplies: MatrixQaThreadRepliesMode; }; @@ -271,6 +273,10 @@ function buildMatrixQaChannelAccountConfig(params: { : {}; const streamingConfig = params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming } : {}; + const startupVerificationConfig = + params.snapshot.startupVerification !== undefined + ? { startupVerification: params.snapshot.startupVerification } + : {}; return { accessToken: params.sutAccessToken, @@ -289,6 +295,7 @@ function buildMatrixQaChannelAccountConfig(params: { dangerouslyAllowPrivateNetwork: true, }, replyToMode: params.snapshot.replyToMode, + ...startupVerificationConfig, threadReplies: params.snapshot.threadReplies, userId: params.sutUserId, ...autoJoinConfig, @@ -318,6 +325,7 @@ export function buildMatrixQaConfigSnapshot(params: { topology: params.topology, }), replyToMode: params.overrides?.replyToMode ?? "off", + startupVerification: params.overrides?.startupVerification, streaming: resolveMatrixQaStreamingMode(params.overrides?.streaming), threadReplies: params.overrides?.threadReplies ?? "inbound", }; @@ -335,6 +343,7 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot `blockStreaming=${formatMatrixQaBoolean(snapshot.blockStreaming)}`, `autoJoin=${snapshot.autoJoin}`, `encryption=${formatMatrixQaBoolean(snapshot.encryption)}`, + `startupVerification=${snapshot.startupVerification ?? ""}`, ].join(", "); } diff --git a/extensions/qa-matrix/src/substrate/e2ee-client.test.ts b/extensions/qa-matrix/src/substrate/e2ee-client.test.ts new file mode 100644 index 00000000000..3e0a2f0ebf3 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/e2ee-client.test.ts @@ -0,0 +1,30 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { __testing } from "./e2ee-client.js"; + +describe("matrix qa e2ee client storage", () => { + it("scopes persisted crypto state to the account actor", () => { + const first = __testing.buildMatrixQaE2eeStoragePaths({ + actorId: "driver", + outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", + scenarioId: "matrix-e2ee-basic-reply", + }); + const second = __testing.buildMatrixQaE2eeStoragePaths({ + actorId: "driver", + outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", + scenarioId: "matrix-e2ee-qr-verification", + }); + + expect(first.accountDir).toBe( + path.join( + "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", + "matrix-e2ee", + "accounts", + "driver", + "account", + ), + ); + expect(first.cryptoDatabasePrefix).toBe(second.cryptoDatabasePrefix); + expect(first.recoveryKeyPath).toBe(path.join(first.accountDir, "recovery-key.json")); + }); +}); diff --git a/extensions/qa-matrix/src/substrate/e2ee-client.ts b/extensions/qa-matrix/src/substrate/e2ee-client.ts new file mode 100644 index 00000000000..36a585146a4 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/e2ee-client.ts @@ -0,0 +1,399 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { + EncryptedFile, + MatrixDeviceVerificationStatus, + MatrixClient, + MatrixOwnDeviceDeleteResult, + MatrixOwnDeviceInfo, + MatrixRawEvent, + MatrixRecoveryKeyVerificationResult, + MatrixRoomKeyBackupResetResult, + MatrixRoomKeyBackupRestoreResult, + MatrixVerificationBootstrapResult, + MatrixVerificationMethod, + MatrixVerificationSummary, + MessageEventContent, +} from "@openclaw/matrix/test-api.js"; +import { buildMatrixQaMessageContent } from "./client.js"; +import { normalizeMatrixQaObservedEvent } from "./events.js"; +import type { MatrixQaObservedEvent } from "./events.js"; +import type { MatrixQaRoomEventWaitResult } from "./sync.js"; + +type MatrixQaE2eeActorId = "driver" | "observer" | `driver-${string}`; + +type MatrixQaE2eeRuntime = typeof import("@openclaw/matrix/test-api.js"); + +type MatrixQaE2eeClientParams = { + accessToken: string; + actorId: MatrixQaE2eeActorId; + baseUrl: string; + deviceId?: string; + outputDir: string; + password?: string; + scenarioId: string; + timeoutMs: number; + userId: string; +}; + +export type MatrixQaE2eeScenarioClient = { + acceptVerification(id: string): Promise; + bootstrapOwnDeviceVerification(params?: { + forceResetCrossSigning?: boolean; + recoveryKey?: string; + }): Promise; + confirmVerificationReciprocateQr(id: string): Promise; + confirmVerificationSas(id: string): Promise; + deleteOwnDevices(deviceIds: string[]): Promise; + generateVerificationQr(id: string): Promise<{ qrDataBase64: string }>; + getDeviceVerificationStatus( + userId: string, + deviceId: string, + ): Promise; + getRecoveryKey(): Promise<{ + encodedPrivateKey?: string; + keyId?: string | null; + createdAt?: string; + } | null>; + listOwnDevices(): Promise; + listVerifications(): Promise; + prime(): Promise; + requestVerification(params: { + deviceId?: string; + ownUser?: boolean; + roomId?: string; + userId?: string; + }): Promise; + resetRoomKeyBackup(): Promise; + restoreRoomKeyBackup(params?: { + recoveryKey?: string; + }): Promise; + scanVerificationQr(id: string, qrDataBase64: string): Promise; + verifyWithRecoveryKey(rawRecoveryKey: string): Promise; + sendTextMessage(opts: { + body: string; + mentionUserIds?: string[]; + replyToEventId?: string; + roomId: string; + threadRootEventId?: string; + }): Promise; + sendNoticeMessage(opts: { + body: string; + mentionUserIds?: string[]; + roomId: string; + }): Promise; + sendImageMessage(opts: { + body: string; + buffer: Buffer; + contentType: string; + fileName: string; + mentionUserIds?: string[]; + roomId: string; + }): Promise; + startVerification( + id: string, + method?: MatrixVerificationMethod, + ): Promise; + stop(): Promise; + waitForOptionalRoomEvent(params: { + predicate: (event: MatrixQaObservedEvent) => boolean; + roomId: string; + timeoutMs: number; + }): Promise; + waitForRoomEvent(params: { + predicate: (event: MatrixQaObservedEvent) => boolean; + roomId: string; + timeoutMs: number; + }): Promise<{ + event: MatrixQaObservedEvent; + since?: string; + }>; +}; + +async function loadMatrixQaE2eeRuntime(): Promise { + const { loadQaRunnerBundledPluginTestApi } = + await import("openclaw/plugin-sdk/qa-runner-runtime"); + return loadQaRunnerBundledPluginTestApi("matrix"); +} + +function buildMatrixQaE2eeStoragePaths(params: { + actorId: MatrixQaE2eeActorId; + outputDir: string; + scenarioId: string; +}) { + void params.scenarioId; + const rootDir = path.join(params.outputDir, "matrix-e2ee", "accounts", params.actorId); + const accountDir = path.join(rootDir, "account"); + const runKey = path + .basename(params.outputDir) + .replace(/[^A-Za-z0-9_-]/g, "-") + .slice(-80); + const actorKey = params.actorId.replace(/[^A-Za-z0-9_-]/g, "-").slice(-40); + return { + accountDir, + cryptoDatabasePrefix: `qa-matrix-${runKey || "run"}-${actorKey || "actor"}`, + idbSnapshotPath: path.join(accountDir, "crypto-idb-snapshot.json"), + recoveryKeyPath: path.join(accountDir, "recovery-key.json"), + rootDir, + storagePath: path.join(accountDir, "sync-store.json"), + }; +} + +async function prepareMatrixQaE2eeStorage(params: { + actorId: MatrixQaE2eeActorId; + outputDir: string; + scenarioId: string; +}) { + const storage = buildMatrixQaE2eeStoragePaths(params); + await fs.mkdir(storage.rootDir, { recursive: true }); + await fs.mkdir(storage.accountDir, { recursive: true }); + await fs.writeFile(storage.idbSnapshotPath, "[]\n", { flag: "wx" }).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + }); + return storage; +} + +async function createMatrixQaE2eeMatrixClient(params: MatrixQaE2eeClientParams) { + const runtime = await loadMatrixQaE2eeRuntime(); + const storage = await prepareMatrixQaE2eeStorage({ + actorId: params.actorId, + outputDir: params.outputDir, + scenarioId: params.scenarioId, + }); + return new runtime.MatrixClient(params.baseUrl, params.accessToken, { + autoBootstrapCrypto: false, + cryptoDatabasePrefix: storage.cryptoDatabasePrefix, + deviceId: params.deviceId, + encryption: true, + idbSnapshotPath: storage.idbSnapshotPath, + localTimeoutMs: Math.max(10_000, params.timeoutMs), + password: params.password, + recoveryKeyPath: storage.recoveryKeyPath, + ssrfPolicy: { allowPrivateNetwork: true }, + storagePath: storage.storagePath, + userId: params.userId, + }); +} + +function findMatrixQaObservedEventMatch(params: { + cursorIndex: number; + events: MatrixQaObservedEvent[]; + predicate: (event: MatrixQaObservedEvent) => boolean; + roomId: string; +}) { + for (let index = params.cursorIndex; index < params.events.length; index += 1) { + const event = params.events[index]; + if (event?.roomId !== params.roomId) { + continue; + } + if (params.predicate(event)) { + return { + event, + nextCursorIndex: index + 1, + }; + } + } + return undefined; +} + +export async function createMatrixQaE2eeScenarioClient( + params: MatrixQaE2eeClientParams & { + observedEvents: MatrixQaObservedEvent[]; + }, +): Promise { + const client: MatrixClient = await createMatrixQaE2eeMatrixClient(params); + const localEvents: MatrixQaObservedEvent[] = []; + const verificationSummaries: MatrixVerificationSummary[] = []; + const observedEventIds = new Set(); + let cursorIndex = 0; + + const recordEvent = (roomId: string, event: MatrixRawEvent) => { + const normalized = normalizeMatrixQaObservedEvent(roomId, event); + if (!normalized || observedEventIds.has(normalized.eventId)) { + return; + } + observedEventIds.add(normalized.eventId); + localEvents.push(normalized); + params.observedEvents.push(normalized); + }; + client.on("room.message", recordEvent); + const recordVerificationSummary = (summary: MatrixVerificationSummary) => { + verificationSummaries.push(summary); + }; + client.on("verification.summary", recordVerificationSummary); + + try { + await client.start({ readyTimeoutMs: Math.min(45_000, Math.max(15_000, params.timeoutMs)) }); + } catch (error) { + await client.stopAndPersist().catch(() => undefined); + throw error; + } + + const prime = async () => { + cursorIndex = Math.max(cursorIndex, localEvents.length); + return `e2ee:${cursorIndex}`; + }; + const waitForOptionalRoomEvent: MatrixQaE2eeScenarioClient["waitForOptionalRoomEvent"] = async ( + waitParams, + ) => { + const startSince = `e2ee:${cursorIndex}`; + const startedAt = Date.now(); + let scanIndex = cursorIndex; + while (Date.now() - startedAt < waitParams.timeoutMs) { + const matched = findMatrixQaObservedEventMatch({ + cursorIndex: scanIndex, + events: localEvents, + predicate: waitParams.predicate, + roomId: waitParams.roomId, + }); + if (matched) { + cursorIndex = Math.max(cursorIndex, matched.nextCursorIndex); + return { + event: matched.event, + matched: true, + since: `e2ee:${cursorIndex}`, + }; + } + scanIndex = localEvents.length; + await sleep(Math.min(250, Math.max(25, waitParams.timeoutMs - (Date.now() - startedAt)))); + } + cursorIndex = Math.max(cursorIndex, scanIndex); + return { + matched: false, + since: startSince, + }; + }; + + const requireCrypto = () => { + if (!client.crypto) { + throw new Error("Matrix E2EE scenario requires Matrix crypto"); + } + return client.crypto; + }; + + return { + async acceptVerification(id) { + return await requireCrypto().acceptVerification(id); + }, + async bootstrapOwnDeviceVerification(opts) { + return await client.bootstrapOwnDeviceVerification(opts); + }, + async confirmVerificationReciprocateQr(id) { + return await requireCrypto().confirmVerificationReciprocateQr(id); + }, + async confirmVerificationSas(id) { + return await requireCrypto().confirmVerificationSas(id); + }, + async deleteOwnDevices(deviceIds) { + return await client.deleteOwnDevices(deviceIds); + }, + async generateVerificationQr(id) { + return await requireCrypto().generateVerificationQr(id); + }, + async getDeviceVerificationStatus(userId, deviceId) { + return await client.getDeviceVerificationStatus(userId, deviceId); + }, + async getRecoveryKey() { + return await requireCrypto().getRecoveryKey(); + }, + async listOwnDevices() { + return await client.listOwnDevices(); + }, + async listVerifications() { + const current = await requireCrypto().listVerifications(); + return [...verificationSummaries, ...current].toSorted((a, b) => + b.updatedAt.localeCompare(a.updatedAt), + ); + }, + prime, + async requestVerification(opts) { + return await requireCrypto().requestVerification(opts); + }, + async resetRoomKeyBackup() { + return await client.resetRoomKeyBackup(); + }, + async restoreRoomKeyBackup(opts) { + return await client.restoreRoomKeyBackup(opts); + }, + async scanVerificationQr(id, qrDataBase64) { + return await requireCrypto().scanVerificationQr(id, qrDataBase64); + }, + async sendTextMessage(opts) { + return await client.sendMessage( + opts.roomId, + buildMatrixQaMessageContent(opts) as MessageEventContent, + ); + }, + async sendNoticeMessage(opts) { + return await client.sendMessage(opts.roomId, { + ...buildMatrixQaMessageContent(opts), + msgtype: "m.notice", + } as MessageEventContent); + }, + async sendImageMessage(opts) { + const encrypted = await requireCrypto().encryptMedia(opts.buffer); + const contentUri = await client.uploadContent( + encrypted.buffer, + opts.contentType, + opts.fileName, + ); + const file: EncryptedFile = { url: contentUri, ...encrypted.file }; + return await client.sendMessage(opts.roomId, { + ...buildMatrixQaMessageContent({ + body: opts.body, + mentionUserIds: opts.mentionUserIds, + }), + file, + filename: opts.fileName, + info: { + mimetype: opts.contentType, + size: opts.buffer.byteLength, + }, + msgtype: "m.image", + } as MessageEventContent); + }, + async startVerification(id, method) { + return await requireCrypto().startVerification(id, method); + }, + async stop() { + client.off("room.message", recordEvent); + client.off("verification.summary", recordVerificationSummary); + await client.drainPendingDecryptions().catch(() => undefined); + await client.stopAndPersist(); + }, + waitForOptionalRoomEvent, + async waitForRoomEvent(waitParams) { + const result = await waitForOptionalRoomEvent(waitParams); + if (result.matched) { + return { + event: result.event, + since: result.since, + }; + } + throw new Error(`timed out after ${waitParams.timeoutMs}ms waiting for Matrix E2EE event`); + }, + async verifyWithRecoveryKey(rawRecoveryKey) { + return await client.verifyWithRecoveryKey(rawRecoveryKey); + }, + }; +} + +export async function runMatrixQaE2eeBootstrap( + params: MatrixQaE2eeClientParams, +): Promise { + const client: MatrixClient = await createMatrixQaE2eeMatrixClient(params); + + try { + return await client.bootstrapOwnDeviceVerification(); + } finally { + await client.stopAndPersist().catch(() => undefined); + } +} + +export const __testing = { + buildMatrixQaE2eeStoragePaths, + findMatrixQaObservedEventMatch, +}; diff --git a/extensions/qa-matrix/src/substrate/fault-proxy.test.ts b/extensions/qa-matrix/src/substrate/fault-proxy.test.ts new file mode 100644 index 00000000000..b90ec4f0a32 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/fault-proxy.test.ts @@ -0,0 +1,127 @@ +import { createServer } from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import { startMatrixQaFaultProxy, type MatrixQaFaultProxy } from "./fault-proxy.js"; + +const servers: Array<{ close(): Promise }> = []; + +async function startTargetServer() { + const requests: Array<{ + authorization?: string; + body: string; + method: string; + url: string; + }> = []; + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + requests.push({ + ...(req.headers.authorization ? { authorization: req.headers.authorization } : {}), + body: Buffer.concat(chunks).toString("utf8"), + method: req.method ?? "GET", + url: req.url ?? "/", + }); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ forwarded: true })); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("target server did not bind to a TCP port"); + } + const handle = { + baseUrl: `http://127.0.0.1:${address.port}`, + close: async () => { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); + }, + requests, + }; + servers.push(handle); + return handle; +} + +describe("Matrix QA fault proxy", () => { + let proxy: MatrixQaFaultProxy | undefined; + + afterEach(async () => { + await proxy?.stop(); + proxy = undefined; + while (servers.length > 0) { + await servers.pop()?.close(); + } + }); + + it("faults matching Matrix requests and forwards everything else", async () => { + const target = await startTargetServer(); + proxy = await startMatrixQaFaultProxy({ + targetBaseUrl: target.baseUrl, + rules: [ + { + id: "room-key-backup-version-unavailable", + match: (request) => + request.method === "GET" && + request.path === "/_matrix/client/v3/room_keys/version" && + request.bearerToken === "driver-token", + response: () => ({ + body: { + errcode: "M_NOT_FOUND", + error: "No current key backup", + }, + status: 404, + }), + }, + ], + }); + + const faulted = await fetch(`${proxy.baseUrl}/_matrix/client/v3/room_keys/version`, { + headers: { authorization: "Bearer driver-token" }, + }); + expect(faulted.status).toBe(404); + await expect(faulted.json()).resolves.toEqual({ + errcode: "M_NOT_FOUND", + error: "No current key backup", + }); + + const forwarded = await fetch(`${proxy.baseUrl}/_matrix/client/v3/sync?timeout=0`, { + body: JSON.stringify({ ok: true }), + headers: { + authorization: "Bearer driver-token", + "content-type": "application/json", + }, + method: "POST", + }); + expect(forwarded.status).toBe(200); + await expect(forwarded.json()).resolves.toEqual({ forwarded: true }); + + expect(proxy.hits()).toEqual([ + { + method: "GET", + path: "/_matrix/client/v3/room_keys/version", + ruleId: "room-key-backup-version-unavailable", + }, + ]); + expect(target.requests).toEqual([ + { + authorization: "Bearer driver-token", + body: '{"ok":true}', + method: "POST", + url: "/_matrix/client/v3/sync?timeout=0", + }, + ]); + }); +}); diff --git a/extensions/qa-matrix/src/substrate/fault-proxy.ts b/extensions/qa-matrix/src/substrate/fault-proxy.ts new file mode 100644 index 00000000000..1c423bd70b1 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/fault-proxy.ts @@ -0,0 +1,216 @@ +import { + createServer, + type IncomingHttpHeaders, + type IncomingMessage, + type ServerResponse, +} from "node:http"; + +const HOP_BY_HOP_HEADERS = new Set([ + "connection", + "content-length", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); + +export type MatrixQaFaultProxyRequest = { + bearerToken?: string; + headers: IncomingHttpHeaders; + method: string; + path: string; + search: string; +}; + +export type MatrixQaFaultProxyResponse = { + body?: unknown; + headers?: Record; + status: number; +}; + +export type MatrixQaFaultProxyRule = { + id: string; + match(request: MatrixQaFaultProxyRequest): boolean; + response(request: MatrixQaFaultProxyRequest): MatrixQaFaultProxyResponse; +}; + +export type MatrixQaFaultProxyHit = { + method: string; + path: string; + ruleId: string; +}; + +export type MatrixQaFaultProxy = { + baseUrl: string; + hits(): MatrixQaFaultProxyHit[]; + stop(): Promise; +}; + +function normalizeHeaderValue(value: string | string[] | undefined) { + if (Array.isArray(value)) { + return value.join(", "); + } + return value; +} + +function extractBearerToken(headers: IncomingHttpHeaders) { + const value = normalizeHeaderValue(headers.authorization)?.trim(); + const match = /^Bearer\s+(.+)$/i.exec(value ?? ""); + return match?.[1]; +} + +function buildFetchHeaders(headers: IncomingHttpHeaders) { + const result = new Headers(); + for (const [key, rawValue] of Object.entries(headers)) { + if (HOP_BY_HOP_HEADERS.has(key.toLowerCase()) || key.toLowerCase() === "host") { + continue; + } + const value = normalizeHeaderValue(rawValue); + if (value !== undefined) { + result.set(key, value); + } + } + return result; +} + +async function readRequestBody(req: IncomingMessage) { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks); +} + +function bufferToArrayBuffer(buffer: Buffer) { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength, + ) as ArrayBuffer; +} + +function writeJsonResponse(res: ServerResponse, response: MatrixQaFaultProxyResponse) { + const body = response.body === undefined ? "" : JSON.stringify(response.body); + res.writeHead(response.status, { + "content-type": "application/json", + ...response.headers, + }); + res.end(body); +} + +async function forwardMatrixQaFaultProxyRequest(params: { + body: Buffer; + req: IncomingMessage; + targetUrl: URL; +}) { + const method = params.req.method ?? "GET"; + const init: RequestInit = { + headers: buildFetchHeaders(params.req.headers), + method, + redirect: "manual", + }; + if (method !== "GET" && method !== "HEAD") { + init.body = bufferToArrayBuffer(params.body); + } + const response = await fetch(params.targetUrl, init); + return { + body: Buffer.from(await response.arrayBuffer()), + headers: response.headers, + status: response.status, + }; +} + +function writeForwardedResponse( + res: ServerResponse, + response: Awaited>, +) { + const headers: Record = {}; + for (const [key, value] of response.headers) { + if (!HOP_BY_HOP_HEADERS.has(key.toLowerCase())) { + headers[key] = value; + } + } + res.writeHead(response.status, headers); + res.end(response.body); +} + +export async function startMatrixQaFaultProxy(params: { + rules: MatrixQaFaultProxyRule[]; + targetBaseUrl: string; +}): Promise { + const targetBaseUrl = new URL(params.targetBaseUrl); + const hits: MatrixQaFaultProxyHit[] = []; + const server = createServer(async (req, res) => { + try { + const requestUrl = new URL(req.url ?? "/", targetBaseUrl); + const path = requestUrl.pathname; + const bearerToken = extractBearerToken(req.headers); + const request: MatrixQaFaultProxyRequest = { + ...(bearerToken ? { bearerToken } : {}), + headers: req.headers, + method: req.method ?? "GET", + path, + search: requestUrl.search, + }; + const body = await readRequestBody(req); + const rule = params.rules.find((candidate) => candidate.match(request)); + if (rule) { + hits.push({ + method: request.method, + path: request.path, + ruleId: rule.id, + }); + writeJsonResponse(res, rule.response(request)); + return; + } + writeForwardedResponse( + res, + await forwardMatrixQaFaultProxyRequest({ + body, + req, + targetUrl: requestUrl, + }), + ); + } catch (error) { + writeJsonResponse(res, { + body: { + errcode: "MATRIX_QA_FAULT_PROXY_ERROR", + error: error instanceof Error ? error.message : String(error), + }, + status: 502, + }); + } + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Matrix QA fault proxy did not bind to a TCP port"); + } + + return { + baseUrl: `http://127.0.0.1:${address.port}`, + hits: () => [...hits], + stop: async () => { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); + }, + }; +} diff --git a/extensions/qa-matrix/src/substrate/harness.runtime.test.ts b/extensions/qa-matrix/src/substrate/harness.runtime.test.ts index 784a1005b88..c97f00d866d 100644 --- a/extensions/qa-matrix/src/substrate/harness.runtime.test.ts +++ b/extensions/qa-matrix/src/substrate/harness.runtime.test.ts @@ -26,6 +26,7 @@ describe("matrix harness runtime", () => { expect(compose).toContain(`image: ${__testing.MATRIX_QA_DEFAULT_IMAGE}`); expect(compose).toContain(' - "127.0.0.1:28008:8008"'); + expect(compose).toContain('TUWUNEL_ALLOW_ENCRYPTION: "true"'); expect(compose).toContain('TUWUNEL_ALLOW_REGISTRATION: "true"'); expect(compose).toContain('TUWUNEL_REGISTRATION_TOKEN: "secret-token"'); expect(compose).toContain('TUWUNEL_SERVER_NAME: "matrix-qa.test"'); diff --git a/extensions/qa-matrix/src/substrate/harness.runtime.ts b/extensions/qa-matrix/src/substrate/harness.runtime.ts index 069c81c5d2e..bece8d275f9 100644 --- a/extensions/qa-matrix/src/substrate/harness.runtime.ts +++ b/extensions/qa-matrix/src/substrate/harness.runtime.ts @@ -111,7 +111,7 @@ function renderMatrixQaCompose(params: { - "127.0.0.1:${params.homeserverPort}:${MATRIX_QA_INTERNAL_PORT}" environment: TUWUNEL_ADDRESS: "0.0.0.0" - TUWUNEL_ALLOW_ENCRYPTION: "false" + TUWUNEL_ALLOW_ENCRYPTION: "true" TUWUNEL_ALLOW_FEDERATION: "false" TUWUNEL_ALLOW_REGISTRATION: "true" TUWUNEL_DATABASE_PATH: "/var/lib/tuwunel" diff --git a/extensions/qa-matrix/src/substrate/topology.ts b/extensions/qa-matrix/src/substrate/topology.ts index bced2982666..5865e2ec899 100644 --- a/extensions/qa-matrix/src/substrate/topology.ts +++ b/extensions/qa-matrix/src/substrate/topology.ts @@ -3,6 +3,7 @@ export type MatrixQaParticipantRole = "driver" | "observer" | "sut"; export type MatrixQaRoomKind = "dm" | "group"; export type MatrixQaTopologyRoomSpec = { + encrypted?: boolean; key: string; kind: MatrixQaRoomKind; members: MatrixQaParticipantRole[]; @@ -16,6 +17,7 @@ export type MatrixQaTopologySpec = { }; export type MatrixQaProvisionedRoom = { + encrypted?: boolean; key: string; kind: MatrixQaRoomKind; memberRoles: MatrixQaParticipantRole[]; @@ -34,6 +36,7 @@ export type MatrixQaProvisionedTopology = { function matrixQaRoomSpecsEqual(left: MatrixQaTopologyRoomSpec, right: MatrixQaTopologyRoomSpec) { return ( left.key === right.key && + (left.encrypted === true) === (right.encrypted === true) && left.kind === right.kind && left.name === right.name && left.requireMention === right.requireMention && @@ -49,6 +52,7 @@ export function buildDefaultMatrixQaTopologySpec(params: { defaultRoomKey: "main", rooms: [ { + encrypted: false, key: "main", kind: "group", members: ["driver", "observer", "sut"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae9620cca86..b1cd1e474dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -998,6 +998,9 @@ importers: extensions/qa-matrix: devDependencies: + '@openclaw/matrix': + specifier: workspace:* + version: link:../matrix '@openclaw/plugin-sdk': specifier: workspace:* version: link:../../packages/plugin-sdk