vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 05:30:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

QA Matrix: expand contract coverage

Browse Source
This commit is contained in:
Gustavo Madeira Santana
2026-04-16 16:14:21 -04:00
parent 0f7c40e508
commit 988447ca24
26 changed files with 4407 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/qa-matrix/package.json
View File

@@ -5,6 +5,7 @@
"description": "OpenClaw Matrix QA runner plugin",
"type": "module",
"devDependencies": {
"@openclaw/matrix": "workspace:*",
"@openclaw/plugin-sdk": "workspace:*",
"openclaw": "workspace:*"
},

7
extensions/qa-matrix/src/runners/contract/runtime.ts
View File

@@ -609,6 +609,8 @@ export async function runMatrixQaLive(params: {
baseUrl: harness.baseUrl,
canary: canaryArtifact,
driverAccessToken: provisioning.driver.accessToken,
driverDeviceId: provisioning.driver.deviceId,
driverPassword: provisioning.driver.password,
driverUserId: provisioning.driver.userId,
interruptTransport: async () => {
writeMatrixQaProgress(`transport interrupt start ${scenario.id}`);
@@ -626,7 +628,10 @@ export async function runMatrixQaLive(params: {
},
observedEvents,
observerAccessToken: provisioning.observer.accessToken,
observerDeviceId: provisioning.observer.deviceId,
observerPassword: provisioning.observer.password,
observerUserId: provisioning.observer.userId,
outputDir,
restartGateway: async () => {
if (!gatewayHarness) {
throw new Error("Matrix restart scenario requires a live gateway");
@@ -644,6 +649,8 @@ export async function runMatrixQaLive(params: {
},
roomId: provisioning.roomId,
sutAccessToken: provisioning.sut.accessToken,
sutDeviceId: provisioning.sut.deviceId,
sutPassword: provisioning.sut.password,
syncState,
syncStreams,
sutUserId: provisioning.sut.userId,

202
extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
View File

@@ -23,6 +23,9 @@ export type MatrixQaScenarioId =
| "matrix-room-block-streaming"
| "matrix-room-image-understanding-attachment"
| "matrix-room-generated-image-delivery"
| "matrix-media-type-coverage"
| "matrix-attachment-only-ignored"
| "matrix-unsupported-media-safe"
| "matrix-dm-reply-shape"
| "matrix-dm-shared-session-notice"
| "matrix-dm-thread-reply-override"
@@ -33,12 +36,31 @@ export type MatrixQaScenarioId =
| "matrix-reaction-notification"
| "matrix-reaction-threaded"
| "matrix-reaction-not-a-reply"
| "matrix-reaction-redaction-observed"
| "matrix-restart-resume"
| "matrix-post-restart-room-continue"
| "matrix-room-membership-loss"
| "matrix-homeserver-restart-resume"
| "matrix-mention-gating"
| "matrix-mention-metadata-spoof-block"
| "matrix-observer-allowlist-override"
| "matrix-allowlist-block";
| "matrix-allowlist-block"
| "matrix-multi-actor-ordering"
| "matrix-inbound-edit-ignored"
| "matrix-inbound-edit-no-duplicate-trigger"
| "matrix-e2ee-basic-reply"
| "matrix-e2ee-thread-follow-up"
| "matrix-e2ee-bootstrap-success"
| "matrix-e2ee-recovery-key-lifecycle"
| "matrix-e2ee-device-sas-verification"
| "matrix-e2ee-qr-verification"
| "matrix-e2ee-stale-device-hygiene"
| "matrix-e2ee-dm-sas-verification"
| "matrix-e2ee-restart-resume"
| "matrix-e2ee-verification-notice-no-trigger"
| "matrix-e2ee-artifact-redaction"
| "matrix-e2ee-media-image"
| "matrix-e2ee-key-bootstrap-failure";
export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQaScenarioId> & {
configOverrides?: MatrixQaConfigOverrides;
@@ -48,6 +70,8 @@ export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQ
export const MATRIX_QA_BLOCK_ROOM_KEY = "block";
export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm";
export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared";
export const MATRIX_QA_E2EE_ROOM_KEY = "e2ee";
export const MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY = "e2ee-verification-dm";
export const MATRIX_QA_HOMESERVER_ROOM_KEY = "homeserver";
export const MATRIX_QA_MEDIA_ROOM_KEY = "media";
export const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership";
@@ -72,6 +96,7 @@ function buildMatrixQaDmTopology(
}
function buildMatrixQaSingleGroupTopology(params: {
encrypted?: boolean;
key: string;
name: string;
requireMention: boolean;
@@ -80,6 +105,7 @@ function buildMatrixQaSingleGroupTopology(params: {
defaultRoomKey: "main",
rooms: [
{
encrypted: params.encrypted === true,
key: params.key,
kind: "group",
members: ["driver", "observer", "sut"],
@@ -144,6 +170,31 @@ const MATRIX_QA_HOMESERVER_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({
requireMention: true,
});
const MATRIX_QA_E2EE_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({
encrypted: true,
key: MATRIX_QA_E2EE_ROOM_KEY,
name: "Matrix QA E2EE Room",
requireMention: true,
});
const MATRIX_QA_E2EE_VERIFICATION_DM_TOPOLOGY: MatrixQaTopologySpec = {
defaultRoomKey: "main",
rooms: [
{
encrypted: true,
key: MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY,
kind: "dm",
members: ["driver", "observer"],
name: "Matrix QA E2EE Verification DM",
},
],
};
const MATRIX_QA_E2EE_CONFIG = {
encryption: true,
startupVerification: "off",
} satisfies MatrixQaConfigOverrides;
export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
{
id: "matrix-thread-follow-up",
@@ -214,13 +265,34 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
{
id: "matrix-room-image-understanding-attachment",
timeoutMs: 60_000,
title: "Matrix image attachments reach the model vision path",
title: "Matrix captioned image attachments reach the model vision path",
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
},
{
id: "matrix-room-generated-image-delivery",
timeoutMs: 60_000,
title: "Matrix generated images deliver as real image attachments",
title: "Matrix generated images deliver as real image attachments while streaming",
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
configOverrides: {
streaming: "quiet",
},
},
{
id: "matrix-media-type-coverage",
timeoutMs: 90_000,
title: "Matrix media attachments cover image, audio, video, PDF, and EPUB transport",
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
},
{
id: "matrix-attachment-only-ignored",
timeoutMs: 8_000,
title: "Matrix attachment-only group media does not bypass mention gating",
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
},
{
id: "matrix-unsupported-media-safe",
timeoutMs: 45_000,
title: "Matrix unsupported media attachments do not block caption replies",
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
},
{
@@ -302,6 +374,11 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
timeoutMs: 8_000,
title: "Matrix reactions do not trigger a fresh bot reply",
},
{
id: "matrix-reaction-redaction-observed",
timeoutMs: 45_000,
title: "Matrix reaction removals are observed as redactions",
},
{
id: "matrix-restart-resume",
standardId: "restart-resume",
@@ -309,6 +386,12 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
title: "Matrix lane resumes cleanly after gateway restart",
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
},
{
id: "matrix-post-restart-room-continue",
timeoutMs: 75_000,
title: "Matrix restarted room continues after the first recovered reply",
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
},
{
id: "matrix-room-membership-loss",
timeoutMs: 75_000,
@@ -327,6 +410,11 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
timeoutMs: 8_000,
title: "Matrix room message without mention does not trigger",
},
{
id: "matrix-mention-metadata-spoof-block",
timeoutMs: 8_000,
title: "Matrix metadata-only mention spoof does not trigger",
},
{
id: "matrix-observer-allowlist-override",
timeoutMs: 45_000,
@@ -339,7 +427,113 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
id: "matrix-allowlist-block",
standardId: "allowlist-block",
timeoutMs: 8_000,
title: "Matrix allowlist blocks non-driver replies",
title: "Matrix sender allowlist blocks observer replies",
},
{
id: "matrix-multi-actor-ordering",
timeoutMs: 60_000,
title: "Matrix blocked observer traffic does not poison later driver replies",
},
{
id: "matrix-inbound-edit-ignored",
timeoutMs: 8_000,
title: "Matrix inbound edits cannot turn ignored messages into triggers",
},
{
id: "matrix-inbound-edit-no-duplicate-trigger",
timeoutMs: 45_000,
title: "Matrix inbound edits do not duplicate already handled triggers",
},
{
id: "matrix-e2ee-basic-reply",
timeoutMs: 75_000,
title: "Matrix E2EE encrypted room replies decrypt end-to-end",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-thread-follow-up",
timeoutMs: 75_000,
title: "Matrix E2EE encrypted threads preserve reply shape",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-bootstrap-success",
timeoutMs: 90_000,
title: "Matrix E2EE bootstrap verifies the owner device and backup",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-recovery-key-lifecycle",
timeoutMs: 90_000,
title: "Matrix E2EE recovery key restores and resets room-key backup",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-device-sas-verification",
timeoutMs: 90_000,
title: "Matrix E2EE device verification completes SAS emoji compare",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-qr-verification",
timeoutMs: 90_000,
title: "Matrix E2EE QR verification completes identity scan",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-stale-device-hygiene",
timeoutMs: 90_000,
title: "Matrix E2EE stale own devices can be removed without deleting the current device",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-dm-sas-verification",
timeoutMs: 90_000,
title: "Matrix E2EE DM verification notices stay scoped and complete SAS",
topology: MATRIX_QA_E2EE_VERIFICATION_DM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-restart-resume",
timeoutMs: 90_000,
title: "Matrix E2EE encrypted rooms resume after gateway restart",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-verification-notice-no-trigger",
timeoutMs: 30_000,
title: "Matrix E2EE verification notices do not trigger replies",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-artifact-redaction",
timeoutMs: 75_000,
title: "Matrix E2EE decrypted payloads stay out of default event artifacts",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-media-image",
timeoutMs: 90_000,
title: "Matrix E2EE encrypted image attachments reach the model vision path",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
{
id: "matrix-e2ee-key-bootstrap-failure",
timeoutMs: 90_000,
title: "Matrix E2EE bootstrap reports room-key backup failures",
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
configOverrides: MATRIX_QA_E2EE_CONFIG,
},
];

146
extensions/qa-matrix/src/runners/contract/scenario-media-fixtures.ts Normal file
View File

@@ -0,0 +1,146 @@
import { encodePngRgba, fillPixel } from "openclaw/plugin-sdk/media-runtime";
export const MATRIX_QA_IMAGE_ATTACHMENT_FILENAME = "red-top-blue-bottom.png";
export type MatrixQaMediaTypeCoverageCase = {
contentType: string;
createBuffer: () => Buffer;
expectedAttachmentKind: "audio" | "file" | "image" | "video";
expectedMsgtype: "m.audio" | "m.file" | "m.image" | "m.video";
fileName: string;
kind: "audio" | "file" | "image" | "video";
label: string;
tokenPrefix: string;
};
const MATRIX_QA_IMAGE_COLOR_GROUPS = [["red"], ["blue"]] as const;
const MATRIX_QA_ONE_PIXEL_JPEG_BASE64 =
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEBUQEBAVFRUVFRUVFRUVFRUVFRUVFRUXFhUVFRUYHSggGBolHRUVITEhJSkrLi4uFx8zODMsNygtLisBCgoKDg0OGhAQGi0fHyUtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAAEAAQMBIgACEQEDEQH/xAAXAAEBAQEAAAAAAAAAAAAAAAAAAQID/8QAFhEBAQEAAAAAAAAAAAAAAAAAAAER/9oADAMBAAIQAxAAAAH2AP/EABgQAQEAAwAAAAAAAAAAAAAAAAEAEQIS/9oACAEBAAEFAk1o7//EABYRAQEBAAAAAAAAAAAAAAAAAAABEf/aAAgBAwEBPwGn/8QAFhEBAQEAAAAAAAAAAAAAAAAAABEB/9oACAECAQE/AYf/xAAaEAACAgMAAAAAAAAAAAAAAAABEQAhMUFh/9oACAEBAAY/AjK9cY2f/8QAGhABAQACAwAAAAAAAAAAAAAAAAERITFBUf/aAAgBAQABPyGQk7W5jVYkA//Z";
export function createMatrixQaSplitColorImagePng() {
const width = 16;
const height = 16;
const rgba = Buffer.alloc(width * height * 4);
for (let y = 0; y < height; y += 1) {
const isTopHalf = y < height / 2;
for (let x = 0; x < width; x += 1) {
const [red, green, blue] = isTopHalf ? [255, 0, 0] : [0, 0, 255];
fillPixel(rgba, x, y, width, red, green, blue);
}
}
return encodePngRgba(rgba, width, height);
}
function createMatrixQaOnePixelJpeg() {
return Buffer.from(MATRIX_QA_ONE_PIXEL_JPEG_BASE64, "base64");
}
function createMatrixQaPdfFixture() {
return Buffer.from(
[
"%PDF-1.4",
"1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj",
"2 0 obj << /Type /Pages /Count 0 >> endobj",
"trailer << /Root 1 0 R >>",
"%%EOF",
].join("\n"),
"utf8",
);
}
function createMatrixQaEpubFixture() {
return Buffer.from("PK\u0003\u0004mimetypeapplication/epub+zip\n", "utf8");
}
function createMatrixQaWavFixture() {
const header = Buffer.alloc(44);
header.write("RIFF", 0);
header.writeUInt32LE(36, 4);
header.write("WAVE", 8);
header.write("fmt ", 12);
header.writeUInt32LE(16, 16);
header.writeUInt16LE(1, 20);
header.writeUInt16LE(1, 22);
header.writeUInt32LE(8_000, 24);
header.writeUInt32LE(16_000, 28);
header.writeUInt16LE(2, 32);
header.writeUInt16LE(16, 34);
header.write("data", 36);
header.writeUInt32LE(0, 40);
return header;
}
function createMatrixQaMp4Fixture() {
return Buffer.from([
0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, 0x00, 0x00, 0x02, 0x00,
0x69, 0x73, 0x6f, 0x6d, 0x6d, 0x70, 0x34, 0x31,
]);
}
export const MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES: MatrixQaMediaTypeCoverageCase[] = [
{
contentType: "image/jpeg",
createBuffer: createMatrixQaOnePixelJpeg,
expectedAttachmentKind: "image",
expectedMsgtype: "m.image",
fileName: "matrix-qa-one-pixel.jpg",
kind: "image",
label: "jpeg image",
tokenPrefix: "MATRIX_QA_MEDIA_JPEG",
},
{
contentType: "application/pdf",
createBuffer: createMatrixQaPdfFixture,
expectedAttachmentKind: "file",
expectedMsgtype: "m.file",
fileName: "matrix-qa-document.pdf",
kind: "file",
label: "pdf file",
tokenPrefix: "MATRIX_QA_MEDIA_PDF",
},
{
contentType: "application/epub+zip",
createBuffer: createMatrixQaEpubFixture,
expectedAttachmentKind: "file",
expectedMsgtype: "m.file",
fileName: "matrix-qa-book.epub",
kind: "file",
label: "epub file",
tokenPrefix: "MATRIX_QA_MEDIA_EPUB",
},
{
contentType: "audio/wav",
createBuffer: createMatrixQaWavFixture,
expectedAttachmentKind: "audio",
expectedMsgtype: "m.audio",
fileName: "matrix-qa-audio.wav",
kind: "audio",
label: "wav audio",
tokenPrefix: "MATRIX_QA_MEDIA_AUDIO",
},
{
contentType: "video/mp4",
createBuffer: createMatrixQaMp4Fixture,
expectedAttachmentKind: "video",
expectedMsgtype: "m.video",
fileName: "matrix-qa-video.mp4",
kind: "video",
label: "mp4 video",
tokenPrefix: "MATRIX_QA_MEDIA_VIDEO",
},
];
export function buildMatrixQaImageUnderstandingPrompt(sutUserId: string) {
return `${sutUserId} Image understanding check: describe the top and bottom colors in the attached image in one short sentence.`;
}
export function buildMatrixQaImageGenerationPrompt(sutUserId: string) {
return `${sutUserId} Image generation check: generate a QA lighthouse image and summarize it in one short sentence.`;
}
export function hasMatrixQaExpectedColorReply(body: string | undefined) {
const normalizedBody = body?.toLowerCase() ?? "";
return MATRIX_QA_IMAGE_COLOR_GROUPS.every((group) =>
group.some((color) => normalizedBody.includes(color)),
);
}

1228
extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts Normal file
View File

File diff suppressed because it is too large Load Diff

90
extensions/qa-matrix/src/runners/contract/scenario-runtime-edit.ts Normal file
View File

@@ -0,0 +1,90 @@
import { randomUUID } from "node:crypto";
import {
assertNoSutReplyWindow,
buildExactMarkerPrompt,
buildMatrixReplyDetails,
buildMentionPrompt,
primeMatrixQaDriverScenarioClient,
runAssertedDriverTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
export async function runInboundEditIgnoredScenario(context: MatrixQaScenarioContext) {
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const ignoredToken = `MATRIX_QA_EDIT_IGNORED_SOURCE_${randomUUID().slice(0, 8).toUpperCase()}`;
const editedToken = `MATRIX_QA_EDIT_IGNORED_${randomUUID().slice(0, 8).toUpperCase()}`;
const rootEventId = await client.sendTextMessage({
body: buildExactMarkerPrompt(ignoredToken),
roomId: context.roomId,
});
const editEventId = await client.sendReplacementMessage({
body: buildMentionPrompt(context.sutUserId, editedToken),
mentionUserIds: [context.sutUserId],
roomId: context.roomId,
targetEventId: rootEventId,
});
const { noReplyWindowMs } = await assertNoSutReplyWindow({
actorId: "driver",
client,
context,
roomId: context.roomId,
since: startSince,
startSince,
unexpectedMessage: "unexpected SUT reply after Matrix edit-to-mention event",
});
return {
artifacts: {
editEventId,
editedToken,
expectedNoReplyWindowMs: noReplyWindowMs,
rootEventId,
},
details: [
`root event: ${rootEventId}`,
`edit event: ${editEventId}`,
`waited ${noReplyWindowMs}ms with no SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runInboundEditNoDuplicateTriggerScenario(context: MatrixQaScenarioContext) {
const first = await runAssertedDriverTopLevelScenario({
context,
label: "pre-edit reply",
tokenPrefix: "MATRIX_QA_EDIT_ORIGINAL",
});
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const editedToken = `MATRIX_QA_EDIT_DUPLICATE_${randomUUID().slice(0, 8).toUpperCase()}`;
const editEventId = await client.sendReplacementMessage({
body: buildMentionPrompt(context.sutUserId, editedToken),
mentionUserIds: [context.sutUserId],
roomId: context.roomId,
targetEventId: first.driverEventId,
});
const { noReplyWindowMs } = await assertNoSutReplyWindow({
actorId: "driver",
client,
context,
roomId: context.roomId,
since: startSince,
startSince,
unexpectedMessage: "unexpected duplicate SUT reply after Matrix edit",
});
return {
artifacts: {
editEventId,
editedToken,
expectedNoReplyWindowMs: noReplyWindowMs,
originalDriverEventId: first.driverEventId,
originalReply: first.reply,
originalToken: first.token,
},
details: [
`original driver event: ${first.driverEventId}`,
...buildMatrixReplyDetails("original reply", first.reply),
`edit event: ${editEventId}`,
`waited ${noReplyWindowMs}ms with no duplicate SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

368
extensions/qa-matrix/src/runners/contract/scenario-runtime-media.ts Normal file
View File

@@ -0,0 +1,368 @@
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
import { MATRIX_QA_MEDIA_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js";
import {
buildMatrixQaImageGenerationPrompt,
buildMatrixQaImageUnderstandingPrompt,
createMatrixQaSplitColorImagePng,
hasMatrixQaExpectedColorReply,
MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES,
} from "./scenario-media-fixtures.js";
import {
advanceMatrixQaActorCursor,
assertNoSutReplyWindow,
buildMatrixQaToken,
buildMatrixReplyArtifact,
buildMatrixReplyDetails,
isMatrixQaExactMarkerReply,
isMatrixQaMessageLikeKind,
primeMatrixQaActorCursor,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
function requireMatrixQaImageAttachment(event: MatrixQaObservedEvent, scenarioLabel: string) {
if (event.msgtype !== "m.image" || event.attachment?.kind !== "image") {
throw new Error(
`${scenarioLabel} expected an m.image attachment but saw ${event.msgtype ?? "<none>"}`,
);
}
return event.attachment;
}
function buildMatrixQaAttachmentDetailLines(params: {
attachmentEvent: MatrixQaObservedEvent;
label: string;
}) {
return [
`${params.label} event: ${params.attachmentEvent.eventId}`,
`${params.label} msgtype: ${params.attachmentEvent.msgtype ?? "<none>"}`,
`${params.label} attachment kind: ${params.attachmentEvent.attachment?.kind ?? "<none>"}`,
`${params.label} attachment filename: ${params.attachmentEvent.attachment?.filename ?? "<none>"}`,
`${params.label} body preview: ${params.attachmentEvent.body?.slice(0, 200) ?? "<none>"}`,
];
}
async function primeMatrixQaDriverMediaClient(context: MatrixQaScenarioContext) {
return await primeMatrixQaActorCursor({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
syncState: context.syncState,
syncStreams: context.syncStreams,
});
}
function buildMatrixQaMediaTypeCoveragePrompt(params: {
label: string;
sutUserId: string;
token: string;
}) {
return `${params.sutUserId} Matrix media type coverage (${params.label}): ignore the attachment content and reply with only this exact marker: ${params.token}`;
}
export async function runImageUnderstandingAttachmentScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId);
const driverEventId = await client.sendMediaMessage({
body: triggerBody,
buffer: createMatrixQaSplitColorImagePng(),
contentType: "image/png",
fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
kind: "image",
mentionUserIds: [context.sutUserId],
roomId,
});
const attachmentEvent = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.eventId === driverEventId &&
event.sender === context.driverUserId &&
event.attachment?.kind === "image" &&
event.attachment.caption === triggerBody,
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message" &&
event.relatesTo === undefined &&
isMatrixQaMessageLikeKind(event.kind) &&
hasMatrixQaExpectedColorReply(event.body),
roomId,
since: attachmentEvent.since,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
const reply = buildMatrixReplyArtifact(matched.event);
return {
artifacts: {
attachmentCaptionPreview: attachmentEvent.event.attachment?.caption?.slice(0, 200),
attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
driverEventId,
reply,
roomId,
triggerBody,
},
details: [
`room id: ${roomId}`,
`driver attachment event: ${driverEventId}`,
`sent attachment filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`,
`sent attachment caption: ${attachmentEvent.event.attachment?.caption ?? "<none>"}`,
...buildMatrixReplyDetails("reply", reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runMediaTypeCoverageScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
const attachments: NonNullable<MatrixQaScenarioExecution["artifacts"]>["attachments"] = [];
const replies: NonNullable<MatrixQaScenarioExecution["artifacts"]>["replies"] = [];
const details = [`room id: ${roomId}`];
let since = startSince;
for (const mediaCase of MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES) {
const token = buildMatrixQaToken(mediaCase.tokenPrefix);
const triggerBody = buildMatrixQaMediaTypeCoveragePrompt({
label: mediaCase.label,
sutUserId: context.sutUserId,
token,
});
const driverEventId = await client.sendMediaMessage({
body: triggerBody,
buffer: mediaCase.createBuffer(),
contentType: mediaCase.contentType,
fileName: mediaCase.fileName,
kind: mediaCase.kind,
mentionUserIds: [context.sutUserId],
roomId,
});
const attachmentEvent = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.eventId === driverEventId &&
event.sender === context.driverUserId &&
event.msgtype === mediaCase.expectedMsgtype &&
event.attachment?.kind === mediaCase.expectedAttachmentKind &&
event.attachment.filename === mediaCase.fileName &&
event.attachment.caption === triggerBody,
roomId,
since,
timeoutMs: context.timeoutMs,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
isMatrixQaExactMarkerReply(event, {
roomId,
sutUserId: context.sutUserId,
token,
}) && event.relatesTo === undefined,
roomId,
since: attachmentEvent.since,
timeoutMs: context.timeoutMs,
});
since = matched.since ?? since;
const reply = buildMatrixReplyArtifact(matched.event, token);
attachments.push({
eventId: driverEventId,
filename: mediaCase.fileName,
kind: attachmentEvent.event.attachment?.kind,
label: mediaCase.label,
msgtype: attachmentEvent.event.msgtype,
});
replies.push({
eventId: reply.eventId,
label: mediaCase.label,
token,
tokenMatched: reply.tokenMatched,
});
details.push(
`${mediaCase.label} event: ${driverEventId}`,
`${mediaCase.label} msgtype: ${attachmentEvent.event.msgtype ?? "<none>"}`,
`${mediaCase.label} attachment kind: ${attachmentEvent.event.attachment?.kind ?? "<none>"}`,
`${mediaCase.label} attachment filename: ${attachmentEvent.event.attachment?.filename ?? "<none>"}`,
...buildMatrixReplyDetails(`${mediaCase.label} reply`, reply),
);
}
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: since,
startSince,
});
return {
artifacts: {
attachments,
replies,
roomId,
},
details: details.join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runAttachmentOnlyIgnoredScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
const driverEventId = await client.sendMediaMessage({
buffer: createMatrixQaSplitColorImagePng(),
contentType: "image/png",
fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
kind: "image",
roomId,
});
const attachmentEvent = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.eventId === driverEventId &&
event.sender === context.driverUserId &&
event.attachment?.kind === "image" &&
event.attachment.caption === undefined,
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
const { noReplyWindowMs } = await assertNoSutReplyWindow({
actorId: "driver",
client,
context,
roomId,
since: attachmentEvent.since,
startSince,
unexpectedMessage: "unexpected SUT reply to attachment-only group media",
});
return {
artifacts: {
attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
driverEventId,
expectedNoReplyWindowMs: noReplyWindowMs,
roomId,
},
details: [
`room id: ${roomId}`,
`driver attachment event: ${driverEventId}`,
`waited ${noReplyWindowMs}ms with no SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runUnsupportedMediaSafeScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
const token = buildMatrixQaToken("MATRIX_QA_UNSUPPORTED_MEDIA");
const triggerBody = `${context.sutUserId} Unsupported media QA check: ignore the attached text file and reply with only this exact marker: ${token}`;
const driverEventId = await client.sendMediaMessage({
body: triggerBody,
buffer: Buffer.from("unsupported Matrix QA attachment body\n", "utf8"),
contentType: "text/plain",
fileName: "unsupported-matrix-qa.txt",
kind: "file",
mentionUserIds: [context.sutUserId],
roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
isMatrixQaExactMarkerReply(event, {
roomId,
sutUserId: context.sutUserId,
token,
}) && event.relatesTo === undefined,
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
const reply = buildMatrixReplyArtifact(matched.event, token);
return {
artifacts: {
attachmentFilename: "unsupported-matrix-qa.txt",
attachmentKind: "file",
driverEventId,
reply,
roomId,
token,
triggerBody,
},
details: [
`room id: ${roomId}`,
`driver file event: ${driverEventId}`,
...buildMatrixReplyDetails("reply", reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId);
const driverEventId = await client.sendTextMessage({
body: triggerBody,
mentionUserIds: [context.sutUserId],
roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message" &&
event.relatesTo === undefined &&
event.msgtype === "m.image" &&
event.attachment?.kind === "image",
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
const attachment = requireMatrixQaImageAttachment(
matched.event,
"Matrix generated image delivery scenario",
);
return {
artifacts: {
attachmentBodyPreview: matched.event.body?.slice(0, 200),
attachmentEventId: matched.event.eventId,
attachmentFilename: attachment.filename,
attachmentKind: attachment.kind,
attachmentMsgtype: matched.event.msgtype,
driverEventId,
roomId,
triggerBody,
},
details: [
`room id: ${roomId}`,
`driver event: ${driverEventId}`,
...buildMatrixQaAttachmentDetailLines({
attachmentEvent: matched.event,
label: "generated image",
}),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

239
extensions/qa-matrix/src/runners/contract/scenario-runtime-reaction.ts Normal file
View File

@@ -0,0 +1,239 @@
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
import {
advanceMatrixQaActorCursor,
assertNoSutReplyWindow,
createMatrixQaDriverScenarioClient,
primeMatrixQaActorCursor,
type MatrixQaActorId,
type MatrixQaScenarioContext,
type MatrixQaSyncState,
} from "./scenario-runtime-shared.js";
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
export function buildMatrixQaReactionDetailLines(params: {
actorUserId?: string;
observedReactionKey?: string;
reactionEmoji: string;
reactionEventId: string;
reactionTargetEventId: string;
}) {
return [
`reaction event: ${params.reactionEventId}`,
`reaction target: ${params.reactionTargetEventId}`,
`reaction emoji: ${params.reactionEmoji}`,
...(params.actorUserId ? [`reaction sender: ${params.actorUserId}`] : []),
...(params.observedReactionKey ? [`observed reaction key: ${params.observedReactionKey}`] : []),
];
}
function requireMatrixQaReactionTargetEventId(
reactionTargetEventId: string | undefined,
scenarioLabel: string,
) {
const normalizedReactionTargetEventId = reactionTargetEventId?.trim();
if (!normalizedReactionTargetEventId) {
throw new Error(`${scenarioLabel} requires a canary reply event id`);
}
return normalizedReactionTargetEventId;
}
export async function observeReactionScenario(params: {
actorId: MatrixQaActorId;
actorUserId: string;
accessToken: string;
baseUrl: string;
observedEvents: MatrixQaObservedEvent[];
reactionEmoji?: string;
reactionTargetEventId: string;
roomId: string;
syncState: MatrixQaSyncState;
syncStreams?: MatrixQaScenarioContext["syncStreams"];
timeoutMs: number;
}) {
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: params.accessToken,
actorId: params.actorId,
baseUrl: params.baseUrl,
observedEvents: params.observedEvents,
syncState: params.syncState,
syncStreams: params.syncStreams,
});
const reactionEmoji = params.reactionEmoji ?? "👍";
const reactionEventId = await client.sendReaction({
emoji: reactionEmoji,
messageId: params.reactionTargetEventId,
roomId: params.roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.actorUserId &&
event.type === "m.reaction" &&
event.eventId === reactionEventId &&
event.reaction?.eventId === params.reactionTargetEventId &&
event.reaction?.key === reactionEmoji,
roomId: params.roomId,
since: startSince,
timeoutMs: params.timeoutMs,
});
return {
actorId: params.actorId,
actorUserId: params.actorUserId,
event: matched.event,
reactionEmoji,
reactionEventId,
reactionTargetEventId: params.reactionTargetEventId,
since: matched.since,
startSince,
};
}
export function buildMatrixQaReactionArtifacts(params: {
actorUserId?: string;
expectedNoReplyWindowMs?: number;
reaction: Awaited<ReturnType<typeof observeReactionScenario>>;
}) {
return {
...(params.actorUserId ? { actorUserId: params.actorUserId } : {}),
...(params.expectedNoReplyWindowMs === undefined
? {}
: { expectedNoReplyWindowMs: params.expectedNoReplyWindowMs }),
reactionEmoji: params.reaction.reactionEmoji,
reactionEventId: params.reaction.reactionEventId,
reactionTargetEventId: params.reaction.reactionTargetEventId,
};
}
export async function runReactionNotificationScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = requireMatrixQaReactionTargetEventId(
context.canary?.reply.eventId,
"Matrix reaction scenario",
);
const result = await observeReactionScenario({
actorId: "driver",
actorUserId: context.driverUserId,
accessToken: context.driverAccessToken,
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
reactionTargetEventId,
roomId: context.roomId,
syncState: context.syncState,
syncStreams: context.syncStreams,
timeoutMs: context.timeoutMs,
});
return {
artifacts: buildMatrixQaReactionArtifacts({ reaction: result }),
details: buildMatrixQaReactionDetailLines({
actorUserId: result.actorUserId,
observedReactionKey: result.event.reaction?.key,
reactionEmoji: result.reactionEmoji,
reactionEventId: result.reactionEventId,
reactionTargetEventId: result.reactionTargetEventId,
}).join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runReactionNotAReplyScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = requireMatrixQaReactionTargetEventId(
context.canary?.reply.eventId,
"Matrix reaction no-reply scenario",
);
const reaction = await observeReactionScenario({
actorId: "driver",
actorUserId: context.driverUserId,
accessToken: context.driverAccessToken,
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
reactionTargetEventId,
roomId: context.roomId,
syncState: context.syncState,
syncStreams: context.syncStreams,
timeoutMs: context.timeoutMs,
});
const client = createMatrixQaDriverScenarioClient(context);
const { noReplyWindowMs } = await assertNoSutReplyWindow({
actorId: reaction.actorId,
client,
context,
roomId: context.roomId,
since: reaction.since,
startSince: reaction.startSince,
unexpectedLines: [
`reaction target: ${reaction.reactionTargetEventId}`,
`reaction event: ${reaction.reactionEventId}`,
],
unexpectedMessage: `unexpected SUT reply after reaction from ${context.driverUserId}`,
});
return {
artifacts: buildMatrixQaReactionArtifacts({
actorUserId: context.driverUserId,
expectedNoReplyWindowMs: noReplyWindowMs,
reaction,
}),
details: [
...buildMatrixQaReactionDetailLines({
reactionEmoji: reaction.reactionEmoji,
reactionEventId: reaction.reactionEventId,
reactionTargetEventId: reaction.reactionTargetEventId,
}),
`waited ${noReplyWindowMs}ms with no SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runReactionRedactionObservedScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = requireMatrixQaReactionTargetEventId(
context.canary?.reply.eventId,
"Matrix reaction redaction scenario",
);
const reaction = await observeReactionScenario({
actorId: "driver",
actorUserId: context.driverUserId,
accessToken: context.driverAccessToken,
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
reactionTargetEventId,
roomId: context.roomId,
syncState: context.syncState,
syncStreams: context.syncStreams,
timeoutMs: context.timeoutMs,
});
const client = createMatrixQaDriverScenarioClient(context);
const redactionEventId = await client.redactEvent({
eventId: reaction.reactionEventId,
reason: "matrix qa reaction removal",
roomId: context.roomId,
});
const redaction = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === context.roomId &&
event.eventId === redactionEventId &&
event.sender === context.driverUserId &&
event.kind === "redaction",
roomId: context.roomId,
since: reaction.since,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: reaction.actorId,
syncState: context.syncState,
nextSince: redaction.since,
startSince: reaction.startSince,
});
return {
artifacts: {
...buildMatrixQaReactionArtifacts({ reaction }),
redactionEventId,
},
details: [
...buildMatrixQaReactionDetailLines({
reactionEmoji: reaction.reactionEmoji,
reactionEventId: reaction.reactionEventId,
reactionTargetEventId: reaction.reactionTargetEventId,
}),
`redaction event: ${redactionEventId}`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

109
extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts Normal file
View File

@@ -0,0 +1,109 @@
import {
MATRIX_QA_HOMESERVER_ROOM_KEY,
MATRIX_QA_RESTART_ROOM_KEY,
resolveMatrixQaScenarioRoomId,
} from "./scenario-catalog.js";
import {
buildMatrixReplyDetails,
runAssertedDriverTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
export async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) {
if (!context.interruptTransport) {
throw new Error("Matrix homeserver restart scenario requires a transport interruption hook");
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_HOMESERVER_ROOM_KEY);
await context.interruptTransport();
const resumed = await runAssertedDriverTopLevelScenario({
context,
label: "post-homeserver-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_HOMESERVER",
});
return {
artifacts: {
driverEventId: resumed.driverEventId,
reply: resumed.reply,
roomId,
token: resumed.token,
transportInterruption: "homeserver-restart",
},
details: [
`room id: ${roomId}`,
"transport interruption: homeserver-restart",
`driver event: ${resumed.driverEventId}`,
...buildMatrixReplyDetails("reply", resumed.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runRestartResumeScenario(context: MatrixQaScenarioContext) {
if (!context.restartGateway) {
throw new Error("Matrix restart scenario requires a gateway restart callback");
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY);
await context.restartGateway();
const result = await runAssertedDriverTopLevelScenario({
context,
label: "post-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_RESTART",
});
return {
artifacts: {
driverEventId: result.driverEventId,
reply: result.reply,
restartSignal: "SIGUSR1",
roomId,
token: result.token,
},
details: [
`room id: ${roomId}`,
"restart signal: SIGUSR1",
`post-restart driver event: ${result.driverEventId}`,
...buildMatrixReplyDetails("reply", result.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runPostRestartRoomContinueScenario(context: MatrixQaScenarioContext) {
if (!context.restartGateway) {
throw new Error("Matrix post-restart continuity scenario requires a gateway restart callback");
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY);
await context.restartGateway();
const first = await runAssertedDriverTopLevelScenario({
context,
label: "first post-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_RESTART_FIRST",
});
const second = await runAssertedDriverTopLevelScenario({
context,
label: "second post-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_RESTART_SECOND",
});
return {
artifacts: {
firstDriverEventId: first.driverEventId,
firstReply: first.reply,
firstToken: first.token,
restartSignal: "SIGUSR1",
roomId,
secondDriverEventId: second.driverEventId,
secondReply: second.reply,
secondToken: second.token,
},
details: [
`room id: ${roomId}`,
"restart signal: SIGUSR1",
`first post-restart driver event: ${first.driverEventId}`,
...buildMatrixReplyDetails("first reply", first.reply),
`second post-restart driver event: ${second.driverEventId}`,
...buildMatrixReplyDetails("second reply", second.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

488
extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
View File

@@ -1,95 +1,44 @@
import { randomUUID } from "node:crypto";
import { encodePngRgba, fillPixel } from "openclaw/plugin-sdk/media-runtime";
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
import {
MATRIX_QA_BLOCK_ROOM_KEY,
MATRIX_QA_HOMESERVER_ROOM_KEY,
MATRIX_QA_MEDIA_ROOM_KEY,
MATRIX_QA_MEMBERSHIP_ROOM_KEY,
MATRIX_QA_RESTART_ROOM_KEY,
resolveMatrixQaScenarioRoomId,
} from "./scenario-catalog.js";
import {
buildMatrixQaReactionArtifacts,
buildMatrixQaReactionDetailLines,
observeReactionScenario,
} from "./scenario-runtime-reaction.js";
import {
assertThreadReplyArtifact,
assertTopLevelReplyArtifact,
advanceMatrixQaActorCursor,
buildMatrixBlockStreamingPrompt,
buildMatrixQuietStreamingPrompt,
buildMatrixQaToken,
buildMatrixReplyArtifact,
buildMatrixReplyDetails,
buildMentionPrompt,
createMatrixQaDriverScenarioClient,
createMatrixQaScenarioClient,
isMatrixQaExactMarkerReply,
isMatrixQaMessageLikeKind,
NO_REPLY_WINDOW_MS,
primeMatrixQaActorCursor,
primeMatrixQaDriverScenarioClient,
runAssertedDriverTopLevelScenario,
runConfigurableTopLevelScenario,
runDriverTopLevelMentionScenario,
runNoReplyExpectedScenario,
runTopologyScopedTopLevelScenario,
waitForMembershipEvent,
type MatrixQaActorId,
type MatrixQaScenarioContext,
type MatrixQaSyncState,
} from "./scenario-runtime-shared.js";
import type { MatrixQaCanaryArtifact, MatrixQaScenarioExecution } from "./scenario-types.js";
type MatrixQaThreadScenarioResult = Awaited<ReturnType<typeof runThreadScenario>>;
const MATRIX_QA_IMAGE_ATTACHMENT_FILENAME = "red-top-blue-bottom.png";
const MATRIX_QA_IMAGE_COLOR_GROUPS = [["red"], ["blue"]] as const;
function createMatrixQaSplitColorImagePng() {
const width = 16;
const height = 16;
const rgba = Buffer.alloc(width * height * 4);
for (let y = 0; y < height; y += 1) {
const isTopHalf = y < height / 2;
for (let x = 0; x < width; x += 1) {
if (isTopHalf) {
fillPixel(rgba, x, y, width, 255, 0, 0);
continue;
}
fillPixel(rgba, x, y, width, 0, 0, 255);
}
}
return encodePngRgba(rgba, width, height);
}
function buildMatrixQaImageUnderstandingPrompt(sutUserId: string) {
return `${sutUserId} Image understanding check: describe the top and bottom colors in the attached image in one short sentence.`;
}
function buildMatrixQaImageGenerationPrompt(sutUserId: string) {
return `${sutUserId} Image generation check: generate a QA lighthouse image and summarize it in one short sentence.`;
}
function hasMatrixQaExpectedColorReply(body: string | undefined) {
const normalizedBody = body?.toLowerCase() ?? "";
return MATRIX_QA_IMAGE_COLOR_GROUPS.every((group) =>
group.some((color) => normalizedBody.includes(color)),
);
}
function requireMatrixQaImageAttachment(event: MatrixQaObservedEvent, scenarioLabel: string) {
if (event.msgtype !== "m.image" || event.attachment?.kind !== "image") {
throw new Error(
`${scenarioLabel} expected an m.image attachment but saw ${event.msgtype ?? "<none>"}`,
);
}
return event.attachment;
}
function buildMatrixQaAttachmentDetailLines(params: {
attachmentEvent: MatrixQaObservedEvent;
label: string;
}) {
return [
`${params.label} event: ${params.attachmentEvent.eventId}`,
`${params.label} msgtype: ${params.attachmentEvent.msgtype ?? "<none>"}`,
`${params.label} attachment kind: ${params.attachmentEvent.attachment?.kind ?? "<none>"}`,
`${params.label} attachment filename: ${params.attachmentEvent.attachment?.filename ?? "<none>"}`,
`${params.label} body preview: ${params.attachmentEvent.body?.slice(0, 200) ?? "<none>"}`,
];
}
function assertMatrixQaInReplyTarget(params: {
actualEventId?: string;
@@ -139,49 +88,6 @@ function buildMatrixQaThreadDetailLines(params: {
];
}
async function primeMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) {
return await primeMatrixQaActorCursor({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
syncState: context.syncState,
syncStreams: context.syncStreams,
});
}
function createMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) {
return createMatrixQaScenarioClient({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
syncState: context.syncState,
syncStreams: context.syncStreams,
});
}
async function runAssertedDriverTopLevelScenario(params: {
context: MatrixQaScenarioContext;
label: string;
roomId?: string;
tokenPrefix: string;
}) {
const result = await runDriverTopLevelMentionScenario({
baseUrl: params.context.baseUrl,
driverAccessToken: params.context.driverAccessToken,
observedEvents: params.context.observedEvents,
roomId: params.roomId ?? params.context.roomId,
syncState: params.context.syncState,
syncStreams: params.context.syncStreams,
sutUserId: params.context.sutUserId,
timeoutMs: params.context.timeoutMs,
tokenPrefix: params.tokenPrefix,
});
assertTopLevelReplyArtifact(params.label, result.reply);
return result;
}
async function runThreadScenario(
params: MatrixQaScenarioContext,
options?: {
@@ -205,7 +111,7 @@ async function runThreadScenario(
})
: undefined;
const triggerEventId = nestedDriverEventId ?? rootEventId;
const token = `${options?.tokenPrefix ?? "MATRIX_QA_THREAD"}_${randomUUID().slice(0, 8).toUpperCase()}`;
const token = buildMatrixQaToken(options?.tokenPrefix ?? "MATRIX_QA_THREAD");
const driverEventId = await client.sendTextMessage({
body: buildMentionPrompt(params.sutUserId, token),
mentionUserIds: [params.sutUserId],
@@ -216,10 +122,11 @@ async function runThreadScenario(
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
(event.body ?? "").includes(token) &&
isMatrixQaExactMarkerReply(event, {
roomId: params.roomId,
sutUserId: params.sutUserId,
token,
}) &&
event.relatesTo?.relType === "m.thread" &&
event.relatesTo.eventId === rootEventId,
roomId: params.roomId,
@@ -435,7 +342,7 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen
syncState: context.syncState,
syncStreams: context.syncStreams,
});
const token = `MATRIX_QA_OBSERVER_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`;
const token = buildMatrixQaToken("MATRIX_QA_OBSERVER_ALLOWLIST");
const body = buildMentionPrompt(context.sutUserId, token);
const driverEventId = await client.sendTextMessage({
body,
@@ -445,12 +352,11 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === context.roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message" &&
event.relatesTo === undefined &&
typeof event.body === "string" &&
event.body.trim().length > 0,
isMatrixQaExactMarkerReply(event, {
roomId: context.roomId,
sutUserId: context.sutUserId,
token,
}) && event.relatesTo === undefined,
roomId: context.roomId,
since: startSince,
timeoutMs: context.timeoutMs,
@@ -462,6 +368,7 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen
startSince,
});
const reply = buildMatrixReplyArtifact(matched.event, token);
assertTopLevelReplyArtifact("observer allowlist reply", reply);
return {
artifacts: {
actorUserId: context.observerUserId,
@@ -603,110 +510,6 @@ export async function runBlockStreamingScenario(context: MatrixQaScenarioContext
} satisfies MatrixQaScenarioExecution;
}
export async function runImageUnderstandingAttachmentScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId);
const driverEventId = await client.sendMediaMessage({
body: triggerBody,
buffer: createMatrixQaSplitColorImagePng(),
contentType: "image/png",
fileName: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
kind: "image",
mentionUserIds: [context.sutUserId],
roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message" &&
event.relatesTo === undefined &&
isMatrixQaMessageLikeKind(event.kind) &&
hasMatrixQaExpectedColorReply(event.body),
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
const reply = buildMatrixReplyArtifact(matched.event);
return {
artifacts: {
attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME,
driverEventId,
reply,
roomId,
triggerBody,
},
details: [
`room id: ${roomId}`,
`driver attachment event: ${driverEventId}`,
`sent attachment filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`,
...buildMatrixReplyDetails("reply", reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenarioContext) {
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId);
const driverEventId = await client.sendTextMessage({
body: triggerBody,
mentionUserIds: [context.sutUserId],
roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message" &&
event.relatesTo === undefined &&
event.msgtype === "m.image" &&
event.attachment?.kind === "image",
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
const attachment = requireMatrixQaImageAttachment(
matched.event,
"Matrix generated image delivery scenario",
);
return {
artifacts: {
attachmentBodyPreview: matched.event.body?.slice(0, 200),
attachmentEventId: matched.event.eventId,
attachmentFilename: attachment.filename,
attachmentKind: attachment.kind,
attachmentMsgtype: matched.event.msgtype,
driverEventId,
roomId,
triggerBody,
},
details: [
`room id: ${roomId}`,
`driver event: ${driverEventId}`,
...buildMatrixQaAttachmentDetailLines({
attachmentEvent: matched.event,
label: "generated image",
}),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runRoomAutoJoinInviteScenario(context: MatrixQaScenarioContext) {
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const dynamicRoomId = await client.createPrivateRoom({
@@ -857,130 +660,6 @@ export async function runMembershipLossScenario(context: MatrixQaScenarioContext
} satisfies MatrixQaScenarioExecution;
}
export async function runReactionNotificationScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = requireMatrixQaReactionTargetEventId(
context.canary?.reply.eventId,
"Matrix reaction scenario",
);
const result = await observeReactionScenario({
actorId: "driver",
actorUserId: context.driverUserId,
accessToken: context.driverAccessToken,
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
reactionTargetEventId,
roomId: context.roomId,
syncState: context.syncState,
syncStreams: context.syncStreams,
timeoutMs: context.timeoutMs,
});
return {
artifacts: buildMatrixQaReactionArtifacts({ reaction: result }),
details: buildMatrixQaReactionDetailLines({
actorUserId: result.actorUserId,
observedReactionKey: result.event.reaction?.key,
reactionEmoji: result.reactionEmoji,
reactionEventId: result.reactionEventId,
reactionTargetEventId: result.reactionTargetEventId,
}).join("\n"),
} satisfies MatrixQaScenarioExecution;
}
function buildMatrixQaReactionDetailLines(params: {
actorUserId?: string;
observedReactionKey?: string;
reactionEmoji: string;
reactionEventId: string;
reactionTargetEventId: string;
}) {
return [
`reaction event: ${params.reactionEventId}`,
`reaction target: ${params.reactionTargetEventId}`,
`reaction emoji: ${params.reactionEmoji}`,
...(params.actorUserId ? [`reaction sender: ${params.actorUserId}`] : []),
...(params.observedReactionKey ? [`observed reaction key: ${params.observedReactionKey}`] : []),
];
}
function requireMatrixQaReactionTargetEventId(
reactionTargetEventId: string | undefined,
scenarioLabel: string,
) {
const normalizedReactionTargetEventId = reactionTargetEventId?.trim();
if (!normalizedReactionTargetEventId) {
throw new Error(`${scenarioLabel} requires a canary reply event id`);
}
return normalizedReactionTargetEventId;
}
async function observeReactionScenario(params: {
actorId: MatrixQaActorId;
actorUserId: string;
accessToken: string;
baseUrl: string;
observedEvents: MatrixQaObservedEvent[];
reactionEmoji?: string;
reactionTargetEventId: string;
roomId: string;
syncState: MatrixQaSyncState;
syncStreams?: MatrixQaScenarioContext["syncStreams"];
timeoutMs: number;
}) {
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: params.accessToken,
actorId: params.actorId,
baseUrl: params.baseUrl,
observedEvents: params.observedEvents,
syncState: params.syncState,
syncStreams: params.syncStreams,
});
const reactionEmoji = params.reactionEmoji ?? "👍";
const reactionEventId = await client.sendReaction({
emoji: reactionEmoji,
messageId: params.reactionTargetEventId,
roomId: params.roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.actorUserId &&
event.type === "m.reaction" &&
event.eventId === reactionEventId &&
event.reaction?.eventId === params.reactionTargetEventId &&
event.reaction?.key === reactionEmoji,
roomId: params.roomId,
since: startSince,
timeoutMs: params.timeoutMs,
});
return {
actorId: params.actorId,
actorUserId: params.actorUserId,
event: matched.event,
reactionEmoji,
reactionEventId,
reactionTargetEventId: params.reactionTargetEventId,
since: matched.since,
startSince,
};
}
function buildMatrixQaReactionArtifacts(params: {
actorUserId?: string;
expectedNoReplyWindowMs?: number;
reaction: Awaited<ReturnType<typeof observeReactionScenario>>;
}) {
return {
...(params.actorUserId ? { actorUserId: params.actorUserId } : {}),
...(params.expectedNoReplyWindowMs === undefined
? {}
: { expectedNoReplyWindowMs: params.expectedNoReplyWindowMs }),
reactionEmoji: params.reaction.reactionEmoji,
reactionEventId: params.reaction.reactionEventId,
reactionTargetEventId: params.reaction.reactionTargetEventId,
};
}
export async function runReactionThreadedScenario(context: MatrixQaScenarioContext) {
const thread = await runThreadScenario(context, {
createNestedReply: true,
@@ -1031,124 +710,3 @@ export async function runReactionThreadedScenario(context: MatrixQaScenarioConte
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runReactionNotAReplyScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = requireMatrixQaReactionTargetEventId(
context.canary?.reply.eventId,
"Matrix reaction no-reply scenario",
);
const reaction = await observeReactionScenario({
actorId: "driver",
actorUserId: context.driverUserId,
accessToken: context.driverAccessToken,
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
reactionTargetEventId,
roomId: context.roomId,
syncState: context.syncState,
syncStreams: context.syncStreams,
timeoutMs: context.timeoutMs,
});
const client = createMatrixQaDriverScenarioClient(context);
const noReplyWindowMs = Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs);
const noReplyResult = await client.waitForOptionalRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === context.roomId &&
event.sender === context.sutUserId &&
event.type === "m.room.message",
roomId: context.roomId,
since: reaction.since,
timeoutMs: noReplyWindowMs,
});
if (noReplyResult.matched) {
const unexpectedReply = buildMatrixReplyArtifact(noReplyResult.event);
throw new Error(
[
`unexpected SUT reply after reaction from ${context.driverUserId}`,
`reaction target: ${reaction.reactionTargetEventId}`,
`reaction event: ${reaction.reactionEventId}`,
...buildMatrixReplyDetails("unexpected reply", unexpectedReply),
].join("\n"),
);
}
advanceMatrixQaActorCursor({
actorId: reaction.actorId,
syncState: context.syncState,
nextSince: noReplyResult.since,
startSince: reaction.startSince,
});
return {
artifacts: buildMatrixQaReactionArtifacts({
actorUserId: context.driverUserId,
expectedNoReplyWindowMs: noReplyWindowMs,
reaction,
}),
details: [
...buildMatrixQaReactionDetailLines({
reactionEmoji: reaction.reactionEmoji,
reactionEventId: reaction.reactionEventId,
reactionTargetEventId: reaction.reactionTargetEventId,
}),
`waited ${noReplyWindowMs}ms with no SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runHomeserverRestartResumeScenario(context: MatrixQaScenarioContext) {
if (!context.interruptTransport) {
throw new Error("Matrix homeserver restart scenario requires a transport interruption hook");
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_HOMESERVER_ROOM_KEY);
await context.interruptTransport();
const resumed = await runAssertedDriverTopLevelScenario({
context,
label: "post-homeserver-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_HOMESERVER",
});
return {
artifacts: {
driverEventId: resumed.driverEventId,
reply: resumed.reply,
roomId,
token: resumed.token,
transportInterruption: "homeserver-restart",
},
details: [
`room id: ${roomId}`,
"transport interruption: homeserver-restart",
`driver event: ${resumed.driverEventId}`,
...buildMatrixReplyDetails("reply", resumed.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runRestartResumeScenario(context: MatrixQaScenarioContext) {
if (!context.restartGateway) {
throw new Error("Matrix restart scenario requires a gateway restart callback");
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY);
await context.restartGateway();
const result = await runAssertedDriverTopLevelScenario({
context,
label: "post-restart reply",
roomId,
tokenPrefix: "MATRIX_QA_RESTART",
});
return {
artifacts: {
driverEventId: result.driverEventId,
reply: result.reply,
restartSignal: "SIGUSR1",
roomId,
token: result.token,
},
details: [
`room id: ${roomId}`,
"restart signal: SIGUSR1",
`post-restart driver event: ${result.driverEventId}`,
...buildMatrixReplyDetails("reply", result.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

132
extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
View File

@@ -19,14 +19,21 @@ export type MatrixQaScenarioContext = {
baseUrl: string;
canary?: MatrixQaCanaryArtifact;
driverAccessToken: string;
driverDeviceId?: string;
driverPassword?: string;
driverUserId: string;
observedEvents: MatrixQaObservedEvent[];
observerAccessToken: string;
observerDeviceId?: string;
observerPassword?: string;
observerUserId: string;
outputDir?: string;
restartGateway?: () => Promise<void>;
roomId: string;
interruptTransport?: () => Promise<void>;
sutAccessToken: string;
sutDeviceId?: string;
sutPassword?: string;
syncState: MatrixQaSyncState;
syncStreams?: MatrixQaSyncStreams;
sutUserId: string;
@@ -44,6 +51,10 @@ export function buildExactMarkerPrompt(token: string) {
return `reply with only this exact marker: ${token}`;
}
export function buildMatrixQaToken(prefix: string) {
return `${prefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
}
export function buildMatrixQuietStreamingPrompt(sutUserId: string, text: string) {
return `${sutUserId} Matrix quiet streaming QA check: reply exactly \`${text}\`.`;
}
@@ -66,6 +77,27 @@ export function isMatrixQaMessageLikeKind(kind: MatrixQaObservedEvent["kind"]) {
return kind === "message" || kind === "notice";
}
export function doesMatrixQaReplyBodyMatchToken(event: MatrixQaObservedEvent, token: string) {
return event.body?.trim() === token;
}
export function isMatrixQaExactMarkerReply(
event: MatrixQaObservedEvent,
params: {
roomId: string;
sutUserId: string;
token: string;
},
) {
return (
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
isMatrixQaMessageLikeKind(event.kind) &&
doesMatrixQaReplyBodyMatchToken(event, params.token)
);
}
export function buildMatrixReplyArtifact(
event: MatrixQaObservedEvent,
token?: string,
@@ -77,7 +109,7 @@ export function buildMatrixReplyArtifact(
mentions: event.mentions,
relatesTo: event.relatesTo,
sender: event.sender,
...(token ? { tokenMatched: replyBody === token } : {}),
...(token ? { tokenMatched: doesMatrixQaReplyBodyMatchToken(event, token) } : {}),
};
}
@@ -200,6 +232,17 @@ export function createMatrixQaScenarioClient(params: {
});
}
export function createMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) {
return createMatrixQaScenarioClient({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
syncState: context.syncState,
syncStreams: context.syncStreams,
});
}
export async function primeMatrixQaActorCursor(params: {
accessToken: string;
actorId: MatrixQaActorId;
@@ -227,6 +270,17 @@ export async function primeMatrixQaActorCursor(params: {
return { client, startSince };
}
export async function primeMatrixQaDriverScenarioClient(context: MatrixQaScenarioContext) {
return await primeMatrixQaActorCursor({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
syncState: context.syncState,
syncStreams: context.syncStreams,
});
}
export function advanceMatrixQaActorCursor(params: {
actorId: MatrixQaActorId;
syncState: MatrixQaSyncState;
@@ -236,6 +290,50 @@ export function advanceMatrixQaActorCursor(params: {
writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince);
}
type MatrixQaScenarioClient = ReturnType<typeof createMatrixQaScenarioClient>;
export async function assertNoSutReplyWindow(params: {
actorId: MatrixQaActorId;
client: MatrixQaScenarioClient;
context: MatrixQaScenarioContext;
roomId: string;
since?: string;
startSince: string;
unexpectedLines?: string[];
unexpectedMessage: string;
}) {
const noReplyWindowMs = Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs);
const result = await params.client.waitForOptionalRoomEvent({
observedEvents: params.context.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.context.sutUserId &&
event.type === "m.room.message",
roomId: params.roomId,
since: params.since,
timeoutMs: noReplyWindowMs,
});
if (result.matched) {
throw new Error(
[
params.unexpectedMessage,
...(params.unexpectedLines ?? []),
...buildMatrixReplyDetails("unexpected reply", buildMatrixReplyArtifact(result.event)),
].join("\n"),
);
}
advanceMatrixQaActorCursor({
actorId: params.actorId,
syncState: params.context.syncState,
nextSince: result.since,
startSince: params.startSince,
});
return {
noReplyWindowMs,
since: result.since,
};
}
export async function runConfigurableTopLevelScenario(params: {
accessToken: string;
actorId: MatrixQaActorId;
@@ -261,7 +359,7 @@ export async function runConfigurableTopLevelScenario(params: {
syncState: params.syncState,
syncStreams: params.syncStreams,
});
const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
const token = buildMatrixQaToken(params.tokenPrefix);
const body =
params.withMention === false
? buildExactMarkerPrompt(token)
@@ -274,10 +372,11 @@ export async function runConfigurableTopLevelScenario(params: {
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
(event.body ?? "").includes(token) &&
isMatrixQaExactMarkerReply(event, {
roomId: params.roomId,
sutUserId: params.sutUserId,
token,
}) &&
(params.replyPredicate?.(event, { driverEventId, token }) ?? event.relatesTo === undefined),
roomId: params.roomId,
since: startSince,
@@ -338,6 +437,27 @@ export async function runDriverTopLevelMentionScenario(params: {
});
}
export async function runAssertedDriverTopLevelScenario(params: {
context: MatrixQaScenarioContext;
label: string;
roomId?: string;
tokenPrefix: string;
}) {
const result = await runDriverTopLevelMentionScenario({
baseUrl: params.context.baseUrl,
driverAccessToken: params.context.driverAccessToken,
observedEvents: params.context.observedEvents,
roomId: params.roomId ?? params.context.roomId,
syncState: params.context.syncState,
syncStreams: params.context.syncStreams,
sutUserId: params.context.sutUserId,
timeoutMs: params.context.timeoutMs,
tokenPrefix: params.tokenPrefix,
});
assertTopLevelReplyArtifact(params.label, result.reply);
return result;
}
export async function waitForMembershipEvent(params: {
accessToken: string;
actorId: MatrixQaActorId;

131
extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
View File

@@ -1,4 +1,3 @@
import { randomUUID } from "node:crypto";
import {
MATRIX_QA_DRIVER_DM_ROOM_KEY,
MATRIX_QA_SECONDARY_ROOM_KEY,
@@ -10,18 +9,48 @@ import {
runDmThreadReplyOverrideScenario,
} from "./scenario-runtime-dm.js";
import {
runBlockStreamingScenario,
runMatrixQaE2eeArtifactRedactionScenario,
runMatrixQaE2eeBasicReplyScenario,
runMatrixQaE2eeBootstrapSuccessScenario,
runMatrixQaE2eeDeviceSasVerificationScenario,
runMatrixQaE2eeDmSasVerificationScenario,
runMatrixQaE2eeKeyBootstrapFailureScenario,
runMatrixQaE2eeMediaImageScenario,
runMatrixQaE2eeQrVerificationScenario,
runMatrixQaE2eeRecoveryKeyLifecycleScenario,
runMatrixQaE2eeRestartResumeScenario,
runMatrixQaE2eeStaleDeviceHygieneScenario,
runMatrixQaE2eeThreadFollowUpScenario,
runMatrixQaE2eeVerificationNoticeNoTriggerScenario,
} from "./scenario-runtime-e2ee.js";
import {
runInboundEditIgnoredScenario,
runInboundEditNoDuplicateTriggerScenario,
} from "./scenario-runtime-edit.js";
import {
runAttachmentOnlyIgnoredScenario,
runGeneratedImageDeliveryScenario,
runHomeserverRestartResumeScenario,
runImageUnderstandingAttachmentScenario,
runMediaTypeCoverageScenario,
runUnsupportedMediaSafeScenario,
} from "./scenario-runtime-media.js";
import {
runReactionNotAReplyScenario,
runReactionNotificationScenario,
runReactionRedactionObservedScenario,
} from "./scenario-runtime-reaction.js";
import {
runHomeserverRestartResumeScenario,
runPostRestartRoomContinueScenario,
runRestartResumeScenario,
} from "./scenario-runtime-restart.js";
import {
runBlockStreamingScenario,
runMatrixQaCanary,
runMembershipLossScenario,
runObserverAllowlistOverrideScenario,
runQuietStreamingPreviewScenario,
runReactionNotAReplyScenario,
runReactionNotificationScenario,
runReactionThreadedScenario,
runRestartResumeScenario,
runRoomAutoJoinInviteScenario,
runRoomThreadReplyOverrideScenario,
runThreadFollowUpScenario,
@@ -32,9 +61,11 @@ import {
} from "./scenario-runtime-room.js";
import {
buildExactMarkerPrompt,
buildMatrixQaToken,
buildMatrixReplyArtifact,
buildMatrixReplyDetails,
buildMentionPrompt,
NO_REPLY_WINDOW_MS,
readMatrixQaSyncCursor,
runNoReplyExpectedScenario,
runTopologyScopedTopLevelScenario,
@@ -71,10 +102,6 @@ async function runDriverTopologyScopedScenario(params: {
});
}
function buildMatrixQaToken(prefix: string) {
return `${prefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
}
async function runNoReplyScenario(params: {
accessToken: string;
actorId: "driver" | "observer";
@@ -82,8 +109,10 @@ async function runNoReplyScenario(params: {
body: string;
context: MatrixQaScenarioContext;
mentionUserIds?: string[];
timeoutMs?: number;
token: string;
}) {
const timeoutMs = params.timeoutMs ?? params.context.timeoutMs;
return await runNoReplyExpectedScenario({
accessToken: params.accessToken,
actorId: params.actorId,
@@ -95,11 +124,37 @@ async function runNoReplyScenario(params: {
roomId: params.context.roomId,
syncState: params.context.syncState,
sutUserId: params.context.sutUserId,
timeoutMs: params.context.timeoutMs,
timeoutMs,
token: params.token,
});
}
async function runMultiActorOrderingScenario(context: MatrixQaScenarioContext) {
const blockedToken = buildMatrixQaToken("MATRIX_QA_MULTI_BLOCKED");
const blocked = await runNoReplyScenario({
accessToken: context.observerAccessToken,
actorId: "observer",
actorUserId: context.observerUserId,
body: buildMentionPrompt(context.sutUserId, blockedToken),
mentionUserIds: [context.sutUserId],
context,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
token: blockedToken,
});
const accepted = await runDriverTopologyScopedScenario({
context,
roomKey: context.topology.defaultRoomKey,
tokenPrefix: "MATRIX_QA_MULTI_DRIVER",
});
return {
artifacts: {
accepted: accepted.artifacts ?? {},
blocked: blocked.artifacts ?? {},
},
details: [blocked.details, accepted.details].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runMatrixQaScenario(
scenario: MatrixQaScenarioDefinition,
context: MatrixQaScenarioContext,
@@ -125,6 +180,12 @@ export async function runMatrixQaScenario(
return await runImageUnderstandingAttachmentScenario(context);
case "matrix-room-generated-image-delivery":
return await runGeneratedImageDeliveryScenario(context);
case "matrix-media-type-coverage":
return await runMediaTypeCoverageScenario(context);
case "matrix-attachment-only-ignored":
return await runAttachmentOnlyIgnoredScenario(context);
case "matrix-unsupported-media-safe":
return await runUnsupportedMediaSafeScenario(context);
case "matrix-dm-reply-shape":
return await runDriverTopologyScopedScenario({
context,
@@ -159,8 +220,12 @@ export async function runMatrixQaScenario(
return await runReactionThreadedScenario(context);
case "matrix-reaction-not-a-reply":
return await runReactionNotAReplyScenario(context);
case "matrix-reaction-redaction-observed":
return await runReactionRedactionObservedScenario(context);
case "matrix-restart-resume":
return await runRestartResumeScenario(context);
case "matrix-post-restart-room-continue":
return await runPostRestartRoomContinueScenario(context);
case "matrix-room-membership-loss":
return await runMembershipLossScenario(context);
case "matrix-homeserver-restart-resume":
@@ -176,6 +241,18 @@ export async function runMatrixQaScenario(
token,
});
}
case "matrix-mention-metadata-spoof-block": {
const token = buildMatrixQaToken("MATRIX_QA_METADATA_SPOOF");
return await runNoReplyScenario({
accessToken: context.driverAccessToken,
actorId: "driver",
actorUserId: context.driverUserId,
body: buildExactMarkerPrompt(token),
mentionUserIds: [context.sutUserId],
context,
token,
});
}
case "matrix-observer-allowlist-override":
return await runObserverAllowlistOverrideScenario(context);
case "matrix-allowlist-block": {
@@ -190,6 +267,38 @@ export async function runMatrixQaScenario(
token,
});
}
case "matrix-multi-actor-ordering":
return await runMultiActorOrderingScenario(context);
case "matrix-inbound-edit-ignored":
return await runInboundEditIgnoredScenario(context);
case "matrix-inbound-edit-no-duplicate-trigger":
return await runInboundEditNoDuplicateTriggerScenario(context);
case "matrix-e2ee-basic-reply":
return await runMatrixQaE2eeBasicReplyScenario(context);
case "matrix-e2ee-thread-follow-up":
return await runMatrixQaE2eeThreadFollowUpScenario(context);
case "matrix-e2ee-bootstrap-success":
return await runMatrixQaE2eeBootstrapSuccessScenario(context);
case "matrix-e2ee-recovery-key-lifecycle":
return await runMatrixQaE2eeRecoveryKeyLifecycleScenario(context);
case "matrix-e2ee-device-sas-verification":
return await runMatrixQaE2eeDeviceSasVerificationScenario(context);
case "matrix-e2ee-qr-verification":
return await runMatrixQaE2eeQrVerificationScenario(context);
case "matrix-e2ee-stale-device-hygiene":
return await runMatrixQaE2eeStaleDeviceHygieneScenario(context);
case "matrix-e2ee-dm-sas-verification":
return await runMatrixQaE2eeDmSasVerificationScenario(context);
case "matrix-e2ee-restart-resume":
return await runMatrixQaE2eeRestartResumeScenario(context);
case "matrix-e2ee-verification-notice-no-trigger":
return await runMatrixQaE2eeVerificationNoticeNoTriggerScenario(context);
case "matrix-e2ee-artifact-redaction":
return await runMatrixQaE2eeArtifactRedactionScenario(context);
case "matrix-e2ee-media-image":
return await runMatrixQaE2eeMediaImageScenario(context);
case "matrix-e2ee-key-bootstrap-failure":
return await runMatrixQaE2eeKeyBootstrapFailureScenario(context);
default: {
const exhaustiveScenarioId: never = scenario.id;
return exhaustiveScenarioId;

49
extensions/qa-matrix/src/runners/contract/scenario-types.ts
View File

@@ -16,24 +16,52 @@ export type MatrixQaCanaryArtifact = {
};
export type MatrixQaScenarioArtifacts = {
accepted?: MatrixQaScenarioArtifacts;
attachments?: Array<{
eventId: string;
filename?: string;
kind?: string;
label: string;
msgtype?: string;
}>;
attachmentCaptionPreview?: string;
attachmentBodyPreview?: string;
attachmentEventId?: string;
attachmentFilename?: string;
attachmentKind?: string;
attachmentMsgtype?: string;
actorUserId?: string;
blocked?: MatrixQaScenarioArtifacts;
driverEventId?: string;
editEventId?: string;
editedToken?: string;
expectedNoReplyWindowMs?: number;
firstDriverEventId?: string;
firstReply?: MatrixQaReplyArtifact;
firstToken?: string;
originalDriverEventId?: string;
originalReply?: MatrixQaReplyArtifact;
originalToken?: string;
reactionEmoji?: string;
reactionEventId?: string;
reactionTargetEventId?: string;
redactionEventId?: string;
reply?: MatrixQaReplyArtifact;
replies?: Array<{
eventId: string;
label: string;
token: string;
tokenMatched?: boolean;
}>;
recoveredDriverEventId?: string;
recoveredReply?: MatrixQaReplyArtifact;
rootEventId?: string;
roomKey?: string;
roomId?: string;
restartSignal?: string;
rootEventId?: string;
secondDriverEventId?: string;
secondReply?: MatrixQaReplyArtifact;
secondToken?: string;
threadDriverEventId?: string;
threadReply?: MatrixQaReplyArtifact;
threadRootEventId?: string;
@@ -50,7 +78,26 @@ export type MatrixQaScenarioArtifacts = {
previewBodyPreview?: string;
previewEventId?: string;
blockEventIds?: string[];
bootstrapActor?: "driver" | "observer" | "sut";
bootstrapErrorPreview?: string;
bootstrapSuccess?: boolean;
backupCreatedVersion?: string | null;
backupRestored?: boolean;
backupReset?: boolean;
completedVerificationIds?: string[];
currentDeviceId?: string | null;
deletedDeviceIds?: string[];
faultedEndpoint?: string;
faultHitCount?: number;
faultRuleId?: string;
qrBytes?: number;
recoveryKeyId?: string | null;
recoveryKeyStored?: boolean;
remainingDeviceIds?: string[];
sasEmoji?: string[];
secondaryDeviceId?: string;
transportInterruption?: string;
verificationRoomId?: string;
joinedRoomId?: string;
};

599
extensions/qa-matrix/src/runners/contract/scenarios.test.ts
View File

@@ -2,15 +2,29 @@ import { describe, expect, it, beforeEach, vi } from "vitest";
const { createMatrixQaClient } = vi.hoisted(() => ({
createMatrixQaClient: vi.fn(),
}));
const { createMatrixQaE2eeScenarioClient, runMatrixQaE2eeBootstrap, startMatrixQaFaultProxy } =
vi.hoisted(() => ({
createMatrixQaE2eeScenarioClient: vi.fn(),
runMatrixQaE2eeBootstrap: vi.fn(),
startMatrixQaFaultProxy: vi.fn(),
}));
vi.mock("../../substrate/client.js", () => ({
createMatrixQaClient,
}));
vi.mock("../../substrate/e2ee-client.js", () => ({
createMatrixQaE2eeScenarioClient,
runMatrixQaE2eeBootstrap,
}));
vi.mock("../../substrate/fault-proxy.js", () => ({
startMatrixQaFaultProxy,
}));
import {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
findMissingLiveTransportStandardScenarios,
} from "../../shared/live-transport-scenarios.js";
import { MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES } from "./scenario-media-fixtures.js";
import {
__testing as scenarioTesting,
MATRIX_QA_SCENARIOS,
@@ -20,6 +34,9 @@ import {
describe("matrix live qa scenarios", () => {
beforeEach(() => {
createMatrixQaClient.mockReset();
createMatrixQaE2eeScenarioClient.mockReset();
runMatrixQaE2eeBootstrap.mockReset();
startMatrixQaFaultProxy.mockReset();
});
it("ships the Matrix live QA scenario set by default", () => {
@@ -34,6 +51,9 @@ describe("matrix live qa scenarios", () => {
"matrix-room-block-streaming",
"matrix-room-image-understanding-attachment",
"matrix-room-generated-image-delivery",
"matrix-media-type-coverage",
"matrix-attachment-only-ignored",
"matrix-unsupported-media-safe",
"matrix-dm-reply-shape",
"matrix-dm-shared-session-notice",
"matrix-dm-thread-reply-override",
@@ -44,12 +64,31 @@ describe("matrix live qa scenarios", () => {
"matrix-reaction-notification",
"matrix-reaction-threaded",
"matrix-reaction-not-a-reply",
"matrix-reaction-redaction-observed",
"matrix-restart-resume",
"matrix-post-restart-room-continue",
"matrix-room-membership-loss",
"matrix-homeserver-restart-resume",
"matrix-mention-gating",
"matrix-mention-metadata-spoof-block",
"matrix-observer-allowlist-override",
"matrix-allowlist-block",
"matrix-multi-actor-ordering",
"matrix-inbound-edit-ignored",
"matrix-inbound-edit-no-duplicate-trigger",
"matrix-e2ee-basic-reply",
"matrix-e2ee-thread-follow-up",
"matrix-e2ee-bootstrap-success",
"matrix-e2ee-recovery-key-lifecycle",
"matrix-e2ee-device-sas-verification",
"matrix-e2ee-qr-verification",
"matrix-e2ee-stale-device-hygiene",
"matrix-e2ee-dm-sas-verification",
"matrix-e2ee-restart-resume",
"matrix-e2ee-verification-notice-no-trigger",
"matrix-e2ee-artifact-redaction",
"matrix-e2ee-media-image",
"matrix-e2ee-key-bootstrap-failure",
]);
});
@@ -149,6 +188,7 @@ describe("matrix live qa scenarios", () => {
defaultRoomKey: "main",
rooms: [
{
encrypted: false,
key: "main",
kind: "group",
members: ["driver", "observer", "sut"],
@@ -342,7 +382,10 @@ describe("matrix live qa scenarios", () => {
eventId: "$sut-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "observer sender accepted",
body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace(
"@sut:matrix-qa.test reply with only this exact marker: ",
"",
),
},
since: "observer-sync-next",
}));
@@ -384,7 +427,7 @@ describe("matrix live qa scenarios", () => {
actorUserId: "@observer:matrix-qa.test",
driverEventId: "$observer-allow-trigger",
reply: {
tokenMatched: false,
tokenMatched: true,
},
},
});
@@ -753,17 +796,35 @@ describe("matrix live qa scenarios", () => {
it("sends a real Matrix image attachment for image-understanding prompts", async () => {
const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
const sendMediaMessage = vi.fn().mockResolvedValue("$image-understanding-trigger");
const waitForRoomEvent = vi.fn().mockResolvedValue({
event: {
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: "$sut-image-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.",
},
since: "driver-sync-next",
});
const waitForRoomEvent = vi
.fn()
.mockImplementationOnce(async () => ({
event: {
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: "$image-understanding-trigger",
sender: "@driver:matrix-qa.test",
type: "m.room.message",
attachment: {
kind: "image",
filename: "red-top-blue-bottom.png",
caption:
"@sut:matrix-qa.test Image understanding check: describe the top and bottom colors in the attached image in one short sentence.",
},
},
since: "driver-sync-attachment",
}))
.mockImplementationOnce(async () => ({
event: {
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: "$sut-image-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom.",
},
since: "driver-sync-next",
}));
createMatrixQaClient.mockReturnValue({
primeRoom,
@@ -823,6 +884,7 @@ describe("matrix live qa scenarios", () => {
expect(sendMediaMessage).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.stringContaining("Image understanding check"),
contentType: "image/png",
fileName: "red-top-blue-bottom.png",
kind: "image",
@@ -830,6 +892,12 @@ describe("matrix live qa scenarios", () => {
roomId: "!media:matrix-qa.test",
}),
);
expect(waitForRoomEvent).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
since: "driver-sync-attachment",
}),
);
});
it("waits for a real Matrix image attachment after image generation", async () => {
@@ -915,6 +983,148 @@ describe("matrix live qa scenarios", () => {
});
});
it("covers every Matrix media msgtype with caption-triggered replies", async () => {
const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
const mediaCases = MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES.map((mediaCase) => ({
...mediaCase,
eventId: `$media-${mediaCase.fileName}`,
}));
const sendMediaMessage = vi.fn().mockImplementation(async (opts: { fileName: string }) => {
const mediaCase = mediaCases.find((entry) => entry.fileName === opts.fileName);
return mediaCase?.eventId ?? "$unknown-media";
});
const waitForRoomEvent = vi.fn().mockImplementation(async () => {
const callIndex = waitForRoomEvent.mock.calls.length - 1;
const mediaCaseIndex = Math.floor(callIndex / 2);
const mediaCase = mediaCases[mediaCaseIndex];
const sendOpts = sendMediaMessage.mock.calls[mediaCaseIndex]?.[0];
if (callIndex % 2 === 0) {
return {
event: {
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: mediaCase.eventId,
sender: "@driver:matrix-qa.test",
type: "m.room.message",
msgtype: mediaCase.expectedMsgtype,
attachment: {
kind: mediaCase.expectedAttachmentKind,
filename: mediaCase.fileName,
caption: sendOpts?.body,
},
},
since: `driver-sync-attachment-${callIndex}`,
};
}
const token = String(sendOpts?.body).match(/MATRIX_QA_MEDIA_[A-Z]+_[A-Z0-9]+/)?.[0] ?? "";
return {
event: {
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: `$reply-${mediaCase.fileName}`,
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: token,
},
since: `driver-sync-reply-${callIndex}`,
};
});
createMatrixQaClient.mockReturnValue({
primeRoom,
sendMediaMessage,
waitForRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find((entry) => entry.id === "matrix-media-type-coverage");
expect(scenario).toBeDefined();
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
roomId: "!main:matrix-qa.test",
restartGateway: undefined,
syncState: {},
sutAccessToken: "sut-token",
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
topology: {
defaultRoomId: "!main:matrix-qa.test",
defaultRoomKey: "main",
rooms: [
{
key: scenarioTesting.MATRIX_QA_MEDIA_ROOM_KEY,
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "Media",
requireMention: true,
roomId: "!media:matrix-qa.test",
},
],
},
}),
).resolves.toMatchObject({
artifacts: {
attachments: mediaCases.map((mediaCase) => ({
eventId: mediaCase.eventId,
filename: mediaCase.fileName,
kind: mediaCase.expectedAttachmentKind,
msgtype: mediaCase.expectedMsgtype,
})),
roomId: "!media:matrix-qa.test",
},
});
expect(sendMediaMessage).toHaveBeenCalledTimes(mediaCases.length);
for (const [index, mediaCase] of MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES.entries()) {
expect(sendMediaMessage).toHaveBeenNthCalledWith(
index + 1,
expect.objectContaining({
contentType: mediaCase.contentType,
fileName: mediaCase.fileName,
kind: mediaCase.kind,
mentionUserIds: ["@sut:matrix-qa.test"],
}),
);
}
const firstReplyWait = waitForRoomEvent.mock.calls[1]?.[0];
const firstToken =
String(sendMediaMessage.mock.calls[0]?.[0]?.body).match(
/MATRIX_QA_MEDIA_[A-Z]+_[A-Z0-9]+/,
)?.[0] ?? "";
expect(
firstReplyWait.predicate({
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: "$verbose-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: `Sure, ${firstToken}`,
}),
).toBe(false);
expect(
firstReplyWait.predicate({
kind: "message",
roomId: "!media:matrix-qa.test",
eventId: "$exact-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: ` ${firstToken}\n`,
}),
).toBe(true);
});
it("uses DM thread override scenarios against the provisioned DM room", async () => {
const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$dm-thread-trigger");
@@ -1414,4 +1624,367 @@ describe("matrix live qa scenarios", () => {
}),
);
});
it("ignores stale E2EE replies when checking a verification notice", async () => {
let noticeToken = "";
const sendNoticeMessage = vi.fn().mockImplementation(async ({ body }) => {
noticeToken = body.match(/MATRIX_QA_E2EE_VERIFY_NOTICE_[A-Z0-9]+/)?.[0] ?? "";
return "$verification-notice";
});
const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => {
expect(
params.predicate({
body: "MATRIX_QA_E2EE_AFTER_RESTART_STALE",
eventId: "$stale-reply",
originServerTs: Date.now() - 60_000,
roomId: "!e2ee:matrix-qa.test",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
}),
).toBe(false);
expect(
params.predicate({
body: noticeToken,
eventId: "$token-reply",
roomId: "!e2ee:matrix-qa.test",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
}),
).toBe(true);
expect(
params.predicate({
eventId: "$related-reply",
relatesTo: {
inReplyToId: "$verification-notice",
},
roomId: "!e2ee:matrix-qa.test",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
}),
).toBe(true);
expect(
params.predicate({
eventId: "$new-unrelated-reply",
originServerTs: Date.now() + 1_000,
roomId: "!e2ee:matrix-qa.test",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
}),
).toBe(true);
return {
matched: false,
since: "e2ee:next",
};
});
createMatrixQaE2eeScenarioClient.mockResolvedValue({
prime: vi.fn().mockResolvedValue("e2ee:start"),
sendNoticeMessage,
stop: vi.fn().mockResolvedValue(undefined),
waitForOptionalRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-e2ee-verification-notice-no-trigger",
);
expect(scenario).toBeDefined();
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverDeviceId: "DRIVERDEVICE",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
outputDir: "/tmp/matrix-qa",
roomId: "!main:matrix-qa.test",
restartGateway: undefined,
syncState: {},
sutAccessToken: "sut-token",
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
topology: {
defaultRoomId: "!main:matrix-qa.test",
defaultRoomKey: "main",
rooms: [
{
key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY,
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "E2EE",
requireMention: true,
roomId: "!e2ee:matrix-qa.test",
},
],
},
}),
).resolves.toMatchObject({
artifacts: {
noticeEventId: "$verification-notice",
roomId: "!e2ee:matrix-qa.test",
},
});
expect(noticeToken).toMatch(/^MATRIX_QA_E2EE_VERIFY_NOTICE_[A-Z0-9]+$/);
expect(waitForOptionalRoomEvent).toHaveBeenCalledWith(
expect.objectContaining({
roomId: "!e2ee:matrix-qa.test",
}),
);
});
it("applies a recovery key before restoring backed up room keys", async () => {
const verifyWithRecoveryKey = vi.fn().mockResolvedValue({
backup: {
keyLoadError: null,
serverVersion: "backup-v1",
trusted: true,
},
error:
"Matrix device is still not verified by its owner after applying the recovery key. Ensure cross-signing is available and the device is signed.",
success: false,
});
const restoreRoomKeyBackup = vi.fn().mockResolvedValue({
imported: 1,
loadedFromSecretStorage: true,
success: true,
total: 1,
});
const resetRoomKeyBackup = vi.fn().mockResolvedValue({
createdVersion: "backup-v2",
deletedVersion: "backup-v1",
previousVersion: "backup-v1",
success: true,
});
const driverStop = vi.fn().mockResolvedValue(undefined);
const recoveryStop = vi.fn().mockResolvedValue(undefined);
createMatrixQaClient.mockReturnValue({
loginWithPassword: vi.fn().mockResolvedValue({
accessToken: "recovery-token",
deviceId: "RECOVERYDEVICE",
password: "driver-password",
userId: "@driver:matrix-qa.test",
}),
});
createMatrixQaE2eeScenarioClient
.mockResolvedValueOnce({
bootstrapOwnDeviceVerification: vi.fn().mockResolvedValue({
crossSigning: {
published: true,
},
success: true,
verification: {
backupVersion: "backup-v1",
recoveryKeyStored: true,
signedByOwner: true,
verified: true,
},
}),
deleteOwnDevices: vi.fn().mockResolvedValue(undefined),
getRecoveryKey: vi.fn().mockResolvedValue({
encodedPrivateKey: "encoded-recovery-key",
keyId: "SSSS",
}),
sendTextMessage: vi.fn().mockResolvedValue("$seeded-event"),
stop: driverStop,
})
.mockResolvedValueOnce({
resetRoomKeyBackup,
restoreRoomKeyBackup,
stop: recoveryStop,
verifyWithRecoveryKey,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-e2ee-recovery-key-lifecycle",
);
expect(scenario).toBeDefined();
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverDeviceId: "DRIVERDEVICE",
driverPassword: "driver-password",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
outputDir: "/tmp/matrix-qa",
roomId: "!main:matrix-qa.test",
restartGateway: undefined,
syncState: {},
sutAccessToken: "sut-token",
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
topology: {
defaultRoomId: "!main:matrix-qa.test",
defaultRoomKey: "main",
rooms: [
{
encrypted: true,
key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY,
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "E2EE",
requireMention: true,
roomId: "!e2ee:matrix-qa.test",
},
],
},
}),
).resolves.toMatchObject({
artifacts: {
backupRestored: true,
recoveryDeviceId: "RECOVERYDEVICE",
recoveryKeyUsable: true,
recoveryVerified: false,
restoreImported: 1,
restoreTotal: 1,
},
});
expect(verifyWithRecoveryKey).toHaveBeenCalledWith("encoded-recovery-key");
expect(verifyWithRecoveryKey.mock.invocationCallOrder[0]).toBeLessThan(
restoreRoomKeyBackup.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER,
);
});
it("runs Matrix E2EE bootstrap failure through a real faulted homeserver endpoint", async () => {
const stop = vi.fn().mockResolvedValue(undefined);
const hits = vi.fn().mockReturnValue([
{
method: "GET",
path: "/_matrix/client/v3/room_keys/version",
ruleId: "room-key-backup-version-unavailable",
},
]);
startMatrixQaFaultProxy.mockResolvedValue({
baseUrl: "http://127.0.0.1:39876",
hits,
stop,
});
runMatrixQaE2eeBootstrap.mockResolvedValue({
crossSigning: {
masterKeyPublished: true,
published: true,
selfSigningKeyPublished: true,
userId: "@driver:matrix-qa.test",
userSigningKeyPublished: true,
},
cryptoBootstrap: null,
error: "Matrix room key backup is still missing after bootstrap",
pendingVerifications: 0,
success: false,
verification: {
backup: {
activeVersion: null,
enabled: false,
keyCached: false,
trusted: false,
},
deviceId: "DRIVERDEVICE",
userId: "@driver:matrix-qa.test",
verified: true,
},
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-e2ee-key-bootstrap-failure",
);
expect(scenario).toBeDefined();
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverDeviceId: "DRIVERDEVICE",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
outputDir: "/tmp/matrix-qa",
roomId: "!main:matrix-qa.test",
restartGateway: undefined,
syncState: {},
sutAccessToken: "sut-token",
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
topology: {
defaultRoomId: "!main:matrix-qa.test",
defaultRoomKey: "main",
rooms: [
{
key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY,
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "E2EE",
requireMention: true,
roomId: "!e2ee:matrix-qa.test",
},
],
},
}),
).resolves.toMatchObject({
artifacts: {
bootstrapActor: "driver",
bootstrapSuccess: false,
faultedEndpoint: "/_matrix/client/v3/room_keys/version",
faultHitCount: 1,
faultRuleId: "room-key-backup-version-unavailable",
},
});
const proxyArgs = startMatrixQaFaultProxy.mock.calls[0]?.[0];
expect(proxyArgs).toBeDefined();
if (!proxyArgs) {
throw new Error("expected Matrix QA fault proxy to start");
}
const [faultRule] = proxyArgs.rules;
expect(faultRule).toBeDefined();
if (!faultRule) {
throw new Error("expected Matrix QA fault proxy rule");
}
expect(proxyArgs.targetBaseUrl).toBe("http://127.0.0.1:28008/");
expect(
faultRule.match({
bearerToken: "driver-token",
headers: {},
method: "GET",
path: "/_matrix/client/v3/room_keys/version",
search: "",
}),
).toBe(true);
expect(runMatrixQaE2eeBootstrap).toHaveBeenCalledWith({
accessToken: "driver-token",
actorId: "driver",
baseUrl: "http://127.0.0.1:39876",
deviceId: "DRIVERDEVICE",
outputDir: "/tmp/matrix-qa",
scenarioId: "matrix-e2ee-key-bootstrap-failure",
timeoutMs: 8_000,
userId: "@driver:matrix-qa.test",
});
expect(stop).toHaveBeenCalledTimes(1);
});
});

2
extensions/qa-matrix/src/runners/contract/scenarios.ts
View File

@@ -1,6 +1,7 @@
import {
MATRIX_QA_DRIVER_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY,
MATRIX_QA_E2EE_ROOM_KEY,
MATRIX_QA_MEDIA_ROOM_KEY,
MATRIX_QA_MEMBERSHIP_ROOM_KEY,
MATRIX_QA_SCENARIOS,
@@ -55,6 +56,7 @@ export type { MatrixQaScenarioContext, MatrixQaSyncState };
export const __testing = {
MATRIX_QA_DRIVER_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY,
MATRIX_QA_E2EE_ROOM_KEY,
MATRIX_QA_MEDIA_ROOM_KEY,
MATRIX_QA_MEMBERSHIP_ROOM_KEY,
MATRIX_QA_SECONDARY_ROOM_KEY,

165
extensions/qa-matrix/src/substrate/client.test.ts
View File

@@ -63,6 +63,33 @@ describe("matrix driver client", () => {
});
});
it("builds Matrix replacement messages with replacement-local mention metadata", () => {
expect(
__testing.buildMatrixQaReplacementMessageContent({
body: "@sut:matrix-qa.test updated prompt",
mentionUserIds: ["@sut:matrix-qa.test"],
targetEventId: " $msg-1 ",
}),
).toEqual({
body: "* @sut:matrix-qa.test updated prompt",
msgtype: "m.text",
"m.new_content": {
body: "@sut:matrix-qa.test updated prompt",
msgtype: "m.text",
format: "org.matrix.custom.html",
formatted_body:
'<a href="https://matrix.to/#/%40sut%3Amatrix-qa.test">@sut:matrix-qa.test</a> updated prompt',
"m.mentions": {
user_ids: ["@sut:matrix-qa.test"],
},
},
"m.relates_to": {
rel_type: "m.replace",
event_id: "$msg-1",
},
});
});
it("advances Matrix registration through token then dummy auth stages", () => {
const firstStage = __testing.resolveNextRegistrationAuth({
registrationToken: "reg-token",
@@ -105,6 +132,57 @@ describe("matrix driver client", () => {
).toThrow("Matrix registration requires unsupported auth stages:");
});
it("logs in with Matrix password auth to create a secondary QA device", async () => {
const requests: Array<{ body: Record<string, unknown>; url: string }> = [];
const fetchImpl: typeof fetch = async (input, init) => {
requests.push({
body: parseJsonRequestBody(init),
url: resolveRequestUrl(input),
});
return new Response(
JSON.stringify({
access_token: "secondary-token",
device_id: "SECONDARYDEVICE",
user_id: "@qa-driver:matrix-qa.test",
}),
{ status: 200, headers: { "content-type": "application/json" } },
);
};
const client = createMatrixQaClient({
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
await expect(
client.loginWithPassword({
deviceName: "OpenClaw Matrix QA Stale Device",
password: "driver-password",
userId: "@qa-driver:matrix-qa.test",
}),
).resolves.toMatchObject({
accessToken: "secondary-token",
deviceId: "SECONDARYDEVICE",
password: "driver-password",
userId: "@qa-driver:matrix-qa.test",
});
expect(requests).toEqual([
{
url: "http://127.0.0.1:28008/_matrix/client/v3/login",
body: {
type: "m.login.password",
identifier: {
type: "m.id.user",
user: "@qa-driver:matrix-qa.test",
},
initial_device_display_name: "OpenClaw Matrix QA Stale Device",
password: "driver-password",
},
},
]);
});
it("issues Matrix room membership control requests for QA topology changes", async () => {
const requests: Array<{ body: Record<string, unknown>; url: string }> = [];
const fetchImpl: typeof fetch = async (input, init) => {
@@ -189,6 +267,61 @@ describe("matrix driver client", () => {
).resolves.toBe("$reaction-1");
});
it("sends Matrix replacements and redactions through protocol endpoints", async () => {
const requests: Array<{ body: Record<string, unknown>; url: string }> = [];
const fetchImpl: typeof fetch = async (input, init) => {
requests.push({
body: parseJsonRequestBody(init),
url: resolveRequestUrl(input),
});
const eventId = requests.length === 1 ? "$replacement-1" : "$redaction-1";
return new Response(JSON.stringify({ event_id: eventId }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
const client = createMatrixQaClient({
accessToken: "token",
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
await expect(
client.sendReplacementMessage({
body: "@sut:matrix-qa.test updated prompt",
mentionUserIds: ["@sut:matrix-qa.test"],
roomId: "!room:matrix-qa.test",
targetEventId: "$msg-1",
}),
).resolves.toBe("$replacement-1");
await expect(
client.redactEvent({
eventId: "$reaction-1",
reason: "qa cleanup",
roomId: "!room:matrix-qa.test",
}),
).resolves.toBe("$redaction-1");
expect(requests[0]?.url).toContain(
"/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.room.message/",
);
expect(requests[0]?.body).toMatchObject({
"m.relates_to": {
rel_type: "m.replace",
event_id: "$msg-1",
},
});
expect(requests[1]).toEqual({
url: expect.stringContaining(
"/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/redact/%24reaction-1/",
),
body: {
reason: "qa cleanup",
},
});
});
it("uploads Matrix media before sending the room event", async () => {
const requests: Array<{
body: RequestInit["body"];
@@ -262,6 +395,38 @@ describe("matrix driver client", () => {
});
});
it("adds Matrix room encryption state when provisioning encrypted QA rooms", async () => {
const createRoomBodies: Array<Record<string, unknown>> = [];
const fetchImpl: typeof fetch = async (input, init) => {
createRoomBodies.push(parseJsonRequestBody(init));
expect(resolveRequestUrl(input)).toBe("http://127.0.0.1:28008/_matrix/client/v3/createRoom");
return new Response(JSON.stringify({ room_id: "!encrypted:matrix-qa.test" }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
const client = createMatrixQaClient({
accessToken: "token",
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
await expect(
client.createPrivateRoom({
encrypted: true,
inviteUserIds: ["@sut:matrix-qa.test"],
name: "Encrypted QA Room",
}),
).resolves.toBe("!encrypted:matrix-qa.test");
expect(createRoomBodies[0]?.initial_state).toContainEqual({
type: "m.room.encryption",
state_key: "",
content: { algorithm: "m.megolm.v1.aes-sha2" },
});
});
it("provisions a three-member room so Matrix QA runs in a group context", async () => {
const createRoomBodies: Array<Record<string, unknown>> = [];
const fetchImpl: typeof fetch = async (input, init) => {

208
extensions/qa-matrix/src/substrate/client.ts
View File

@@ -30,6 +30,8 @@ type MatrixQaRegisterResponse = {
user_id?: string;
};
type MatrixQaLoginResponse = MatrixQaRegisterResponse;
type MatrixQaRoomCreateResponse = {
room_id?: string;
};
@@ -38,17 +40,23 @@ type MatrixQaSendMessageContent = {
body: string;
format?: "org.matrix.custom.html";
formatted_body?: string;
"m.new_content"?: MatrixQaSendMessageContent;
"m.mentions"?: {
user_ids?: string[];
};
"m.relates_to"?: {
rel_type: "m.thread";
event_id: string;
is_falling_back: true;
"m.in_reply_to": {
event_id: string;
};
};
"m.relates_to"?:
| {
rel_type: "m.thread";
event_id: string;
is_falling_back: true;
"m.in_reply_to": {
event_id: string;
};
}
| {
rel_type: "m.replace";
event_id: string;
};
msgtype: "m.text";
};
@@ -72,6 +80,12 @@ type MatrixQaSendReactionContent = {
};
};
type MatrixQaRoomInitialState = Array<{
content: Record<string, unknown>;
state_key: string;
type: string;
}>;
type MatrixQaUiaaResponse = {
completed?: string[];
flows?: Array<{ stages?: string[] }>;
@@ -107,6 +121,19 @@ function buildMatrixThreadRelation(threadRootEventId: string, replyToEventId?: s
};
}
function buildMatrixReplacementRelation(targetEventId: string) {
const normalizedTargetEventId = targetEventId.trim();
if (!normalizedTargetEventId) {
throw new Error("Matrix replacement requires a target event id");
}
return {
"m.relates_to": {
rel_type: "m.replace" as const,
event_id: normalizedTargetEventId,
},
};
}
function buildMatrixReactionRelation(
messageId: string,
emoji: string,
@@ -128,6 +155,24 @@ function buildMatrixReactionRelation(
};
}
function buildMatrixQaRoomInitialState(encrypted?: boolean): MatrixQaRoomInitialState {
const initialState: MatrixQaRoomInitialState = [
{
type: "m.room.history_visibility",
state_key: "",
content: { history_visibility: "joined" },
},
];
if (encrypted === true) {
initialState.push({
type: "m.room.encryption",
state_key: "",
content: { algorithm: "m.megolm.v1.aes-sha2" },
});
}
return initialState;
}
function escapeMatrixHtml(value: string): string {
return value.replace(/[&<>"']/g, (char) => {
switch (char) {
@@ -153,7 +198,7 @@ function buildMatrixMentionLink(userId: string) {
return `<a href="${href}">${label}</a>`;
}
function buildMatrixQaMessageContent(params: {
export function buildMatrixQaMessageContent(params: {
body: string;
mentionUserIds?: string[];
replyToEventId?: string;
@@ -201,6 +246,23 @@ function buildMatrixQaMessageContent(params: {
};
}
function buildMatrixQaReplacementMessageContent(params: {
body: string;
mentionUserIds?: string[];
targetEventId: string;
}): MatrixQaSendMessageContent {
const newContent = buildMatrixQaMessageContent({
body: params.body,
mentionUserIds: params.mentionUserIds,
});
return {
body: `* ${params.body}`,
msgtype: "m.text",
"m.new_content": newContent,
...buildMatrixReplacementRelation(params.targetEventId),
};
}
function resolveMatrixQaMediaMsgtype(params: {
contentType?: string;
kind?: "audio" | "file" | "image" | "video";
@@ -361,6 +423,14 @@ function buildRegisteredAccount(params: {
} satisfies MatrixQaRegisteredAccount;
}
function resolveMatrixQaLoginUser(params: { localpart?: string; userId?: string }) {
const user = params.userId?.trim() || params.localpart?.trim();
if (!user) {
throw new Error("Matrix password login requires a localpart or userId.");
}
return user;
}
export function createMatrixQaClient(params: {
accessToken?: string;
baseUrl: string;
@@ -369,21 +439,35 @@ export function createMatrixQaClient(params: {
}) {
const fetchImpl = params.fetchImpl ?? fetch;
const syncObserver = params.syncObserver;
const sendEvent = async (opts: { body: unknown; endpoint: string; errorLabel: string }) => {
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: opts.body,
endpoint: opts.endpoint,
fetchImpl,
method: "PUT",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error(`Matrix ${opts.errorLabel} did not return event_id.`);
}
return eventId;
};
return {
async createPrivateRoom(opts: { inviteUserIds: string[]; isDirect?: boolean; name: string }) {
async createPrivateRoom(opts: {
encrypted?: boolean;
inviteUserIds: string[];
isDirect?: boolean;
name: string;
}) {
const result = await requestMatrixJson<MatrixQaRoomCreateResponse>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: {
creation_content: { "m.federate": false },
initial_state: [
{
type: "m.room.history_visibility",
state_key: "",
content: { history_visibility: "joined" },
},
],
initial_state: buildMatrixQaRoomInitialState(opts.encrypted),
invite: opts.inviteUserIds,
is_direct: opts.isDirect === true,
name: opts.name,
@@ -451,6 +535,34 @@ export function createMatrixQaClient(params: {
`Matrix registration for ${opts.localpart} did not complete after 4 attempts.`,
);
},
async loginWithPassword(opts: {
deviceName: string;
localpart?: string;
password: string;
userId?: string;
}) {
const result = await requestMatrixJson<MatrixQaLoginResponse>({
baseUrl: params.baseUrl,
body: {
type: "m.login.password",
identifier: {
type: "m.id.user",
user: resolveMatrixQaLoginUser(opts),
},
initial_device_display_name: opts.deviceName,
password: opts.password,
},
endpoint: "/_matrix/client/v3/login",
fetchImpl,
method: "POST",
timeoutMs: 30_000,
});
return buildRegisteredAccount({
localpart: opts.localpart ?? opts.userId ?? "",
password: opts.password,
response: result.body,
});
},
async sendTextMessage(opts: {
body: string;
mentionUserIds?: string[];
@@ -459,19 +571,24 @@ export function createMatrixQaClient(params: {
threadRootEventId?: string;
}) {
const txnId = randomUUID();
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
return await sendEvent({
body: buildMatrixQaMessageContent(opts),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
fetchImpl,
method: "PUT",
errorLabel: "sendMessage",
});
},
async sendReplacementMessage(opts: {
body: string;
mentionUserIds?: string[];
roomId: string;
targetEventId: string;
}) {
const txnId = randomUUID();
return await sendEvent({
body: buildMatrixQaReplacementMessageContent(opts),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
errorLabel: "sendReplacementMessage",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error("Matrix sendMessage did not return event_id.");
}
return eventId;
},
async sendMediaMessage(opts: {
body?: string;
@@ -493,9 +610,7 @@ export function createMatrixQaClient(params: {
fileName: opts.fileName,
});
const txnId = randomUUID();
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
return await sendEvent({
body: buildMatrixQaMediaMessageContent({
body: opts.body,
contentType: opts.contentType,
@@ -508,30 +623,25 @@ export function createMatrixQaClient(params: {
url: contentUri,
}),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
fetchImpl,
method: "PUT",
errorLabel: "sendMediaMessage",
});
},
async redactEvent(opts: { eventId: string; reason?: string; roomId: string }) {
const txnId = randomUUID();
const reason = opts.reason?.trim();
return await sendEvent({
body: reason ? { reason } : {},
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/redact/${encodeURIComponent(opts.eventId)}/${encodeURIComponent(txnId)}`,
errorLabel: "redactEvent",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error("Matrix sendMediaMessage did not return event_id.");
}
return eventId;
},
async sendReaction(opts: { emoji: string; messageId: string; roomId: string }) {
const txnId = randomUUID();
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
return await sendEvent({
body: buildMatrixReactionRelation(opts.messageId, opts.emoji),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.reaction/${encodeURIComponent(txnId)}`,
fetchImpl,
method: "PUT",
errorLabel: "sendReaction",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error("Matrix sendReaction did not return event_id.");
}
return eventId;
},
async joinRoom(roomId: string) {
const result = await requestMatrixJson<{ room_id?: string }>({
@@ -684,6 +794,7 @@ async function provisionMatrixQaTopology(params: {
fetchImpl: params.fetchImpl,
});
const roomId = await creatorClient.createPrivateRoom({
encrypted: room.encrypted === true,
inviteUserIds: invitees.map((entry) => entry.account.userId),
isDirect: room.kind === "dm",
name: room.name,
@@ -699,6 +810,7 @@ async function provisionMatrixQaTopology(params: {
),
);
rooms.push({
encrypted: room.encrypted === true,
key: room.key,
kind: room.kind,
memberRoles: members.map((entry) => entry.role),
@@ -793,7 +905,9 @@ export async function provisionMatrixQaRoom(params: {
export const __testing = {
buildMatrixQaMessageContent,
buildMatrixQaReplacementMessageContent,
buildMatrixReactionRelation,
buildMatrixReplacementRelation,
buildMatrixThreadRelation,
createMatrixQaRoomObserver,
resolveNextRegistrationAuth,

9
extensions/qa-matrix/src/substrate/config.ts
View File

@@ -47,6 +47,7 @@ export type MatrixQaConfigOverrides = {
groupPolicy?: MatrixQaGroupPolicy;
groupsByKey?: Record<string, MatrixQaGroupConfigOverrides>;
replyToMode?: MatrixQaReplyToMode;
startupVerification?: "if-unverified" | "off";
streaming?: "off" | "partial" | "quiet" | boolean;
threadReplies?: MatrixQaThreadRepliesMode;
};
@@ -74,6 +75,7 @@ export type MatrixQaConfigSnapshot = {
}
>;
replyToMode: MatrixQaReplyToMode;
startupVerification?: "if-unverified" | "off";
streaming: MatrixQaStreamingMode;
threadReplies: MatrixQaThreadRepliesMode;
};
@@ -271,6 +273,10 @@ function buildMatrixQaChannelAccountConfig(params: {
: {};
const streamingConfig =
params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming } : {};
const startupVerificationConfig =
params.snapshot.startupVerification !== undefined
? { startupVerification: params.snapshot.startupVerification }
: {};
return {
accessToken: params.sutAccessToken,
@@ -289,6 +295,7 @@ function buildMatrixQaChannelAccountConfig(params: {
dangerouslyAllowPrivateNetwork: true,
},
replyToMode: params.snapshot.replyToMode,
...startupVerificationConfig,
threadReplies: params.snapshot.threadReplies,
userId: params.sutUserId,
...autoJoinConfig,
@@ -318,6 +325,7 @@ export function buildMatrixQaConfigSnapshot(params: {
topology: params.topology,
}),
replyToMode: params.overrides?.replyToMode ?? "off",
startupVerification: params.overrides?.startupVerification,
streaming: resolveMatrixQaStreamingMode(params.overrides?.streaming),
threadReplies: params.overrides?.threadReplies ?? "inbound",
};
@@ -335,6 +343,7 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot
`blockStreaming=${formatMatrixQaBoolean(snapshot.blockStreaming)}`,
`autoJoin=${snapshot.autoJoin}`,
`encryption=${formatMatrixQaBoolean(snapshot.encryption)}`,
`startupVerification=${snapshot.startupVerification ?? "<default>"}`,
].join(", ");
}

30
extensions/qa-matrix/src/substrate/e2ee-client.test.ts Normal file
View File

@@ -0,0 +1,30 @@
import path from "node:path";
import { describe, expect, it } from "vitest";
import { __testing } from "./e2ee-client.js";
describe("matrix qa e2ee client storage", () => {
it("scopes persisted crypto state to the account actor", () => {
const first = __testing.buildMatrixQaE2eeStoragePaths({
actorId: "driver",
outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix-run",
scenarioId: "matrix-e2ee-basic-reply",
});
const second = __testing.buildMatrixQaE2eeStoragePaths({
actorId: "driver",
outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix-run",
scenarioId: "matrix-e2ee-qr-verification",
});
expect(first.accountDir).toBe(
path.join(
"/tmp/openclaw/.artifacts/qa-e2e/matrix-run",
"matrix-e2ee",
"accounts",
"driver",
"account",
),
);
expect(first.cryptoDatabasePrefix).toBe(second.cryptoDatabasePrefix);
expect(first.recoveryKeyPath).toBe(path.join(first.accountDir, "recovery-key.json"));
});
});

399
extensions/qa-matrix/src/substrate/e2ee-client.ts Normal file
View File

@@ -0,0 +1,399 @@
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import type {
EncryptedFile,
MatrixDeviceVerificationStatus,
MatrixClient,
MatrixOwnDeviceDeleteResult,
MatrixOwnDeviceInfo,
MatrixRawEvent,
MatrixRecoveryKeyVerificationResult,
MatrixRoomKeyBackupResetResult,
MatrixRoomKeyBackupRestoreResult,
MatrixVerificationBootstrapResult,
MatrixVerificationMethod,
MatrixVerificationSummary,
MessageEventContent,
} from "@openclaw/matrix/test-api.js";
import { buildMatrixQaMessageContent } from "./client.js";
import { normalizeMatrixQaObservedEvent } from "./events.js";
import type { MatrixQaObservedEvent } from "./events.js";
import type { MatrixQaRoomEventWaitResult } from "./sync.js";
type MatrixQaE2eeActorId = "driver" | "observer" | `driver-${string}`;
type MatrixQaE2eeRuntime = typeof import("@openclaw/matrix/test-api.js");
type MatrixQaE2eeClientParams = {
accessToken: string;
actorId: MatrixQaE2eeActorId;
baseUrl: string;
deviceId?: string;
outputDir: string;
password?: string;
scenarioId: string;
timeoutMs: number;
userId: string;
};
export type MatrixQaE2eeScenarioClient = {
acceptVerification(id: string): Promise<MatrixVerificationSummary>;
bootstrapOwnDeviceVerification(params?: {
forceResetCrossSigning?: boolean;
recoveryKey?: string;
}): Promise<MatrixVerificationBootstrapResult>;
confirmVerificationReciprocateQr(id: string): Promise<MatrixVerificationSummary>;
confirmVerificationSas(id: string): Promise<MatrixVerificationSummary>;
deleteOwnDevices(deviceIds: string[]): Promise<MatrixOwnDeviceDeleteResult>;
generateVerificationQr(id: string): Promise<{ qrDataBase64: string }>;
getDeviceVerificationStatus(
userId: string,
deviceId: string,
): Promise<MatrixDeviceVerificationStatus>;
getRecoveryKey(): Promise<{
encodedPrivateKey?: string;
keyId?: string | null;
createdAt?: string;
} | null>;
listOwnDevices(): Promise<MatrixOwnDeviceInfo[]>;
listVerifications(): Promise<MatrixVerificationSummary[]>;
prime(): Promise<string | undefined>;
requestVerification(params: {
deviceId?: string;
ownUser?: boolean;
roomId?: string;
userId?: string;
}): Promise<MatrixVerificationSummary>;
resetRoomKeyBackup(): Promise<MatrixRoomKeyBackupResetResult>;
restoreRoomKeyBackup(params?: {
recoveryKey?: string;
}): Promise<MatrixRoomKeyBackupRestoreResult>;
scanVerificationQr(id: string, qrDataBase64: string): Promise<MatrixVerificationSummary>;
verifyWithRecoveryKey(rawRecoveryKey: string): Promise<MatrixRecoveryKeyVerificationResult>;
sendTextMessage(opts: {
body: string;
mentionUserIds?: string[];
replyToEventId?: string;
roomId: string;
threadRootEventId?: string;
}): Promise<string>;
sendNoticeMessage(opts: {
body: string;
mentionUserIds?: string[];
roomId: string;
}): Promise<string>;
sendImageMessage(opts: {
body: string;
buffer: Buffer;
contentType: string;
fileName: string;
mentionUserIds?: string[];
roomId: string;
}): Promise<string>;
startVerification(
id: string,
method?: MatrixVerificationMethod,
): Promise<MatrixVerificationSummary>;
stop(): Promise<void>;
waitForOptionalRoomEvent(params: {
predicate: (event: MatrixQaObservedEvent) => boolean;
roomId: string;
timeoutMs: number;
}): Promise<MatrixQaRoomEventWaitResult>;
waitForRoomEvent(params: {
predicate: (event: MatrixQaObservedEvent) => boolean;
roomId: string;
timeoutMs: number;
}): Promise<{
event: MatrixQaObservedEvent;
since?: string;
}>;
};
async function loadMatrixQaE2eeRuntime(): Promise<MatrixQaE2eeRuntime> {
const { loadQaRunnerBundledPluginTestApi } =
await import("openclaw/plugin-sdk/qa-runner-runtime");
return loadQaRunnerBundledPluginTestApi<MatrixQaE2eeRuntime>("matrix");
}
function buildMatrixQaE2eeStoragePaths(params: {
actorId: MatrixQaE2eeActorId;
outputDir: string;
scenarioId: string;
}) {
void params.scenarioId;
const rootDir = path.join(params.outputDir, "matrix-e2ee", "accounts", params.actorId);
const accountDir = path.join(rootDir, "account");
const runKey = path
.basename(params.outputDir)
.replace(/[^A-Za-z0-9_-]/g, "-")
.slice(-80);
const actorKey = params.actorId.replace(/[^A-Za-z0-9_-]/g, "-").slice(-40);
return {
accountDir,
cryptoDatabasePrefix: `qa-matrix-${runKey || "run"}-${actorKey || "actor"}`,
idbSnapshotPath: path.join(accountDir, "crypto-idb-snapshot.json"),
recoveryKeyPath: path.join(accountDir, "recovery-key.json"),
rootDir,
storagePath: path.join(accountDir, "sync-store.json"),
};
}
async function prepareMatrixQaE2eeStorage(params: {
actorId: MatrixQaE2eeActorId;
outputDir: string;
scenarioId: string;
}) {
const storage = buildMatrixQaE2eeStoragePaths(params);
await fs.mkdir(storage.rootDir, { recursive: true });
await fs.mkdir(storage.accountDir, { recursive: true });
await fs.writeFile(storage.idbSnapshotPath, "[]\n", { flag: "wx" }).catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
throw error;
}
});
return storage;
}
async function createMatrixQaE2eeMatrixClient(params: MatrixQaE2eeClientParams) {
const runtime = await loadMatrixQaE2eeRuntime();
const storage = await prepareMatrixQaE2eeStorage({
actorId: params.actorId,
outputDir: params.outputDir,
scenarioId: params.scenarioId,
});
return new runtime.MatrixClient(params.baseUrl, params.accessToken, {
autoBootstrapCrypto: false,
cryptoDatabasePrefix: storage.cryptoDatabasePrefix,
deviceId: params.deviceId,
encryption: true,
idbSnapshotPath: storage.idbSnapshotPath,
localTimeoutMs: Math.max(10_000, params.timeoutMs),
password: params.password,
recoveryKeyPath: storage.recoveryKeyPath,
ssrfPolicy: { allowPrivateNetwork: true },
storagePath: storage.storagePath,
userId: params.userId,
});
}
function findMatrixQaObservedEventMatch(params: {
cursorIndex: number;
events: MatrixQaObservedEvent[];
predicate: (event: MatrixQaObservedEvent) => boolean;
roomId: string;
}) {
for (let index = params.cursorIndex; index < params.events.length; index += 1) {
const event = params.events[index];
if (event?.roomId !== params.roomId) {
continue;
}
if (params.predicate(event)) {
return {
event,
nextCursorIndex: index + 1,
};
}
}
return undefined;
}
export async function createMatrixQaE2eeScenarioClient(
params: MatrixQaE2eeClientParams & {
observedEvents: MatrixQaObservedEvent[];
},
): Promise<MatrixQaE2eeScenarioClient> {
const client: MatrixClient = await createMatrixQaE2eeMatrixClient(params);
const localEvents: MatrixQaObservedEvent[] = [];
const verificationSummaries: MatrixVerificationSummary[] = [];
const observedEventIds = new Set<string>();
let cursorIndex = 0;
const recordEvent = (roomId: string, event: MatrixRawEvent) => {
const normalized = normalizeMatrixQaObservedEvent(roomId, event);
if (!normalized || observedEventIds.has(normalized.eventId)) {
return;
}
observedEventIds.add(normalized.eventId);
localEvents.push(normalized);
params.observedEvents.push(normalized);
};
client.on("room.message", recordEvent);
const recordVerificationSummary = (summary: MatrixVerificationSummary) => {
verificationSummaries.push(summary);
};
client.on("verification.summary", recordVerificationSummary);
try {
await client.start({ readyTimeoutMs: Math.min(45_000, Math.max(15_000, params.timeoutMs)) });
} catch (error) {
await client.stopAndPersist().catch(() => undefined);
throw error;
}
const prime = async () => {
cursorIndex = Math.max(cursorIndex, localEvents.length);
return `e2ee:${cursorIndex}`;
};
const waitForOptionalRoomEvent: MatrixQaE2eeScenarioClient["waitForOptionalRoomEvent"] = async (
waitParams,
) => {
const startSince = `e2ee:${cursorIndex}`;
const startedAt = Date.now();
let scanIndex = cursorIndex;
while (Date.now() - startedAt < waitParams.timeoutMs) {
const matched = findMatrixQaObservedEventMatch({
cursorIndex: scanIndex,
events: localEvents,
predicate: waitParams.predicate,
roomId: waitParams.roomId,
});
if (matched) {
cursorIndex = Math.max(cursorIndex, matched.nextCursorIndex);
return {
event: matched.event,
matched: true,
since: `e2ee:${cursorIndex}`,
};
}
scanIndex = localEvents.length;
await sleep(Math.min(250, Math.max(25, waitParams.timeoutMs - (Date.now() - startedAt))));
}
cursorIndex = Math.max(cursorIndex, scanIndex);
return {
matched: false,
since: startSince,
};
};
const requireCrypto = () => {
if (!client.crypto) {
throw new Error("Matrix E2EE scenario requires Matrix crypto");
}
return client.crypto;
};
return {
async acceptVerification(id) {
return await requireCrypto().acceptVerification(id);
},
async bootstrapOwnDeviceVerification(opts) {
return await client.bootstrapOwnDeviceVerification(opts);
},
async confirmVerificationReciprocateQr(id) {
return await requireCrypto().confirmVerificationReciprocateQr(id);
},
async confirmVerificationSas(id) {
return await requireCrypto().confirmVerificationSas(id);
},
async deleteOwnDevices(deviceIds) {
return await client.deleteOwnDevices(deviceIds);
},
async generateVerificationQr(id) {
return await requireCrypto().generateVerificationQr(id);
},
async getDeviceVerificationStatus(userId, deviceId) {
return await client.getDeviceVerificationStatus(userId, deviceId);
},
async getRecoveryKey() {
return await requireCrypto().getRecoveryKey();
},
async listOwnDevices() {
return await client.listOwnDevices();
},
async listVerifications() {
const current = await requireCrypto().listVerifications();
return [...verificationSummaries, ...current].toSorted((a, b) =>
b.updatedAt.localeCompare(a.updatedAt),
);
},
prime,
async requestVerification(opts) {
return await requireCrypto().requestVerification(opts);
},
async resetRoomKeyBackup() {
return await client.resetRoomKeyBackup();
},
async restoreRoomKeyBackup(opts) {
return await client.restoreRoomKeyBackup(opts);
},
async scanVerificationQr(id, qrDataBase64) {
return await requireCrypto().scanVerificationQr(id, qrDataBase64);
},
async sendTextMessage(opts) {
return await client.sendMessage(
opts.roomId,
buildMatrixQaMessageContent(opts) as MessageEventContent,
);
},
async sendNoticeMessage(opts) {
return await client.sendMessage(opts.roomId, {
...buildMatrixQaMessageContent(opts),
msgtype: "m.notice",
} as MessageEventContent);
},
async sendImageMessage(opts) {
const encrypted = await requireCrypto().encryptMedia(opts.buffer);
const contentUri = await client.uploadContent(
encrypted.buffer,
opts.contentType,
opts.fileName,
);
const file: EncryptedFile = { url: contentUri, ...encrypted.file };
return await client.sendMessage(opts.roomId, {
...buildMatrixQaMessageContent({
body: opts.body,
mentionUserIds: opts.mentionUserIds,
}),
file,
filename: opts.fileName,
info: {
mimetype: opts.contentType,
size: opts.buffer.byteLength,
},
msgtype: "m.image",
} as MessageEventContent);
},
async startVerification(id, method) {
return await requireCrypto().startVerification(id, method);
},
async stop() {
client.off("room.message", recordEvent);
client.off("verification.summary", recordVerificationSummary);
await client.drainPendingDecryptions().catch(() => undefined);
await client.stopAndPersist();
},
waitForOptionalRoomEvent,
async waitForRoomEvent(waitParams) {
const result = await waitForOptionalRoomEvent(waitParams);
if (result.matched) {
return {
event: result.event,
since: result.since,
};
}
throw new Error(`timed out after ${waitParams.timeoutMs}ms waiting for Matrix E2EE event`);
},
async verifyWithRecoveryKey(rawRecoveryKey) {
return await client.verifyWithRecoveryKey(rawRecoveryKey);
},
};
}
export async function runMatrixQaE2eeBootstrap(
params: MatrixQaE2eeClientParams,
): Promise<MatrixVerificationBootstrapResult> {
const client: MatrixClient = await createMatrixQaE2eeMatrixClient(params);
try {
return await client.bootstrapOwnDeviceVerification();
} finally {
await client.stopAndPersist().catch(() => undefined);
}
}
export const __testing = {
buildMatrixQaE2eeStoragePaths,
findMatrixQaObservedEventMatch,
};

127
extensions/qa-matrix/src/substrate/fault-proxy.test.ts Normal file
View File

@@ -0,0 +1,127 @@
import { createServer } from "node:http";
import { afterEach, describe, expect, it } from "vitest";
import { startMatrixQaFaultProxy, type MatrixQaFaultProxy } from "./fault-proxy.js";
const servers: Array<{ close(): Promise<void> }> = [];
async function startTargetServer() {
const requests: Array<{
authorization?: string;
body: string;
method: string;
url: string;
}> = [];
const server = createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
}
requests.push({
...(req.headers.authorization ? { authorization: req.headers.authorization } : {}),
body: Buffer.concat(chunks).toString("utf8"),
method: req.method ?? "GET",
url: req.url ?? "/",
});
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ forwarded: true }));
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
server.off("error", reject);
resolve();
});
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("target server did not bind to a TCP port");
}
const handle = {
baseUrl: `http://127.0.0.1:${address.port}`,
close: async () => {
await new Promise<void>((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
},
requests,
};
servers.push(handle);
return handle;
}
describe("Matrix QA fault proxy", () => {
let proxy: MatrixQaFaultProxy | undefined;
afterEach(async () => {
await proxy?.stop();
proxy = undefined;
while (servers.length > 0) {
await servers.pop()?.close();
}
});
it("faults matching Matrix requests and forwards everything else", async () => {
const target = await startTargetServer();
proxy = await startMatrixQaFaultProxy({
targetBaseUrl: target.baseUrl,
rules: [
{
id: "room-key-backup-version-unavailable",
match: (request) =>
request.method === "GET" &&
request.path === "/_matrix/client/v3/room_keys/version" &&
request.bearerToken === "driver-token",
response: () => ({
body: {
errcode: "M_NOT_FOUND",
error: "No current key backup",
},
status: 404,
}),
},
],
});
const faulted = await fetch(`${proxy.baseUrl}/_matrix/client/v3/room_keys/version`, {
headers: { authorization: "Bearer driver-token" },
});
expect(faulted.status).toBe(404);
await expect(faulted.json()).resolves.toEqual({
errcode: "M_NOT_FOUND",
error: "No current key backup",
});
const forwarded = await fetch(`${proxy.baseUrl}/_matrix/client/v3/sync?timeout=0`, {
body: JSON.stringify({ ok: true }),
headers: {
authorization: "Bearer driver-token",
"content-type": "application/json",
},
method: "POST",
});
expect(forwarded.status).toBe(200);
await expect(forwarded.json()).resolves.toEqual({ forwarded: true });
expect(proxy.hits()).toEqual([
{
method: "GET",
path: "/_matrix/client/v3/room_keys/version",
ruleId: "room-key-backup-version-unavailable",
},
]);
expect(target.requests).toEqual([
{
authorization: "Bearer driver-token",
body: '{"ok":true}',
method: "POST",
url: "/_matrix/client/v3/sync?timeout=0",
},
]);
});
});

216
extensions/qa-matrix/src/substrate/fault-proxy.ts Normal file
View File

@@ -0,0 +1,216 @@
import {
createServer,
type IncomingHttpHeaders,
type IncomingMessage,
type ServerResponse,
} from "node:http";
const HOP_BY_HOP_HEADERS = new Set([
"connection",
"content-length",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
]);
export type MatrixQaFaultProxyRequest = {
bearerToken?: string;
headers: IncomingHttpHeaders;
method: string;
path: string;
search: string;
};
export type MatrixQaFaultProxyResponse = {
body?: unknown;
headers?: Record<string, string>;
status: number;
};
export type MatrixQaFaultProxyRule = {
id: string;
match(request: MatrixQaFaultProxyRequest): boolean;
response(request: MatrixQaFaultProxyRequest): MatrixQaFaultProxyResponse;
};
export type MatrixQaFaultProxyHit = {
method: string;
path: string;
ruleId: string;
};
export type MatrixQaFaultProxy = {
baseUrl: string;
hits(): MatrixQaFaultProxyHit[];
stop(): Promise<void>;
};
function normalizeHeaderValue(value: string | string[] | undefined) {
if (Array.isArray(value)) {
return value.join(", ");
}
return value;
}
function extractBearerToken(headers: IncomingHttpHeaders) {
const value = normalizeHeaderValue(headers.authorization)?.trim();
const match = /^Bearer\s+(.+)$/i.exec(value ?? "");
return match?.[1];
}
function buildFetchHeaders(headers: IncomingHttpHeaders) {
const result = new Headers();
for (const [key, rawValue] of Object.entries(headers)) {
if (HOP_BY_HOP_HEADERS.has(key.toLowerCase()) || key.toLowerCase() === "host") {
continue;
}
const value = normalizeHeaderValue(rawValue);
if (value !== undefined) {
result.set(key, value);
}
}
return result;
}
async function readRequestBody(req: IncomingMessage) {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
}
return Buffer.concat(chunks);
}
function bufferToArrayBuffer(buffer: Buffer) {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength,
) as ArrayBuffer;
}
function writeJsonResponse(res: ServerResponse, response: MatrixQaFaultProxyResponse) {
const body = response.body === undefined ? "" : JSON.stringify(response.body);
res.writeHead(response.status, {
"content-type": "application/json",
...response.headers,
});
res.end(body);
}
async function forwardMatrixQaFaultProxyRequest(params: {
body: Buffer;
req: IncomingMessage;
targetUrl: URL;
}) {
const method = params.req.method ?? "GET";
const init: RequestInit = {
headers: buildFetchHeaders(params.req.headers),
method,
redirect: "manual",
};
if (method !== "GET" && method !== "HEAD") {
init.body = bufferToArrayBuffer(params.body);
}
const response = await fetch(params.targetUrl, init);
return {
body: Buffer.from(await response.arrayBuffer()),
headers: response.headers,
status: response.status,
};
}
function writeForwardedResponse(
res: ServerResponse,
response: Awaited<ReturnType<typeof forwardMatrixQaFaultProxyRequest>>,
) {
const headers: Record<string, string> = {};
for (const [key, value] of response.headers) {
if (!HOP_BY_HOP_HEADERS.has(key.toLowerCase())) {
headers[key] = value;
}
}
res.writeHead(response.status, headers);
res.end(response.body);
}
export async function startMatrixQaFaultProxy(params: {
rules: MatrixQaFaultProxyRule[];
targetBaseUrl: string;
}): Promise<MatrixQaFaultProxy> {
const targetBaseUrl = new URL(params.targetBaseUrl);
const hits: MatrixQaFaultProxyHit[] = [];
const server = createServer(async (req, res) => {
try {
const requestUrl = new URL(req.url ?? "/", targetBaseUrl);
const path = requestUrl.pathname;
const bearerToken = extractBearerToken(req.headers);
const request: MatrixQaFaultProxyRequest = {
...(bearerToken ? { bearerToken } : {}),
headers: req.headers,
method: req.method ?? "GET",
path,
search: requestUrl.search,
};
const body = await readRequestBody(req);
const rule = params.rules.find((candidate) => candidate.match(request));
if (rule) {
hits.push({
method: request.method,
path: request.path,
ruleId: rule.id,
});
writeJsonResponse(res, rule.response(request));
return;
}
writeForwardedResponse(
res,
await forwardMatrixQaFaultProxyRequest({
body,
req,
targetUrl: requestUrl,
}),
);
} catch (error) {
writeJsonResponse(res, {
body: {
errcode: "MATRIX_QA_FAULT_PROXY_ERROR",
error: error instanceof Error ? error.message : String(error),
},
status: 502,
});
}
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
server.off("error", reject);
resolve();
});
});
const address = server.address();
if (!address || typeof address === "string") {
server.close();
throw new Error("Matrix QA fault proxy did not bind to a TCP port");
}
return {
baseUrl: `http://127.0.0.1:${address.port}`,
hits: () => [...hits],
stop: async () => {
await new Promise<void>((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
},
};
}

1
extensions/qa-matrix/src/substrate/harness.runtime.test.ts
View File

@@ -26,6 +26,7 @@ describe("matrix harness runtime", () => {
expect(compose).toContain(`image: ${__testing.MATRIX_QA_DEFAULT_IMAGE}`);
expect(compose).toContain(' - "127.0.0.1:28008:8008"');
expect(compose).toContain('TUWUNEL_ALLOW_ENCRYPTION: "true"');
expect(compose).toContain('TUWUNEL_ALLOW_REGISTRATION: "true"');
expect(compose).toContain('TUWUNEL_REGISTRATION_TOKEN: "secret-token"');
expect(compose).toContain('TUWUNEL_SERVER_NAME: "matrix-qa.test"');

2
extensions/qa-matrix/src/substrate/harness.runtime.ts
View File

@@ -111,7 +111,7 @@ function renderMatrixQaCompose(params: {
- "127.0.0.1:${params.homeserverPort}:${MATRIX_QA_INTERNAL_PORT}"
environment:
TUWUNEL_ADDRESS: "0.0.0.0"
TUWUNEL_ALLOW_ENCRYPTION: "false"
TUWUNEL_ALLOW_ENCRYPTION: "true"
TUWUNEL_ALLOW_FEDERATION: "false"
TUWUNEL_ALLOW_REGISTRATION: "true"
TUWUNEL_DATABASE_PATH: "/var/lib/tuwunel"

4
extensions/qa-matrix/src/substrate/topology.ts
View File

@@ -3,6 +3,7 @@ export type MatrixQaParticipantRole = "driver" | "observer" | "sut";
export type MatrixQaRoomKind = "dm" | "group";
export type MatrixQaTopologyRoomSpec = {
encrypted?: boolean;
key: string;
kind: MatrixQaRoomKind;
members: MatrixQaParticipantRole[];
@@ -16,6 +17,7 @@ export type MatrixQaTopologySpec = {
};
export type MatrixQaProvisionedRoom = {
encrypted?: boolean;
key: string;
kind: MatrixQaRoomKind;
memberRoles: MatrixQaParticipantRole[];
@@ -34,6 +36,7 @@ export type MatrixQaProvisionedTopology = {
function matrixQaRoomSpecsEqual(left: MatrixQaTopologyRoomSpec, right: MatrixQaTopologyRoomSpec) {
return (
left.key === right.key &&
(left.encrypted === true) === (right.encrypted === true) &&
left.kind === right.kind &&
left.name === right.name &&
left.requireMention === right.requireMention &&
@@ -49,6 +52,7 @@ export function buildDefaultMatrixQaTopologySpec(params: {
defaultRoomKey: "main",
rooms: [
{
encrypted: false,
key: "main",
kind: "group",
members: ["driver", "observer", "sut"],

3
pnpm-lock.yaml generated
View File

@@ -998,6 +998,9 @@ importers:
extensions/qa-matrix:
devDependencies:
'@openclaw/matrix':
specifier: workspace:*
version: link:../matrix
'@openclaw/plugin-sdk':
specifier: workspace:*
version: link:../../packages/plugin-sdk