From 3aae0fb16d6077e1a4607e1840080169530bc61f Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Tue, 14 Apr 2026 20:05:33 -0400 Subject: [PATCH] QA: tighten Matrix substrate coverage --- .../src/runners/contract/runtime.test.ts | 84 ------- .../src/runners/contract/scenarios.test.ts | 2 + .../qa-matrix/src/substrate/artifacts.test.ts | 92 ++++++++ .../qa-matrix/src/substrate/client.test.ts | 206 +----------------- .../qa-matrix/src/substrate/events.test.ts | 141 ++++++++++++ .../qa-matrix/src/substrate/sync.test.ts | 144 ++++++++++++ 6 files changed, 380 insertions(+), 289 deletions(-) create mode 100644 extensions/qa-matrix/src/substrate/artifacts.test.ts create mode 100644 extensions/qa-matrix/src/substrate/events.test.ts create mode 100644 extensions/qa-matrix/src/substrate/sync.test.ts diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 08511811da7..d4a0952092a 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -144,90 +144,6 @@ describe("matrix live qa runtime", () => { }); }); - it("redacts Matrix observed event content by default in artifacts", () => { - expect( - liveTesting.buildObservedEventsArtifact({ - includeContent: false, - observedEvents: [ - { - roomId: "!room:matrix-qa.test", - eventId: "$event", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - body: "secret", - formattedBody: "

secret

", - msgtype: "m.text", - originServerTs: 1_700_000_000_000, - relatesTo: { - relType: "m.thread", - eventId: "$root", - inReplyToId: "$driver", - isFallingBack: true, - }, - }, - ], - }), - ).toEqual([ - { - roomId: "!room:matrix-qa.test", - eventId: "$event", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - msgtype: "m.text", - originServerTs: 1_700_000_000_000, - relatesTo: { - relType: "m.thread", - eventId: "$root", - inReplyToId: "$driver", - isFallingBack: true, - }, - }, - ]); - }); - - it("keeps reaction metadata in redacted Matrix observed-event artifacts", () => { - expect( - liveTesting.buildObservedEventsArtifact({ - includeContent: false, - observedEvents: [ - { - roomId: "!room:matrix-qa.test", - eventId: "$reaction", - sender: "@driver:matrix-qa.test", - type: "m.reaction", - reaction: { - eventId: "$reply", - key: "👍", - }, - relatesTo: { - relType: "m.annotation", - eventId: "$reply", - }, - }, - ], - }), - ).toEqual([ - { - roomId: "!room:matrix-qa.test", - eventId: "$reaction", - sender: "@driver:matrix-qa.test", - type: "m.reaction", - originServerTs: undefined, - msgtype: undefined, - membership: undefined, - relatesTo: { - relType: "m.annotation", - eventId: "$reply", - }, - mentions: undefined, - reaction: { - eventId: "$reply", - key: "👍", - }, - }, - ]); - }); - it("preserves negative-scenario artifacts in the Matrix summary", () => { expect( liveTesting.buildMatrixQaSummary({ diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 6df694b055d..18c099aef0e 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -49,6 +49,7 @@ describe("matrix live qa scenarios", () => { expect( scenarioTesting.buildMatrixReplyArtifact( { + kind: "message", roomId: "!room:matrix-qa.test", eventId: "$event", sender: "@sut:matrix-qa.test", @@ -61,6 +62,7 @@ describe("matrix live qa scenarios", () => { expect( scenarioTesting.buildMatrixReplyArtifact( { + kind: "message", roomId: "!room:matrix-qa.test", eventId: "$event-2", sender: "@sut:matrix-qa.test", diff --git a/extensions/qa-matrix/src/substrate/artifacts.test.ts b/extensions/qa-matrix/src/substrate/artifacts.test.ts new file mode 100644 index 00000000000..c8a5a5f3e55 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/artifacts.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { buildMatrixQaObservedEventsArtifact } from "./artifacts.js"; + +describe("matrix observed event artifacts", () => { + it("redacts Matrix observed event content by default in artifacts", () => { + expect( + buildMatrixQaObservedEventsArtifact({ + includeContent: false, + observedEvents: [ + { + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$event", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: "secret", + formattedBody: "

secret

", + msgtype: "m.text", + originServerTs: 1_700_000_000_000, + relatesTo: { + relType: "m.thread", + eventId: "$root", + inReplyToId: "$driver", + isFallingBack: true, + }, + }, + ], + }), + ).toEqual([ + { + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$event", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + msgtype: "m.text", + originServerTs: 1_700_000_000_000, + relatesTo: { + relType: "m.thread", + eventId: "$root", + inReplyToId: "$driver", + isFallingBack: true, + }, + }, + ]); + }); + + it("keeps reaction metadata in redacted Matrix observed-event artifacts", () => { + expect( + buildMatrixQaObservedEventsArtifact({ + includeContent: false, + observedEvents: [ + { + kind: "reaction", + roomId: "!room:matrix-qa.test", + eventId: "$reaction", + sender: "@driver:matrix-qa.test", + type: "m.reaction", + reaction: { + eventId: "$reply", + key: "👍", + }, + relatesTo: { + relType: "m.annotation", + eventId: "$reply", + }, + }, + ], + }), + ).toEqual([ + { + kind: "reaction", + roomId: "!room:matrix-qa.test", + eventId: "$reaction", + sender: "@driver:matrix-qa.test", + type: "m.reaction", + originServerTs: undefined, + msgtype: undefined, + membership: undefined, + relatesTo: { + relType: "m.annotation", + eventId: "$reply", + }, + mentions: undefined, + reaction: { + eventId: "$reply", + key: "👍", + }, + }, + ]); + }); +}); diff --git a/extensions/qa-matrix/src/substrate/client.test.ts b/extensions/qa-matrix/src/substrate/client.test.ts index fd1711f03f9..0a9aedba443 100644 --- a/extensions/qa-matrix/src/substrate/client.test.ts +++ b/extensions/qa-matrix/src/substrate/client.test.ts @@ -1,10 +1,5 @@ import { describe, expect, it } from "vitest"; -import { - __testing, - createMatrixQaClient, - provisionMatrixQaRoom, - type MatrixQaObservedEvent, -} from "./client.js"; +import { __testing, createMatrixQaClient, provisionMatrixQaRoom } from "./client.js"; import { buildDefaultMatrixQaTopologySpec } from "./topology.js"; function resolveRequestUrl(input: RequestInfo | URL) { @@ -58,49 +53,6 @@ describe("matrix driver client", () => { }); }); - it("normalizes message events with thread metadata", () => { - expect( - __testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { - event_id: "$event", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - origin_server_ts: 1_700_000_000_000, - content: { - body: "hello", - msgtype: "m.text", - "m.mentions": { - user_ids: ["@sut:matrix-qa.test"], - }, - "m.relates_to": { - rel_type: "m.thread", - event_id: "$root", - is_falling_back: true, - "m.in_reply_to": { - event_id: "$driver", - }, - }, - }, - }), - ).toEqual({ - roomId: "!room:matrix-qa.test", - eventId: "$event", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - originServerTs: 1_700_000_000_000, - body: "hello", - msgtype: "m.text", - relatesTo: { - relType: "m.thread", - eventId: "$root", - inReplyToId: "$driver", - isFallingBack: true, - }, - mentions: { - userIds: ["@sut:matrix-qa.test"], - }, - }); - }); - it("builds trimmed Matrix reaction relations for QA driver events", () => { expect(__testing.buildMatrixReactionRelation(" $msg-1 ", " 👍 ")).toEqual({ "m.relates_to": { @@ -111,38 +63,6 @@ describe("matrix driver client", () => { }); }); - it("normalizes Matrix reaction events with target metadata", () => { - expect( - __testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { - event_id: "$reaction", - sender: "@driver:matrix-qa.test", - type: "m.reaction", - origin_server_ts: 1_700_000_000_000, - content: { - "m.relates_to": { - rel_type: "m.annotation", - event_id: "$msg", - key: "👍", - }, - }, - }), - ).toEqual({ - roomId: "!room:matrix-qa.test", - eventId: "$reaction", - sender: "@driver:matrix-qa.test", - type: "m.reaction", - originServerTs: 1_700_000_000_000, - relatesTo: { - eventId: "$msg", - relType: "m.annotation", - }, - reaction: { - eventId: "$msg", - key: "👍", - }, - }); - }); - it("advances Matrix registration through token then dummy auth stages", () => { const firstStage = __testing.resolveNextRegistrationAuth({ registrationToken: "reg-token", @@ -185,130 +105,6 @@ describe("matrix driver client", () => { ).toThrow("Matrix registration requires unsupported auth stages:"); }); - it("returns a typed no-match result while preserving the latest sync token", async () => { - const fetchImpl: typeof fetch = async () => - new Response( - JSON.stringify({ - next_batch: "next-batch-2", - rooms: { - join: { - "!room:matrix-qa.test": { - timeline: { - events: [ - { - event_id: "$driver", - sender: "@driver:matrix-qa.test", - type: "m.room.message", - content: { body: "hello", msgtype: "m.text" }, - }, - ], - }, - }, - }, - }, - }), - { status: 200, headers: { "content-type": "application/json" } }, - ); - - const client = createMatrixQaClient({ - accessToken: "token", - baseUrl: "http://127.0.0.1:28008/", - fetchImpl, - }); - const observedEvents: MatrixQaObservedEvent[] = []; - - const result = await client.waitForOptionalRoomEvent({ - observedEvents, - predicate: (event) => event.sender === "@sut:matrix-qa.test", - roomId: "!room:matrix-qa.test", - since: "start-batch", - timeoutMs: 1, - }); - - expect(result).toEqual({ - matched: false, - since: "next-batch-2", - }); - expect(observedEvents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - body: "hello", - eventId: "$driver", - roomId: "!room:matrix-qa.test", - sender: "@driver:matrix-qa.test", - type: "m.room.message", - }), - ]), - ); - }); - - it("keeps recording later same-batch events after the first match", async () => { - const fetchImpl: typeof fetch = async () => - new Response( - JSON.stringify({ - next_batch: "next-batch-2", - rooms: { - join: { - "!room:matrix-qa.test": { - timeline: { - events: [ - { - event_id: "$sut", - sender: "@sut:matrix-qa.test", - type: "m.room.message", - content: { body: "target", msgtype: "m.text" }, - }, - { - event_id: "$driver", - sender: "@driver:matrix-qa.test", - type: "m.room.message", - content: { body: "trailing event", msgtype: "m.text" }, - }, - ], - }, - }, - }, - }, - }), - { status: 200, headers: { "content-type": "application/json" } }, - ); - - const client = createMatrixQaClient({ - accessToken: "token", - baseUrl: "http://127.0.0.1:28008/", - fetchImpl, - }); - const observedEvents: MatrixQaObservedEvent[] = []; - - const result = await client.waitForOptionalRoomEvent({ - observedEvents, - predicate: (event) => event.eventId === "$sut", - roomId: "!room:matrix-qa.test", - since: "start-batch", - timeoutMs: 1, - }); - - expect(result).toEqual({ - event: expect.objectContaining({ - eventId: "$sut", - }), - matched: true, - since: "next-batch-2", - }); - expect(observedEvents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - body: "target", - eventId: "$sut", - }), - expect.objectContaining({ - body: "trailing event", - eventId: "$driver", - }), - ]), - ); - }); - it("issues Matrix room membership control requests for QA topology changes", async () => { const requests: Array<{ body: Record; url: string }> = []; const fetchImpl: typeof fetch = async (input, init) => { diff --git a/extensions/qa-matrix/src/substrate/events.test.ts b/extensions/qa-matrix/src/substrate/events.test.ts new file mode 100644 index 00000000000..6b9b82dd635 --- /dev/null +++ b/extensions/qa-matrix/src/substrate/events.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import { normalizeMatrixQaObservedEvent } from "./events.js"; + +describe("matrix observed event normalization", () => { + it("normalizes message events with thread metadata", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$event", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + origin_server_ts: 1_700_000_000_000, + content: { + body: "hello", + msgtype: "m.text", + "m.mentions": { + user_ids: ["@sut:matrix-qa.test"], + }, + "m.relates_to": { + rel_type: "m.thread", + event_id: "$root", + is_falling_back: true, + "m.in_reply_to": { + event_id: "$driver", + }, + }, + }, + }), + ).toEqual({ + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$event", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + originServerTs: 1_700_000_000_000, + body: "hello", + msgtype: "m.text", + relatesTo: { + relType: "m.thread", + eventId: "$root", + inReplyToId: "$driver", + isFallingBack: true, + }, + mentions: { + userIds: ["@sut:matrix-qa.test"], + }, + }); + }); + + it("classifies Matrix notices separately from regular messages", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$notice", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + content: { + body: "notice", + msgtype: "m.notice", + }, + }), + ).toEqual( + expect.objectContaining({ + kind: "notice", + eventId: "$notice", + msgtype: "m.notice", + type: "m.room.message", + }), + ); + }); + + it("normalizes Matrix reaction events with target metadata", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$reaction", + sender: "@driver:matrix-qa.test", + type: "m.reaction", + origin_server_ts: 1_700_000_000_000, + content: { + "m.relates_to": { + rel_type: "m.annotation", + event_id: "$msg", + key: "👍", + }, + }, + }), + ).toEqual({ + kind: "reaction", + roomId: "!room:matrix-qa.test", + eventId: "$reaction", + sender: "@driver:matrix-qa.test", + type: "m.reaction", + originServerTs: 1_700_000_000_000, + relatesTo: { + eventId: "$msg", + relType: "m.annotation", + }, + reaction: { + eventId: "$msg", + key: "👍", + }, + }); + }); + + it("normalizes membership events with explicit membership kind", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$membership", + sender: "@driver:matrix-qa.test", + state_key: "@sut:matrix-qa.test", + type: "m.room.member", + content: { + membership: "leave", + }, + }), + ).toEqual( + expect.objectContaining({ + kind: "membership", + eventId: "$membership", + membership: "leave", + stateKey: "@sut:matrix-qa.test", + type: "m.room.member", + }), + ); + }); + + it("classifies Matrix redactions without needing raw event inspection", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$redaction", + sender: "@driver:matrix-qa.test", + type: "m.room.redaction", + content: {}, + }), + ).toEqual( + expect.objectContaining({ + kind: "redaction", + eventId: "$redaction", + type: "m.room.redaction", + }), + ); + }); +}); diff --git a/extensions/qa-matrix/src/substrate/sync.test.ts b/extensions/qa-matrix/src/substrate/sync.test.ts new file mode 100644 index 00000000000..e5042ec841f --- /dev/null +++ b/extensions/qa-matrix/src/substrate/sync.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; +import type { MatrixQaObservedEvent } from "./events.js"; +import { primeMatrixQaRoom, waitForOptionalMatrixQaRoomEvent } from "./sync.js"; + +describe("matrix sync helpers", () => { + it("primes the Matrix sync cursor without recording observed events", async () => { + const fetchImpl: typeof fetch = async () => + new Response(JSON.stringify({ next_batch: "primed-sync-cursor" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + + await expect( + primeMatrixQaRoom({ + accessToken: "token", + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + }), + ).resolves.toBe("primed-sync-cursor"); + }); + + it("returns a typed no-match result while preserving the latest sync token", async () => { + const fetchImpl: typeof fetch = async () => + new Response( + JSON.stringify({ + next_batch: "next-batch-2", + rooms: { + join: { + "!room:matrix-qa.test": { + timeline: { + events: [ + { + event_id: "$driver", + sender: "@driver:matrix-qa.test", + type: "m.room.message", + content: { body: "hello", msgtype: "m.text" }, + }, + ], + }, + }, + }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + + const observedEvents: MatrixQaObservedEvent[] = []; + + const result = await waitForOptionalMatrixQaRoomEvent({ + accessToken: "token", + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + observedEvents, + predicate: (event) => event.sender === "@sut:matrix-qa.test", + roomId: "!room:matrix-qa.test", + since: "start-batch", + timeoutMs: 1, + }); + + expect(result).toEqual({ + matched: false, + since: "next-batch-2", + }); + expect(observedEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "message", + body: "hello", + eventId: "$driver", + roomId: "!room:matrix-qa.test", + sender: "@driver:matrix-qa.test", + type: "m.room.message", + }), + ]), + ); + }); + + it("keeps recording later same-batch events after the first match", async () => { + const fetchImpl: typeof fetch = async () => + new Response( + JSON.stringify({ + next_batch: "next-batch-2", + rooms: { + join: { + "!room:matrix-qa.test": { + timeline: { + events: [ + { + event_id: "$sut", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + content: { body: "target", msgtype: "m.text" }, + }, + { + event_id: "$driver", + sender: "@driver:matrix-qa.test", + type: "m.room.message", + content: { body: "trailing event", msgtype: "m.text" }, + }, + ], + }, + }, + }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + + const observedEvents: MatrixQaObservedEvent[] = []; + + const result = await waitForOptionalMatrixQaRoomEvent({ + accessToken: "token", + baseUrl: "http://127.0.0.1:28008/", + fetchImpl, + observedEvents, + predicate: (event) => event.eventId === "$sut", + roomId: "!room:matrix-qa.test", + since: "start-batch", + timeoutMs: 1, + }); + + expect(result).toEqual({ + event: expect.objectContaining({ + eventId: "$sut", + }), + matched: true, + since: "next-batch-2", + }); + expect(observedEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "message", + body: "target", + eventId: "$sut", + }), + expect.objectContaining({ + kind: "message", + body: "trailing event", + eventId: "$driver", + }), + ]), + ); + }); +});