From 7e659e168bbc3c772d5f7715e7c180eb22f10b35 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Thu, 16 Apr 2026 21:23:40 -0400 Subject: [PATCH] QA Matrix: isolate scenario coverage Add the Matrix subagent-thread scenario and route it through the contract runner while preserving the current missing-hook failure as an explicit scenario result. Give E2EE scenarios isolated rooms and storage keys so lifecycle tests do not reuse stale encrypted state across scenarios. --- .../src/runners/contract/scenario-catalog.ts | 110 +++++++-- .../runners/contract/scenario-runtime-e2ee.ts | 74 +++--- .../runners/contract/scenario-runtime-room.ts | 85 +++++++ .../contract/scenario-runtime-shared.ts | 4 +- .../src/runners/contract/scenario-runtime.ts | 3 + .../src/runners/contract/scenario-types.ts | 2 + .../src/runners/contract/scenarios.test.ts | 227 +++++++++++++++++- .../src/runners/contract/scenarios.ts | 3 + .../qa-matrix/src/substrate/config.test.ts | 26 +- extensions/qa-matrix/src/substrate/config.ts | 56 ++++- .../src/substrate/e2ee-client.test.ts | 32 ++- .../qa-matrix/src/substrate/e2ee-client.ts | 13 +- 12 files changed, 558 insertions(+), 77 deletions(-) diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index 658c6c05bcc..b352c2dd7bf 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -17,6 +17,7 @@ export type MatrixQaScenarioId = | "matrix-thread-root-preservation" | "matrix-thread-nested-reply-shape" | "matrix-thread-isolation" + | "matrix-subagent-thread-spawn" | "matrix-top-level-reply-shape" | "matrix-room-thread-reply-override" | "matrix-room-quiet-streaming-preview" @@ -61,6 +62,7 @@ export type MatrixQaScenarioId = | "matrix-e2ee-artifact-redaction" | "matrix-e2ee-media-image" | "matrix-e2ee-key-bootstrap-failure"; +export type MatrixQaE2eeScenarioId = Extract; export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition & { configOverrides?: MatrixQaConfigOverrides; @@ -73,6 +75,7 @@ export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared"; export const MATRIX_QA_E2EE_ROOM_KEY = "e2ee"; export const MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY = "e2ee-verification-dm"; export const MATRIX_QA_HOMESERVER_ROOM_KEY = "homeserver"; +export const MATRIX_QA_MAIN_ROOM_KEY = "main"; export const MATRIX_QA_MEDIA_ROOM_KEY = "media"; export const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership"; export const MATRIX_QA_RESTART_ROOM_KEY = "restart"; @@ -85,7 +88,7 @@ function buildMatrixQaDmTopology( }>, ): MatrixQaTopologySpec { return { - defaultRoomKey: "main", + defaultRoomKey: MATRIX_QA_MAIN_ROOM_KEY, rooms: rooms.map((room) => ({ key: room.key, kind: "dm" as const, @@ -102,7 +105,7 @@ function buildMatrixQaSingleGroupTopology(params: { requireMention: boolean; }): MatrixQaTopologySpec { return { - defaultRoomKey: "main", + defaultRoomKey: MATRIX_QA_MAIN_ROOM_KEY, rooms: [ { encrypted: params.encrypted === true, @@ -116,6 +119,23 @@ function buildMatrixQaSingleGroupTopology(params: { }; } +export function buildMatrixQaE2eeScenarioRoomKey(scenarioId: MatrixQaE2eeScenarioId) { + const suffix = scenarioId.replace(/^matrix-e2ee-/, "").replace(/[^A-Za-z0-9_-]/g, "-"); + return `${MATRIX_QA_E2EE_ROOM_KEY}-${suffix}`; +} + +function buildMatrixQaE2eeScenarioTopology(params: { + scenarioId: MatrixQaE2eeScenarioId; + name: string; +}): MatrixQaTopologySpec { + return buildMatrixQaSingleGroupTopology({ + encrypted: true, + key: buildMatrixQaE2eeScenarioRoomKey(params.scenarioId), + name: params.name, + requireMention: true, + }); +} + const MATRIX_QA_DRIVER_DM_TOPOLOGY = buildMatrixQaDmTopology([ { key: MATRIX_QA_DRIVER_DM_ROOM_KEY, @@ -170,13 +190,6 @@ const MATRIX_QA_HOMESERVER_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({ requireMention: true, }); -const MATRIX_QA_E2EE_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({ - encrypted: true, - key: MATRIX_QA_E2EE_ROOM_KEY, - name: "Matrix QA E2EE Room", - requireMention: true, -}); - const MATRIX_QA_E2EE_VERIFICATION_DM_TOPOLOGY: MatrixQaTopologySpec = { defaultRoomKey: "main", rooms: [ @@ -218,6 +231,25 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 75_000, title: "Matrix top-level reply stays out of prior thread", }, + { + id: "matrix-subagent-thread-spawn", + timeoutMs: 75_000, + title: "Matrix sessions_spawn thread=true creates a bound child thread", + configOverrides: { + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + tools: { + allow: ["sessions_spawn"], + }, + }, + }, + threadBindings: { + enabled: true, + spawnSubagentSessions: true, + }, + toolProfile: "coding", + }, + }, { id: "matrix-top-level-reply-shape", standardId: "top-level-reply-shape", @@ -448,49 +480,70 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ id: "matrix-e2ee-basic-reply", timeoutMs: 75_000, title: "Matrix E2EE encrypted room replies decrypt end-to-end", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-basic-reply", + name: "Matrix QA E2EE Basic Reply Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-thread-follow-up", timeoutMs: 75_000, title: "Matrix E2EE encrypted threads preserve reply shape", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-thread-follow-up", + name: "Matrix QA E2EE Thread Follow-up Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-bootstrap-success", timeoutMs: 90_000, title: "Matrix E2EE bootstrap verifies the owner device and backup", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-bootstrap-success", + name: "Matrix QA E2EE Bootstrap Success Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-recovery-key-lifecycle", timeoutMs: 90_000, title: "Matrix E2EE recovery key restores and resets room-key backup", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-recovery-key-lifecycle", + name: "Matrix QA E2EE Recovery Key Lifecycle Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-device-sas-verification", timeoutMs: 90_000, title: "Matrix E2EE device verification completes SAS emoji compare", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-device-sas-verification", + name: "Matrix QA E2EE Device SAS Verification Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-qr-verification", timeoutMs: 90_000, title: "Matrix E2EE QR verification completes identity scan", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-qr-verification", + name: "Matrix QA E2EE QR Verification Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-stale-device-hygiene", timeoutMs: 90_000, title: "Matrix E2EE stale own devices can be removed without deleting the current device", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-stale-device-hygiene", + name: "Matrix QA E2EE Stale Device Hygiene Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { @@ -504,35 +557,50 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ id: "matrix-e2ee-restart-resume", timeoutMs: 90_000, title: "Matrix E2EE encrypted rooms resume after gateway restart", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-restart-resume", + name: "Matrix QA E2EE Restart Resume Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-verification-notice-no-trigger", timeoutMs: 30_000, title: "Matrix E2EE verification notices do not trigger replies", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-verification-notice-no-trigger", + name: "Matrix QA E2EE Verification Notice Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-artifact-redaction", timeoutMs: 75_000, title: "Matrix E2EE decrypted payloads stay out of default event artifacts", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-artifact-redaction", + name: "Matrix QA E2EE Artifact Redaction Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-media-image", timeoutMs: 90_000, title: "Matrix E2EE encrypted image attachments reach the model vision path", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-media-image", + name: "Matrix QA E2EE Media Image Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, { id: "matrix-e2ee-key-bootstrap-failure", timeoutMs: 90_000, title: "Matrix E2EE bootstrap reports room-key backup failures", - topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY, + topology: buildMatrixQaE2eeScenarioTopology({ + scenarioId: "matrix-e2ee-key-bootstrap-failure", + name: "Matrix QA E2EE Key Bootstrap Failure Room", + }), configOverrides: MATRIX_QA_E2EE_CONFIG, }, ]; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts index d139053d31f..f07438c3f88 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts @@ -14,7 +14,8 @@ import { type MatrixQaFaultProxyRule, } from "../../substrate/fault-proxy.js"; import { - MATRIX_QA_E2EE_ROOM_KEY, + buildMatrixQaE2eeScenarioRoomKey, + type MatrixQaE2eeScenarioId, MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY, resolveMatrixQaScenarioRoomId, } from "./scenario-catalog.js"; @@ -37,21 +38,6 @@ import { } from "./scenario-runtime-shared.js"; import type { MatrixQaReplyArtifact, MatrixQaScenarioExecution } from "./scenario-types.js"; -type MatrixQaE2eeScenarioId = - | "matrix-e2ee-artifact-redaction" - | "matrix-e2ee-basic-reply" - | "matrix-e2ee-bootstrap-success" - | "matrix-e2ee-device-sas-verification" - | "matrix-e2ee-dm-sas-verification" - | "matrix-e2ee-key-bootstrap-failure" - | "matrix-e2ee-media-image" - | "matrix-e2ee-qr-verification" - | "matrix-e2ee-recovery-key-lifecycle" - | "matrix-e2ee-restart-resume" - | "matrix-e2ee-stale-device-hygiene" - | "matrix-e2ee-thread-follow-up" - | "matrix-e2ee-verification-notice-no-trigger"; - const MATRIX_QA_ROOM_KEY_BACKUP_VERSION_ENDPOINT = "/_matrix/client/v3/room_keys/version"; const MATRIX_QA_ROOM_KEY_BACKUP_FAULT_RULE_ID = "room-key-backup-version-unavailable"; @@ -72,6 +58,17 @@ function requireMatrixQaPassword(context: MatrixQaScenarioContext, actor: "drive return password; } +function resolveMatrixQaE2eeScenarioGroupRoom( + context: MatrixQaScenarioContext, + scenarioId: MatrixQaE2eeScenarioId, +) { + const roomKey = buildMatrixQaE2eeScenarioRoomKey(scenarioId); + return { + roomKey, + roomId: resolveMatrixQaScenarioRoomId(context, roomKey), + }; +} + function assertMatrixQaBootstrapSucceeded(label: string, result: MatrixQaE2eeBootstrapResult) { if (!result.success) { throw new Error(`${label} bootstrap failed: ${result.error ?? "unknown error"}`); @@ -476,7 +473,7 @@ async function runMatrixQaE2eeTopLevelScenario( tokenPrefix: string; }, ) { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom(context, params.scenarioId); return await withMatrixQaE2eeDriver(context, params.scenarioId, async (client) => { const startSince = await client.prime(); const token = buildMatrixQaToken(params.tokenPrefix); @@ -502,6 +499,7 @@ async function runMatrixQaE2eeTopLevelScenario( driverEventId, reply, roomId, + roomKey, since: matched.since ?? startSince, token, }; @@ -519,11 +517,11 @@ export async function runMatrixQaE2eeBasicReplyScenario( artifacts: { driverEventId: result.driverEventId, reply: result.reply, - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey: result.roomKey, roomId: result.roomId, }, details: [ - `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room key: ${result.roomKey}`, `encrypted room id: ${result.roomId}`, `driver event: ${result.driverEventId}`, ...buildMatrixReplyDetails("E2EE reply", result.reply), @@ -534,7 +532,10 @@ export async function runMatrixQaE2eeBasicReplyScenario( export async function runMatrixQaE2eeThreadFollowUpScenario( context: MatrixQaScenarioContext, ): Promise { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-thread-follow-up", + ); const result = await withMatrixQaE2eeDriver( context, "matrix-e2ee-thread-follow-up", @@ -582,11 +583,11 @@ export async function runMatrixQaE2eeThreadFollowUpScenario( driverEventId: result.driverEventId, reply: result.reply, rootEventId: result.rootEventId, - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey, roomId, }, details: [ - `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room key: ${roomKey}`, `encrypted room id: ${roomId}`, `thread root event: ${result.rootEventId}`, `mention trigger event: ${result.driverEventId}`, @@ -633,7 +634,10 @@ export async function runMatrixQaE2eeRecoveryKeyLifecycleScenario( context, "matrix-e2ee-recovery-key-lifecycle", async (client) => { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const { roomId } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-recovery-key-lifecycle", + ); const ready = await ensureMatrixQaE2eeOwnDeviceVerified({ client, label: "driver", @@ -1042,11 +1046,11 @@ export async function runMatrixQaE2eeRestartResumeScenario( recoveredDriverEventId: recovered.driverEventId, recoveredReply: recovered.reply, restartSignal: "gateway-restart", - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey: recovered.roomKey, roomId: recovered.roomId, }, details: [ - `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room key: ${recovered.roomKey}`, `encrypted room id: ${recovered.roomId}`, `pre-restart event: ${first.driverEventId}`, ...buildMatrixReplyDetails("pre-restart reply", first.reply), @@ -1059,7 +1063,10 @@ export async function runMatrixQaE2eeRestartResumeScenario( export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario( context: MatrixQaScenarioContext, ): Promise { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-verification-notice-no-trigger", + ); return await withMatrixQaE2eeDriver( context, "matrix-e2ee-verification-notice-no-trigger", @@ -1096,11 +1103,11 @@ export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario( artifacts: { expectedNoReplyWindowMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), noticeEventId, - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey, roomId, }, details: [ - `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room key: ${roomKey}`, `encrypted room id: ${roomId}`, `verification notice event: ${noticeEventId}`, `waited ${Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs)}ms with no SUT reply`, @@ -1129,7 +1136,7 @@ export async function runMatrixQaE2eeArtifactRedactionScenario( artifacts: { driverEventId: result.driverEventId, reply: result.reply, - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey: result.roomKey, roomId: result.roomId, }, details: [ @@ -1144,7 +1151,10 @@ export async function runMatrixQaE2eeArtifactRedactionScenario( export async function runMatrixQaE2eeMediaImageScenario( context: MatrixQaScenarioContext, ): Promise { - const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_E2EE_ROOM_KEY); + const { roomId, roomKey } = resolveMatrixQaE2eeScenarioGroupRoom( + context, + "matrix-e2ee-media-image", + ); return await withMatrixQaE2eeDriver(context, "matrix-e2ee-media-image", async (client) => { const startSince = await client.prime(); const triggerBody = buildMatrixQaImageUnderstandingPrompt(context.sutUserId); @@ -1187,11 +1197,11 @@ export async function runMatrixQaE2eeMediaImageScenario( attachmentFilename: MATRIX_QA_IMAGE_ATTACHMENT_FILENAME, driverEventId, reply, - roomKey: MATRIX_QA_E2EE_ROOM_KEY, + roomKey, roomId, }, details: [ - `encrypted room key: ${MATRIX_QA_E2EE_ROOM_KEY}`, + `encrypted room key: ${roomKey}`, `encrypted room id: ${roomId}`, `driver encrypted image event: ${driverEventId}`, `driver encrypted image filename: ${MATRIX_QA_IMAGE_ATTACHMENT_FILENAME}`, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index 4f373c912d0..477c920707b 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -40,6 +40,9 @@ import type { MatrixQaCanaryArtifact, MatrixQaScenarioExecution } from "./scenar type MatrixQaThreadScenarioResult = Awaited>; +const MATRIX_SUBAGENT_THREAD_HOOK_ERROR_RE = + /thread=true is unavailable because no channel plugin registered subagent_spawning hooks/i; + function assertMatrixQaInReplyTarget(params: { actualEventId?: string; expectedEventId: string; @@ -71,6 +74,14 @@ function buildMatrixQaThreadArtifacts(result: MatrixQaThreadScenarioResult) { }; } +function failIfMatrixSubagentThreadHookError(event: MatrixQaObservedEvent) { + if (MATRIX_SUBAGENT_THREAD_HOOK_ERROR_RE.test(event.body ?? "")) { + throw new Error( + `Matrix subagent thread spawn hit missing hook error: ${event.body ?? ""}`, + ); + } +} + function buildMatrixQaThreadDetailLines(params: { result: MatrixQaThreadScenarioResult; includeNestedTrigger?: boolean; @@ -281,6 +292,80 @@ export async function runThreadIsolationScenario(context: MatrixQaScenarioContex } satisfies MatrixQaScenarioExecution; } +export async function runSubagentThreadSpawnScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const childToken = buildMatrixQaToken("MATRIX_QA_SUBAGENT_CHILD"); + const triggerBody = [ + `${context.sutUserId} Use sessions_spawn for this QA check.`, + `task="Reply exactly \`${childToken}\`. This is the marker."`, + "label=matrix-thread-subagent thread=true mode=session runTimeoutSeconds=30", + ].join(" "); + const driverEventId = await client.sendTextMessage({ + body: triggerBody, + mentionUserIds: [context.sutUserId], + roomId: context.roomId, + }); + const intro = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => { + failIfMatrixSubagentThreadHookError(event); + return ( + event.roomId === context.roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + isMatrixQaMessageLikeKind(event.kind) && + /\bsession active\b/i.test(event.body ?? "") && + /Messages here go directly to this session/i.test(event.body ?? "") + ); + }, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + const completion = await client.waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => { + failIfMatrixSubagentThreadHookError(event); + return ( + event.roomId === context.roomId && + event.sender === context.sutUserId && + event.type === "m.room.message" && + isMatrixQaMessageLikeKind(event.kind) && + (event.body ?? "").includes(childToken) && + event.relatesTo?.relType === "m.thread" && + event.relatesTo.eventId === intro.event.eventId + ); + }, + roomId: context.roomId, + since: intro.since, + timeoutMs: context.timeoutMs, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + nextSince: completion.since, + startSince, + }); + const subagentIntro = buildMatrixReplyArtifact(intro.event); + const subagentCompletion = buildMatrixReplyArtifact(completion.event, childToken); + return { + artifacts: { + driverEventId, + subagentCompletion, + subagentIntro, + threadRootEventId: intro.event.eventId, + threadToken: childToken, + triggerBody, + }, + details: [ + `driver event: ${driverEventId}`, + `subagent thread root event: ${intro.event.eventId}`, + ...buildMatrixReplyDetails("subagent intro", subagentIntro), + ...buildMatrixReplyDetails("subagent completion", subagentCompletion), + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + export async function runTopLevelReplyShapeScenario(context: MatrixQaScenarioContext) { const result = await runAssertedDriverTopLevelScenario({ context, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index f9e51854bfd..7847381c008 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -56,7 +56,7 @@ export function buildMatrixQaToken(prefix: string) { } export function buildMatrixQuietStreamingPrompt(sutUserId: string, text: string) { - return `${sutUserId} Matrix quiet streaming QA check: reply exactly \`${text}\`.`; + return `${sutUserId} Quiet streaming QA check: reply exactly \`${text}\`.`; } export function buildMatrixBlockStreamingPrompt( @@ -66,7 +66,7 @@ export function buildMatrixBlockStreamingPrompt( ) { return [ sutUserId, - "Matrix block streaming QA check:", + "Block streaming QA check:", "emit exactly two assistant message blocks in order.", `First exact marker: \`${firstText}\`.`, `Second exact marker: \`${secondText}\`.`, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index a9cad400cd3..10bea7d9e36 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -53,6 +53,7 @@ import { runReactionThreadedScenario, runRoomAutoJoinInviteScenario, runRoomThreadReplyOverrideScenario, + runSubagentThreadSpawnScenario, runThreadFollowUpScenario, runThreadIsolationScenario, runThreadNestedReplyShapeScenario, @@ -168,6 +169,8 @@ export async function runMatrixQaScenario( return await runThreadNestedReplyShapeScenario(context); case "matrix-thread-isolation": return await runThreadIsolationScenario(context); + case "matrix-subagent-thread-spawn": + return await runSubagentThreadSpawnScenario(context); case "matrix-top-level-reply-shape": return await runTopLevelReplyShapeScenario(context); case "matrix-room-thread-reply-override": diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index 84c56d0c895..d2b5958ee62 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -62,6 +62,8 @@ export type MatrixQaScenarioArtifacts = { secondDriverEventId?: string; secondReply?: MatrixQaReplyArtifact; secondToken?: string; + subagentCompletion?: MatrixQaReplyArtifact; + subagentIntro?: MatrixQaReplyArtifact; threadDriverEventId?: string; threadReply?: MatrixQaReplyArtifact; threadRootEventId?: string; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index b37f37a60f9..d14b31cb031 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -24,13 +24,47 @@ import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, findMissingLiveTransportStandardScenarios, } from "../../shared/live-transport-scenarios.js"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; import { MATRIX_QA_MEDIA_TYPE_COVERAGE_CASES } from "./scenario-media-fixtures.js"; import { __testing as scenarioTesting, MATRIX_QA_SCENARIOS, runMatrixQaScenario, + type MatrixQaScenarioContext, } from "./scenarios.js"; +const MATRIX_SUBAGENT_MISSING_HOOK_ERROR = + "thread=true is unavailable because no channel plugin registered subagent_spawning hooks."; + +function matrixQaScenarioContext(): MatrixQaScenarioContext { + return { + baseUrl: "http://127.0.0.1:28008/", + canary: undefined, + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents: [], + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId: "!main:matrix-qa.test", + restartGateway: undefined, + syncState: {}, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + timeoutMs: 8_000, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [], + }, + }; +} + +function matrixQaE2eeRoomKey( + scenarioId: Parameters[0], +) { + return scenarioTesting.buildMatrixQaE2eeScenarioRoomKey(scenarioId); +} + describe("matrix live qa scenarios", () => { beforeEach(() => { createMatrixQaClient.mockReset(); @@ -45,6 +79,7 @@ describe("matrix live qa scenarios", () => { "matrix-thread-root-preservation", "matrix-thread-nested-reply-shape", "matrix-thread-isolation", + "matrix-subagent-thread-spawn", "matrix-top-level-reply-shape", "matrix-room-thread-reply-override", "matrix-room-quiet-streaming-preview", @@ -258,6 +293,43 @@ describe("matrix live qa scenarios", () => { ).toThrow('Matrix QA topology room "ops" has conflicting definitions'); }); + it("provisions isolated encrypted rooms for each E2EE scenario", () => { + const topology = scenarioTesting.buildMatrixQaTopologyForScenarios({ + defaultRoomName: "OpenClaw Matrix QA run", + scenarios: [ + MATRIX_QA_SCENARIOS.find((scenario) => scenario.id === "matrix-e2ee-basic-reply")!, + MATRIX_QA_SCENARIOS.find((scenario) => scenario.id === "matrix-e2ee-thread-follow-up")!, + ], + }); + + expect(topology.rooms).toEqual([ + { + encrypted: false, + key: "main", + kind: "group", + members: ["driver", "observer", "sut"], + name: "OpenClaw Matrix QA run", + requireMention: true, + }, + { + encrypted: true, + key: "e2ee-basic-reply", + kind: "group", + members: ["driver", "observer", "sut"], + name: "Matrix QA E2EE Basic Reply Room", + requireMention: true, + }, + { + encrypted: true, + key: "e2ee-thread-follow-up", + kind: "group", + members: ["driver", "observer", "sut"], + name: "Matrix QA E2EE Thread Follow-up Room", + requireMention: true, + }, + ]); + }); + it("resolves scenario room ids from provisioned topology keys", () => { expect( scenarioTesting.resolveMatrixQaScenarioRoomId( @@ -565,6 +637,70 @@ describe("matrix live qa scenarios", () => { ); expect(scenario).toBeDefined(); + await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({ + artifacts: { + driverEventId: "$room-thread-trigger", + reply: { + relatesTo: { + relType: "m.thread", + eventId: "$room-thread-trigger", + }, + }, + }, + }); + }); + + it("runs the subagent thread spawn scenario against a child thread", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$subagent-spawn-trigger"); + const waitForRoomEvent = vi + .fn() + .mockImplementationOnce(async () => ({ + event: { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$subagent-thread-root", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: "qa session active. Messages here go directly to this session.", + }, + since: "driver-sync-intro", + })) + .mockImplementationOnce(async () => { + const childToken = + /task="Reply exactly `([^`]+)`/.exec( + String(sendTextMessage.mock.calls[0]?.[0]?.body), + )?.[1] ?? "MATRIX_QA_SUBAGENT_CHILD_FIXED"; + return { + event: { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$subagent-completion", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: childToken, + relatesTo: { + relType: "m.thread", + eventId: "$subagent-thread-root", + inReplyToId: "$subagent-thread-root", + isFallingBack: true, + }, + }, + since: "driver-sync-next", + }; + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-subagent-thread-spawn", + ); + expect(scenario).toBeDefined(); + await expect( runMatrixQaScenario(scenario!, { baseUrl: "http://127.0.0.1:28008/", @@ -588,15 +724,90 @@ describe("matrix live qa scenarios", () => { }), ).resolves.toMatchObject({ artifacts: { - driverEventId: "$room-thread-trigger", - reply: { + driverEventId: "$subagent-spawn-trigger", + subagentCompletion: { + eventId: "$subagent-completion", relatesTo: { relType: "m.thread", - eventId: "$room-thread-trigger", + eventId: "$subagent-thread-root", }, + tokenMatched: true, }, + subagentIntro: { + eventId: "$subagent-thread-root", + }, + threadRootEventId: "$subagent-thread-root", }, }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("Use sessions_spawn for this QA check"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + since: "driver-sync-start", + }), + ); + expect(waitForRoomEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + predicate: expect.any(Function), + since: "driver-sync-intro", + }), + ); + const introPredicate = waitForRoomEvent.mock.calls[0]?.[0]?.predicate as + | ((event: MatrixQaObservedEvent) => boolean) + | undefined; + expect(() => + introPredicate?.({ + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$missing-hook-error", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: MATRIX_SUBAGENT_MISSING_HOOK_ERROR, + }), + ).toThrow("missing hook error"); + }); + + it("fails the subagent thread spawn scenario when Matrix lacks subagent hooks", async () => { + const primeRoom = vi.fn().mockResolvedValue("driver-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$subagent-spawn-trigger"); + const waitForRoomEvent = vi.fn().mockImplementationOnce(async (options) => { + const event = { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$missing-hook-error", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: MATRIX_SUBAGENT_MISSING_HOOK_ERROR, + } satisfies MatrixQaObservedEvent; + options.predicate(event); + return { + event, + since: "driver-sync-error", + }; + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-subagent-thread-spawn", + ); + expect(scenario).toBeDefined(); + + await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).rejects.toThrow( + "missing hook error", + ); + + expect(waitForRoomEvent).toHaveBeenCalledTimes(1); }); it("captures quiet preview notices before the finalized Matrix reply", async () => { @@ -676,7 +887,7 @@ describe("matrix live qa scenarios", () => { }); expect(sendTextMessage).toHaveBeenCalledWith({ - body: expect.stringContaining("Matrix quiet streaming QA check"), + body: expect.stringContaining("Quiet streaming QA check"), mentionUserIds: ["@sut:matrix-qa.test"], roomId: "!main:matrix-qa.test", }); @@ -781,7 +992,7 @@ describe("matrix live qa scenarios", () => { }); expect(sendTextMessage).toHaveBeenCalledWith({ - body: expect.stringContaining("Matrix block streaming QA check"), + body: expect.stringContaining("Block streaming QA check"), mentionUserIds: ["@sut:matrix-qa.test"], roomId: "!block:matrix-qa.test", }); @@ -1711,7 +1922,7 @@ describe("matrix live qa scenarios", () => { defaultRoomKey: "main", rooms: [ { - key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + key: matrixQaE2eeRoomKey("matrix-e2ee-verification-notice-no-trigger"), kind: "group", memberRoles: ["driver", "observer", "sut"], memberUserIds: [ @@ -1832,7 +2043,7 @@ describe("matrix live qa scenarios", () => { rooms: [ { encrypted: true, - key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + key: matrixQaE2eeRoomKey("matrix-e2ee-recovery-key-lifecycle"), kind: "group", memberRoles: ["driver", "observer", "sut"], memberUserIds: [ @@ -1930,7 +2141,7 @@ describe("matrix live qa scenarios", () => { defaultRoomKey: "main", rooms: [ { - key: scenarioTesting.MATRIX_QA_E2EE_ROOM_KEY, + key: matrixQaE2eeRoomKey("matrix-e2ee-key-bootstrap-failure"), kind: "group", memberRoles: ["driver", "observer", "sut"], memberUserIds: [ diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.ts b/extensions/qa-matrix/src/runners/contract/scenarios.ts index 661257fa77e..1bd2c12f71c 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.ts @@ -7,6 +7,7 @@ import { MATRIX_QA_SCENARIOS, MATRIX_QA_SECONDARY_ROOM_KEY, MATRIX_QA_STANDARD_SCENARIO_IDS, + buildMatrixQaE2eeScenarioRoomKey, buildMatrixQaTopologyForScenarios, findMatrixQaScenarios, resolveMatrixQaScenarioRoomId, @@ -37,6 +38,7 @@ export { MATRIX_QA_STANDARD_SCENARIO_IDS, buildMatrixReplyArtifact, buildMatrixReplyDetails, + buildMatrixQaE2eeScenarioRoomKey, buildMatrixQaTopologyForScenarios, buildMentionPrompt, findMatrixQaScenarios, @@ -61,6 +63,7 @@ export const __testing = { MATRIX_QA_MEMBERSHIP_ROOM_KEY, MATRIX_QA_SECONDARY_ROOM_KEY, MATRIX_QA_STANDARD_SCENARIO_IDS, + buildMatrixQaE2eeScenarioRoomKey, buildMatrixQaTopologyForScenarios, buildMatrixReplyDetails, buildMatrixReplyArtifact, diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index 5187165efdf..995969af053 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -108,11 +108,20 @@ describe("matrix qa config", () => { groupsByKey: { secondary: { requireMention: false, + tools: { + allow: ["sessions_spawn"], + }, }, }, replyToMode: "all", streaming: "quiet", + threadBindings: { + enabled: true, + idleHours: 1, + spawnSubagentSessions: true, + }, threadReplies: "always", + toolProfile: "coding", }, sutAccessToken: "sut-token", sutAccountId: "sut", @@ -132,6 +141,9 @@ describe("matrix qa config", () => { minChars: 1, }, }); + expect(next.tools).toMatchObject({ + profile: "coding", + }); expect(next.channels?.matrix?.accounts?.sut).toMatchObject({ autoJoin: "allowlist", autoJoinAllowlist: ["!dm:matrix-qa.test", "#ops:matrix-qa.test"], @@ -144,10 +156,21 @@ describe("matrix qa config", () => { groupAllowFrom: ["@driver:matrix-qa.test", "@observer:matrix-qa.test"], groups: { "!main:matrix-qa.test": { enabled: true, requireMention: true }, - "!secondary:matrix-qa.test": { enabled: true, requireMention: false }, + "!secondary:matrix-qa.test": { + enabled: true, + requireMention: false, + tools: { + allow: ["sessions_spawn"], + }, + }, }, replyToMode: "all", streaming: "quiet", + threadBindings: { + enabled: true, + idleHours: 1, + spawnSubagentSessions: true, + }, threadReplies: "always", }); }); @@ -231,6 +254,7 @@ describe("matrix qa config", () => { }, replyToMode: "off", streaming: "partial", + threadBindings: {}, threadReplies: "inbound", }); expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("autoJoin=allowlist"); diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index 8a523118ef8..7c8dee83993 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -22,9 +22,15 @@ export type MatrixQaAgentDefaultsOverrides = { }; }; +export type MatrixQaToolConfigOverrides = { + allow?: string[]; + deny?: string[]; +}; + export type MatrixQaGroupConfigOverrides = { enabled?: boolean; requireMention?: boolean; + tools?: MatrixQaToolConfigOverrides; }; export type MatrixQaDmConfigOverrides = { @@ -35,6 +41,14 @@ export type MatrixQaDmConfigOverrides = { threadReplies?: MatrixQaThreadRepliesMode; }; +export type MatrixQaThreadBindingsConfigOverrides = { + enabled?: boolean; + idleHours?: number; + maxAgeHours?: number; + spawnAcpSessions?: boolean; + spawnSubagentSessions?: boolean; +}; + export type MatrixQaConfigOverrides = { agentDefaults?: MatrixQaAgentDefaultsOverrides; autoJoin?: MatrixQaAutoJoinMode; @@ -49,7 +63,9 @@ export type MatrixQaConfigOverrides = { replyToMode?: MatrixQaReplyToMode; startupVerification?: "if-unverified" | "off"; streaming?: "off" | "partial" | "quiet" | boolean; + threadBindings?: MatrixQaThreadBindingsConfigOverrides; threadReplies?: MatrixQaThreadRepliesMode; + toolProfile?: "coding" | "messaging" | "minimal"; }; export type MatrixQaConfigSnapshot = { @@ -66,20 +82,22 @@ export type MatrixQaConfigSnapshot = { encryption: boolean; groupAllowFrom: string[]; groupPolicy: MatrixQaGroupPolicy; - groupsByKey: Record< - string, - { - enabled: boolean; - requireMention: boolean; - roomId: string; - } - >; + groupsByKey: Record; replyToMode: MatrixQaReplyToMode; startupVerification?: "if-unverified" | "off"; streaming: MatrixQaStreamingMode; + threadBindings: MatrixQaThreadBindingsConfigOverrides; threadReplies: MatrixQaThreadRepliesMode; }; +type MatrixQaGroupSnapshot = { + enabled: boolean; + requireMention: boolean; + roomId: string; + tools?: MatrixQaToolConfigOverrides; +}; + +type MatrixQaGroupEntry = Omit; type MatrixQaChannelConfig = NonNullable["matrix"]; type MatrixQaChannelAccountConfig = NonNullable< NonNullable["accounts"] @@ -122,6 +140,7 @@ function resolveMatrixQaGroupSnapshots(params: { roomId: room.roomId, enabled: override?.enabled ?? true, requireMention: override?.requireMention ?? room.requireMention, + ...(override?.tools ? { tools: override.tools } : {}), }, ]; }), @@ -130,13 +149,14 @@ function resolveMatrixQaGroupSnapshots(params: { function buildMatrixQaGroupEntries( groupsByKey: MatrixQaConfigSnapshot["groupsByKey"], -): Record { +): Record { return Object.fromEntries( Object.values(groupsByKey).map((group) => [ group.roomId, { enabled: group.enabled, requireMention: group.requireMention, + ...(group.tools ? { tools: group.tools } : {}), }, ]), ); @@ -252,7 +272,7 @@ function buildMatrixQaAccountDmConfig(params: { } function buildMatrixQaChannelAccountConfig(params: { - groups: Record; + groups: Record; homeserver: string; overrides?: MatrixQaConfigOverrides; snapshot: MatrixQaConfigSnapshot; @@ -277,6 +297,10 @@ function buildMatrixQaChannelAccountConfig(params: { params.snapshot.startupVerification !== undefined ? { startupVerification: params.snapshot.startupVerification } : {}; + const threadBindingsConfig = + params.overrides?.threadBindings !== undefined + ? { threadBindings: params.snapshot.threadBindings } + : {}; return { accessToken: params.sutAccessToken, @@ -296,6 +320,7 @@ function buildMatrixQaChannelAccountConfig(params: { }, replyToMode: params.snapshot.replyToMode, ...startupVerificationConfig, + ...threadBindingsConfig, threadReplies: params.snapshot.threadReplies, userId: params.sutUserId, ...autoJoinConfig, @@ -327,6 +352,7 @@ export function buildMatrixQaConfigSnapshot(params: { replyToMode: params.overrides?.replyToMode ?? "off", startupVerification: params.overrides?.startupVerification, streaming: resolveMatrixQaStreamingMode(params.overrides?.streaming), + threadBindings: { ...params.overrides?.threadBindings }, threadReplies: params.overrides?.threadReplies ?? "inbound", }; } @@ -344,6 +370,8 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot `autoJoin=${snapshot.autoJoin}`, `encryption=${formatMatrixQaBoolean(snapshot.encryption)}`, `startupVerification=${snapshot.startupVerification ?? ""}`, + `threadBindings.enabled=${snapshot.threadBindings.enabled ?? ""}`, + `threadBindings.spawnSubagentSessions=${snapshot.threadBindings.spawnSubagentSessions ?? ""}`, ].join(", "); } @@ -373,6 +401,14 @@ export function buildMatrixQaConfig( return { ...baseCfg, + ...(params.overrides?.toolProfile + ? { + tools: { + ...baseCfg.tools, + profile: params.overrides.toolProfile, + }, + } + : {}), ...(params.overrides?.agentDefaults ? { agents: { diff --git a/extensions/qa-matrix/src/substrate/e2ee-client.test.ts b/extensions/qa-matrix/src/substrate/e2ee-client.test.ts index 3e0a2f0ebf3..077309b2ff3 100644 --- a/extensions/qa-matrix/src/substrate/e2ee-client.test.ts +++ b/extensions/qa-matrix/src/substrate/e2ee-client.test.ts @@ -3,7 +3,15 @@ import { describe, expect, it } from "vitest"; import { __testing } from "./e2ee-client.js"; describe("matrix qa e2ee client storage", () => { - it("scopes persisted crypto state to the account actor", () => { + it("filters receipt noise without suppressing room state or timeline events", () => { + expect(__testing.MATRIX_QA_E2EE_SYNC_FILTER).toEqual({ + room: { + ephemeral: { not_types: ["m.receipt"] }, + }, + }); + }); + + it("shares persisted crypto by actor and scopes sync replay by scenario", () => { const first = __testing.buildMatrixQaE2eeStoragePaths({ actorId: "driver", outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", @@ -26,5 +34,27 @@ describe("matrix qa e2ee client storage", () => { ); expect(first.cryptoDatabasePrefix).toBe(second.cryptoDatabasePrefix); expect(first.recoveryKeyPath).toBe(path.join(first.accountDir, "recovery-key.json")); + expect(first.storagePath).toBe( + path.join( + "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", + "matrix-e2ee", + "accounts", + "driver", + "scenarios", + "matrix-e2ee-basic-reply", + "sync-store.json", + ), + ); + expect(second.storagePath).toBe( + path.join( + "/tmp/openclaw/.artifacts/qa-e2e/matrix-run", + "matrix-e2ee", + "accounts", + "driver", + "scenarios", + "matrix-e2ee-qr-verification", + "sync-store.json", + ), + ); }); }); diff --git a/extensions/qa-matrix/src/substrate/e2ee-client.ts b/extensions/qa-matrix/src/substrate/e2ee-client.ts index 36a585146a4..1ec6057174b 100644 --- a/extensions/qa-matrix/src/substrate/e2ee-client.ts +++ b/extensions/qa-matrix/src/substrate/e2ee-client.ts @@ -37,6 +37,12 @@ type MatrixQaE2eeClientParams = { userId: string; }; +const MATRIX_QA_E2EE_SYNC_FILTER = { + room: { + ephemeral: { not_types: ["m.receipt"] }, + }, +}; + export type MatrixQaE2eeScenarioClient = { acceptVerification(id: string): Promise; bootstrapOwnDeviceVerification(params?: { @@ -122,9 +128,9 @@ function buildMatrixQaE2eeStoragePaths(params: { outputDir: string; scenarioId: string; }) { - void params.scenarioId; const rootDir = path.join(params.outputDir, "matrix-e2ee", "accounts", params.actorId); const accountDir = path.join(rootDir, "account"); + const scenarioKey = params.scenarioId.replace(/[^A-Za-z0-9_-]/g, "-").slice(-80); const runKey = path .basename(params.outputDir) .replace(/[^A-Za-z0-9_-]/g, "-") @@ -136,7 +142,7 @@ function buildMatrixQaE2eeStoragePaths(params: { idbSnapshotPath: path.join(accountDir, "crypto-idb-snapshot.json"), recoveryKeyPath: path.join(accountDir, "recovery-key.json"), rootDir, - storagePath: path.join(accountDir, "sync-store.json"), + storagePath: path.join(rootDir, "scenarios", scenarioKey || "scenario", "sync-store.json"), }; } @@ -148,6 +154,7 @@ async function prepareMatrixQaE2eeStorage(params: { const storage = buildMatrixQaE2eeStoragePaths(params); await fs.mkdir(storage.rootDir, { recursive: true }); await fs.mkdir(storage.accountDir, { recursive: true }); + await fs.mkdir(path.dirname(storage.storagePath), { recursive: true }); await fs.writeFile(storage.idbSnapshotPath, "[]\n", { flag: "wx" }).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code !== "EEXIST") { throw error; @@ -174,6 +181,7 @@ async function createMatrixQaE2eeMatrixClient(params: MatrixQaE2eeClientParams) recoveryKeyPath: storage.recoveryKeyPath, ssrfPolicy: { allowPrivateNetwork: true }, storagePath: storage.storagePath, + syncFilter: MATRIX_QA_E2EE_SYNC_FILTER, userId: params.userId, }); } @@ -394,6 +402,7 @@ export async function runMatrixQaE2eeBootstrap( } export const __testing = { + MATRIX_QA_E2EE_SYNC_FILTER, buildMatrixQaE2eeStoragePaths, findMatrixQaObservedEventMatch, };