diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 8f35aaf9053..36c3e9065ad 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -128,11 +128,11 @@ The doctor checks Convex broker env, validates endpoint settings, and verifies a Live transport lanes share one contract instead of each inventing their own scenario list shape. `qa-channel` is the broad synthetic product-behavior suite and is not part of the live transport coverage matrix. -| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration | -| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- | -| Matrix | x | x | x | x | x | x | x | x | | | -| Telegram | x | x | | | | | | | x | | -| Discord | x | x | | | | | | | | x | +| Lane | Canary | Mention gating | Bot-to-bot | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration | +| -------- | ------ | -------------- | ---------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- | +| Matrix | x | x | x | x | x | x | x | x | x | | | +| Telegram | x | x | x | | | | | | | x | | +| Discord | x | x | x | | | | | | | | x | This keeps `qa-channel` as the broad product-behavior suite while Matrix, Telegram, and future live transports share one explicit transport-contract diff --git a/docs/concepts/qa-matrix.md b/docs/concepts/qa-matrix.md index 13cf49da864..3e323e333cd 100644 --- a/docs/concepts/qa-matrix.md +++ b/docs/concepts/qa-matrix.md @@ -88,7 +88,7 @@ The full scenario id list is the `MatrixQaScenarioId` union in `extensions/qa-ma - reactions — `matrix-reaction-*` - approvals — `matrix-approval-*` (exec/plugin metadata, chunked fallback, deny reactions, threads, and `target: "both"` routing) - restart and replay — `matrix-restart-*`, `matrix-stale-sync-replay-dedupe`, `matrix-room-membership-loss`, `matrix-homeserver-restart-resume`, `matrix-initial-catchup-then-incremental` -- mention gating and allowlists — `matrix-mention-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override` +- mention gating, bot-to-bot, and allowlists — `matrix-mention-*`, `matrix-allowbots-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override` - E2EE — `matrix-e2ee-*` (basic reply, thread follow-up, bootstrap, recovery key lifecycle, state-loss variants, server backup behavior, device hygiene, SAS / QR / DM verification, restart, artifact redaction) - E2EE CLI — `matrix-e2ee-cli-*` (encryption setup, idempotent setup, bootstrap failure, recovery-key lifecycle, multi-account, gateway-reply round-trip, self-verification) diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 87cf542aab4..f042d9cfe5e 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -259,8 +259,10 @@ function formatMatrixQaScenarioDetails(params: { details: string; configSummary? function buildMatrixQaScenarioConfigEntry(params: { gatewayConfigParams: { + driverAccessToken?: string; driverUserId: string; homeserver: string; + observerAccessToken?: string; observerUserId: string; sutAccessToken: string; sutAccountId: string; @@ -628,8 +630,10 @@ export async function runMatrixQaLive(params: { let scenarioTransportInterruptMs = 0; const scenarioTimings: MatrixQaScenarioTiming[] = []; const gatewayConfigParams = { + driverAccessToken: provisioning.driver.accessToken, driverUserId: provisioning.driver.userId, homeserver: harness.baseUrl, + observerAccessToken: provisioning.observer.accessToken, observerUserId: provisioning.observer.userId, sutAccessToken: provisioning.sut.accessToken, sutAccountId, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index 9c73b074c07..827d43a5769 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -57,6 +57,14 @@ export type MatrixQaScenarioId = | "matrix-room-membership-loss" | "matrix-homeserver-restart-resume" | "matrix-mention-gating" + | "matrix-allowbots-default-block" + | "matrix-allowbots-true-unmentioned-open-room" + | "matrix-allowbots-mentions-mentioned-room" + | "matrix-allowbots-mentions-unmentioned-open-room-block" + | "matrix-allowbots-mentions-dm-unmentioned" + | "matrix-allowbots-room-override-blocks-account-true" + | "matrix-allowbots-room-override-enables-account-off" + | "matrix-allowbots-self-sender-ignored" | "matrix-mxid-prefixed-command-block" | "matrix-mention-metadata-spoof-block" | "matrix-observer-allowlist-override" @@ -117,6 +125,7 @@ export type MatrixQaProfile = | "transport"; export const MATRIX_QA_BLOCK_ROOM_KEY = "block"; +export const MATRIX_QA_BOT_DM_ROOM_KEY = "bot-dm"; export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm"; export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared"; export const MATRIX_QA_E2EE_ROOM_KEY = "e2ee"; @@ -137,6 +146,7 @@ const MATRIX_QA_E2EE_MEDIA_TIMEOUT_MS = 180_000; function buildMatrixQaDmTopology( rooms: Array<{ key: string; + members?: ["driver" | "observer", "sut"]; name: string; }>, ): MatrixQaTopologySpec { @@ -145,7 +155,7 @@ function buildMatrixQaDmTopology( rooms: rooms.map((room) => ({ key: room.key, kind: "dm" as const, - members: ["driver", "sut"], + members: room.members ?? ["driver", "sut"], name: room.name, })), }; @@ -207,6 +217,14 @@ const MATRIX_QA_SHARED_DM_TOPOLOGY = buildMatrixQaDmTopology([ }, ]); +const MATRIX_QA_BOT_DM_TOPOLOGY = buildMatrixQaDmTopology([ + { + key: MATRIX_QA_BOT_DM_ROOM_KEY, + members: ["observer", "sut"], + name: "Matrix QA Observer/SUT Bot DM", + }, +]); + const MATRIX_QA_SECONDARY_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({ key: MATRIX_QA_SECONDARY_ROOM_KEY, name: "Matrix QA Secondary Room", @@ -655,6 +673,110 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 8_000, title: "Matrix room message without mention does not trigger", }, + { + id: "matrix-allowbots-default-block", + timeoutMs: 8_000, + title: "Matrix allowBots default blocks configured bot senders", + configOverrides: { + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + }, + }, + { + id: "matrix-allowbots-true-unmentioned-open-room", + timeoutMs: 45_000, + title: "Matrix allowBots=true accepts unmentioned configured bot messages in open rooms", + configOverrides: { + allowBots: true, + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + requireMention: false, + }, + }, + }, + }, + { + id: "matrix-allowbots-mentions-mentioned-room", + timeoutMs: 45_000, + title: "Matrix allowBots=mentions accepts mentioned configured bot messages", + configOverrides: { + allowBots: "mentions", + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + }, + }, + { + id: "matrix-allowbots-mentions-unmentioned-open-room-block", + timeoutMs: 8_000, + title: "Matrix allowBots=mentions blocks unmentioned configured bot messages in open rooms", + configOverrides: { + allowBots: "mentions", + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + requireMention: false, + }, + }, + }, + }, + { + id: "matrix-allowbots-mentions-dm-unmentioned", + timeoutMs: 45_000, + title: "Matrix allowBots=mentions accepts unmentioned configured bot DMs", + topology: MATRIX_QA_BOT_DM_TOPOLOGY, + configOverrides: { + allowBots: "mentions", + configuredBotRoles: ["observer"], + }, + }, + { + id: "matrix-allowbots-room-override-blocks-account-true", + timeoutMs: 8_000, + title: "Matrix room allowBots=false overrides account allowBots=true", + configOverrides: { + allowBots: true, + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + allowBots: false, + requireMention: false, + }, + }, + }, + }, + { + id: "matrix-allowbots-room-override-enables-account-off", + timeoutMs: 45_000, + title: "Matrix room allowBots=mentions overrides account allowBots off", + configOverrides: { + configuredBotRoles: ["observer"], + groupAllowRoles: ["driver", "observer"], + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + allowBots: "mentions", + requireMention: true, + }, + }, + }, + }, + { + id: "matrix-allowbots-self-sender-ignored", + timeoutMs: 8_000, + title: "Matrix allowBots=true still ignores messages from the SUT user id", + configOverrides: { + allowBots: true, + groupAllowRoles: ["driver", "observer", "sut"], + groupsByKey: { + [MATRIX_QA_MAIN_ROOM_KEY]: { + requireMention: false, + }, + }, + }, + }, { id: "matrix-mxid-prefixed-command-block", timeoutMs: 8_000, @@ -1077,6 +1199,8 @@ const MATRIX_QA_FAST_PROFILE_SCENARIO_IDS = [ "matrix-approval-exec-metadata-chunked", "matrix-restart-resume", "matrix-mention-gating", + "matrix-allowbots-default-block", + "matrix-allowbots-mentions-mentioned-room", "matrix-allowlist-block", "matrix-e2ee-basic-reply", ] satisfies MatrixQaScenarioId[]; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-allowbots.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-allowbots.ts new file mode 100644 index 00000000000..ca228b10690 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-allowbots.ts @@ -0,0 +1,143 @@ +import { MATRIX_QA_BOT_DM_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js"; +import { + buildExactMarkerPrompt, + buildMatrixQaToken, + buildMentionPrompt, + createMatrixQaScenarioClient, + resolveMatrixQaNoReplyWindowMs, + runNoReplyExpectedScenario, + runTopologyScopedTopLevelScenario, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +async function runObserverBotReplyScenario(params: { + context: MatrixQaScenarioContext; + roomKey?: string; + tokenPrefix: string; + withMention?: boolean; +}) { + return await runTopologyScopedTopLevelScenario({ + accessToken: params.context.observerAccessToken, + actorId: "observer", + actorUserId: params.context.observerUserId, + context: params.context, + roomKey: params.roomKey ?? params.context.topology.defaultRoomKey, + tokenPrefix: params.tokenPrefix, + ...(params.withMention === undefined ? {} : { withMention: params.withMention }), + }); +} + +async function runObserverBotNoReplyScenario(params: { + context: MatrixQaScenarioContext; + roomKey?: string; + tokenPrefix: string; + withMention?: boolean; +}) { + const token = buildMatrixQaToken(params.tokenPrefix); + const withMention = params.withMention !== false; + return await runNoReplyExpectedScenario({ + accessToken: params.context.observerAccessToken, + actorId: "observer", + actorUserId: params.context.observerUserId, + baseUrl: params.context.baseUrl, + body: withMention + ? buildMentionPrompt(params.context.sutUserId, token) + : buildExactMarkerPrompt(token), + ...(withMention ? { mentionUserIds: [params.context.sutUserId] } : {}), + observedEvents: params.context.observedEvents, + roomId: resolveMatrixQaScenarioRoomId(params.context, params.roomKey), + syncState: params.context.syncState, + syncStreams: params.context.syncStreams, + sutUserId: params.context.sutUserId, + timeoutMs: resolveMatrixQaNoReplyWindowMs(params.context.timeoutMs), + token, + }); +} + +export async function runAllowBotsDefaultBlockScenario(context: MatrixQaScenarioContext) { + return await runObserverBotNoReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_DEFAULT_BLOCK", + }); +} + +export async function runAllowBotsTrueUnmentionedOpenRoomScenario( + context: MatrixQaScenarioContext, +) { + return await runObserverBotReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_TRUE_OPEN", + withMention: false, + }); +} + +export async function runAllowBotsMentionsMentionedRoomScenario(context: MatrixQaScenarioContext) { + return await runObserverBotReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_MENTIONED", + }); +} + +export async function runAllowBotsMentionsUnmentionedOpenRoomBlockScenario( + context: MatrixQaScenarioContext, +) { + return await runObserverBotNoReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_OPEN_BLOCK", + withMention: false, + }); +} + +export async function runAllowBotsMentionsDmUnmentionedScenario(context: MatrixQaScenarioContext) { + return await runObserverBotReplyScenario({ + context, + roomKey: MATRIX_QA_BOT_DM_ROOM_KEY, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_DM", + withMention: false, + }); +} + +export async function runAllowBotsRoomOverrideBlocksAccountTrueScenario( + context: MatrixQaScenarioContext, +) { + return await runObserverBotNoReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_ROOM_BLOCK", + withMention: false, + }); +} + +export async function runAllowBotsRoomOverrideEnablesAccountOffScenario( + context: MatrixQaScenarioContext, +) { + return await runObserverBotReplyScenario({ + context, + tokenPrefix: "MATRIX_QA_ALLOWBOTS_ROOM_ENABLE", + }); +} + +export async function runAllowBotsSelfSenderIgnoredScenario( + context: MatrixQaScenarioContext, +): Promise { + const sutSender = createMatrixQaScenarioClient({ + accessToken: context.sutAccessToken, + baseUrl: context.baseUrl, + }); + const token = buildMatrixQaToken("MATRIX_QA_ALLOWBOTS_SELF_IGNORED"); + return await runNoReplyExpectedScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + actorUserId: context.sutUserId, + baseUrl: context.baseUrl, + body: buildExactMarkerPrompt(token), + observedEvents: context.observedEvents, + roomId: context.roomId, + sendClient: sutSender, + syncState: context.syncState, + syncStreams: context.syncStreams, + sutUserId: context.sutUserId, + timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs), + token, + }); +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index d5dad267d89..ddc11c109fc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -340,7 +340,7 @@ export function advanceMatrixQaActorCursor(params: { writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince); } -type MatrixQaScenarioClient = ReturnType; +export type MatrixQaScenarioClient = ReturnType; export async function assertNoSutReplyWindow(params: { actorId: MatrixQaActorId; @@ -600,6 +600,7 @@ export async function runNoReplyExpectedScenario(params: { mentionUserIds?: string[]; observedEvents: MatrixQaObservedEvent[]; roomId: string; + sendClient?: MatrixQaScenarioClient; syncState: MatrixQaSyncState; syncStreams?: MatrixQaSyncStreams; sutUserId: string; @@ -618,7 +619,8 @@ export async function runNoReplyExpectedScenario(params: { syncState: params.syncState, syncStreams: params.syncStreams, }); - const driverEventId = await client.sendTextMessage({ + const sendClient = params.sendClient ?? client; + const triggerEventId = await sendClient.sendTextMessage({ body: params.body, ...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}), roomId: params.roomId, @@ -630,7 +632,7 @@ export async function runNoReplyExpectedScenario(params: { if (event.roomId !== params.roomId) { return false; } - if (event.eventId === driverEventId) { + if (event.eventId === triggerEventId) { observedTriggerEvent = true; return false; } @@ -638,7 +640,8 @@ export async function runNoReplyExpectedScenario(params: { observedTriggerEvent && event.sender === params.sutUserId && event.type === "m.room.message" && - (params.replyPredicate?.(event, { driverEventId, token: params.token }) ?? true) + (params.replyPredicate?.(event, { driverEventId: triggerEventId, token: params.token }) ?? + true) ); }, roomId: params.roomId, @@ -664,13 +667,13 @@ export async function runNoReplyExpectedScenario(params: { return { artifacts: { actorUserId: params.actorUserId, - driverEventId, + driverEventId: triggerEventId, expectedNoReplyWindowMs: params.timeoutMs, token: params.token, triggerBody: params.body, }, details: [ - `trigger event: ${driverEventId}`, + `trigger event: ${triggerEventId}`, `trigger sender: ${params.actorUserId}`, `waited ${params.timeoutMs}ms with no SUT reply`, ].join("\n"), diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index e82fe2a2001..5ea4249b800 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -3,6 +3,16 @@ import { MATRIX_QA_SECONDARY_ROOM_KEY, type MatrixQaScenarioDefinition, } from "./scenario-catalog.js"; +import { + runAllowBotsDefaultBlockScenario, + runAllowBotsMentionsDmUnmentionedScenario, + runAllowBotsMentionsMentionedRoomScenario, + runAllowBotsMentionsUnmentionedOpenRoomBlockScenario, + runAllowBotsRoomOverrideBlocksAccountTrueScenario, + runAllowBotsRoomOverrideEnablesAccountOffScenario, + runAllowBotsSelfSenderIgnoredScenario, + runAllowBotsTrueUnmentionedOpenRoomScenario, +} from "./scenario-runtime-allowbots.js"; import { runApprovalChannelTargetBothScenario, runApprovalDenyReactionScenario, @@ -165,6 +175,7 @@ async function runNoReplyScenario(params: { observedEvents: params.context.observedEvents, roomId: params.context.roomId, syncState: params.context.syncState, + syncStreams: params.context.syncStreams, sutUserId: params.context.sutUserId, timeoutMs, token: params.token, @@ -313,6 +324,22 @@ export async function runMatrixQaScenario( token, }); } + case "matrix-allowbots-default-block": + return await runAllowBotsDefaultBlockScenario(context); + case "matrix-allowbots-true-unmentioned-open-room": + return await runAllowBotsTrueUnmentionedOpenRoomScenario(context); + case "matrix-allowbots-mentions-mentioned-room": + return await runAllowBotsMentionsMentionedRoomScenario(context); + case "matrix-allowbots-mentions-unmentioned-open-room-block": + return await runAllowBotsMentionsUnmentionedOpenRoomBlockScenario(context); + case "matrix-allowbots-mentions-dm-unmentioned": + return await runAllowBotsMentionsDmUnmentionedScenario(context); + case "matrix-allowbots-room-override-blocks-account-true": + return await runAllowBotsRoomOverrideBlocksAccountTrueScenario(context); + case "matrix-allowbots-room-override-enables-account-off": + return await runAllowBotsRoomOverrideEnablesAccountOffScenario(context); + case "matrix-allowbots-self-sender-ignored": + return await runAllowBotsSelfSenderIgnoredScenario(context); case "matrix-mxid-prefixed-command-block": { const token = buildMatrixQaToken("MATRIX_QA_MXID_COMMAND"); return await runNoReplyScenario({ diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 4dc9b2624fc..f7e99537d5b 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -79,7 +79,21 @@ function matrixQaScenarioContext(): MatrixQaScenarioContext { topology: { defaultRoomId: "!main:matrix-qa.test", defaultRoomKey: "main", - rooms: [], + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Main", + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + ], }, }; } @@ -244,6 +258,14 @@ describe("matrix live qa scenarios", () => { "matrix-room-membership-loss", "matrix-homeserver-restart-resume", "matrix-mention-gating", + "matrix-allowbots-default-block", + "matrix-allowbots-true-unmentioned-open-room", + "matrix-allowbots-mentions-mentioned-room", + "matrix-allowbots-mentions-unmentioned-open-room-block", + "matrix-allowbots-mentions-dm-unmentioned", + "matrix-allowbots-room-override-blocks-account-true", + "matrix-allowbots-room-override-enables-account-off", + "matrix-allowbots-self-sender-ignored", "matrix-mxid-prefixed-command-block", "matrix-mention-metadata-spoof-block", "matrix-observer-allowlist-override", @@ -329,6 +351,8 @@ describe("matrix live qa scenarios", () => { "matrix-approval-exec-metadata-chunked", "matrix-restart-resume", "matrix-mention-gating", + "matrix-allowbots-default-block", + "matrix-allowbots-mentions-mentioned-room", "matrix-allowlist-block", "matrix-e2ee-basic-reply", ]); @@ -799,6 +823,134 @@ describe("matrix live qa scenarios", () => { }); }); + it("runs mentioned allowBots=mentions room traffic through the observer bot account", async () => { + const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$observer-bot-trigger"); + const waitForRoomEvent = vi.fn().mockImplementation(async () => ({ + event: { + kind: "message", + roomId: "!main:matrix-qa.test", + eventId: "$sut-bot-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace( + "@sut:matrix-qa.test reply with only this exact marker: ", + "", + ), + }, + since: "observer-sync-next", + })); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-allowbots-mentions-mentioned-room", + ); + expect(scenario).toBeDefined(); + + await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({ + artifacts: { + actorUserId: "@observer:matrix-qa.test", + driverEventId: "$observer-bot-trigger", + reply: { + tokenMatched: true, + }, + }, + }); + + expect(createMatrixQaClient).toHaveBeenCalledWith({ + accessToken: "observer-token", + baseUrl: "http://127.0.0.1:28008/", + }); + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("@sut:matrix-qa.test reply with only this exact marker:"), + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }); + }); + + it("blocks unmentioned allowBots=mentions room traffic even when the room is open", async () => { + const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); + const sendTextMessage = vi.fn().mockResolvedValue("$observer-bot-unmentioned"); + const waitForOptionalRoomEvent = vi.fn().mockResolvedValue({ + matched: false, + since: "observer-sync-next", + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForOptionalRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-allowbots-mentions-unmentioned-open-room-block", + ); + expect(scenario).toBeDefined(); + + await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({ + artifacts: { + actorUserId: "@observer:matrix-qa.test", + driverEventId: "$observer-bot-unmentioned", + }, + }); + + expect(sendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("reply with only this exact marker:"), + roomId: "!main:matrix-qa.test", + }); + }); + + it("uses the SUT account as the sender for the self-sender allowBots loop guard", async () => { + const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); + const observerWaitForOptionalRoomEvent = vi.fn().mockResolvedValue({ + matched: false, + since: "observer-sync-next", + }); + const observerSendTextMessage = vi.fn(); + const sutSendTextMessage = vi.fn().mockResolvedValue("$sut-self-trigger"); + + createMatrixQaClient + .mockReturnValueOnce({ + sendTextMessage: sutSendTextMessage, + }) + .mockReturnValueOnce({ + primeRoom, + sendTextMessage: observerSendTextMessage, + waitForOptionalRoomEvent: observerWaitForOptionalRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-allowbots-self-sender-ignored", + ); + expect(scenario).toBeDefined(); + + await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({ + artifacts: { + actorUserId: "@sut:matrix-qa.test", + driverEventId: "$sut-self-trigger", + }, + }); + + expect(createMatrixQaClient).toHaveBeenNthCalledWith(1, { + accessToken: "sut-token", + baseUrl: "http://127.0.0.1:28008/", + }); + expect(createMatrixQaClient).toHaveBeenNthCalledWith(2, { + accessToken: "observer-token", + baseUrl: "http://127.0.0.1:28008/", + }); + expect(observerSendTextMessage).not.toHaveBeenCalled(); + expect(sutSendTextMessage).toHaveBeenCalledWith({ + body: expect.stringContaining("reply with only this exact marker:"), + roomId: "!main:matrix-qa.test", + }); + }); + it("blocks MXID-prefixed Matrix control commands from non-allowlisted observers", async () => { const primeRoom = vi.fn().mockResolvedValue("observer-sync-start"); const sendTextMessage = vi.fn().mockResolvedValue("$observer-command-trigger"); diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.ts b/extensions/qa-matrix/src/runners/contract/scenarios.ts index 304a14cb45f..0bc076ebefc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.ts @@ -1,4 +1,5 @@ import { + MATRIX_QA_BOT_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, MATRIX_QA_E2EE_ROOM_KEY, @@ -61,6 +62,7 @@ export type { export type { MatrixQaScenarioContext, MatrixQaSyncState }; export const __testing = { + MATRIX_QA_BOT_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_ROOM_KEY, MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY, MATRIX_QA_E2EE_ROOM_KEY, diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index 0537072642e..5ab9715766b 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -104,9 +104,12 @@ describe("matrix qa config", () => { threadReplies: "off", }, encryption: true, + allowBots: "mentions", + configuredBotRoles: ["observer"], groupAllowFrom: ["@driver:matrix-qa.test", "@observer:matrix-qa.test"], groupsByKey: { secondary: { + allowBots: false, requireMention: false, tools: { allow: ["sessions_spawn"], @@ -123,6 +126,7 @@ describe("matrix qa config", () => { threadReplies: "always", toolProfile: "coding", }, + observerAccessToken: "observer-token", sutAccessToken: "sut-token", sutAccountId: "sut", sutUserId: "@sut:matrix-qa.test", @@ -144,7 +148,14 @@ describe("matrix qa config", () => { expect(next.tools).toMatchObject({ profile: "coding", }); + expect(next.channels?.matrix?.accounts?.["qa-observer-bot-source"]).toMatchObject({ + accessToken: "observer-token", + enabled: false, + homeserver: "http://127.0.0.1:28008/", + userId: "@observer:matrix-qa.test", + }); expect(next.channels?.matrix?.accounts?.sut).toMatchObject({ + allowBots: "mentions", autoJoin: "allowlist", autoJoinAllowlist: ["!dm:matrix-qa.test", "#ops:matrix-qa.test"], blockStreaming: true, @@ -157,6 +168,7 @@ describe("matrix qa config", () => { groups: { "!main:matrix-qa.test": { enabled: true, requireMention: true }, "!secondary:matrix-qa.test": { + allowBots: false, enabled: true, requireMention: false, tools: { @@ -231,6 +243,7 @@ describe("matrix qa config", () => { exec: false, plugin: false, }, + allowBots: undefined, autoJoin: "allowlist", autoJoinAllowlist: ["!ops:matrix-qa.test"], blockStreaming: true, @@ -244,6 +257,7 @@ describe("matrix qa config", () => { }, encryption: false, execApprovals: undefined, + configuredBotRoles: [], groupAllowFrom: ["@driver:matrix-qa.test"], groupPolicy: "open", groupsByKey: { @@ -265,6 +279,8 @@ describe("matrix qa config", () => { threadBindings: {}, threadReplies: "inbound", }); + expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("allowBots="); + expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("configuredBotRoles="); expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("autoJoin=allowlist"); expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("streaming=partial"); expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain( @@ -354,6 +370,40 @@ describe("matrix qa config", () => { expect(snapshot.groupAllowFrom).toEqual(["@driver:matrix-qa.test", "@observer:matrix-qa.test"]); }); + it("rejects configured bot roles without matching side-account auth", () => { + expect(() => + buildMatrixQaConfig({} as OpenClawConfig, { + driverUserId: "@driver:matrix-qa.test", + homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", + overrides: { + configuredBotRoles: ["observer"], + }, + sutAccessToken: "sut-token", + sutAccountId: "sut", + sutUserId: "@sut:matrix-qa.test", + topology, + }), + ).toThrow('Matrix QA configured bot role "observer" requires an access token'); + }); + + it("rejects the SUT role as a configured bot source", () => { + expect(() => + buildMatrixQaConfig({} as OpenClawConfig, { + driverUserId: "@driver:matrix-qa.test", + homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", + overrides: { + configuredBotRoles: ["sut"], + }, + sutAccessToken: "sut-token", + sutAccountId: "sut", + sutUserId: "@sut:matrix-qa.test", + topology, + }), + ).toThrow('Matrix QA configured bot role "sut" would match the SUT account itself'); + }); + it("rejects unknown room-key overrides", () => { expect(() => buildMatrixQaConfig({} as OpenClawConfig, { diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index ee6c292d9b4..abd3cbbacd7 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -11,6 +11,7 @@ export type MatrixQaActorRole = "driver" | "observer" | "sut"; export type MatrixQaChunkMode = "length" | "newline"; export type MatrixQaExecApprovalTarget = "both" | "channel" | "dm"; export type MatrixQaExecApprovalsEnabled = boolean | "auto"; +export type MatrixQaAllowBotsMode = boolean | "mentions"; export type MatrixQaStreamingConfig = { mode?: MatrixQaStreamingMode; @@ -38,6 +39,7 @@ export type MatrixQaToolConfigOverrides = { }; export type MatrixQaGroupConfigOverrides = { + allowBots?: MatrixQaAllowBotsMode; enabled?: boolean; requireMention?: boolean; tools?: MatrixQaToolConfigOverrides; @@ -73,6 +75,7 @@ export type MatrixQaConfigOverrides = { plugin?: boolean; }; agentDefaults?: MatrixQaAgentDefaultsOverrides; + allowBots?: MatrixQaAllowBotsMode; autoJoin?: MatrixQaAutoJoinMode; autoJoinAllowlist?: string[]; blockStreaming?: boolean; @@ -83,6 +86,7 @@ export type MatrixQaConfigOverrides = { groupAllowFrom?: string[]; groupAllowRoles?: MatrixQaActorRole[]; groupPolicy?: MatrixQaGroupPolicy; + configuredBotRoles?: MatrixQaActorRole[]; groupsByKey?: Record; replyToMode?: MatrixQaReplyToMode; startupVerification?: "if-unverified" | "off"; @@ -100,6 +104,7 @@ export type MatrixQaConfigSnapshot = { }; autoJoin: MatrixQaAutoJoinMode; autoJoinAllowlist: string[]; + allowBots?: MatrixQaAllowBotsMode; blockStreaming: boolean; chunkMode?: MatrixQaChunkMode; dm: { @@ -111,6 +116,7 @@ export type MatrixQaConfigSnapshot = { }; encryption: boolean; execApprovals?: MatrixQaExecApprovalsConfigOverrides; + configuredBotRoles: MatrixQaActorRole[]; groupAllowFrom: string[]; groupPolicy: MatrixQaGroupPolicy; groupsByKey: Record; @@ -124,6 +130,7 @@ export type MatrixQaConfigSnapshot = { }; type MatrixQaGroupSnapshot = { + allowBots?: MatrixQaAllowBotsMode; enabled: boolean; requireMention: boolean; roomId: string; @@ -180,6 +187,9 @@ function resolveMatrixQaGroupSnapshots(params: { { roomId: room.roomId, enabled: override?.enabled ?? true, + ...(override && Object.hasOwn(override, "allowBots") + ? { allowBots: override.allowBots } + : {}), requireMention: override?.requireMention ?? room.requireMention, ...(override?.tools ? { tools: override.tools } : {}), }, @@ -195,6 +205,7 @@ function buildMatrixQaGroupEntries( Object.values(groupsByKey).map((group) => [ group.roomId, { + ...(group.allowBots !== undefined ? { allowBots: group.allowBots } : {}), enabled: group.enabled, requireMention: group.requireMention, ...(group.tools ? { tools: group.tools } : {}), @@ -347,6 +358,59 @@ function buildMatrixQaAccountExecApprovalsConfig( }; } +function buildMatrixQaConfiguredBotAccounts(params: { + driverAccessToken: string | undefined; + driverUserId: string; + homeserver: string; + observerAccessToken: string | undefined; + observerUserId: string; + roles: MatrixQaActorRole[]; +}): Record { + const selectedRoles = new Set(params.roles); + if (selectedRoles.has("sut")) { + throw new Error('Matrix QA configured bot role "sut" would match the SUT account itself'); + } + + const botSources: Record< + Exclude, + { + accessToken: string | undefined; + accountId: string; + userId: string; + } + > = { + driver: { + accessToken: params.driverAccessToken, + accountId: "qa-driver-bot-source", + userId: params.driverUserId, + }, + observer: { + accessToken: params.observerAccessToken, + accountId: "qa-observer-bot-source", + userId: params.observerUserId, + }, + }; + + const accounts: Record = {}; + for (const role of selectedRoles) { + if (role !== "driver" && role !== "observer") { + continue; + } + const source = botSources[role]; + if (!source.accessToken) { + throw new Error(`Matrix QA configured bot role "${role}" requires an access token`); + } + accounts[source.accountId] = { + accessToken: source.accessToken, + enabled: false, + homeserver: params.homeserver, + userId: source.userId, + }; + } + + return accounts; +} + function buildMatrixQaChannelAccountConfig(params: { groups: Record; homeserver: string; @@ -394,6 +458,7 @@ function buildMatrixQaChannelAccountConfig(params: { dmOverrides: params.overrides?.dm, snapshot: params.snapshot, }), + ...(params.snapshot.allowBots !== undefined ? { allowBots: params.snapshot.allowBots } : {}), enabled: true, encryption: params.snapshot.encryption, groupAllowFrom: params.snapshot.groupAllowFrom, @@ -426,6 +491,7 @@ export function buildMatrixQaConfigSnapshot(params: { topology: MatrixQaProvisionedTopology; }): MatrixQaConfigSnapshot { return { + allowBots: params.overrides?.allowBots, autoJoin: params.overrides?.autoJoin ?? "off", autoJoinAllowlist: resolveMatrixQaAutoJoinAllowlist(params), blockStreaming: params.overrides?.blockStreaming ?? false, @@ -433,6 +499,7 @@ export function buildMatrixQaConfigSnapshot(params: { dm: resolveMatrixQaDmConfigSnapshot(params), encryption: params.overrides?.encryption ?? false, execApprovals: params.overrides?.execApprovals, + configuredBotRoles: [...(params.overrides?.configuredBotRoles ?? [])], groupAllowFrom: resolveMatrixQaGroupAllowFrom(params), groupPolicy: params.overrides?.groupPolicy ?? "allowlist", groupsByKey: resolveMatrixQaGroupSnapshots({ @@ -458,6 +525,8 @@ export function buildMatrixQaConfigSnapshot(params: { export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot) { return [ + `allowBots=${snapshot.allowBots ?? ""}`, + `configuredBotRoles=${snapshot.configuredBotRoles.length > 0 ? snapshot.configuredBotRoles.join("|") : ""}`, `replyToMode=${snapshot.replyToMode}`, `threadReplies=${snapshot.threadReplies}`, `dm.enabled=${formatMatrixQaBoolean(snapshot.dm.enabled)}`, @@ -484,8 +553,10 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot export function buildMatrixQaConfig( baseCfg: OpenClawConfig, params: { + driverAccessToken?: string; driverUserId: string; homeserver: string; + observerAccessToken?: string; observerUserId: string; overrides?: MatrixQaConfigOverrides; sutAccessToken: string; @@ -504,6 +575,14 @@ export function buildMatrixQaConfig( topology: params.topology, }); const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey); + const configuredBotAccounts = buildMatrixQaConfiguredBotAccounts({ + driverAccessToken: params.driverAccessToken, + driverUserId: params.driverUserId, + homeserver: params.homeserver, + observerAccessToken: params.observerAccessToken, + observerUserId: params.observerUserId, + roles: snapshot.configuredBotRoles, + }); const approvalForwardingConfig = snapshot.approvalForwarding.exec || snapshot.approvalForwarding.plugin ? { @@ -569,6 +648,7 @@ export function buildMatrixQaConfig( defaultAccount: params.sutAccountId, accounts: { ...baseCfg.channels?.matrix?.accounts, + ...configuredBotAccounts, [params.sutAccountId]: buildMatrixQaChannelAccountConfig({ groups, homeserver: params.homeserver,