vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 06:00:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(qa-matrix): cover allowBots modes

Browse Source
This commit is contained in:
Gustavo Madeira Santana
2026-04-28 00:16:39 -04:00
parent 6d7901f5c8
commit d59f001507
11 changed files with 599 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/concepts/qa-e2e-automation.md
View File

@@ -128,11 +128,11 @@ The doctor checks Convex broker env, validates endpoint settings, and verifies a
Live transport lanes share one contract instead of each inventing their own scenario list shape. `qa-channel` is the broad synthetic product-behavior suite and is not part of the live transport coverage matrix.
| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration |
| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- |
| Matrix | x | x | x | x | x | x | x | x | | |
| Telegram | x | x | | | | | | | x | |
| Discord | x | x | | | | | | | | x |
| Lane | Canary | Mention gating | Bot-to-bot | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration |
| -------- | ------ | -------------- | ---------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- |
| Matrix | x | x | x | x | x | x | x | x | x | | |
| Telegram | x | x | x | | | | | | | x | |
| Discord | x | x | x | | | | | | | | x |
This keeps `qa-channel` as the broad product-behavior suite while Matrix,
Telegram, and future live transports share one explicit transport-contract

2
docs/concepts/qa-matrix.md
View File

@@ -88,7 +88,7 @@ The full scenario id list is the `MatrixQaScenarioId` union in `extensions/qa-ma
- reactions — `matrix-reaction-*`
- approvals — `matrix-approval-*` (exec/plugin metadata, chunked fallback, deny reactions, threads, and `target: "both"` routing)
- restart and replay — `matrix-restart-*`, `matrix-stale-sync-replay-dedupe`, `matrix-room-membership-loss`, `matrix-homeserver-restart-resume`, `matrix-initial-catchup-then-incremental`
- mention gating and allowlists — `matrix-mention-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override`
- mention gating, bot-to-bot, and allowlists — `matrix-mention-*`, `matrix-allowbots-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override`
- E2EE — `matrix-e2ee-*` (basic reply, thread follow-up, bootstrap, recovery key lifecycle, state-loss variants, server backup behavior, device hygiene, SAS / QR / DM verification, restart, artifact redaction)
- E2EE CLI — `matrix-e2ee-cli-*` (encryption setup, idempotent setup, bootstrap failure, recovery-key lifecycle, multi-account, gateway-reply round-trip, self-verification)

4
extensions/qa-matrix/src/runners/contract/runtime.ts
View File

@@ -259,8 +259,10 @@ function formatMatrixQaScenarioDetails(params: { details: string; configSummary?
function buildMatrixQaScenarioConfigEntry(params: {
gatewayConfigParams: {
driverAccessToken?: string;
driverUserId: string;
homeserver: string;
observerAccessToken?: string;
observerUserId: string;
sutAccessToken: string;
sutAccountId: string;
@@ -628,8 +630,10 @@ export async function runMatrixQaLive(params: {
let scenarioTransportInterruptMs = 0;
const scenarioTimings: MatrixQaScenarioTiming[] = [];
const gatewayConfigParams = {
driverAccessToken: provisioning.driver.accessToken,
driverUserId: provisioning.driver.userId,
homeserver: harness.baseUrl,
observerAccessToken: provisioning.observer.accessToken,
observerUserId: provisioning.observer.userId,
sutAccessToken: provisioning.sut.accessToken,
sutAccountId,

126
extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
View File

@@ -57,6 +57,14 @@ export type MatrixQaScenarioId =
| "matrix-room-membership-loss"
| "matrix-homeserver-restart-resume"
| "matrix-mention-gating"
| "matrix-allowbots-default-block"
| "matrix-allowbots-true-unmentioned-open-room"
| "matrix-allowbots-mentions-mentioned-room"
| "matrix-allowbots-mentions-unmentioned-open-room-block"
| "matrix-allowbots-mentions-dm-unmentioned"
| "matrix-allowbots-room-override-blocks-account-true"
| "matrix-allowbots-room-override-enables-account-off"
| "matrix-allowbots-self-sender-ignored"
| "matrix-mxid-prefixed-command-block"
| "matrix-mention-metadata-spoof-block"
| "matrix-observer-allowlist-override"
@@ -117,6 +125,7 @@ export type MatrixQaProfile =
| "transport";
export const MATRIX_QA_BLOCK_ROOM_KEY = "block";
export const MATRIX_QA_BOT_DM_ROOM_KEY = "bot-dm";
export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm";
export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared";
export const MATRIX_QA_E2EE_ROOM_KEY = "e2ee";
@@ -137,6 +146,7 @@ const MATRIX_QA_E2EE_MEDIA_TIMEOUT_MS = 180_000;
function buildMatrixQaDmTopology(
rooms: Array<{
key: string;
members?: ["driver" | "observer", "sut"];
name: string;
}>,
): MatrixQaTopologySpec {
@@ -145,7 +155,7 @@ function buildMatrixQaDmTopology(
rooms: rooms.map((room) => ({
key: room.key,
kind: "dm" as const,
members: ["driver", "sut"],
members: room.members ?? ["driver", "sut"],
name: room.name,
})),
};
@@ -207,6 +217,14 @@ const MATRIX_QA_SHARED_DM_TOPOLOGY = buildMatrixQaDmTopology([
},
]);
const MATRIX_QA_BOT_DM_TOPOLOGY = buildMatrixQaDmTopology([
{
key: MATRIX_QA_BOT_DM_ROOM_KEY,
members: ["observer", "sut"],
name: "Matrix QA Observer/SUT Bot DM",
},
]);
const MATRIX_QA_SECONDARY_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({
key: MATRIX_QA_SECONDARY_ROOM_KEY,
name: "Matrix QA Secondary Room",
@@ -655,6 +673,110 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
timeoutMs: 8_000,
title: "Matrix room message without mention does not trigger",
},
{
id: "matrix-allowbots-default-block",
timeoutMs: 8_000,
title: "Matrix allowBots default blocks configured bot senders",
configOverrides: {
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
},
},
{
id: "matrix-allowbots-true-unmentioned-open-room",
timeoutMs: 45_000,
title: "Matrix allowBots=true accepts unmentioned configured bot messages in open rooms",
configOverrides: {
allowBots: true,
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
groupsByKey: {
[MATRIX_QA_MAIN_ROOM_KEY]: {
requireMention: false,
},
},
},
},
{
id: "matrix-allowbots-mentions-mentioned-room",
timeoutMs: 45_000,
title: "Matrix allowBots=mentions accepts mentioned configured bot messages",
configOverrides: {
allowBots: "mentions",
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
},
},
{
id: "matrix-allowbots-mentions-unmentioned-open-room-block",
timeoutMs: 8_000,
title: "Matrix allowBots=mentions blocks unmentioned configured bot messages in open rooms",
configOverrides: {
allowBots: "mentions",
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
groupsByKey: {
[MATRIX_QA_MAIN_ROOM_KEY]: {
requireMention: false,
},
},
},
},
{
id: "matrix-allowbots-mentions-dm-unmentioned",
timeoutMs: 45_000,
title: "Matrix allowBots=mentions accepts unmentioned configured bot DMs",
topology: MATRIX_QA_BOT_DM_TOPOLOGY,
configOverrides: {
allowBots: "mentions",
configuredBotRoles: ["observer"],
},
},
{
id: "matrix-allowbots-room-override-blocks-account-true",
timeoutMs: 8_000,
title: "Matrix room allowBots=false overrides account allowBots=true",
configOverrides: {
allowBots: true,
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
groupsByKey: {
[MATRIX_QA_MAIN_ROOM_KEY]: {
allowBots: false,
requireMention: false,
},
},
},
},
{
id: "matrix-allowbots-room-override-enables-account-off",
timeoutMs: 45_000,
title: "Matrix room allowBots=mentions overrides account allowBots off",
configOverrides: {
configuredBotRoles: ["observer"],
groupAllowRoles: ["driver", "observer"],
groupsByKey: {
[MATRIX_QA_MAIN_ROOM_KEY]: {
allowBots: "mentions",
requireMention: true,
},
},
},
},
{
id: "matrix-allowbots-self-sender-ignored",
timeoutMs: 8_000,
title: "Matrix allowBots=true still ignores messages from the SUT user id",
configOverrides: {
allowBots: true,
groupAllowRoles: ["driver", "observer", "sut"],
groupsByKey: {
[MATRIX_QA_MAIN_ROOM_KEY]: {
requireMention: false,
},
},
},
},
{
id: "matrix-mxid-prefixed-command-block",
timeoutMs: 8_000,
@@ -1077,6 +1199,8 @@ const MATRIX_QA_FAST_PROFILE_SCENARIO_IDS = [
"matrix-approval-exec-metadata-chunked",
"matrix-restart-resume",
"matrix-mention-gating",
"matrix-allowbots-default-block",
"matrix-allowbots-mentions-mentioned-room",
"matrix-allowlist-block",
"matrix-e2ee-basic-reply",
] satisfies MatrixQaScenarioId[];

143
extensions/qa-matrix/src/runners/contract/scenario-runtime-allowbots.ts Normal file
View File

@@ -0,0 +1,143 @@
import { MATRIX_QA_BOT_DM_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js";
import {
buildExactMarkerPrompt,
buildMatrixQaToken,
buildMentionPrompt,
createMatrixQaScenarioClient,
resolveMatrixQaNoReplyWindowMs,
runNoReplyExpectedScenario,
runTopologyScopedTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
async function runObserverBotReplyScenario(params: {
context: MatrixQaScenarioContext;
roomKey?: string;
tokenPrefix: string;
withMention?: boolean;
}) {
return await runTopologyScopedTopLevelScenario({
accessToken: params.context.observerAccessToken,
actorId: "observer",
actorUserId: params.context.observerUserId,
context: params.context,
roomKey: params.roomKey ?? params.context.topology.defaultRoomKey,
tokenPrefix: params.tokenPrefix,
...(params.withMention === undefined ? {} : { withMention: params.withMention }),
});
}
async function runObserverBotNoReplyScenario(params: {
context: MatrixQaScenarioContext;
roomKey?: string;
tokenPrefix: string;
withMention?: boolean;
}) {
const token = buildMatrixQaToken(params.tokenPrefix);
const withMention = params.withMention !== false;
return await runNoReplyExpectedScenario({
accessToken: params.context.observerAccessToken,
actorId: "observer",
actorUserId: params.context.observerUserId,
baseUrl: params.context.baseUrl,
body: withMention
? buildMentionPrompt(params.context.sutUserId, token)
: buildExactMarkerPrompt(token),
...(withMention ? { mentionUserIds: [params.context.sutUserId] } : {}),
observedEvents: params.context.observedEvents,
roomId: resolveMatrixQaScenarioRoomId(params.context, params.roomKey),
syncState: params.context.syncState,
syncStreams: params.context.syncStreams,
sutUserId: params.context.sutUserId,
timeoutMs: resolveMatrixQaNoReplyWindowMs(params.context.timeoutMs),
token,
});
}
export async function runAllowBotsDefaultBlockScenario(context: MatrixQaScenarioContext) {
return await runObserverBotNoReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_DEFAULT_BLOCK",
});
}
export async function runAllowBotsTrueUnmentionedOpenRoomScenario(
context: MatrixQaScenarioContext,
) {
return await runObserverBotReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_TRUE_OPEN",
withMention: false,
});
}
export async function runAllowBotsMentionsMentionedRoomScenario(context: MatrixQaScenarioContext) {
return await runObserverBotReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_MENTIONED",
});
}
export async function runAllowBotsMentionsUnmentionedOpenRoomBlockScenario(
context: MatrixQaScenarioContext,
) {
return await runObserverBotNoReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_OPEN_BLOCK",
withMention: false,
});
}
export async function runAllowBotsMentionsDmUnmentionedScenario(context: MatrixQaScenarioContext) {
return await runObserverBotReplyScenario({
context,
roomKey: MATRIX_QA_BOT_DM_ROOM_KEY,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_MENTIONS_DM",
withMention: false,
});
}
export async function runAllowBotsRoomOverrideBlocksAccountTrueScenario(
context: MatrixQaScenarioContext,
) {
return await runObserverBotNoReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_ROOM_BLOCK",
withMention: false,
});
}
export async function runAllowBotsRoomOverrideEnablesAccountOffScenario(
context: MatrixQaScenarioContext,
) {
return await runObserverBotReplyScenario({
context,
tokenPrefix: "MATRIX_QA_ALLOWBOTS_ROOM_ENABLE",
});
}
export async function runAllowBotsSelfSenderIgnoredScenario(
context: MatrixQaScenarioContext,
): Promise<MatrixQaScenarioExecution> {
const sutSender = createMatrixQaScenarioClient({
accessToken: context.sutAccessToken,
baseUrl: context.baseUrl,
});
const token = buildMatrixQaToken("MATRIX_QA_ALLOWBOTS_SELF_IGNORED");
return await runNoReplyExpectedScenario({
accessToken: context.observerAccessToken,
actorId: "observer",
actorUserId: context.sutUserId,
baseUrl: context.baseUrl,
body: buildExactMarkerPrompt(token),
observedEvents: context.observedEvents,
roomId: context.roomId,
sendClient: sutSender,
syncState: context.syncState,
syncStreams: context.syncStreams,
sutUserId: context.sutUserId,
timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
token,
});
}

15
extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
View File

@@ -340,7 +340,7 @@ export function advanceMatrixQaActorCursor(params: {
writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince);
}
type MatrixQaScenarioClient = ReturnType<typeof createMatrixQaScenarioClient>;
export type MatrixQaScenarioClient = ReturnType<typeof createMatrixQaScenarioClient>;
export async function assertNoSutReplyWindow(params: {
actorId: MatrixQaActorId;
@@ -600,6 +600,7 @@ export async function runNoReplyExpectedScenario(params: {
mentionUserIds?: string[];
observedEvents: MatrixQaObservedEvent[];
roomId: string;
sendClient?: MatrixQaScenarioClient;
syncState: MatrixQaSyncState;
syncStreams?: MatrixQaSyncStreams;
sutUserId: string;
@@ -618,7 +619,8 @@ export async function runNoReplyExpectedScenario(params: {
syncState: params.syncState,
syncStreams: params.syncStreams,
});
const driverEventId = await client.sendTextMessage({
const sendClient = params.sendClient ?? client;
const triggerEventId = await sendClient.sendTextMessage({
body: params.body,
...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}),
roomId: params.roomId,
@@ -630,7 +632,7 @@ export async function runNoReplyExpectedScenario(params: {
if (event.roomId !== params.roomId) {
return false;
}
if (event.eventId === driverEventId) {
if (event.eventId === triggerEventId) {
observedTriggerEvent = true;
return false;
}
@@ -638,7 +640,8 @@ export async function runNoReplyExpectedScenario(params: {
observedTriggerEvent &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
(params.replyPredicate?.(event, { driverEventId, token: params.token }) ?? true)
(params.replyPredicate?.(event, { driverEventId: triggerEventId, token: params.token }) ??
true)
);
},
roomId: params.roomId,
@@ -664,13 +667,13 @@ export async function runNoReplyExpectedScenario(params: {
return {
artifacts: {
actorUserId: params.actorUserId,
driverEventId,
driverEventId: triggerEventId,
expectedNoReplyWindowMs: params.timeoutMs,
token: params.token,
triggerBody: params.body,
},
details: [
`trigger event: ${driverEventId}`,
`trigger event: ${triggerEventId}`,
`trigger sender: ${params.actorUserId}`,
`waited ${params.timeoutMs}ms with no SUT reply`,
].join("\n"),

27
extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
View File

@@ -3,6 +3,16 @@ import {
MATRIX_QA_SECONDARY_ROOM_KEY,
type MatrixQaScenarioDefinition,
} from "./scenario-catalog.js";
import {
runAllowBotsDefaultBlockScenario,
runAllowBotsMentionsDmUnmentionedScenario,
runAllowBotsMentionsMentionedRoomScenario,
runAllowBotsMentionsUnmentionedOpenRoomBlockScenario,
runAllowBotsRoomOverrideBlocksAccountTrueScenario,
runAllowBotsRoomOverrideEnablesAccountOffScenario,
runAllowBotsSelfSenderIgnoredScenario,
runAllowBotsTrueUnmentionedOpenRoomScenario,
} from "./scenario-runtime-allowbots.js";
import {
runApprovalChannelTargetBothScenario,
runApprovalDenyReactionScenario,
@@ -165,6 +175,7 @@ async function runNoReplyScenario(params: {
observedEvents: params.context.observedEvents,
roomId: params.context.roomId,
syncState: params.context.syncState,
syncStreams: params.context.syncStreams,
sutUserId: params.context.sutUserId,
timeoutMs,
token: params.token,
@@ -313,6 +324,22 @@ export async function runMatrixQaScenario(
token,
});
}
case "matrix-allowbots-default-block":
return await runAllowBotsDefaultBlockScenario(context);
case "matrix-allowbots-true-unmentioned-open-room":
return await runAllowBotsTrueUnmentionedOpenRoomScenario(context);
case "matrix-allowbots-mentions-mentioned-room":
return await runAllowBotsMentionsMentionedRoomScenario(context);
case "matrix-allowbots-mentions-unmentioned-open-room-block":
return await runAllowBotsMentionsUnmentionedOpenRoomBlockScenario(context);
case "matrix-allowbots-mentions-dm-unmentioned":
return await runAllowBotsMentionsDmUnmentionedScenario(context);
case "matrix-allowbots-room-override-blocks-account-true":
return await runAllowBotsRoomOverrideBlocksAccountTrueScenario(context);
case "matrix-allowbots-room-override-enables-account-off":
return await runAllowBotsRoomOverrideEnablesAccountOffScenario(context);
case "matrix-allowbots-self-sender-ignored":
return await runAllowBotsSelfSenderIgnoredScenario(context);
case "matrix-mxid-prefixed-command-block": {
const token = buildMatrixQaToken("MATRIX_QA_MXID_COMMAND");
return await runNoReplyScenario({

154
extensions/qa-matrix/src/runners/contract/scenarios.test.ts
View File

@@ -79,7 +79,21 @@ function matrixQaScenarioContext(): MatrixQaScenarioContext {
topology: {
defaultRoomId: "!main:matrix-qa.test",
defaultRoomKey: "main",
rooms: [],
rooms: [
{
key: "main",
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "Main",
requireMention: true,
roomId: "!main:matrix-qa.test",
},
],
},
};
}
@@ -244,6 +258,14 @@ describe("matrix live qa scenarios", () => {
"matrix-room-membership-loss",
"matrix-homeserver-restart-resume",
"matrix-mention-gating",
"matrix-allowbots-default-block",
"matrix-allowbots-true-unmentioned-open-room",
"matrix-allowbots-mentions-mentioned-room",
"matrix-allowbots-mentions-unmentioned-open-room-block",
"matrix-allowbots-mentions-dm-unmentioned",
"matrix-allowbots-room-override-blocks-account-true",
"matrix-allowbots-room-override-enables-account-off",
"matrix-allowbots-self-sender-ignored",
"matrix-mxid-prefixed-command-block",
"matrix-mention-metadata-spoof-block",
"matrix-observer-allowlist-override",
@@ -329,6 +351,8 @@ describe("matrix live qa scenarios", () => {
"matrix-approval-exec-metadata-chunked",
"matrix-restart-resume",
"matrix-mention-gating",
"matrix-allowbots-default-block",
"matrix-allowbots-mentions-mentioned-room",
"matrix-allowlist-block",
"matrix-e2ee-basic-reply",
]);
@@ -799,6 +823,134 @@ describe("matrix live qa scenarios", () => {
});
});
it("runs mentioned allowBots=mentions room traffic through the observer bot account", async () => {
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$observer-bot-trigger");
const waitForRoomEvent = vi.fn().mockImplementation(async () => ({
event: {
kind: "message",
roomId: "!main:matrix-qa.test",
eventId: "$sut-bot-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: String(sendTextMessage.mock.calls[0]?.[0]?.body).replace(
"@sut:matrix-qa.test reply with only this exact marker: ",
"",
),
},
since: "observer-sync-next",
}));
createMatrixQaClient.mockReturnValue({
primeRoom,
sendTextMessage,
waitForRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-allowbots-mentions-mentioned-room",
);
expect(scenario).toBeDefined();
await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({
artifacts: {
actorUserId: "@observer:matrix-qa.test",
driverEventId: "$observer-bot-trigger",
reply: {
tokenMatched: true,
},
},
});
expect(createMatrixQaClient).toHaveBeenCalledWith({
accessToken: "observer-token",
baseUrl: "http://127.0.0.1:28008/",
});
expect(sendTextMessage).toHaveBeenCalledWith({
body: expect.stringContaining("@sut:matrix-qa.test reply with only this exact marker:"),
mentionUserIds: ["@sut:matrix-qa.test"],
roomId: "!main:matrix-qa.test",
});
});
it("blocks unmentioned allowBots=mentions room traffic even when the room is open", async () => {
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$observer-bot-unmentioned");
const waitForOptionalRoomEvent = vi.fn().mockResolvedValue({
matched: false,
since: "observer-sync-next",
});
createMatrixQaClient.mockReturnValue({
primeRoom,
sendTextMessage,
waitForOptionalRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-allowbots-mentions-unmentioned-open-room-block",
);
expect(scenario).toBeDefined();
await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({
artifacts: {
actorUserId: "@observer:matrix-qa.test",
driverEventId: "$observer-bot-unmentioned",
},
});
expect(sendTextMessage).toHaveBeenCalledWith({
body: expect.stringContaining("reply with only this exact marker:"),
roomId: "!main:matrix-qa.test",
});
});
it("uses the SUT account as the sender for the self-sender allowBots loop guard", async () => {
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
const observerWaitForOptionalRoomEvent = vi.fn().mockResolvedValue({
matched: false,
since: "observer-sync-next",
});
const observerSendTextMessage = vi.fn();
const sutSendTextMessage = vi.fn().mockResolvedValue("$sut-self-trigger");
createMatrixQaClient
.mockReturnValueOnce({
sendTextMessage: sutSendTextMessage,
})
.mockReturnValueOnce({
primeRoom,
sendTextMessage: observerSendTextMessage,
waitForOptionalRoomEvent: observerWaitForOptionalRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-allowbots-self-sender-ignored",
);
expect(scenario).toBeDefined();
await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({
artifacts: {
actorUserId: "@sut:matrix-qa.test",
driverEventId: "$sut-self-trigger",
},
});
expect(createMatrixQaClient).toHaveBeenNthCalledWith(1, {
accessToken: "sut-token",
baseUrl: "http://127.0.0.1:28008/",
});
expect(createMatrixQaClient).toHaveBeenNthCalledWith(2, {
accessToken: "observer-token",
baseUrl: "http://127.0.0.1:28008/",
});
expect(observerSendTextMessage).not.toHaveBeenCalled();
expect(sutSendTextMessage).toHaveBeenCalledWith({
body: expect.stringContaining("reply with only this exact marker:"),
roomId: "!main:matrix-qa.test",
});
});
it("blocks MXID-prefixed Matrix control commands from non-allowlisted observers", async () => {
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$observer-command-trigger");

2
extensions/qa-matrix/src/runners/contract/scenarios.ts
View File

@@ -1,4 +1,5 @@
import {
MATRIX_QA_BOT_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY,
MATRIX_QA_E2EE_ROOM_KEY,
@@ -61,6 +62,7 @@ export type {
export type { MatrixQaScenarioContext, MatrixQaSyncState };
export const __testing = {
MATRIX_QA_BOT_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_ROOM_KEY,
MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY,
MATRIX_QA_E2EE_ROOM_KEY,

50
extensions/qa-matrix/src/substrate/config.test.ts
View File

@@ -104,9 +104,12 @@ describe("matrix qa config", () => {
threadReplies: "off",
},
encryption: true,
allowBots: "mentions",
configuredBotRoles: ["observer"],
groupAllowFrom: ["@driver:matrix-qa.test", "@observer:matrix-qa.test"],
groupsByKey: {
secondary: {
allowBots: false,
requireMention: false,
tools: {
allow: ["sessions_spawn"],
@@ -123,6 +126,7 @@ describe("matrix qa config", () => {
threadReplies: "always",
toolProfile: "coding",
},
observerAccessToken: "observer-token",
sutAccessToken: "sut-token",
sutAccountId: "sut",
sutUserId: "@sut:matrix-qa.test",
@@ -144,7 +148,14 @@ describe("matrix qa config", () => {
expect(next.tools).toMatchObject({
profile: "coding",
});
expect(next.channels?.matrix?.accounts?.["qa-observer-bot-source"]).toMatchObject({
accessToken: "observer-token",
enabled: false,
homeserver: "http://127.0.0.1:28008/",
userId: "@observer:matrix-qa.test",
});
expect(next.channels?.matrix?.accounts?.sut).toMatchObject({
allowBots: "mentions",
autoJoin: "allowlist",
autoJoinAllowlist: ["!dm:matrix-qa.test", "#ops:matrix-qa.test"],
blockStreaming: true,
@@ -157,6 +168,7 @@ describe("matrix qa config", () => {
groups: {
"!main:matrix-qa.test": { enabled: true, requireMention: true },
"!secondary:matrix-qa.test": {
allowBots: false,
enabled: true,
requireMention: false,
tools: {
@@ -231,6 +243,7 @@ describe("matrix qa config", () => {
exec: false,
plugin: false,
},
allowBots: undefined,
autoJoin: "allowlist",
autoJoinAllowlist: ["!ops:matrix-qa.test"],
blockStreaming: true,
@@ -244,6 +257,7 @@ describe("matrix qa config", () => {
},
encryption: false,
execApprovals: undefined,
configuredBotRoles: [],
groupAllowFrom: ["@driver:matrix-qa.test"],
groupPolicy: "open",
groupsByKey: {
@@ -265,6 +279,8 @@ describe("matrix qa config", () => {
threadBindings: {},
threadReplies: "inbound",
});
expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("allowBots=<default>");
expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("configuredBotRoles=<none>");
expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("autoJoin=allowlist");
expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain("streaming=partial");
expect(summarizeMatrixQaConfigSnapshot(snapshot)).toContain(
@@ -354,6 +370,40 @@ describe("matrix qa config", () => {
expect(snapshot.groupAllowFrom).toEqual(["@driver:matrix-qa.test", "@observer:matrix-qa.test"]);
});
it("rejects configured bot roles without matching side-account auth", () => {
expect(() =>
buildMatrixQaConfig({} as OpenClawConfig, {
driverUserId: "@driver:matrix-qa.test",
homeserver: "http://127.0.0.1:28008/",
observerUserId: "@observer:matrix-qa.test",
overrides: {
configuredBotRoles: ["observer"],
},
sutAccessToken: "sut-token",
sutAccountId: "sut",
sutUserId: "@sut:matrix-qa.test",
topology,
}),
).toThrow('Matrix QA configured bot role "observer" requires an access token');
});
it("rejects the SUT role as a configured bot source", () => {
expect(() =>
buildMatrixQaConfig({} as OpenClawConfig, {
driverUserId: "@driver:matrix-qa.test",
homeserver: "http://127.0.0.1:28008/",
observerUserId: "@observer:matrix-qa.test",
overrides: {
configuredBotRoles: ["sut"],
},
sutAccessToken: "sut-token",
sutAccountId: "sut",
sutUserId: "@sut:matrix-qa.test",
topology,
}),
).toThrow('Matrix QA configured bot role "sut" would match the SUT account itself');
});
it("rejects unknown room-key overrides", () => {
expect(() =>
buildMatrixQaConfig({} as OpenClawConfig, {

80
extensions/qa-matrix/src/substrate/config.ts
View File

@@ -11,6 +11,7 @@ export type MatrixQaActorRole = "driver" | "observer" | "sut";
export type MatrixQaChunkMode = "length" | "newline";
export type MatrixQaExecApprovalTarget = "both" | "channel" | "dm";
export type MatrixQaExecApprovalsEnabled = boolean | "auto";
export type MatrixQaAllowBotsMode = boolean | "mentions";
export type MatrixQaStreamingConfig = {
mode?: MatrixQaStreamingMode;
@@ -38,6 +39,7 @@ export type MatrixQaToolConfigOverrides = {
};
export type MatrixQaGroupConfigOverrides = {
allowBots?: MatrixQaAllowBotsMode;
enabled?: boolean;
requireMention?: boolean;
tools?: MatrixQaToolConfigOverrides;
@@ -73,6 +75,7 @@ export type MatrixQaConfigOverrides = {
plugin?: boolean;
};
agentDefaults?: MatrixQaAgentDefaultsOverrides;
allowBots?: MatrixQaAllowBotsMode;
autoJoin?: MatrixQaAutoJoinMode;
autoJoinAllowlist?: string[];
blockStreaming?: boolean;
@@ -83,6 +86,7 @@ export type MatrixQaConfigOverrides = {
groupAllowFrom?: string[];
groupAllowRoles?: MatrixQaActorRole[];
groupPolicy?: MatrixQaGroupPolicy;
configuredBotRoles?: MatrixQaActorRole[];
groupsByKey?: Record<string, MatrixQaGroupConfigOverrides>;
replyToMode?: MatrixQaReplyToMode;
startupVerification?: "if-unverified" | "off";
@@ -100,6 +104,7 @@ export type MatrixQaConfigSnapshot = {
};
autoJoin: MatrixQaAutoJoinMode;
autoJoinAllowlist: string[];
allowBots?: MatrixQaAllowBotsMode;
blockStreaming: boolean;
chunkMode?: MatrixQaChunkMode;
dm: {
@@ -111,6 +116,7 @@ export type MatrixQaConfigSnapshot = {
};
encryption: boolean;
execApprovals?: MatrixQaExecApprovalsConfigOverrides;
configuredBotRoles: MatrixQaActorRole[];
groupAllowFrom: string[];
groupPolicy: MatrixQaGroupPolicy;
groupsByKey: Record<string, MatrixQaGroupSnapshot>;
@@ -124,6 +130,7 @@ export type MatrixQaConfigSnapshot = {
};
type MatrixQaGroupSnapshot = {
allowBots?: MatrixQaAllowBotsMode;
enabled: boolean;
requireMention: boolean;
roomId: string;
@@ -180,6 +187,9 @@ function resolveMatrixQaGroupSnapshots(params: {
{
roomId: room.roomId,
enabled: override?.enabled ?? true,
...(override && Object.hasOwn(override, "allowBots")
? { allowBots: override.allowBots }
: {}),
requireMention: override?.requireMention ?? room.requireMention,
...(override?.tools ? { tools: override.tools } : {}),
},
@@ -195,6 +205,7 @@ function buildMatrixQaGroupEntries(
Object.values(groupsByKey).map((group) => [
group.roomId,
{
...(group.allowBots !== undefined ? { allowBots: group.allowBots } : {}),
enabled: group.enabled,
requireMention: group.requireMention,
...(group.tools ? { tools: group.tools } : {}),
@@ -347,6 +358,59 @@ function buildMatrixQaAccountExecApprovalsConfig(
};
}
function buildMatrixQaConfiguredBotAccounts(params: {
driverAccessToken: string | undefined;
driverUserId: string;
homeserver: string;
observerAccessToken: string | undefined;
observerUserId: string;
roles: MatrixQaActorRole[];
}): Record<string, MatrixQaChannelAccountConfig> {
const selectedRoles = new Set(params.roles);
if (selectedRoles.has("sut")) {
throw new Error('Matrix QA configured bot role "sut" would match the SUT account itself');
}
const botSources: Record<
Exclude<MatrixQaActorRole, "sut">,
{
accessToken: string | undefined;
accountId: string;
userId: string;
}
> = {
driver: {
accessToken: params.driverAccessToken,
accountId: "qa-driver-bot-source",
userId: params.driverUserId,
},
observer: {
accessToken: params.observerAccessToken,
accountId: "qa-observer-bot-source",
userId: params.observerUserId,
},
};
const accounts: Record<string, MatrixQaChannelAccountConfig> = {};
for (const role of selectedRoles) {
if (role !== "driver" && role !== "observer") {
continue;
}
const source = botSources[role];
if (!source.accessToken) {
throw new Error(`Matrix QA configured bot role "${role}" requires an access token`);
}
accounts[source.accountId] = {
accessToken: source.accessToken,
enabled: false,
homeserver: params.homeserver,
userId: source.userId,
};
}
return accounts;
}
function buildMatrixQaChannelAccountConfig(params: {
groups: Record<string, MatrixQaGroupEntry>;
homeserver: string;
@@ -394,6 +458,7 @@ function buildMatrixQaChannelAccountConfig(params: {
dmOverrides: params.overrides?.dm,
snapshot: params.snapshot,
}),
...(params.snapshot.allowBots !== undefined ? { allowBots: params.snapshot.allowBots } : {}),
enabled: true,
encryption: params.snapshot.encryption,
groupAllowFrom: params.snapshot.groupAllowFrom,
@@ -426,6 +491,7 @@ export function buildMatrixQaConfigSnapshot(params: {
topology: MatrixQaProvisionedTopology;
}): MatrixQaConfigSnapshot {
return {
allowBots: params.overrides?.allowBots,
autoJoin: params.overrides?.autoJoin ?? "off",
autoJoinAllowlist: resolveMatrixQaAutoJoinAllowlist(params),
blockStreaming: params.overrides?.blockStreaming ?? false,
@@ -433,6 +499,7 @@ export function buildMatrixQaConfigSnapshot(params: {
dm: resolveMatrixQaDmConfigSnapshot(params),
encryption: params.overrides?.encryption ?? false,
execApprovals: params.overrides?.execApprovals,
configuredBotRoles: [...(params.overrides?.configuredBotRoles ?? [])],
groupAllowFrom: resolveMatrixQaGroupAllowFrom(params),
groupPolicy: params.overrides?.groupPolicy ?? "allowlist",
groupsByKey: resolveMatrixQaGroupSnapshots({
@@ -458,6 +525,8 @@ export function buildMatrixQaConfigSnapshot(params: {
export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot) {
return [
`allowBots=${snapshot.allowBots ?? "<default>"}`,
`configuredBotRoles=${snapshot.configuredBotRoles.length > 0 ? snapshot.configuredBotRoles.join("|") : "<none>"}`,
`replyToMode=${snapshot.replyToMode}`,
`threadReplies=${snapshot.threadReplies}`,
`dm.enabled=${formatMatrixQaBoolean(snapshot.dm.enabled)}`,
@@ -484,8 +553,10 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot
export function buildMatrixQaConfig(
baseCfg: OpenClawConfig,
params: {
driverAccessToken?: string;
driverUserId: string;
homeserver: string;
observerAccessToken?: string;
observerUserId: string;
overrides?: MatrixQaConfigOverrides;
sutAccessToken: string;
@@ -504,6 +575,14 @@ export function buildMatrixQaConfig(
topology: params.topology,
});
const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey);
const configuredBotAccounts = buildMatrixQaConfiguredBotAccounts({
driverAccessToken: params.driverAccessToken,
driverUserId: params.driverUserId,
homeserver: params.homeserver,
observerAccessToken: params.observerAccessToken,
observerUserId: params.observerUserId,
roles: snapshot.configuredBotRoles,
});
const approvalForwardingConfig =
snapshot.approvalForwarding.exec || snapshot.approvalForwarding.plugin
? {
@@ -569,6 +648,7 @@ export function buildMatrixQaConfig(
defaultAccount: params.sutAccountId,
accounts: {
...baseCfg.channels?.matrix?.accounts,
...configuredBotAccounts,
[params.sutAccountId]: buildMatrixQaChannelAccountConfig({
groups,
homeserver: params.homeserver,