diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 3a26a6fbcf1..853810f0ce9 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -361,6 +361,28 @@ async function waitForMatrixChannelReady( throw new Error(`matrix account "${accountId}" did not become ready`); } +async function patchMatrixQaGatewayConfig(params: { + gateway: MatrixQaGatewayChild; + patch: Record; + restartDelayMs?: number; +}) { + const snapshot = (await params.gateway.call("config.get", {}, { timeoutMs: 60_000 })) as { + hash?: string; + }; + if (!snapshot.hash) { + throw new Error("Matrix QA config patch requires config.get hash"); + } + await params.gateway.call( + "config.patch", + { + raw: JSON.stringify(params.patch, null, 2), + baseHash: snapshot.hash, + restartDelayMs: params.restartDelayMs ?? 0, + }, + { timeoutMs: 60_000 }, + ); +} + async function startMatrixQaLiveLaneGateway(params: { repoRoot: string; transport: { @@ -665,6 +687,7 @@ export async function runMatrixQaLive(params: { ); }, roomId: provisioning.roomId, + sutAccountId, sutAccessToken: provisioning.sut.accessToken, sutDeviceId: provisioning.sut.deviceId, sutPassword: provisioning.sut.password, @@ -673,6 +696,13 @@ export async function runMatrixQaLive(params: { sutUserId: provisioning.sut.userId, timeoutMs: scenario.timeoutMs, topology: provisioning.topology, + patchGatewayConfig: async (patch, opts) => { + await patchMatrixQaGatewayConfig({ + gateway: scenarioGateway.harness.gateway, + patch, + restartDelayMs: opts?.restartDelayMs, + }); + }, }), ); const result = measuredScenario.result; @@ -881,6 +911,7 @@ export const __testing = { buildMatrixQaConfigSnapshot, findMatrixQaScenarios, isMatrixAccountReady, + patchMatrixQaGatewayConfig, resolveMatrixQaModels, summarizeMatrixQaConfigSnapshot, waitForMatrixChannelReady, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index ba33718867c..270b99797bc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -48,6 +48,7 @@ export type MatrixQaScenarioId = | "matrix-mention-metadata-spoof-block" | "matrix-observer-allowlist-override" | "matrix-allowlist-block" + | "matrix-allowlist-hot-reload" | "matrix-multi-actor-ordering" | "matrix-inbound-edit-ignored" | "matrix-inbound-edit-no-duplicate-trigger" @@ -477,6 +478,14 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 8_000, title: "Matrix sender allowlist blocks observer replies", }, + { + id: "matrix-allowlist-hot-reload", + timeoutMs: 60_000, + title: "Matrix group sender allowlist removals hot-reload without gateway restart", + configOverrides: { + groupAllowRoles: ["driver", "observer"], + }, + }, { id: "matrix-multi-actor-ordering", timeoutMs: 60_000, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index 477c920707b..dc691062b77 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -42,6 +42,7 @@ type MatrixQaThreadScenarioResult = Awaited const MATRIX_SUBAGENT_THREAD_HOOK_ERROR_RE = /thread=true is unavailable because no channel plugin registered subagent_spawning hooks/i; +const MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS = 300_000; function assertMatrixQaInReplyTarget(params: { actualEventId?: string; @@ -470,6 +471,85 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen } satisfies MatrixQaScenarioExecution; } +export async function runAllowlistHotReloadScenario(context: MatrixQaScenarioContext) { + if (!context.patchGatewayConfig) { + throw new Error("Matrix allowlist hot-reload scenario requires gateway config patching"); + } + const accepted = await runTopologyScopedTopLevelScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + actorUserId: context.observerUserId, + context, + roomKey: context.topology.defaultRoomKey, + tokenPrefix: "MATRIX_QA_GROUP_RELOAD_ACCEPTED", + }); + const accountId = context.sutAccountId ?? "sut"; + + await context.patchGatewayConfig( + { + channels: { + matrix: { + accounts: { + [accountId]: { + groupAllowFrom: [context.driverUserId], + }, + }, + }, + }, + gateway: { + // Isolate the Matrix handler's per-message config read from generic channel reload. + reload: { + mode: "off", + }, + }, + }, + { + restartDelayMs: MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS, + }, + ); + + const blockedToken = buildMatrixQaToken("MATRIX_QA_GROUP_RELOAD_REMOVED"); + const removed = await runNoReplyExpectedScenario({ + accessToken: context.observerAccessToken, + actorId: "observer", + actorUserId: context.observerUserId, + baseUrl: context.baseUrl, + body: buildMentionPrompt(context.sutUserId, blockedToken), + mentionUserIds: [context.sutUserId], + observedEvents: context.observedEvents, + roomId: context.roomId, + syncState: context.syncState, + syncStreams: context.syncStreams, + sutUserId: context.sutUserId, + replyPredicate: (event) => + isMatrixQaExactMarkerReply(event, { + roomId: context.roomId, + sutUserId: context.sutUserId, + token: blockedToken, + }), + timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs), + token: blockedToken, + }); + + return { + artifacts: { + accepted: accepted.artifacts ?? {}, + blocked: removed.artifacts ?? {}, + driverEventId: accepted.artifacts?.driverEventId, + secondDriverEventId: removed.artifacts?.driverEventId, + firstReply: accepted.artifacts?.reply, + token: accepted.artifacts?.token, + triggerBody: accepted.artifacts?.triggerBody, + }, + details: [ + "group allowlist before removal:", + accepted.details, + "group allowlist after hot reload removal:", + removed.details, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + export async function runQuietStreamingPreviewScenario(context: MatrixQaScenarioContext) { const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); const finalText = `MATRIX_QA_QUIET_STREAM_${randomUUID().slice(0, 8).toUpperCase()} preview complete`; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index f6a216e5973..e9bb84997bb 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -33,6 +33,7 @@ export type MatrixQaScenarioContext = { roomId: string; interruptTransport?: () => Promise; sutAccessToken: string; + sutAccountId?: string; sutDeviceId?: string; sutPassword?: string; syncState: MatrixQaSyncState; @@ -40,6 +41,10 @@ export type MatrixQaScenarioContext = { sutUserId: string; timeoutMs: number; topology: MatrixQaProvisionedTopology; + patchGatewayConfig?: ( + patch: Record, + opts?: { restartDelayMs?: number }, + ) => Promise; }; export const NO_REPLY_WINDOW_MS = 8_000; @@ -554,6 +559,10 @@ export async function runNoReplyExpectedScenario(params: { syncState: MatrixQaSyncState; syncStreams?: MatrixQaSyncStreams; sutUserId: string; + replyPredicate?: ( + event: MatrixQaObservedEvent, + match: { driverEventId: string; token: string }, + ) => boolean; timeoutMs: number; token: string; }) { @@ -575,7 +584,8 @@ export async function runNoReplyExpectedScenario(params: { predicate: (event) => event.roomId === params.roomId && event.sender === params.sutUserId && - event.type === "m.room.message", + event.type === "m.room.message" && + (params.replyPredicate?.(event, { driverEventId, token: params.token }) ?? true), roomId: params.roomId, since: startSince, timeoutMs: params.timeoutMs, diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index dc9147b9d13..34b64cdceb9 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -46,6 +46,7 @@ import { runRestartResumeScenario, } from "./scenario-runtime-restart.js"; import { + runAllowlistHotReloadScenario, runBlockStreamingScenario, runMatrixQaCanary, runMembershipLossScenario, @@ -285,6 +286,8 @@ export async function runMatrixQaScenario( token, }); } + case "matrix-allowlist-hot-reload": + return await runAllowlistHotReloadScenario(context); case "matrix-multi-actor-ordering": return await runMultiActorOrderingScenario(context); case "matrix-inbound-edit-ignored": diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 81686ef4c8d..c91cc6a38c2 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -35,6 +35,7 @@ import { const MATRIX_SUBAGENT_MISSING_HOOK_ERROR = "thread=true is unavailable because no channel plugin registered subagent_spawning hooks."; +const MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS = 300_000; function matrixQaScenarioContext(): MatrixQaScenarioContext { return { @@ -110,6 +111,7 @@ describe("matrix live qa scenarios", () => { "matrix-mention-metadata-spoof-block", "matrix-observer-allowlist-override", "matrix-allowlist-block", + "matrix-allowlist-hot-reload", "matrix-multi-actor-ordering", "matrix-inbound-edit-ignored", "matrix-inbound-edit-no-duplicate-trigger", @@ -562,6 +564,118 @@ describe("matrix live qa scenarios", () => { ); }); + it("hot-reloads group allowlist removals inside one running Matrix gateway", async () => { + const patchGatewayConfig = vi.fn(async () => {}); + const primeRoom = vi.fn().mockResolvedValue("sync-start"); + const sendTextMessage = vi + .fn() + .mockResolvedValueOnce("$group-accepted") + .mockResolvedValueOnce("$group-removed"); + const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => ({ + matched: false, + since: `${params.roomId}:no-reply`, + })); + const waitForRoomEvent = vi.fn().mockImplementation(async (params) => { + const sentBody = String(sendTextMessage.mock.calls.at(-1)?.[0]?.body ?? ""); + const token = sentBody + .replace("@sut:matrix-qa.test reply with only this exact marker: ", "") + .replace("reply with only this exact marker: ", ""); + return { + event: { + kind: "message", + roomId: params.roomId, + eventId: "$group-reply", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: token, + }, + since: `${params.roomId}:reply`, + }; + }); + + createMatrixQaClient.mockReturnValue({ + primeRoom, + sendTextMessage, + waitForOptionalRoomEvent, + waitForRoomEvent, + }); + + const scenario = MATRIX_QA_SCENARIOS.find( + (entry) => entry.id === "matrix-allowlist-hot-reload", + ); + expect(scenario).toBeDefined(); + + await expect( + runMatrixQaScenario(scenario!, { + ...matrixQaScenarioContext(), + patchGatewayConfig, + topology: { + defaultRoomId: "!main:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Main", + requireMention: true, + roomId: "!main:matrix-qa.test", + }, + ], + }, + }), + ).resolves.toMatchObject({ + artifacts: { + secondDriverEventId: "$group-removed", + firstReply: { + eventId: "$group-reply", + tokenMatched: true, + }, + }, + }); + + expect(patchGatewayConfig).toHaveBeenCalledWith( + { + channels: { + matrix: { + accounts: { + sut: { + groupAllowFrom: ["@driver:matrix-qa.test"], + }, + }, + }, + }, + gateway: { + reload: { + mode: "off", + }, + }, + }, + { + restartDelayMs: MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS, + }, + ); + expect(sendTextMessage).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }), + ); + expect(sendTextMessage).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + mentionUserIds: ["@sut:matrix-qa.test"], + roomId: "!main:matrix-qa.test", + }), + ); + }); + it("queues a Matrix trigger during restart before proving incremental sync continues", async () => { const callOrder: string[] = []; const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");