qa-matrix: cover Matrix allowlist hot reload

Add a Matrix QA scenario that removes an observer from the running account group allowlist and verifies the existing gateway stops replying without relying on a channel restart. The scenario disables generic config reload and defers restart during the probe so it specifically covers the Matrix handler per-message live allowlist read.
2026-05-06 13:20:43 +00:00 · 2026-04-19 18:10:51 -04:00
parent f309656325
commit f9a1875127
6 changed files with 248 additions and 1 deletions
--- a/extensions/qa-matrix/src/runners/contract/runtime.ts
+++ b/extensions/qa-matrix/src/runners/contract/runtime.ts
@@ -361,6 +361,28 @@ async function waitForMatrixChannelReady(
  throw new Error(`matrix account "${accountId}" did not become ready`);
 }

+async function patchMatrixQaGatewayConfig(params: {
+  gateway: MatrixQaGatewayChild;
+  patch: Record<string, unknown>;
+  restartDelayMs?: number;
+}) {
+  const snapshot = (await params.gateway.call("config.get", {}, { timeoutMs: 60_000 })) as {
+    hash?: string;
+  };
+  if (!snapshot.hash) {
+    throw new Error("Matrix QA config patch requires config.get hash");
+  }
+  await params.gateway.call(
+    "config.patch",
+    {
+      raw: JSON.stringify(params.patch, null, 2),
+      baseHash: snapshot.hash,
+      restartDelayMs: params.restartDelayMs ?? 0,
+    },
+    { timeoutMs: 60_000 },
+  );
+}
+
 async function startMatrixQaLiveLaneGateway(params: {
  repoRoot: string;
  transport: {
@@ -665,6 +687,7 @@ export async function runMatrixQaLive(params: {
                );
              },
              roomId: provisioning.roomId,
+              sutAccountId,
              sutAccessToken: provisioning.sut.accessToken,
              sutDeviceId: provisioning.sut.deviceId,
              sutPassword: provisioning.sut.password,
@@ -673,6 +696,13 @@ export async function runMatrixQaLive(params: {
              sutUserId: provisioning.sut.userId,
              timeoutMs: scenario.timeoutMs,
              topology: provisioning.topology,
+              patchGatewayConfig: async (patch, opts) => {
+                await patchMatrixQaGatewayConfig({
+                  gateway: scenarioGateway.harness.gateway,
+                  patch,
+                  restartDelayMs: opts?.restartDelayMs,
+                });
+              },
            }),
          );
          const result = measuredScenario.result;
@@ -881,6 +911,7 @@ export const __testing = {
  buildMatrixQaConfigSnapshot,
  findMatrixQaScenarios,
  isMatrixAccountReady,
+  patchMatrixQaGatewayConfig,
  resolveMatrixQaModels,
  summarizeMatrixQaConfigSnapshot,
  waitForMatrixChannelReady,
--- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
@@ -48,6 +48,7 @@ export type MatrixQaScenarioId =
  | "matrix-mention-metadata-spoof-block"
  | "matrix-observer-allowlist-override"
  | "matrix-allowlist-block"
+  | "matrix-allowlist-hot-reload"
  | "matrix-multi-actor-ordering"
  | "matrix-inbound-edit-ignored"
  | "matrix-inbound-edit-no-duplicate-trigger"
@@ -477,6 +478,14 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
    timeoutMs: 8_000,
    title: "Matrix sender allowlist blocks observer replies",
  },
+  {
+    id: "matrix-allowlist-hot-reload",
+    timeoutMs: 60_000,
+    title: "Matrix group sender allowlist removals hot-reload without gateway restart",
+    configOverrides: {
+      groupAllowRoles: ["driver", "observer"],
+    },
+  },
  {
    id: "matrix-multi-actor-ordering",
    timeoutMs: 60_000,
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
@@ -42,6 +42,7 @@ type MatrixQaThreadScenarioResult = Awaited<ReturnType<typeof runThreadScenario>

 const MATRIX_SUBAGENT_THREAD_HOOK_ERROR_RE =
  /thread=true is unavailable because no channel plugin registered subagent_spawning hooks/i;
+const MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS = 300_000;

 function assertMatrixQaInReplyTarget(params: {
  actualEventId?: string;
@@ -470,6 +471,85 @@ export async function runObserverAllowlistOverrideScenario(context: MatrixQaScen
  } satisfies MatrixQaScenarioExecution;
 }

+export async function runAllowlistHotReloadScenario(context: MatrixQaScenarioContext) {
+  if (!context.patchGatewayConfig) {
+    throw new Error("Matrix allowlist hot-reload scenario requires gateway config patching");
+  }
+  const accepted = await runTopologyScopedTopLevelScenario({
+    accessToken: context.observerAccessToken,
+    actorId: "observer",
+    actorUserId: context.observerUserId,
+    context,
+    roomKey: context.topology.defaultRoomKey,
+    tokenPrefix: "MATRIX_QA_GROUP_RELOAD_ACCEPTED",
+  });
+  const accountId = context.sutAccountId ?? "sut";
+
+  await context.patchGatewayConfig(
+    {
+      channels: {
+        matrix: {
+          accounts: {
+            [accountId]: {
+              groupAllowFrom: [context.driverUserId],
+            },
+          },
+        },
+      },
+      gateway: {
+        // Isolate the Matrix handler's per-message config read from generic channel reload.
+        reload: {
+          mode: "off",
+        },
+      },
+    },
+    {
+      restartDelayMs: MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS,
+    },
+  );
+
+  const blockedToken = buildMatrixQaToken("MATRIX_QA_GROUP_RELOAD_REMOVED");
+  const removed = await runNoReplyExpectedScenario({
+    accessToken: context.observerAccessToken,
+    actorId: "observer",
+    actorUserId: context.observerUserId,
+    baseUrl: context.baseUrl,
+    body: buildMentionPrompt(context.sutUserId, blockedToken),
+    mentionUserIds: [context.sutUserId],
+    observedEvents: context.observedEvents,
+    roomId: context.roomId,
+    syncState: context.syncState,
+    syncStreams: context.syncStreams,
+    sutUserId: context.sutUserId,
+    replyPredicate: (event) =>
+      isMatrixQaExactMarkerReply(event, {
+        roomId: context.roomId,
+        sutUserId: context.sutUserId,
+        token: blockedToken,
+      }),
+    timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
+    token: blockedToken,
+  });
+
+  return {
+    artifacts: {
+      accepted: accepted.artifacts ?? {},
+      blocked: removed.artifacts ?? {},
+      driverEventId: accepted.artifacts?.driverEventId,
+      secondDriverEventId: removed.artifacts?.driverEventId,
+      firstReply: accepted.artifacts?.reply,
+      token: accepted.artifacts?.token,
+      triggerBody: accepted.artifacts?.triggerBody,
+    },
+    details: [
+      "group allowlist before removal:",
+      accepted.details,
+      "group allowlist after hot reload removal:",
+      removed.details,
+    ].join("\n"),
+  } satisfies MatrixQaScenarioExecution;
+}
+
 export async function runQuietStreamingPreviewScenario(context: MatrixQaScenarioContext) {
  const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
  const finalText = `MATRIX_QA_QUIET_STREAM_${randomUUID().slice(0, 8).toUpperCase()} preview complete`;
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
@@ -33,6 +33,7 @@ export type MatrixQaScenarioContext = {
  roomId: string;
  interruptTransport?: () => Promise<void>;
  sutAccessToken: string;
+  sutAccountId?: string;
  sutDeviceId?: string;
  sutPassword?: string;
  syncState: MatrixQaSyncState;
@@ -40,6 +41,10 @@ export type MatrixQaScenarioContext = {
  sutUserId: string;
  timeoutMs: number;
  topology: MatrixQaProvisionedTopology;
+  patchGatewayConfig?: (
+    patch: Record<string, unknown>,
+    opts?: { restartDelayMs?: number },
+  ) => Promise<void>;
 };

 export const NO_REPLY_WINDOW_MS = 8_000;
@@ -554,6 +559,10 @@ export async function runNoReplyExpectedScenario(params: {
  syncState: MatrixQaSyncState;
  syncStreams?: MatrixQaSyncStreams;
  sutUserId: string;
+  replyPredicate?: (
+    event: MatrixQaObservedEvent,
+    match: { driverEventId: string; token: string },
+  ) => boolean;
  timeoutMs: number;
  token: string;
 }) {
@@ -575,7 +584,8 @@ export async function runNoReplyExpectedScenario(params: {
    predicate: (event) =>
      event.roomId === params.roomId &&
      event.sender === params.sutUserId &&
-      event.type === "m.room.message",
+      event.type === "m.room.message" &&
+      (params.replyPredicate?.(event, { driverEventId, token: params.token }) ?? true),
    roomId: params.roomId,
    since: startSince,
    timeoutMs: params.timeoutMs,
--- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
@@ -46,6 +46,7 @@ import {
  runRestartResumeScenario,
 } from "./scenario-runtime-restart.js";
 import {
+  runAllowlistHotReloadScenario,
  runBlockStreamingScenario,
  runMatrixQaCanary,
  runMembershipLossScenario,
@@ -285,6 +286,8 @@ export async function runMatrixQaScenario(
        token,
      });
    }
+    case "matrix-allowlist-hot-reload":
+      return await runAllowlistHotReloadScenario(context);
    case "matrix-multi-actor-ordering":
      return await runMultiActorOrderingScenario(context);
    case "matrix-inbound-edit-ignored":
--- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
+++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
@@ -35,6 +35,7 @@ import {

 const MATRIX_SUBAGENT_MISSING_HOOK_ERROR =
  "thread=true is unavailable because no channel plugin registered subagent_spawning hooks.";
+const MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS = 300_000;

 function matrixQaScenarioContext(): MatrixQaScenarioContext {
  return {
@@ -110,6 +111,7 @@ describe("matrix live qa scenarios", () => {
      "matrix-mention-metadata-spoof-block",
      "matrix-observer-allowlist-override",
      "matrix-allowlist-block",
+      "matrix-allowlist-hot-reload",
      "matrix-multi-actor-ordering",
      "matrix-inbound-edit-ignored",
      "matrix-inbound-edit-no-duplicate-trigger",
@@ -562,6 +564,118 @@ describe("matrix live qa scenarios", () => {
    );
  });

+  it("hot-reloads group allowlist removals inside one running Matrix gateway", async () => {
+    const patchGatewayConfig = vi.fn(async () => {});
+    const primeRoom = vi.fn().mockResolvedValue("sync-start");
+    const sendTextMessage = vi
+      .fn()
+      .mockResolvedValueOnce("$group-accepted")
+      .mockResolvedValueOnce("$group-removed");
+    const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => ({
+      matched: false,
+      since: `${params.roomId}:no-reply`,
+    }));
+    const waitForRoomEvent = vi.fn().mockImplementation(async (params) => {
+      const sentBody = String(sendTextMessage.mock.calls.at(-1)?.[0]?.body ?? "");
+      const token = sentBody
+        .replace("@sut:matrix-qa.test reply with only this exact marker: ", "")
+        .replace("reply with only this exact marker: ", "");
+      return {
+        event: {
+          kind: "message",
+          roomId: params.roomId,
+          eventId: "$group-reply",
+          sender: "@sut:matrix-qa.test",
+          type: "m.room.message",
+          body: token,
+        },
+        since: `${params.roomId}:reply`,
+      };
+    });
+
+    createMatrixQaClient.mockReturnValue({
+      primeRoom,
+      sendTextMessage,
+      waitForOptionalRoomEvent,
+      waitForRoomEvent,
+    });
+
+    const scenario = MATRIX_QA_SCENARIOS.find(
+      (entry) => entry.id === "matrix-allowlist-hot-reload",
+    );
+    expect(scenario).toBeDefined();
+
+    await expect(
+      runMatrixQaScenario(scenario!, {
+        ...matrixQaScenarioContext(),
+        patchGatewayConfig,
+        topology: {
+          defaultRoomId: "!main:matrix-qa.test",
+          defaultRoomKey: "main",
+          rooms: [
+            {
+              key: "main",
+              kind: "group",
+              memberRoles: ["driver", "observer", "sut"],
+              memberUserIds: [
+                "@driver:matrix-qa.test",
+                "@observer:matrix-qa.test",
+                "@sut:matrix-qa.test",
+              ],
+              name: "Main",
+              requireMention: true,
+              roomId: "!main:matrix-qa.test",
+            },
+          ],
+        },
+      }),
+    ).resolves.toMatchObject({
+      artifacts: {
+        secondDriverEventId: "$group-removed",
+        firstReply: {
+          eventId: "$group-reply",
+          tokenMatched: true,
+        },
+      },
+    });
+
+    expect(patchGatewayConfig).toHaveBeenCalledWith(
+      {
+        channels: {
+          matrix: {
+            accounts: {
+              sut: {
+                groupAllowFrom: ["@driver:matrix-qa.test"],
+              },
+            },
+          },
+        },
+        gateway: {
+          reload: {
+            mode: "off",
+          },
+        },
+      },
+      {
+        restartDelayMs: MATRIX_QA_HOT_RELOAD_RESTART_DELAY_MS,
+      },
+    );
+    expect(sendTextMessage).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        mentionUserIds: ["@sut:matrix-qa.test"],
+        roomId: "!main:matrix-qa.test",
+      }),
+    );
+    expect(sendTextMessage).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({
+        mentionUserIds: ["@sut:matrix-qa.test"],
+        roomId: "!main:matrix-qa.test",
+      }),
+    );
+  });
+
  it("queues a Matrix trigger during restart before proving incremental sync continues", async () => {
    const callOrder: string[] = [];
    const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");