vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 07:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QA Matrix: add catchup incremental scenario

Browse Source
This commit is contained in:
Gustavo Madeira Santana
2026-04-17 19:16:58 -04:00
parent 5af1a51f8e
commit 5d8dceb37f
7 changed files with 213 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
extensions/qa-matrix/src/runners/contract/runtime.ts
View File

@@ -647,6 +647,23 @@ export async function runMatrixQaLive(params: {
`gateway restart done ${scenario.id} ${formatMatrixQaDurationMs(measuredRestart.durationMs)}`,
);
},
restartGatewayWithQueuedMessage: async (queueMessage) => {
if (!gatewayHarness) {
throw new Error("Matrix restart catchup scenario requires a live gateway");
}
writeMatrixQaProgress(`gateway restart+queue start ${scenario.id}`);
const measuredRestart = await measureMatrixQaStep(async () => {
await scenarioGateway.harness.gateway.restart();
await sleep(250);
await queueMessage();
await waitForMatrixChannelReady(scenarioGateway.harness.gateway, sutAccountId);
});
gatewayRestartMs += measuredRestart.durationMs;
scenarioRestartGatewayMs += measuredRestart.durationMs;
writeMatrixQaProgress(
`gateway restart+queue done ${scenario.id} ${formatMatrixQaDurationMs(measuredRestart.durationMs)}`,
);
},
roomId: provisioning.roomId,
sutAccessToken: provisioning.sut.accessToken,
sutDeviceId: provisioning.sut.deviceId,

7
extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
View File

@@ -40,6 +40,7 @@ export type MatrixQaScenarioId =
| "matrix-reaction-redaction-observed"
| "matrix-restart-resume"
| "matrix-post-restart-room-continue"
| "matrix-initial-catchup-then-incremental"
| "matrix-room-membership-loss"
| "matrix-homeserver-restart-resume"
| "matrix-mention-gating"
@@ -424,6 +425,12 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
title: "Matrix restarted room continues after the first recovered reply",
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
},
{
id: "matrix-initial-catchup-then-incremental",
timeoutMs: 90_000,
title: "Matrix initial catchup is followed by incremental replies",
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
},
{
id: "matrix-room-membership-loss",
timeoutMs: 75_000,

77
extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts
View File

@@ -5,6 +5,13 @@ import {
} from "./scenario-catalog.js";
import {
buildMatrixReplyDetails,
buildMatrixQaToken,
buildMentionPrompt,
buildMatrixReplyArtifact,
isMatrixQaExactMarkerReply,
assertTopLevelReplyArtifact,
advanceMatrixQaActorCursor,
primeMatrixQaDriverScenarioClient,
runAssertedDriverTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
@@ -107,3 +114,73 @@ export async function runPostRestartRoomContinueScenario(context: MatrixQaScenar
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runInitialCatchupThenIncrementalScenario(context: MatrixQaScenarioContext) {
if (!context.restartGatewayWithQueuedMessage) {
throw new Error(
"Matrix initial catchup scenario requires a queued-message gateway restart callback",
);
}
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_RESTART_ROOM_KEY);
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
const catchupToken = buildMatrixQaToken("MATRIX_QA_CATCHUP");
const catchupBody = buildMentionPrompt(context.sutUserId, catchupToken);
let catchupDriverEventId = "";
await context.restartGatewayWithQueuedMessage(async () => {
catchupDriverEventId = await client.sendTextMessage({
body: catchupBody,
mentionUserIds: [context.sutUserId],
roomId,
});
});
const catchupMatched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
isMatrixQaExactMarkerReply(event, {
roomId,
sutUserId: context.sutUserId,
token: catchupToken,
}) && event.relatesTo === undefined,
roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: catchupMatched.since,
startSince,
});
const catchupReply = buildMatrixReplyArtifact(catchupMatched.event, catchupToken);
assertTopLevelReplyArtifact("catchup reply", catchupReply);
const incremental = await runAssertedDriverTopLevelScenario({
context,
label: "incremental reply after catchup",
roomId,
tokenPrefix: "MATRIX_QA_INCREMENTAL",
});
return {
artifacts: {
catchupDriverEventId,
catchupReply,
catchupToken,
incrementalDriverEventId: incremental.driverEventId,
incrementalReply: incremental.reply,
incrementalToken: incremental.token,
restartSignal: "SIGUSR1",
roomId,
},
details: [
`room id: ${roomId}`,
"restart signal: SIGUSR1",
`catchup driver event: ${catchupDriverEventId}`,
...buildMatrixReplyDetails("catchup reply", catchupReply),
`incremental driver event: ${incremental.driverEventId}`,
...buildMatrixReplyDetails("incremental reply", incremental.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}

1
extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
View File

@@ -29,6 +29,7 @@ export type MatrixQaScenarioContext = {
observerUserId: string;
outputDir?: string;
restartGateway?: () => Promise<void>;
restartGatewayWithQueuedMessage?: (queueMessage: () => Promise<void>) => Promise<void>;
roomId: string;
interruptTransport?: () => Promise<void>;
sutAccessToken: string;

3
extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
View File

@@ -41,6 +41,7 @@ import {
} from "./scenario-runtime-reaction.js";
import {
runHomeserverRestartResumeScenario,
runInitialCatchupThenIncrementalScenario,
runPostRestartRoomContinueScenario,
runRestartResumeScenario,
} from "./scenario-runtime-restart.js";
@@ -229,6 +230,8 @@ export async function runMatrixQaScenario(
return await runRestartResumeScenario(context);
case "matrix-post-restart-room-continue":
return await runPostRestartRoomContinueScenario(context);
case "matrix-initial-catchup-then-incremental":
return await runInitialCatchupThenIncrementalScenario(context);
case "matrix-room-membership-loss":
return await runMembershipLossScenario(context);
case "matrix-homeserver-restart-resume":

6
extensions/qa-matrix/src/runners/contract/scenario-types.ts
View File

@@ -32,6 +32,9 @@ export type MatrixQaScenarioArtifacts = {
attachmentMsgtype?: string;
actorUserId?: string;
blocked?: MatrixQaScenarioArtifacts;
catchupDriverEventId?: string;
catchupReply?: MatrixQaReplyArtifact;
catchupToken?: string;
driverEventId?: string;
editEventId?: string;
editedToken?: string;
@@ -39,6 +42,9 @@ export type MatrixQaScenarioArtifacts = {
firstDriverEventId?: string;
firstReply?: MatrixQaReplyArtifact;
firstToken?: string;
incrementalDriverEventId?: string;
incrementalReply?: MatrixQaReplyArtifact;
incrementalToken?: string;
originalDriverEventId?: string;
originalReply?: MatrixQaReplyArtifact;
originalToken?: string;

102
extensions/qa-matrix/src/runners/contract/scenarios.test.ts
View File

@@ -102,6 +102,7 @@ describe("matrix live qa scenarios", () => {
"matrix-reaction-redaction-observed",
"matrix-restart-resume",
"matrix-post-restart-room-continue",
"matrix-initial-catchup-then-incremental",
"matrix-room-membership-loss",
"matrix-homeserver-restart-resume",
"matrix-mention-gating",
@@ -515,6 +516,107 @@ describe("matrix live qa scenarios", () => {
});
});
it("queues a Matrix trigger during restart before proving incremental sync continues", async () => {
const callOrder: string[] = [];
const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
const sendTextMessage = vi.fn().mockImplementation(async (params) => {
callOrder.push(`send:${String(params.body).includes("CATCHUP") ? "catchup" : "incremental"}`);
return String(params.body).includes("CATCHUP") ? "$catchup-trigger" : "$incremental-trigger";
});
const waitForRoomEvent = vi.fn().mockImplementation(async () => {
const sentBody = String(sendTextMessage.mock.calls.at(-1)?.[0]?.body ?? "");
const token = sentBody.replace("@sut:matrix-qa.test reply with only this exact marker: ", "");
callOrder.push(`wait:${token.includes("CATCHUP") ? "catchup" : "incremental"}`);
return {
event: {
kind: "message",
roomId: "!restart:matrix-qa.test",
eventId: token.includes("CATCHUP") ? "$catchup-reply" : "$incremental-reply",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: token,
},
since: token.includes("CATCHUP")
? "driver-sync-after-catchup"
: "driver-sync-after-incremental",
};
});
createMatrixQaClient.mockReturnValue({
primeRoom,
sendTextMessage,
waitForRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find(
(entry) => entry.id === "matrix-initial-catchup-then-incremental",
);
expect(scenario).toBeDefined();
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
restartGatewayWithQueuedMessage: async (queueMessage) => {
callOrder.push("restart");
await queueMessage();
callOrder.push("ready");
},
roomId: "!room:matrix-qa.test",
syncState: {},
sutAccessToken: "sut-token",
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
topology: {
defaultRoomId: "!room:matrix-qa.test",
defaultRoomKey: "main",
rooms: [
{
key: "restart",
kind: "group",
memberRoles: ["driver", "observer", "sut"],
memberUserIds: [
"@driver:matrix-qa.test",
"@observer:matrix-qa.test",
"@sut:matrix-qa.test",
],
name: "Restart room",
requireMention: true,
roomId: "!restart:matrix-qa.test",
},
],
},
}),
).resolves.toMatchObject({
artifacts: {
catchupDriverEventId: "$catchup-trigger",
catchupReply: {
eventId: "$catchup-reply",
tokenMatched: true,
},
incrementalDriverEventId: "$incremental-trigger",
incrementalReply: {
eventId: "$incremental-reply",
tokenMatched: true,
},
},
});
expect(callOrder).toEqual([
"restart",
"send:catchup",
"ready",
"wait:catchup",
"send:incremental",
"wait:incremental",
]);
});
it("runs the DM scenario against the provisioned DM room without a mention", async () => {
const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$dm-trigger");