diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 84595c2ad4d..f47cd28be1b 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -325,6 +325,26 @@ describe("matrix live qa runtime", () => { }); }); + it("batches Matrix scenarios by config key while preserving stable in-group order", () => { + const scenarios = liveTesting.findMatrixQaScenarios([ + "matrix-top-level-reply-shape", + "matrix-room-thread-reply-override", + "matrix-thread-follow-up", + "matrix-room-quiet-streaming-preview", + "matrix-reaction-notification", + ]); + + expect( + liveTesting.scheduleMatrixQaScenariosByConfig(scenarios).map(({ scenario }) => scenario.id), + ).toEqual([ + "matrix-thread-follow-up", + "matrix-top-level-reply-shape", + "matrix-reaction-notification", + "matrix-room-thread-reply-override", + "matrix-room-quiet-streaming-preview", + ]); + }); + it("treats only connected, healthy Matrix accounts as ready", () => { expect(liveTesting.isMatrixAccountReady({ running: true, connected: true })).toBe(true); expect(liveTesting.isMatrixAccountReady({ running: true, connected: false })).toBe(false); diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 01604162938..19e8d8cad85 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -61,6 +61,11 @@ type MatrixQaScenarioResult = { title: string; }; +type MatrixQaScheduledScenario = { + originalIndex: number; + scenario: (typeof MATRIX_QA_SCENARIOS)[number]; +}; + type MatrixQaScenarioConfigEntry = MatrixQaSummary["config"]["scenarios"][number]; type MatrixQaSummary = { @@ -178,6 +183,25 @@ function buildMatrixQaScenarioResult(params: { }; } +function scheduleMatrixQaScenariosByConfig( + scenarios: readonly (typeof MATRIX_QA_SCENARIOS)[number][], +): MatrixQaScheduledScenario[] { + const grouped = new Map(); + + scenarios.forEach((scenario, originalIndex) => { + const configKey = buildMatrixQaGatewayConfigKey(scenario.configOverrides); + const existing = grouped.get(configKey); + const scheduled = { originalIndex, scenario }; + if (existing) { + existing.push(scheduled); + return; + } + grouped.set(configKey, [scheduled]); + }); + + return [...grouped.values()].flat(); +} + export type MatrixQaRunResult = { observedEventsPath: string; outputDir: string; @@ -368,7 +392,9 @@ export async function runMatrixQaLive(params: { ].join("\n"), }, ]; - const scenarioResults: MatrixQaScenarioResult[] = []; + const scenarioResults: Array = Array.from({ + length: scenarios.length, + }); const cleanupErrors: string[] = []; let canaryArtifact: MatrixQaCanaryArtifact | undefined; let gatewayHarness: MatrixQaLiveLaneGatewayHarness | null = null; @@ -388,6 +414,8 @@ export async function runMatrixQaLive(params: { const defaultConfigSnapshot = buildMatrixQaConfigSnapshot(gatewayConfigParams); const scenarioConfigSnapshots: MatrixQaScenarioConfigEntry[] = []; + const scheduledScenarios = scheduleMatrixQaScenariosByConfig(scenarios); + try { const ensureGatewayHarness = async (overrides?: MatrixQaConfigOverrides) => { const nextKey = buildMatrixQaGatewayConfigKey(overrides); @@ -460,13 +488,13 @@ export async function runMatrixQaLive(params: { } if (!canaryFailed) { - for (const scenario of scenarios) { + for (const { scenario, originalIndex } of scheduledScenarios) { const { entry: scenarioConfigEntry, summary: scenarioConfigSummary } = buildMatrixQaScenarioConfigEntry({ gatewayConfigParams, scenario, }); - scenarioConfigSnapshots.push(scenarioConfigEntry); + scenarioConfigSnapshots[originalIndex] = scenarioConfigEntry; try { const scenarioGateway = await ensureGatewayHarness(scenario.configOverrides); const result = await runMatrixQaScenario(scenario, { @@ -497,24 +525,20 @@ export async function runMatrixQaLive(params: { timeoutMs: scenario.timeoutMs, topology: provisioning.topology, }); - scenarioResults.push( - buildMatrixQaScenarioResult({ - artifacts: result.artifacts, - configSummary: scenarioConfigSummary, - details: result.details, - scenario, - status: "pass", - }), - ); + scenarioResults[originalIndex] = buildMatrixQaScenarioResult({ + artifacts: result.artifacts, + configSummary: scenarioConfigSummary, + details: result.details, + scenario, + status: "pass", + }); } catch (error) { - scenarioResults.push( - buildMatrixQaScenarioResult({ - configSummary: scenarioConfigSummary, - details: formatErrorMessage(error), - scenario, - status: "fail", - }), - ); + scenarioResults[originalIndex] = buildMatrixQaScenarioResult({ + configSummary: scenarioConfigSummary, + details: formatErrorMessage(error), + scenario, + status: "fail", + }); } } } @@ -532,6 +556,9 @@ export async function runMatrixQaLive(params: { appendLiveLaneIssue(cleanupErrors, "Matrix harness cleanup", error); } } + const completedScenarioResults = scenarioResults.filter( + (scenario): scenario is MatrixQaScenarioResult => scenario !== undefined, + ); if (cleanupErrors.length > 0) { checks.push({ name: "Matrix cleanup", @@ -555,7 +582,7 @@ export async function runMatrixQaLive(params: { startedAt: startedAtDate, finishedAt: finishedAtDate, checks, - scenarios: scenarioResults.map((scenario) => ({ + scenarios: completedScenarioResults.map((scenario) => ({ details: scenario.details, name: scenario.title, status: scenario.status, @@ -592,7 +619,7 @@ export async function runMatrixQaLive(params: { serverName: harness.serverName, }, observedEventCount: observedEvents.length, - scenarios: scenarioResults, + scenarios: completedScenarioResults, startedAt, sutAccountId, userIds: { @@ -623,7 +650,7 @@ export async function runMatrixQaLive(params: { const failedChecks = checks.filter( (check) => check.status === "fail" && check.name !== "Matrix cleanup", ); - const failedScenarios = scenarioResults.filter((scenario) => scenario.status === "fail"); + const failedScenarios = completedScenarioResults.filter((scenario) => scenario.status === "fail"); if (failedChecks.length > 0 || failedScenarios.length > 0) { throw new Error( buildLiveLaneArtifactsError({ @@ -651,16 +678,18 @@ export async function runMatrixQaLive(params: { observedEventsPath, outputDir, reportPath, - scenarios: scenarioResults, + scenarios: completedScenarioResults, summaryPath, }; } export const __testing = { buildMatrixQaSummary, + scheduleMatrixQaScenariosByConfig, MATRIX_QA_SCENARIOS, buildMatrixQaConfig, buildMatrixQaConfigSnapshot, + findMatrixQaScenarios, isMatrixAccountReady, resolveMatrixQaModels, summarizeMatrixQaConfigSnapshot, diff --git a/extensions/qa-matrix/src/substrate/client.ts b/extensions/qa-matrix/src/substrate/client.ts index f7e34c08267..8cc5a2db628 100644 --- a/extensions/qa-matrix/src/substrate/client.ts +++ b/extensions/qa-matrix/src/substrate/client.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import type { MatrixQaObservedEvent } from "./events.js"; import { requestMatrixJson, type MatrixQaFetchLike } from "./request.js"; @@ -474,7 +475,7 @@ async function joinRoomWithRetry(params: { return; } catch (error) { lastError = error; - await new Promise((resolve) => setTimeout(resolve, 300 * attempt)); + await sleep(300 * attempt); } } throw new Error(`Matrix join retry failed: ${formatErrorMessage(lastError)}`); @@ -504,40 +505,42 @@ async function provisionMatrixQaTopology(params: { fetchImpl?: MatrixQaFetchLike; spec: MatrixQaTopologySpec; }): Promise { - const rooms = []; - - for (const room of params.spec.rooms) { - const members = resolveTopologyMemberAccounts(params.accounts, room.members); - const creator = members[0]; - const invitees = members.slice(1); - const creatorClient = createMatrixQaClient({ - accessToken: creator.account.accessToken, - baseUrl: params.baseUrl, - fetchImpl: params.fetchImpl, - }); - const roomId = await creatorClient.createPrivateRoom({ - inviteUserIds: invitees.map((entry) => entry.account.userId), - isDirect: room.kind === "dm", - name: room.name, - }); - for (const invitee of invitees) { - await joinRoomWithRetry({ - accessToken: invitee.account.accessToken, + const rooms = await Promise.all( + params.spec.rooms.map(async (room) => { + const members = resolveTopologyMemberAccounts(params.accounts, room.members); + const creator = members[0]; + const invitees = members.slice(1); + const creatorClient = createMatrixQaClient({ + accessToken: creator.account.accessToken, baseUrl: params.baseUrl, fetchImpl: params.fetchImpl, - roomId, }); - } - rooms.push({ - key: room.key, - kind: room.kind, - memberRoles: members.map((entry) => entry.role), - memberUserIds: members.map((entry) => entry.account.userId), - name: room.name, - requireMention: resolveProvisionedRoomRequireMention(room), - roomId, - }); - } + const roomId = await creatorClient.createPrivateRoom({ + inviteUserIds: invitees.map((entry) => entry.account.userId), + isDirect: room.kind === "dm", + name: room.name, + }); + await Promise.all( + invitees.map((invitee) => + joinRoomWithRetry({ + accessToken: invitee.account.accessToken, + baseUrl: params.baseUrl, + fetchImpl: params.fetchImpl, + roomId, + }), + ), + ); + return { + key: room.key, + kind: room.kind, + memberRoles: members.map((entry) => entry.role), + memberUserIds: members.map((entry) => entry.account.userId), + name: room.name, + requireMention: resolveProvisionedRoomRequireMention(room), + roomId, + }; + }), + ); const defaultRoom = findMatrixQaProvisionedRoom( { @@ -569,24 +572,26 @@ export async function provisionMatrixQaRoom(params: { baseUrl: params.baseUrl, fetchImpl: params.fetchImpl, }); - const driver = await anonClient.registerWithToken({ - deviceName: "OpenClaw Matrix QA Driver", - localpart: params.driverLocalpart, - password: `driver-${randomUUID()}`, - registrationToken: params.registrationToken, - }); - const sut = await anonClient.registerWithToken({ - deviceName: "OpenClaw Matrix QA SUT", - localpart: params.sutLocalpart, - password: `sut-${randomUUID()}`, - registrationToken: params.registrationToken, - }); - const observer = await anonClient.registerWithToken({ - deviceName: "OpenClaw Matrix QA Observer", - localpart: params.observerLocalpart, - password: `observer-${randomUUID()}`, - registrationToken: params.registrationToken, - }); + const [driver, sut, observer] = await Promise.all([ + anonClient.registerWithToken({ + deviceName: "OpenClaw Matrix QA Driver", + localpart: params.driverLocalpart, + password: `driver-${randomUUID()}`, + registrationToken: params.registrationToken, + }), + anonClient.registerWithToken({ + deviceName: "OpenClaw Matrix QA SUT", + localpart: params.sutLocalpart, + password: `sut-${randomUUID()}`, + registrationToken: params.registrationToken, + }), + anonClient.registerWithToken({ + deviceName: "OpenClaw Matrix QA Observer", + localpart: params.observerLocalpart, + password: `observer-${randomUUID()}`, + registrationToken: params.registrationToken, + }), + ]); const topology = await provisionMatrixQaTopology({ accounts: { driver,