vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 14:00:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: add Matrix QA profiles

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-27 05:43:10 +01:00
parent 382e03a2d8
commit 6987132aed
23 changed files with 446 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/qa-matrix/src/cli.test.ts
View File

@@ -67,6 +67,8 @@ describe("matrix qa cli registration", () => {
"--alt-model",
"--scenario",
"--fast",
"--profile",
"--fail-fast",
"--sut-account",
]),
);

3
extensions/qa-matrix/src/cli.ts
View File

@@ -56,6 +56,9 @@ export const matrixQaCliRegistration: LiveTransportQaCliRegistration =
commandName: "matrix",
description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver",
outputDirHelp: "Matrix QA artifact directory",
profileHelp:
"Matrix QA profile: all, fast, transport, media, e2ee-smoke, e2ee-deep, or e2ee-cli (default: all)",
failFastHelp: "Stop after the first failed Matrix check or scenario",
scenarioHelp: "Run only the named Matrix QA scenario (repeatable)",
sutAccountHelp: "Temporary Matrix account id inside the QA gateway config",
run: runQaMatrix,

10
extensions/qa-matrix/src/runners/contract/runtime.test.ts
View File

@@ -491,11 +491,14 @@ describe("matrix live qa runtime", () => {
expect(report).toContain("observed events: /tmp/observed.json");
});
it("keeps Matrix scenario execution in catalog order across config changes", () => {
it("groups Matrix scenario execution by gateway config while preserving tail scenarios", () => {
const scenarios = liveTesting.findMatrixQaScenarios([
"matrix-thread-follow-up",
"matrix-e2ee-cli-encryption-setup-multi-account",
"matrix-thread-isolation",
"matrix-e2ee-cli-setup-then-gateway-reply",
"matrix-e2ee-cli-self-verification",
"matrix-e2ee-wrong-account-recovery-key",
]);
expect(
@@ -503,9 +506,12 @@ describe("matrix live qa runtime", () => {
.scheduleMatrixQaScenariosInCatalogOrder(scenarios)
.map(({ scenario }) => scenario.id),
).toEqual([
"matrix-thread-follow-up",
"matrix-thread-isolation",
"matrix-e2ee-cli-self-verification",
"matrix-e2ee-cli-encryption-setup-multi-account",
"matrix-e2ee-cli-setup-then-gateway-reply",
"matrix-e2ee-cli-self-verification",
"matrix-e2ee-wrong-account-recovery-key",
]);
});

34
extensions/qa-matrix/src/runners/contract/runtime.ts
View File

@@ -61,6 +61,8 @@ function buildMatrixQaGatewayConfigKey(overrides?: MatrixQaConfigOverrides) {
return JSON.stringify(overrides ?? null);
}
const MATRIX_QA_EXECUTION_TAIL_SCENARIO_IDS = new Set(["matrix-e2ee-wrong-account-recovery-key"]);
type MatrixQaScenarioResult = {
artifacts?: MatrixQaScenarioArtifacts;
details: string;
@@ -313,7 +315,27 @@ function buildMatrixQaScenarioResult(params: {
function scheduleMatrixQaScenariosInCatalogOrder(
scenarios: readonly (typeof MATRIX_QA_SCENARIOS)[number][],
): MatrixQaScheduledScenario[] {
return scenarios.map((scenario, originalIndex) => ({ originalIndex, scenario }));
const entries = scenarios.map((scenario, originalIndex) => ({ originalIndex, scenario }));
const groupedEntries: MatrixQaScheduledScenario[][] = [];
const groupIndexes = new Map<string, number>();
const tailEntries: MatrixQaScheduledScenario[] = [];
for (const entry of entries) {
if (MATRIX_QA_EXECUTION_TAIL_SCENARIO_IDS.has(entry.scenario.id)) {
tailEntries.push(entry);
continue;
}
const key = buildMatrixQaGatewayConfigKey(entry.scenario.configOverrides);
const existingIndex = groupIndexes.get(key);
if (existingIndex !== undefined) {
groupedEntries[existingIndex]?.push(entry);
continue;
}
groupIndexes.set(key, groupedEntries.length);
groupedEntries.push([entry]);
}
return [...groupedEntries.flat(), ...tailEntries];
}
function getMatrixQaScenarioRestartReadyTimeoutMs(scenario: { timeoutMs: number }): number {
@@ -498,8 +520,10 @@ async function startMatrixQaLiveLaneGateway(params: {
export async function runMatrixQaLive(params: {
fastMode?: boolean;
failFast?: boolean;
outputDir?: string;
primaryModel?: string;
profile?: string;
providerMode?: QaProviderModeInput;
repoRoot?: string;
scenarioIds?: string[];
@@ -518,7 +542,7 @@ export async function runMatrixQaLive(params: {
alternateModel: params.alternateModel,
});
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findMatrixQaScenarios(params.scenarioIds);
const scenarios = findMatrixQaScenarios(params.scenarioIds, params.profile);
const runSuffix = randomUUID().slice(0, 8);
const topology = buildMatrixQaTopologyForScenarios({
defaultRoomName: `OpenClaw Matrix QA ${runSuffix}`,
@@ -531,7 +555,7 @@ export async function runMatrixQaLive(params: {
const runStartedAtMs = Date.now();
const runDeadline = createMatrixQaRunDeadline();
writeMatrixQaProgress(
`suite start scenarios=${scenarios.length} provider=${providerMode} output=${outputDir} timeout=${formatMatrixQaDurationMs(runDeadline.timeoutMs)}`,
`suite start scenarios=${scenarios.length} profile=${params.profile?.trim() || "all"} provider=${providerMode} output=${outputDir} timeout=${formatMatrixQaDurationMs(runDeadline.timeoutMs)}`,
);
const { durationMs: harnessBootMs, result: harness } = await measureMatrixQaStep(() =>
@@ -895,6 +919,10 @@ export async function runMatrixQaLive(params: {
status: "fail",
});
writeMatrixQaProgress(`scenario fail ${scenario.id} ${formatErrorMessage(error)}`);
if (params.failFast) {
writeMatrixQaProgress("fail-fast stop");
break;
}
}
}
}

116
extensions/qa-matrix/src/runners/contract/scenario-catalog.ts
View File

@@ -96,6 +96,15 @@ export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQ
topology?: MatrixQaTopologySpec;
};
export type MatrixQaProfile =
| "all"
| "e2ee-cli"
| "e2ee-deep"
| "e2ee-smoke"
| "fast"
| "media"
| "transport";
export const MATRIX_QA_BLOCK_ROOM_KEY = "block";
export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm";
export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared";
@@ -907,14 +916,117 @@ export const MATRIX_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScena
scenarios: MATRIX_QA_SCENARIOS,
});
export function findMatrixQaScenarios(ids?: string[]) {
export const MATRIX_QA_PROFILE_NAMES: readonly MatrixQaProfile[] = [
"all",
"fast",
"transport",
"media",
"e2ee-smoke",
"e2ee-deep",
"e2ee-cli",
] as const;
const MATRIX_QA_FAST_PROFILE_SCENARIO_IDS = [
"matrix-thread-follow-up",
"matrix-thread-isolation",
"matrix-top-level-reply-shape",
"matrix-reaction-notification",
"matrix-restart-resume",
"matrix-mention-gating",
"matrix-allowlist-block",
"matrix-e2ee-basic-reply",
] satisfies MatrixQaScenarioId[];
const MATRIX_QA_MEDIA_PROFILE_SCENARIO_IDS = [
"matrix-room-image-understanding-attachment",
"matrix-room-generated-image-delivery",
"matrix-media-type-coverage",
"matrix-attachment-only-ignored",
"matrix-unsupported-media-safe",
"matrix-e2ee-media-image",
] satisfies MatrixQaScenarioId[];
const MATRIX_QA_E2EE_SMOKE_PROFILE_SCENARIO_IDS = [
"matrix-e2ee-basic-reply",
"matrix-e2ee-thread-follow-up",
"matrix-e2ee-bootstrap-success",
"matrix-e2ee-recovery-key-lifecycle",
"matrix-e2ee-recovery-owner-verification-required",
"matrix-e2ee-restart-resume",
"matrix-e2ee-artifact-redaction",
"matrix-e2ee-key-bootstrap-failure",
] satisfies MatrixQaScenarioId[];
function isMatrixQaE2eeScenarioId(id: MatrixQaScenarioId): id is MatrixQaE2eeScenarioId {
return id.startsWith("matrix-e2ee-");
}
function isMatrixQaCliE2eeScenarioId(id: MatrixQaScenarioId) {
return id.startsWith("matrix-e2ee-cli-");
}
function buildMatrixQaScenarioIdSet(ids: readonly MatrixQaScenarioId[]) {
return new Set<MatrixQaScenarioId>(ids);
}
function normalizeMatrixQaProfile(profile?: string): MatrixQaProfile {
const normalized = profile?.trim().toLowerCase() || "all";
if (MATRIX_QA_PROFILE_NAMES.includes(normalized as MatrixQaProfile)) {
return normalized as MatrixQaProfile;
}
throw new Error(
`unknown Matrix QA profile "${profile}"; expected one of: ${MATRIX_QA_PROFILE_NAMES.join(", ")}`,
);
}
function getMatrixQaProfileScenarioIds(profile: MatrixQaProfile): MatrixQaScenarioId[] {
const allIds = MATRIX_QA_SCENARIOS.map((scenario) => scenario.id);
const mediaIds = buildMatrixQaScenarioIdSet(MATRIX_QA_MEDIA_PROFILE_SCENARIO_IDS);
const smokeIds = buildMatrixQaScenarioIdSet(MATRIX_QA_E2EE_SMOKE_PROFILE_SCENARIO_IDS);
switch (profile) {
case "all":
return allIds;
case "fast":
return [...MATRIX_QA_FAST_PROFILE_SCENARIO_IDS];
case "transport":
return allIds.filter((id) => !isMatrixQaE2eeScenarioId(id) && !mediaIds.has(id));
case "media":
return [...MATRIX_QA_MEDIA_PROFILE_SCENARIO_IDS];
case "e2ee-smoke":
return [...MATRIX_QA_E2EE_SMOKE_PROFILE_SCENARIO_IDS];
case "e2ee-cli":
return allIds.filter(isMatrixQaCliE2eeScenarioId);
case "e2ee-deep":
return allIds.filter(
(id) =>
isMatrixQaE2eeScenarioId(id) &&
!isMatrixQaCliE2eeScenarioId(id) &&
!mediaIds.has(id) &&
!smokeIds.has(id),
);
default: {
const exhaustiveProfile: never = profile;
return exhaustiveProfile;
}
}
}
export function findMatrixQaScenarios(ids?: string[], profile?: string) {
const normalizedProfile = normalizeMatrixQaProfile(profile);
const selectedIds =
ids && ids.length > 0 ? ids : getMatrixQaProfileScenarioIds(normalizedProfile);
return selectLiveTransportScenarios({
ids,
ids: selectedIds,
laneLabel: "Matrix",
scenarios: MATRIX_QA_SCENARIOS,
});
}
export const __matrixQaProfileTesting = {
getMatrixQaProfileScenarioIds,
normalizeMatrixQaProfile,
};
export function buildMatrixQaTopologyForScenarios(params: {
defaultRoomName: string;
scenarios: MatrixQaScenarioDefinition[];

4
extensions/qa-matrix/src/runners/contract/scenario-runtime-dm.ts
View File

@@ -12,8 +12,8 @@ import {
buildMatrixReplyArtifact,
buildMatrixReplyDetails,
createMatrixQaScenarioClient,
NO_REPLY_WINDOW_MS,
advanceMatrixQaActorCursor,
resolveMatrixQaNoReplyWindowMs,
runConfigurableTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
@@ -98,7 +98,7 @@ async function runDmSharedSessionFlow(params: {
event.body.includes("channels.matrix.dm.sessionScope"),
roomId: secondRoomId,
since: noticeSince,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(params.context.timeoutMs),
}),
]);

8
extensions/qa-matrix/src/runners/contract/scenario-runtime-e2ee.ts
View File

@@ -50,7 +50,7 @@ import {
buildMentionPrompt,
doesMatrixQaReplyBodyMatchToken,
isMatrixQaExactMarkerReply,
NO_REPLY_WINDOW_MS,
resolveMatrixQaNoReplyWindowMs,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
import type { MatrixQaReplyArtifact, MatrixQaScenarioExecution } from "./scenario-types.js";
@@ -3310,14 +3310,14 @@ export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario(
token,
}),
roomId,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
});
if (result.matched) {
throw new Error(`unexpected E2EE verification-notice reply: ${result.event.eventId}`);
}
return {
artifacts: {
expectedNoReplyWindowMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
expectedNoReplyWindowMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
noticeEventId,
roomKey,
roomId,
@@ -3326,7 +3326,7 @@ export async function runMatrixQaE2eeVerificationNoticeNoTriggerScenario(
`encrypted room key: ${roomKey}`,
`encrypted room id: ${roomId}`,
`verification notice event: ${noticeEventId}`,
`waited ${Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs)}ms with no SUT reply`,
`waited ${resolveMatrixQaNoReplyWindowMs(context.timeoutMs)}ms with no SUT reply`,
].join("\n"),
};
},

12
extensions/qa-matrix/src/runners/contract/scenario-runtime-restart.ts
View File

@@ -12,8 +12,8 @@ import {
isMatrixQaExactMarkerReply,
assertTopLevelReplyArtifact,
advanceMatrixQaActorCursor,
NO_REPLY_WINDOW_MS,
primeMatrixQaDriverScenarioClient,
resolveMatrixQaNoReplyWindowMs,
runAssertedDriverTopLevelScenario,
type MatrixQaScenarioContext,
} from "./scenario-runtime-shared.js";
@@ -254,7 +254,7 @@ async function assertNoRestartReplayDuplicate(params: {
token: params.replayToken,
}),
roomId: params.roomId,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(params.context.timeoutMs),
});
if (duplicate.matched) {
throw new Error(
@@ -313,7 +313,7 @@ export async function runRestartReplayDedupeScenario(context: MatrixQaScenarioCo
return {
artifacts: {
duplicateWindowMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
duplicateWindowMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
firstDriverEventId: replayDriverEventId,
firstReply,
firstToken: replayToken,
@@ -328,7 +328,7 @@ export async function runRestartReplayDedupeScenario(context: MatrixQaScenarioCo
"restart signal: SIGUSR1",
`first driver event: ${replayDriverEventId}`,
...buildMatrixReplyDetails("first reply", firstReply),
`duplicate replay window: ${Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs)}ms`,
`duplicate replay window: ${resolveMatrixQaNoReplyWindowMs(context.timeoutMs)}ms`,
`fresh post-restart driver event: ${postRestart.driverEventId}`,
...buildMatrixReplyDetails("fresh reply", postRestart.reply),
].join("\n"),
@@ -401,7 +401,7 @@ export async function runStaleSyncReplayDedupeScenario(context: MatrixQaScenario
return {
artifacts: {
dedupeCommitObserved: true,
duplicateWindowMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
duplicateWindowMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
firstDriverEventId: replayDriverEventId,
firstReply,
firstToken: replayToken,
@@ -418,7 +418,7 @@ export async function runStaleSyncReplayDedupeScenario(context: MatrixQaScenario
`stale sync cursor: ${staleCursor}`,
`first driver event: ${replayDriverEventId}`,
...buildMatrixReplyDetails("first reply", firstReply),
`duplicate replay window: ${Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs)}ms`,
`duplicate replay window: ${resolveMatrixQaNoReplyWindowMs(context.timeoutMs)}ms`,
`fresh post-restart driver event: ${postRestart.driverEventId}`,
...buildMatrixReplyDetails("fresh reply", postRestart.reply),
].join("\n"),

6
extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
View File

@@ -24,9 +24,9 @@ import {
createMatrixQaScenarioClient,
isMatrixQaExactMarkerReply,
isMatrixQaMessageLikeKind,
NO_REPLY_WINDOW_MS,
primeMatrixQaActorCursor,
primeMatrixQaDriverScenarioClient,
resolveMatrixQaNoReplyWindowMs,
runAssertedDriverTopLevelScenario,
runConfigurableTopLevelScenario,
runDriverTopLevelMentionScenario,
@@ -530,7 +530,7 @@ export async function runAllowlistHotReloadScenario(context: MatrixQaScenarioCon
sutUserId: context.sutUserId,
token: blockedToken,
}),
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
token: blockedToken,
});
@@ -767,7 +767,7 @@ export async function runMembershipLossScenario(context: MatrixQaScenarioContext
syncState: context.syncState,
syncStreams: context.syncStreams,
sutUserId: context.sutUserId,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
token: noReplyToken,
});

10
extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
View File

@@ -56,6 +56,14 @@ export type MatrixQaScenarioContext = {
};
export const NO_REPLY_WINDOW_MS = 8_000;
const NO_REPLY_WINDOW_ENV = "OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS";
export function resolveMatrixQaNoReplyWindowMs(timeoutMs: number) {
const raw = process.env[NO_REPLY_WINDOW_ENV];
const parsed = raw === undefined ? NO_REPLY_WINDOW_MS : Number(raw);
const windowMs = Number.isFinite(parsed) && parsed >= 1 ? Math.floor(parsed) : NO_REPLY_WINDOW_MS;
return Math.min(windowMs, timeoutMs);
}
export function buildMentionPrompt(sutUserId: string, token: string) {
return `${sutUserId} reply with only this exact marker: ${token}`;
@@ -316,7 +324,7 @@ export async function assertNoSutReplyWindow(params: {
unexpectedLines?: string[];
unexpectedMessage: string;
}) {
const noReplyWindowMs = Math.min(NO_REPLY_WINDOW_MS, params.context.timeoutMs);
const noReplyWindowMs = resolveMatrixQaNoReplyWindowMs(params.context.timeoutMs);
const result = await params.client.waitForOptionalRoomEvent({
observedEvents: params.context.observedEvents,
predicate: (event) =>

4
extensions/qa-matrix/src/runners/contract/scenario-runtime.ts
View File

@@ -94,8 +94,8 @@ import {
buildMatrixReplyArtifact,
buildMatrixReplyDetails,
buildMentionPrompt,
NO_REPLY_WINDOW_MS,
readMatrixQaSyncCursor,
resolveMatrixQaNoReplyWindowMs,
runNoReplyExpectedScenario,
runTopologyScopedTopLevelScenario,
writeMatrixQaSyncCursor,
@@ -167,7 +167,7 @@ async function runMultiActorOrderingScenario(context: MatrixQaScenarioContext) {
body: buildMentionPrompt(context.sutUserId, blockedToken),
mentionUserIds: [context.sutUserId],
context,
timeoutMs: Math.min(NO_REPLY_WINDOW_MS, context.timeoutMs),
timeoutMs: resolveMatrixQaNoReplyWindowMs(context.timeoutMs),
token: blockedToken,
});
const accepted = await runDriverTopologyScopedScenario({

48
extensions/qa-matrix/src/runners/contract/scenarios.test.ts
View File

@@ -256,6 +256,54 @@ describe("matrix live qa scenarios", () => {
}
});
it("keeps the Matrix CLI default profile on the full catalog", () => {
const allIds = scenarioTesting.findMatrixQaScenarios().map((scenario) => scenario.id);
expect(
scenarioTesting.findMatrixQaScenarios(undefined, "all").map((scenario) => scenario.id),
).toEqual(allIds);
});
it("selects the fast release-critical Matrix profile without media or deep E2EE inventory", () => {
expect(
scenarioTesting.findMatrixQaScenarios(undefined, "fast").map((scenario) => scenario.id),
).toEqual([
"matrix-thread-follow-up",
"matrix-thread-isolation",
"matrix-top-level-reply-shape",
"matrix-reaction-notification",
"matrix-restart-resume",
"matrix-mention-gating",
"matrix-allowlist-block",
"matrix-e2ee-basic-reply",
]);
});
it("keeps the full Matrix shard profiles exhaustive and disjoint", () => {
const allIds = scenarioTesting.findMatrixQaScenarios().map((scenario) => scenario.id);
const shardIds = ["transport", "media", "e2ee-smoke", "e2ee-deep", "e2ee-cli"].flatMap(
(profile) =>
scenarioTesting.findMatrixQaScenarios(undefined, profile).map((scenario) => scenario.id),
);
expect(new Set(shardIds).size).toBe(shardIds.length);
expect(shardIds.toSorted()).toEqual(allIds.toSorted());
});
it("lets explicit Matrix scenario ids override the selected profile", () => {
expect(
scenarioTesting
.findMatrixQaScenarios(["matrix-room-generated-image-delivery"], "fast")
.map((scenario) => scenario.id),
).toEqual(["matrix-room-generated-image-delivery"]);
});
it("fails when the Matrix profile is unknown", () => {
expect(() => scenarioTesting.findMatrixQaScenarios(undefined, "speedy")).toThrow(
'unknown Matrix QA profile "speedy"',
);
});
it("uses the repo-wide exact marker prompt shape for Matrix mentions", () => {
expect(
scenarioTesting.buildMentionPrompt("@sut:matrix-qa.test", "MATRIX_QA_CANARY_TOKEN"),

8
extensions/qa-matrix/src/runners/contract/scenarios.ts
View File

@@ -3,6 +3,7 @@ import {
MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY,
MATRIX_QA_E2EE_ROOM_KEY,
MATRIX_QA_MEDIA_ROOM_KEY,
MATRIX_QA_PROFILE_NAMES,
MATRIX_QA_MEMBERSHIP_ROOM_KEY,
MATRIX_QA_SCENARIOS,
MATRIX_QA_SECONDARY_ROOM_KEY,
@@ -13,6 +14,8 @@ import {
resolveMatrixQaScenarioRoomId,
type MatrixQaScenarioDefinition,
type MatrixQaScenarioId,
type MatrixQaProfile,
__matrixQaProfileTesting,
} from "./scenario-catalog.js";
import {
buildMatrixReplyArtifact,
@@ -34,6 +37,7 @@ import type {
export type { MatrixQaScenarioDefinition, MatrixQaScenarioId };
export {
MATRIX_QA_PROFILE_NAMES,
MATRIX_QA_SCENARIOS,
MATRIX_QA_STANDARD_SCENARIO_IDS,
buildMatrixReplyArtifact,
@@ -46,6 +50,7 @@ export {
runMatrixQaCanary,
runMatrixQaScenario,
};
export type { MatrixQaProfile };
export type {
MatrixQaCanaryArtifact,
MatrixQaReplyArtifact,
@@ -61,6 +66,7 @@ export const __testing = {
MATRIX_QA_E2EE_ROOM_KEY,
MATRIX_QA_MEDIA_ROOM_KEY,
MATRIX_QA_MEMBERSHIP_ROOM_KEY,
MATRIX_QA_PROFILE_NAMES,
MATRIX_QA_SECONDARY_ROOM_KEY,
MATRIX_QA_STANDARD_SCENARIO_IDS,
buildMatrixQaE2eeScenarioRoomKey,
@@ -69,6 +75,8 @@ export const __testing = {
buildMatrixReplyArtifact,
buildMentionPrompt,
findMatrixQaScenarios,
getMatrixQaProfileScenarioIds: __matrixQaProfileTesting.getMatrixQaProfileScenarioIds,
normalizeMatrixQaProfile: __matrixQaProfileTesting.normalizeMatrixQaProfile,
readMatrixQaSyncCursor,
resolveMatrixQaScenarioRoomId,
writeMatrixQaSyncCursor,

2
extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts
View File

@@ -27,6 +27,8 @@ export function resolveLiveTransportQaRunOptions(
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,
fastMode: opts.fastMode,
failFast: opts.failFast,
profile: opts.profile?.trim(),
scenarioIds: opts.scenarioIds,
sutAccountId: opts.sutAccountId,
credentialSource: opts.credentialSource?.trim(),

20
extensions/qa-matrix/src/shared/live-transport-cli.ts
View File

@@ -9,6 +9,8 @@ export type LiveTransportQaCommandOptions = {
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;
failFast?: boolean;
profile?: string;
scenarioIds?: string[];
sutAccountId?: string;
credentialSource?: string;
@@ -23,6 +25,8 @@ type LiveTransportQaCommanderOptions = {
altModel?: string;
scenario?: string[];
fast?: boolean;
failFast?: boolean;
profile?: string;
sutAccount?: string;
credentialSource?: string;
credentialRole?: string;
@@ -56,6 +60,8 @@ export function mapLiveTransportQaCommanderOptions(
primaryModel: opts.model,
alternateModel: opts.altModel,
fastMode: opts.fast,
failFast: opts.failFast,
profile: opts.profile,
scenarioIds: opts.scenario,
sutAccountId: opts.sutAccount,
credentialSource: opts.credentialSource,
@@ -69,6 +75,8 @@ export function registerLiveTransportQaCli(params: {
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
profileHelp?: string;
failFastHelp?: string;
scenarioHelp: string;
sutAccountHelp: string;
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
@@ -89,6 +97,14 @@ export function registerLiveTransportQaCli(params: {
.option("--fast", "Enable provider fast mode where supported", false)
.option("--sut-account <id>", params.sutAccountHelp, "sut");
if (params.profileHelp) {
command.option("--profile <profile>", params.profileHelp);
}
if (params.failFastHelp) {
command.option("--fail-fast", params.failFastHelp, false);
}
if (params.credentialOptions) {
command.option(
"--credential-source <source>",
@@ -110,6 +126,8 @@ export function createLiveTransportQaCliRegistration(params: {
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
profileHelp?: string;
failFastHelp?: string;
scenarioHelp: string;
sutAccountHelp: string;
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
@@ -123,6 +141,8 @@ export function createLiveTransportQaCliRegistration(params: {
credentialOptions: params.credentialOptions,
description: params.description,
outputDirHelp: params.outputDirHelp,
profileHelp: params.profileHelp,
failFastHelp: params.failFastHelp,
scenarioHelp: params.scenarioHelp,
sutAccountHelp: params.sutAccountHelp,
run: params.run,