refactor(qa): split Matrix QA into optional plugin (#66723)

Merged via squash.

Prepared head SHA: 27241bd089
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
Gustavo Madeira Santana
2026-04-14 16:28:57 -04:00
committed by GitHub
parent 3425823dfb
commit 82a2db71e8
69 changed files with 2026 additions and 229 deletions

View File

@@ -1 +1,2 @@
export * from "./src/runtime-api.js";
export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js";

View File

@@ -8,7 +8,6 @@ const {
runQaSuiteFromRuntime,
runQaCharacterEval,
runQaMultipass,
runMatrixQaLive,
runTelegramQaLive,
startQaLabServer,
writeQaDockerHarnessFiles,
@@ -20,7 +19,6 @@ const {
runQaSuiteFromRuntime: vi.fn(),
runQaCharacterEval: vi.fn(),
runQaMultipass: vi.fn(),
runMatrixQaLive: vi.fn(),
runTelegramQaLive: vi.fn(),
startQaLabServer: vi.fn(),
writeQaDockerHarnessFiles: vi.fn(),
@@ -52,10 +50,6 @@ vi.mock("./multipass.runtime.js", () => ({
runQaMultipass,
}));
vi.mock("./live-transports/matrix/matrix-live.runtime.js", () => ({
runMatrixQaLive,
}));
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
runTelegramQaLive,
}));
@@ -88,7 +82,6 @@ import {
runQaParityReportCommand,
runQaSuiteCommand,
} from "./cli.runtime.js";
import { runQaMatrixCommand } from "./live-transports/matrix/cli.runtime.js";
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
describe("qa cli runtime", () => {
@@ -100,7 +93,6 @@ describe("qa cli runtime", () => {
runQaCharacterEval.mockReset();
runQaManualLane.mockReset();
runQaMultipass.mockReset();
runMatrixQaLive.mockReset();
runTelegramQaLive.mockReset();
startQaLabServer.mockReset();
writeQaDockerHarnessFiles.mockReset();
@@ -139,13 +131,6 @@ describe("qa cli runtime", () => {
vmName: "openclaw-qa-test",
scenarioIds: ["channel-chat-baseline"],
});
runMatrixQaLive.mockResolvedValue({
outputDir: "/tmp/matrix",
reportPath: "/tmp/matrix/report.md",
summaryPath: "/tmp/matrix/summary.json",
observedEventsPath: "/tmp/matrix/observed.json",
scenarios: [],
});
runTelegramQaLive.mockResolvedValue({
outputDir: "/tmp/telegram",
reportPath: "/tmp/telegram/report.md",
@@ -226,30 +211,6 @@ describe("qa cli runtime", () => {
});
});
it("resolves matrix qa repo-root-relative paths before dispatching", async () => {
await runQaMatrixCommand({
repoRoot: "/tmp/openclaw-repo",
outputDir: ".artifacts/qa/matrix",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "openai/gpt-5.4",
fastMode: true,
scenarioIds: ["matrix-thread-follow-up"],
sutAccountId: "sut-live",
});
expect(runMatrixQaLive).toHaveBeenCalledWith({
repoRoot: path.resolve("/tmp/openclaw-repo"),
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/matrix"),
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "openai/gpt-5.4",
fastMode: true,
scenarioIds: ["matrix-thread-follow-up"],
sutAccountId: "sut-live",
});
});
it("rejects output dirs that escape the repo root", () => {
expect(() => resolveRepoRelativeOutputDir("/tmp/openclaw-repo", "../outside")).toThrow(
"--output-dir must stay within the repo root.",
@@ -273,20 +234,6 @@ describe("qa cli runtime", () => {
);
});
it("defaults matrix qa runs onto the live provider lane", async () => {
await runQaMatrixCommand({
repoRoot: "/tmp/openclaw-repo",
scenarioIds: ["matrix-thread-follow-up"],
});
expect(runMatrixQaLive).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
providerMode: "live-frontier",
}),
);
});
it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",

View File

@@ -1,22 +1,76 @@
import { Command } from "commander";
import type { QaRunnerCliContribution } from "openclaw/plugin-sdk/qa-runner-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const TEST_QA_RUNNER = {
pluginId: "qa-runner-test",
commandName: "runner-test",
description: "Run the test live QA lane",
npmSpec: "@openclaw/qa-runner-test",
} as const;
function createAvailableQaRunnerContribution() {
return {
pluginId: TEST_QA_RUNNER.pluginId,
commandName: TEST_QA_RUNNER.commandName,
status: "available" as const,
registration: {
commandName: TEST_QA_RUNNER.commandName,
register: vi.fn((qa: Command) => {
qa.command(TEST_QA_RUNNER.commandName).action(() => undefined);
}),
},
} satisfies QaRunnerCliContribution;
}
function createMissingQaRunnerContribution(): QaRunnerCliContribution {
return {
pluginId: TEST_QA_RUNNER.pluginId,
commandName: TEST_QA_RUNNER.commandName,
description: TEST_QA_RUNNER.description,
status: "missing",
npmSpec: TEST_QA_RUNNER.npmSpec,
};
}
function createBlockedQaRunnerContribution(): QaRunnerCliContribution {
return {
pluginId: TEST_QA_RUNNER.pluginId,
commandName: TEST_QA_RUNNER.commandName,
description: TEST_QA_RUNNER.description,
status: "blocked",
};
}
function createConflictingQaRunnerContribution(commandName: string): QaRunnerCliContribution {
return {
pluginId: TEST_QA_RUNNER.pluginId,
commandName,
description: TEST_QA_RUNNER.description,
status: "blocked",
};
}
const {
runQaCredentialsAddCommand,
runQaCredentialsListCommand,
runQaCredentialsRemoveCommand,
runQaMatrixCommand,
runQaTelegramCommand,
} = vi.hoisted(() => ({
runQaCredentialsAddCommand: vi.fn(),
runQaCredentialsListCommand: vi.fn(),
runQaCredentialsRemoveCommand: vi.fn(),
runQaMatrixCommand: vi.fn(),
runQaTelegramCommand: vi.fn(),
}));
vi.mock("./live-transports/matrix/cli.runtime.js", () => ({
runQaMatrixCommand,
const { listQaRunnerCliContributions } = vi.hoisted(() => ({
listQaRunnerCliContributions: vi.fn<() => QaRunnerCliContribution[]>(() => [
createAvailableQaRunnerContribution(),
]),
}));
vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({
listQaRunnerCliContributions,
}));
vi.mock("./live-transports/telegram/cli.runtime.js", () => ({
@@ -36,63 +90,71 @@ describe("qa cli registration", () => {
beforeEach(() => {
program = new Command();
registerQaLabCli(program);
runQaCredentialsAddCommand.mockReset();
runQaCredentialsListCommand.mockReset();
runQaCredentialsRemoveCommand.mockReset();
runQaMatrixCommand.mockReset();
runQaTelegramCommand.mockReset();
listQaRunnerCliContributions
.mockReset()
.mockReturnValue([createAvailableQaRunnerContribution()]);
registerQaLabCli(program);
});
afterEach(() => {
vi.clearAllMocks();
});
it("registers the matrix and telegram live transport subcommands", () => {
it("registers discovered and built-in live transport subcommands", () => {
const qa = program.commands.find((command) => command.name() === "qa");
expect(qa).toBeDefined();
expect(qa?.commands.map((command) => command.name())).toEqual(
expect.arrayContaining(["matrix", "telegram", "credentials"]),
expect.arrayContaining([TEST_QA_RUNNER.commandName, "telegram", "credentials"]),
);
});
it("routes matrix CLI flags into the lane runtime", async () => {
await program.parseAsync([
"node",
"openclaw",
"qa",
"matrix",
"--repo-root",
"/tmp/openclaw-repo",
"--output-dir",
".artifacts/qa/matrix",
"--provider-mode",
"mock-openai",
"--model",
"mock-openai/gpt-5.4",
"--alt-model",
"mock-openai/gpt-5.4-alt",
"--scenario",
"matrix-thread-follow-up",
"--scenario",
"matrix-thread-isolation",
"--fast",
"--sut-account",
"sut-live",
]);
it("delegates discovered qa runner registration through the generic host seam", () => {
const [{ registration }] = listQaRunnerCliContributions.mock.results[0]?.value;
expect(registration.register).toHaveBeenCalledTimes(1);
});
expect(runQaMatrixCommand).toHaveBeenCalledWith({
repoRoot: "/tmp/openclaw-repo",
outputDir: ".artifacts/qa/matrix",
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.4",
alternateModel: "mock-openai/gpt-5.4-alt",
fastMode: true,
scenarioIds: ["matrix-thread-follow-up", "matrix-thread-isolation"],
sutAccountId: "sut-live",
credentialSource: undefined,
credentialRole: undefined,
});
it("keeps Telegram credential flags on the shared host CLI", () => {
const qa = program.commands.find((command) => command.name() === "qa");
const telegram = qa?.commands.find((command) => command.name() === "telegram");
const optionNames = telegram?.options.map((option) => option.long) ?? [];
expect(optionNames).toEqual(
expect.arrayContaining(["--credential-source", "--credential-role"]),
);
});
it("shows an install hint when a discovered runner plugin is unavailable", async () => {
listQaRunnerCliContributions.mockReset().mockReturnValue([createMissingQaRunnerContribution()]);
const missingProgram = new Command();
registerQaLabCli(missingProgram);
await expect(
missingProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
).rejects.toThrow(`openclaw plugins install ${TEST_QA_RUNNER.npmSpec}`);
});
it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
const blockedProgram = new Command();
registerQaLabCli(blockedProgram);
await expect(
blockedProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
).rejects.toThrow(`Enable or allow plugin "${TEST_QA_RUNNER.pluginId}"`);
});
it("rejects discovered runners that collide with built-in qa subcommands", () => {
listQaRunnerCliContributions
.mockReset()
.mockReturnValue([createConflictingQaRunnerContribution("manual")]);
expect(() => registerQaLabCli(new Command())).toThrow(
'QA runner command "manual" conflicts with an existing qa subcommand',
);
});
it("routes telegram CLI defaults into the lane runtime", async () => {

View File

@@ -1,6 +1,6 @@
import type { Command } from "commander";
import { collectString } from "./cli-options.js";
import { LIVE_TRANSPORT_QA_CLI_REGISTRATIONS } from "./live-transports/cli.js";
import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
import type { QaProviderModeInput } from "./run-config.js";
import { hasQaScenarioPack } from "./scenario-catalog.js";
@@ -183,6 +183,12 @@ export function isQaLabCliAvailable(): boolean {
return hasQaScenarioPack();
}
function assertNoQaSubcommandCollision(qa: Command, commandName: string) {
if (qa.commands.some((command) => command.name() === commandName)) {
throw new Error(`QA runner command "${commandName}" conflicts with an existing qa subcommand`);
}
}
export function registerQaLabCli(program: Command) {
const qa = program
.command("qa")
@@ -284,10 +290,6 @@ export function registerQaLabCli(program: Command) {
},
);
for (const lane of LIVE_TRANSPORT_QA_CLI_REGISTRATIONS) {
lane.register(qa);
}
qa.command("character-eval")
.description("Run the character QA scenario across live models and write a judged report")
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
@@ -579,4 +581,9 @@ export function registerQaLabCli(program: Command) {
.action(async (opts: { host?: string; port?: number }) => {
await runQaMockOpenAi(opts);
});
for (const lane of listLiveTransportQaCliRegistrations()) {
assertNoQaSubcommandCollision(qa, lane.commandName);
lane.register(qa);
}
}

View File

@@ -1,8 +1,78 @@
import { matrixQaCliRegistration } from "./matrix/cli.js";
import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
import { telegramQaCliRegistration } from "./telegram/cli.js";
function createMissingQaRunnerCliRegistration(params: {
commandName: string;
description: string;
npmSpec: string;
}): LiveTransportQaCliRegistration {
return {
commandName: params.commandName,
register(qa) {
qa.command(params.commandName)
.description(params.description)
.action(() => {
throw new Error(
`QA runner "${params.commandName}" not installed. Install it with "openclaw plugins install ${params.npmSpec}".`,
);
});
},
};
}
function createBlockedQaRunnerCliRegistration(params: {
commandName: string;
description?: string;
pluginId: string;
}): LiveTransportQaCliRegistration {
return {
commandName: params.commandName,
register(qa) {
qa.command(params.commandName)
.description(params.description ?? `Run the ${params.commandName} live QA lane`)
.action(() => {
throw new Error(
`QA runner "${params.commandName}" is installed but not active. Enable or allow plugin "${params.pluginId}" in your OpenClaw config, then try again.`,
);
});
},
};
}
function createQaRunnerCliRegistration(
runner: ReturnType<typeof listQaRunnerCliContributions>[number],
): LiveTransportQaCliRegistration {
if (runner.status === "available") {
return runner.registration;
}
if (runner.status === "blocked") {
return createBlockedQaRunnerCliRegistration({
commandName: runner.commandName,
description: runner.description,
pluginId: runner.pluginId,
});
}
return createMissingQaRunnerCliRegistration({
commandName: runner.commandName,
description:
runner.description ??
`Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`,
npmSpec: runner.npmSpec,
});
}
export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
telegramQaCliRegistration,
matrixQaCliRegistration,
];
export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
const liveRegistrations = [...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS];
const discoveredRunners = listQaRunnerCliContributions();
for (const runner of discoveredRunners) {
liveRegistrations.push(createQaRunnerCliRegistration(runner));
}
return liveRegistrations;
}

View File

@@ -1,43 +0,0 @@
import { describe, expect, it, vi } from "vitest";
const runMatrixQaLive = vi.hoisted(() => vi.fn());
vi.mock("./matrix-live.runtime.js", () => ({
runMatrixQaLive,
}));
import { runQaMatrixCommand } from "./cli.runtime.js";
describe("matrix qa cli runtime", () => {
it("rejects non-env credential sources for the disposable Matrix lane", async () => {
await expect(
runQaMatrixCommand({
credentialSource: "convex",
}),
).rejects.toThrow("Matrix QA currently supports only --credential-source env");
});
it("passes through default env credential source options", async () => {
runMatrixQaLive.mockResolvedValue({
reportPath: "/tmp/matrix-report.md",
summaryPath: "/tmp/matrix-summary.json",
observedEventsPath: "/tmp/matrix-events.json",
});
await runQaMatrixCommand({
repoRoot: "/tmp/openclaw",
outputDir: ".artifacts/qa-e2e/matrix",
providerMode: "mock-openai",
credentialSource: "env",
});
expect(runMatrixQaLive).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: "/tmp/openclaw",
outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix",
providerMode: "mock-openai",
credentialSource: "env",
}),
);
});
});

View File

@@ -1,23 +0,0 @@
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
import {
printLiveTransportQaArtifacts,
resolveLiveTransportQaRunOptions,
} from "../shared/live-transport-cli.runtime.js";
import { runMatrixQaLive } from "./matrix-live.runtime.js";
export async function runQaMatrixCommand(opts: LiveTransportQaCommandOptions) {
const runOptions = resolveLiveTransportQaRunOptions(opts);
const credentialSource = runOptions.credentialSource?.toLowerCase();
if (credentialSource && credentialSource !== "env") {
throw new Error(
"Matrix QA currently supports only --credential-source env (disposable local harness).",
);
}
const result = await runMatrixQaLive(runOptions);
printLiveTransportQaArtifacts("Matrix QA", {
report: result.reportPath,
summary: result.summaryPath,
"observed events": result.observedEventsPath,
});
}

View File

@@ -1,32 +0,0 @@
import type { Command } from "commander";
import {
createLazyCliRuntimeLoader,
createLiveTransportQaCliRegistration,
type LiveTransportQaCliRegistration,
type LiveTransportQaCommandOptions,
} from "../shared/live-transport-cli.js";
type MatrixQaCliRuntime = typeof import("./cli.runtime.js");
const loadMatrixQaCliRuntime = createLazyCliRuntimeLoader<MatrixQaCliRuntime>(
() => import("./cli.runtime.js"),
);
async function runQaMatrix(opts: LiveTransportQaCommandOptions) {
const runtime = await loadMatrixQaCliRuntime();
await runtime.runQaMatrixCommand(opts);
}
export const matrixQaCliRegistration: LiveTransportQaCliRegistration =
createLiveTransportQaCliRegistration({
commandName: "matrix",
description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver",
outputDirHelp: "Matrix QA artifact directory",
scenarioHelp: "Run only the named Matrix QA scenario (repeatable)",
sutAccountHelp: "Temporary Matrix account id inside the QA gateway config",
run: runQaMatrix,
});
export function registerMatrixQaCli(qa: Command) {
matrixQaCliRegistration.register(qa);
}

View File

@@ -1,416 +0,0 @@
import { describe, expect, it } from "vitest";
import {
__testing,
createMatrixQaClient,
provisionMatrixQaRoom,
type MatrixQaObservedEvent,
} from "./matrix-driver-client.js";
function resolveRequestUrl(input: RequestInfo | URL) {
if (typeof input === "string") {
return input;
}
if (input instanceof URL) {
return input.toString();
}
return input.url;
}
function parseJsonRequestBody(init?: RequestInit) {
if (typeof init?.body !== "string") {
return {};
}
return JSON.parse(init.body) as Record<string, unknown>;
}
describe("matrix driver client", () => {
it("builds Matrix HTML mentions for QA driver messages", () => {
expect(
__testing.buildMatrixQaMessageContent({
body: "@sut:matrix-qa.test reply with exactly: TOKEN",
mentionUserIds: ["@sut:matrix-qa.test"],
}),
).toEqual({
body: "@sut:matrix-qa.test reply with exactly: TOKEN",
msgtype: "m.text",
format: "org.matrix.custom.html",
formatted_body:
'<a href="https://matrix.to/#/%40sut%3Amatrix-qa.test">@sut:matrix-qa.test</a> reply with exactly: TOKEN',
"m.mentions": {
user_ids: ["@sut:matrix-qa.test"],
},
});
});
it("omits Matrix HTML markup when the body has no visible mention token", () => {
expect(
__testing.buildMatrixQaMessageContent({
body: "reply with exactly: TOKEN",
mentionUserIds: ["@sut:matrix-qa.test"],
}),
).toEqual({
body: "reply with exactly: TOKEN",
msgtype: "m.text",
"m.mentions": {
user_ids: ["@sut:matrix-qa.test"],
},
});
});
it("normalizes message events with thread metadata", () => {
expect(
__testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
event_id: "$event",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
origin_server_ts: 1_700_000_000_000,
content: {
body: "hello",
msgtype: "m.text",
"m.mentions": {
user_ids: ["@sut:matrix-qa.test"],
},
"m.relates_to": {
rel_type: "m.thread",
event_id: "$root",
is_falling_back: true,
"m.in_reply_to": {
event_id: "$driver",
},
},
},
}),
).toEqual({
roomId: "!room:matrix-qa.test",
eventId: "$event",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
originServerTs: 1_700_000_000_000,
body: "hello",
msgtype: "m.text",
relatesTo: {
relType: "m.thread",
eventId: "$root",
inReplyToId: "$driver",
isFallingBack: true,
},
mentions: {
userIds: ["@sut:matrix-qa.test"],
},
});
});
it("builds trimmed Matrix reaction relations for QA driver events", () => {
expect(__testing.buildMatrixReactionRelation(" $msg-1 ", " 👍 ")).toEqual({
"m.relates_to": {
rel_type: "m.annotation",
event_id: "$msg-1",
key: "👍",
},
});
});
it("normalizes Matrix reaction events with target metadata", () => {
expect(
__testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
event_id: "$reaction",
sender: "@driver:matrix-qa.test",
type: "m.reaction",
origin_server_ts: 1_700_000_000_000,
content: {
"m.relates_to": {
rel_type: "m.annotation",
event_id: "$msg",
key: "👍",
},
},
}),
).toEqual({
roomId: "!room:matrix-qa.test",
eventId: "$reaction",
sender: "@driver:matrix-qa.test",
type: "m.reaction",
originServerTs: 1_700_000_000_000,
relatesTo: {
eventId: "$msg",
relType: "m.annotation",
},
reaction: {
eventId: "$msg",
key: "👍",
},
});
});
it("advances Matrix registration through token then dummy auth stages", () => {
const firstStage = __testing.resolveNextRegistrationAuth({
registrationToken: "reg-token",
response: {
session: "uiaa-session",
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
},
});
expect(firstStage).toEqual({
session: "uiaa-session",
type: "m.login.registration_token",
token: "reg-token",
});
expect(
__testing.resolveNextRegistrationAuth({
registrationToken: "reg-token",
response: {
session: "uiaa-session",
completed: ["m.login.registration_token"],
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
},
}),
).toEqual({
session: "uiaa-session",
type: "m.login.dummy",
});
});
it("rejects Matrix UIAA flows that require unsupported stages", () => {
expect(() =>
__testing.resolveNextRegistrationAuth({
registrationToken: "reg-token",
response: {
session: "uiaa-session",
flows: [{ stages: ["m.login.registration_token", "m.login.recaptcha", "m.login.dummy"] }],
},
}),
).toThrow("Matrix registration requires unsupported auth stages:");
});
it("returns a typed no-match result while preserving the latest sync token", async () => {
const fetchImpl: typeof fetch = async () =>
new Response(
JSON.stringify({
next_batch: "next-batch-2",
rooms: {
join: {
"!room:matrix-qa.test": {
timeline: {
events: [
{
event_id: "$driver",
sender: "@driver:matrix-qa.test",
type: "m.room.message",
content: { body: "hello", msgtype: "m.text" },
},
],
},
},
},
},
}),
{ status: 200, headers: { "content-type": "application/json" } },
);
const client = createMatrixQaClient({
accessToken: "token",
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
const observedEvents: MatrixQaObservedEvent[] = [];
const result = await client.waitForOptionalRoomEvent({
observedEvents,
predicate: (event) => event.sender === "@sut:matrix-qa.test",
roomId: "!room:matrix-qa.test",
since: "start-batch",
timeoutMs: 1,
});
expect(result).toEqual({
matched: false,
since: "next-batch-2",
});
expect(observedEvents).toEqual(
expect.arrayContaining([
expect.objectContaining({
body: "hello",
eventId: "$driver",
roomId: "!room:matrix-qa.test",
sender: "@driver:matrix-qa.test",
type: "m.room.message",
}),
]),
);
});
it("keeps recording later same-batch events after the first match", async () => {
const fetchImpl: typeof fetch = async () =>
new Response(
JSON.stringify({
next_batch: "next-batch-2",
rooms: {
join: {
"!room:matrix-qa.test": {
timeline: {
events: [
{
event_id: "$sut",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
content: { body: "target", msgtype: "m.text" },
},
{
event_id: "$driver",
sender: "@driver:matrix-qa.test",
type: "m.room.message",
content: { body: "trailing event", msgtype: "m.text" },
},
],
},
},
},
},
}),
{ status: 200, headers: { "content-type": "application/json" } },
);
const client = createMatrixQaClient({
accessToken: "token",
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
const observedEvents: MatrixQaObservedEvent[] = [];
const result = await client.waitForOptionalRoomEvent({
observedEvents,
predicate: (event) => event.eventId === "$sut",
roomId: "!room:matrix-qa.test",
since: "start-batch",
timeoutMs: 1,
});
expect(result).toEqual({
event: expect.objectContaining({
eventId: "$sut",
}),
matched: true,
since: "next-batch-2",
});
expect(observedEvents).toEqual(
expect.arrayContaining([
expect.objectContaining({
body: "target",
eventId: "$sut",
}),
expect.objectContaining({
body: "trailing event",
eventId: "$driver",
}),
]),
);
});
it("sends Matrix reactions through the protocol send endpoint", async () => {
const fetchImpl: typeof fetch = async (input, init) => {
expect(resolveRequestUrl(input)).toContain(
"/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.reaction/",
);
expect(parseJsonRequestBody(init)).toEqual({
"m.relates_to": {
rel_type: "m.annotation",
event_id: "$msg-1",
key: "👍",
},
});
return new Response(JSON.stringify({ event_id: "$reaction-1" }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
const client = createMatrixQaClient({
accessToken: "token",
baseUrl: "http://127.0.0.1:28008/",
fetchImpl,
});
await expect(
client.sendReaction({
emoji: "👍",
messageId: "$msg-1",
roomId: "!room:matrix-qa.test",
}),
).resolves.toBe("$reaction-1");
});
it("provisions a three-member room so Matrix QA runs in a group context", async () => {
const createRoomBodies: Array<Record<string, unknown>> = [];
const fetchImpl: typeof fetch = async (input, init) => {
const url = resolveRequestUrl(input);
const body = parseJsonRequestBody(init);
if (url.endsWith("/_matrix/client/v3/register")) {
const username = typeof body.username === "string" ? body.username : "";
const auth = typeof body.auth === "object" && body.auth ? body.auth : undefined;
if (!auth) {
return new Response(
JSON.stringify({
session: `session-${username}`,
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
}),
{ status: 401, headers: { "content-type": "application/json" } },
);
}
if ((auth as { type?: string }).type === "m.login.registration_token") {
return new Response(
JSON.stringify({
session: `session-${username}`,
completed: ["m.login.registration_token"],
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
}),
{ status: 401, headers: { "content-type": "application/json" } },
);
}
return new Response(
JSON.stringify({
access_token: `token-${username}`,
device_id: `device-${username}`,
user_id: `@${username}:matrix-qa.test`,
}),
{ status: 200, headers: { "content-type": "application/json" } },
);
}
if (url.endsWith("/_matrix/client/v3/createRoom")) {
createRoomBodies.push(body);
return new Response(JSON.stringify({ room_id: "!room:matrix-qa.test" }), {
status: 200,
headers: { "content-type": "application/json" },
});
}
if (url.includes("/_matrix/client/v3/join/")) {
return new Response(JSON.stringify({ room_id: "!room:matrix-qa.test" }), {
status: 200,
headers: { "content-type": "application/json" },
});
}
throw new Error(`unexpected fetch ${url}`);
};
const result = await provisionMatrixQaRoom({
baseUrl: "http://127.0.0.1:28008/",
driverLocalpart: "qa-driver",
observerLocalpart: "qa-observer",
registrationToken: "reg-token",
roomName: "OpenClaw Matrix QA",
sutLocalpart: "qa-sut",
fetchImpl,
});
expect(result.roomId).toBe("!room:matrix-qa.test");
expect(result.observer.userId).toBe("@qa-observer:matrix-qa.test");
expect(createRoomBodies).toEqual([
expect.objectContaining({
invite: ["@qa-sut:matrix-qa.test", "@qa-observer:matrix-qa.test"],
is_direct: false,
preset: "private_chat",
}),
]);
});
});

View File

@@ -1,728 +0,0 @@
import { randomUUID } from "node:crypto";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
type FetchLike = typeof fetch;
type MatrixQaAuthStage = "m.login.dummy" | "m.login.registration_token";
type MatrixQaRequestResult<T> = {
status: number;
body: T;
};
type MatrixQaRegisterResponse = {
access_token?: string;
device_id?: string;
user_id?: string;
};
type MatrixQaRoomCreateResponse = {
room_id?: string;
};
type MatrixQaSendMessageContent = {
body: string;
format?: "org.matrix.custom.html";
formatted_body?: string;
"m.mentions"?: {
user_ids?: string[];
};
"m.relates_to"?: {
rel_type: "m.thread";
event_id: string;
is_falling_back: true;
"m.in_reply_to": {
event_id: string;
};
};
msgtype: "m.text";
};
type MatrixQaSendReactionContent = {
"m.relates_to": {
event_id: string;
key: string;
rel_type: "m.annotation";
};
};
type MatrixQaSyncResponse = {
next_batch?: string;
rooms?: {
join?: Record<
string,
{
timeline?: {
events?: MatrixQaRoomEvent[];
};
}
>;
};
};
type MatrixQaUiaaResponse = {
completed?: string[];
flows?: Array<{ stages?: string[] }>;
session?: string;
};
type MatrixQaRoomEvent = {
content?: Record<string, unknown>;
event_id?: string;
origin_server_ts?: number;
sender?: string;
state_key?: string;
type?: string;
};
export type MatrixQaObservedEvent = {
roomId: string;
eventId: string;
sender?: string;
stateKey?: string;
type: string;
originServerTs?: number;
body?: string;
formattedBody?: string;
msgtype?: string;
membership?: string;
relatesTo?: {
eventId?: string;
inReplyToId?: string;
isFallingBack?: boolean;
relType?: string;
};
mentions?: {
room?: boolean;
userIds?: string[];
};
reaction?: {
eventId?: string;
key?: string;
};
};
export type MatrixQaRegisteredAccount = {
accessToken: string;
deviceId?: string;
localpart: string;
password: string;
userId: string;
};
export type MatrixQaProvisionResult = {
driver: MatrixQaRegisteredAccount;
observer: MatrixQaRegisteredAccount;
roomId: string;
sut: MatrixQaRegisteredAccount;
};
export type MatrixQaRoomEventWaitResult =
| {
event: MatrixQaObservedEvent;
matched: true;
since?: string;
}
| {
matched: false;
since?: string;
};
function buildMatrixThreadRelation(threadRootEventId: string, replyToEventId?: string) {
return {
"m.relates_to": {
rel_type: "m.thread" as const,
event_id: threadRootEventId,
is_falling_back: true as const,
"m.in_reply_to": {
event_id: replyToEventId?.trim() || threadRootEventId,
},
},
};
}
function buildMatrixReactionRelation(
messageId: string,
emoji: string,
): MatrixQaSendReactionContent {
const normalizedMessageId = messageId.trim();
const normalizedEmoji = emoji.trim();
if (!normalizedMessageId) {
throw new Error("Matrix reaction requires a messageId");
}
if (!normalizedEmoji) {
throw new Error("Matrix reaction requires an emoji");
}
return {
"m.relates_to": {
rel_type: "m.annotation",
event_id: normalizedMessageId,
key: normalizedEmoji,
},
};
}
function escapeMatrixHtml(value: string): string {
return value.replace(/[&<>"']/g, (char) => {
switch (char) {
case "&":
return "&amp;";
case "<":
return "&lt;";
case ">":
return "&gt;";
case '"':
return "&quot;";
case "'":
return "&#39;";
default:
return char;
}
});
}
function buildMatrixMentionLink(userId: string) {
const href = `https://matrix.to/#/${encodeURIComponent(userId)}`;
const label = escapeMatrixHtml(userId);
return `<a href="${href}">${label}</a>`;
}
function buildMatrixQaMessageContent(params: {
body: string;
mentionUserIds?: string[];
replyToEventId?: string;
threadRootEventId?: string;
}): MatrixQaSendMessageContent {
const body = params.body;
const uniqueMentionUserIds = [...new Set(params.mentionUserIds?.filter(Boolean) ?? [])];
const formattedParts: string[] = [];
let cursor = 0;
let usedFormattedMention = false;
while (cursor < body.length) {
let matchedUserId: string | null = null;
for (const userId of uniqueMentionUserIds) {
if (body.startsWith(userId, cursor)) {
matchedUserId = userId;
break;
}
}
if (matchedUserId) {
formattedParts.push(buildMatrixMentionLink(matchedUserId));
cursor += matchedUserId.length;
usedFormattedMention = true;
continue;
}
formattedParts.push(escapeMatrixHtml(body[cursor] ?? ""));
cursor += 1;
}
return {
body,
msgtype: "m.text",
...(usedFormattedMention
? {
format: "org.matrix.custom.html" as const,
formatted_body: formattedParts.join(""),
}
: {}),
...(uniqueMentionUserIds.length > 0
? { "m.mentions": { user_ids: uniqueMentionUserIds } }
: {}),
...(params.threadRootEventId
? buildMatrixThreadRelation(params.threadRootEventId, params.replyToEventId)
: {}),
};
}
function normalizeMentionUserIds(value: unknown) {
return Array.isArray(value)
? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
: undefined;
}
export function normalizeMatrixQaObservedEvent(
roomId: string,
event: MatrixQaRoomEvent,
): MatrixQaObservedEvent | null {
const eventId = event.event_id?.trim();
const type = event.type?.trim();
if (!eventId || !type) {
return null;
}
const content = event.content ?? {};
const relatesToRaw = content["m.relates_to"];
const relatesTo =
typeof relatesToRaw === "object" && relatesToRaw !== null
? (relatesToRaw as Record<string, unknown>)
: null;
const inReplyToRaw = relatesTo?.["m.in_reply_to"];
const inReplyTo =
typeof inReplyToRaw === "object" && inReplyToRaw !== null
? (inReplyToRaw as Record<string, unknown>)
: null;
const mentionsRaw = content["m.mentions"];
const mentions =
typeof mentionsRaw === "object" && mentionsRaw !== null
? (mentionsRaw as Record<string, unknown>)
: null;
const mentionUserIds = normalizeMentionUserIds(mentions?.user_ids);
const reactionKey =
type === "m.reaction" && typeof relatesTo?.key === "string" ? relatesTo.key : undefined;
const reactionEventId =
type === "m.reaction" && typeof relatesTo?.event_id === "string"
? relatesTo.event_id
: undefined;
return {
roomId,
eventId,
sender: typeof event.sender === "string" ? event.sender : undefined,
stateKey: typeof event.state_key === "string" ? event.state_key : undefined,
type,
originServerTs:
typeof event.origin_server_ts === "number" ? Math.floor(event.origin_server_ts) : undefined,
body: typeof content.body === "string" ? content.body : undefined,
formattedBody: typeof content.formatted_body === "string" ? content.formatted_body : undefined,
msgtype: typeof content.msgtype === "string" ? content.msgtype : undefined,
membership: typeof content.membership === "string" ? content.membership : undefined,
...(relatesTo
? {
relatesTo: {
eventId: typeof relatesTo.event_id === "string" ? relatesTo.event_id : undefined,
inReplyToId: typeof inReplyTo?.event_id === "string" ? inReplyTo.event_id : undefined,
isFallingBack:
typeof relatesTo.is_falling_back === "boolean"
? relatesTo.is_falling_back
: undefined,
relType: typeof relatesTo.rel_type === "string" ? relatesTo.rel_type : undefined,
},
}
: {}),
...(mentions
? {
mentions: {
...(mentions.room === true ? { room: true } : {}),
...(mentionUserIds ? { userIds: mentionUserIds } : {}),
},
}
: {}),
...(reactionEventId || reactionKey
? {
reaction: {
...(reactionEventId ? { eventId: reactionEventId } : {}),
...(reactionKey ? { key: reactionKey } : {}),
},
}
: {}),
};
}
export function resolveNextRegistrationAuth(params: {
registrationToken: string;
response: MatrixQaUiaaResponse;
}) {
const session = params.response.session?.trim();
if (!session) {
throw new Error("Matrix registration UIAA response did not include a session id.");
}
const completed = new Set(
(params.response.completed ?? []).filter(
(stage): stage is MatrixQaAuthStage =>
stage === "m.login.dummy" || stage === "m.login.registration_token",
),
);
const supportedStages = new Set<MatrixQaAuthStage>([
"m.login.registration_token",
"m.login.dummy",
]);
for (const flow of params.response.flows ?? []) {
const flowStages = flow.stages ?? [];
if (
flowStages.length === 0 ||
flowStages.some((stage) => !supportedStages.has(stage as MatrixQaAuthStage))
) {
continue;
}
const stages = flowStages as MatrixQaAuthStage[];
const nextStage = stages.find((stage) => !completed.has(stage));
if (!nextStage) {
continue;
}
if (nextStage === "m.login.registration_token") {
return {
session,
type: nextStage,
token: params.registrationToken,
};
}
return {
session,
type: nextStage,
};
}
throw new Error(
`Matrix registration requires unsupported auth stages: ${JSON.stringify(params.response.flows ?? [])}`,
);
}
async function requestMatrixJson<T>(params: {
accessToken?: string;
baseUrl: string;
body?: unknown;
endpoint: string;
fetchImpl: FetchLike;
method: "GET" | "POST" | "PUT";
okStatuses?: number[];
query?: Record<string, string | number | undefined>;
timeoutMs?: number;
}) {
const url = new URL(params.endpoint, params.baseUrl);
for (const [key, value] of Object.entries(params.query ?? {})) {
if (value !== undefined) {
url.searchParams.set(key, String(value));
}
}
const response = await params.fetchImpl(url, {
method: params.method,
headers: {
accept: "application/json",
...(params.body !== undefined ? { "content-type": "application/json" } : {}),
...(params.accessToken ? { authorization: `Bearer ${params.accessToken}` } : {}),
},
...(params.body !== undefined ? { body: JSON.stringify(params.body) } : {}),
signal: AbortSignal.timeout(params.timeoutMs ?? 20_000),
});
let body: unknown = {};
try {
body = (await response.json()) as unknown;
} catch {
body = {};
}
const okStatuses = params.okStatuses ?? [200];
if (!okStatuses.includes(response.status)) {
const details =
typeof body === "object" &&
body !== null &&
typeof (body as { error?: unknown }).error === "string"
? (body as { error: string }).error
: `${params.method} ${params.endpoint} failed with status ${response.status}`;
throw new Error(details);
}
return {
status: response.status,
body: body as T,
} satisfies MatrixQaRequestResult<T>;
}
function buildRegisteredAccount(params: {
localpart: string;
password: string;
response: MatrixQaRegisterResponse;
}) {
const userId = params.response.user_id?.trim();
const accessToken = params.response.access_token?.trim();
if (!userId || !accessToken) {
throw new Error("Matrix registration did not return both user_id and access_token.");
}
return {
accessToken,
deviceId: params.response.device_id?.trim() || undefined,
localpart: params.localpart,
password: params.password,
userId,
} satisfies MatrixQaRegisteredAccount;
}
export function createMatrixQaClient(params: {
accessToken?: string;
baseUrl: string;
fetchImpl?: FetchLike;
}) {
const fetchImpl = params.fetchImpl ?? fetch;
async function waitForOptionalRoomEvent(opts: {
observedEvents: MatrixQaObservedEvent[];
predicate: (event: MatrixQaObservedEvent) => boolean;
roomId: string;
since?: string;
timeoutMs: number;
}): Promise<MatrixQaRoomEventWaitResult> {
const startedAt = Date.now();
let since = opts.since;
while (Date.now() - startedAt < opts.timeoutMs) {
const remainingMs = Math.max(1_000, opts.timeoutMs - (Date.now() - startedAt));
const response = await requestMatrixJson<MatrixQaSyncResponse>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
endpoint: "/_matrix/client/v3/sync",
fetchImpl,
method: "GET",
query: {
...(since ? { since } : {}),
timeout: Math.min(10_000, remainingMs),
},
timeoutMs: Math.min(15_000, remainingMs + 5_000),
});
since = response.body.next_batch?.trim() || since;
const roomEvents = response.body.rooms?.join?.[opts.roomId]?.timeline?.events ?? [];
let matchedEvent: MatrixQaObservedEvent | null = null;
for (const event of roomEvents) {
const normalized = normalizeMatrixQaObservedEvent(opts.roomId, event);
if (!normalized) {
continue;
}
opts.observedEvents.push(normalized);
if (matchedEvent === null && opts.predicate(normalized)) {
matchedEvent = normalized;
}
}
if (matchedEvent) {
return { event: matchedEvent, matched: true, since };
}
}
return { matched: false, since };
}
return {
async createPrivateRoom(opts: { inviteUserIds: string[]; name: string }) {
const result = await requestMatrixJson<MatrixQaRoomCreateResponse>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: {
creation_content: { "m.federate": false },
initial_state: [
{
type: "m.room.history_visibility",
state_key: "",
content: { history_visibility: "joined" },
},
],
invite: opts.inviteUserIds,
is_direct: false,
name: opts.name,
preset: "private_chat",
},
endpoint: "/_matrix/client/v3/createRoom",
fetchImpl,
method: "POST",
});
const roomId = result.body.room_id?.trim();
if (!roomId) {
throw new Error("Matrix createRoom did not return room_id.");
}
return roomId;
},
async primeRoom() {
const response = await requestMatrixJson<MatrixQaSyncResponse>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
endpoint: "/_matrix/client/v3/sync",
fetchImpl,
method: "GET",
query: { timeout: 0 },
});
return response.body.next_batch?.trim() || undefined;
},
async registerWithToken(opts: {
deviceName: string;
localpart: string;
password: string;
registrationToken: string;
}) {
let auth: Record<string, unknown> | undefined;
const baseBody = {
inhibit_login: false,
initial_device_display_name: opts.deviceName,
password: opts.password,
username: opts.localpart,
};
for (let attempt = 0; attempt < 4; attempt += 1) {
const response = await requestMatrixJson<MatrixQaRegisterResponse | MatrixQaUiaaResponse>({
baseUrl: params.baseUrl,
body: {
...baseBody,
...(auth ? { auth } : {}),
},
endpoint: "/_matrix/client/v3/register",
fetchImpl,
method: "POST",
okStatuses: [200, 401],
timeoutMs: 30_000,
});
if (response.status === 200) {
return buildRegisteredAccount({
localpart: opts.localpart,
password: opts.password,
response: response.body as MatrixQaRegisterResponse,
});
}
auth = resolveNextRegistrationAuth({
registrationToken: opts.registrationToken,
response: response.body as MatrixQaUiaaResponse,
});
}
throw new Error(
`Matrix registration for ${opts.localpart} did not complete after 4 attempts.`,
);
},
async sendTextMessage(opts: {
body: string;
mentionUserIds?: string[];
replyToEventId?: string;
roomId: string;
threadRootEventId?: string;
}) {
const txnId = randomUUID();
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: buildMatrixQaMessageContent(opts),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
fetchImpl,
method: "PUT",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error("Matrix sendMessage did not return event_id.");
}
return eventId;
},
async sendReaction(opts: { emoji: string; messageId: string; roomId: string }) {
const txnId = randomUUID();
const result = await requestMatrixJson<{ event_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: buildMatrixReactionRelation(opts.messageId, opts.emoji),
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.reaction/${encodeURIComponent(txnId)}`,
fetchImpl,
method: "PUT",
});
const eventId = result.body.event_id?.trim();
if (!eventId) {
throw new Error("Matrix sendReaction did not return event_id.");
}
return eventId;
},
async joinRoom(roomId: string) {
const result = await requestMatrixJson<{ room_id?: string }>({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
body: {},
endpoint: `/_matrix/client/v3/join/${encodeURIComponent(roomId)}`,
fetchImpl,
method: "POST",
});
return result.body.room_id?.trim() || roomId;
},
waitForOptionalRoomEvent,
async waitForRoomEvent(opts: {
observedEvents: MatrixQaObservedEvent[];
predicate: (event: MatrixQaObservedEvent) => boolean;
roomId: string;
since?: string;
timeoutMs: number;
}) {
const result = await waitForOptionalRoomEvent(opts);
if (result.matched) {
return { event: result.event, since: result.since };
}
throw new Error(`timed out after ${opts.timeoutMs}ms waiting for Matrix room event`);
},
};
}
async function joinRoomWithRetry(params: {
accessToken: string;
baseUrl: string;
fetchImpl?: FetchLike;
roomId: string;
}) {
const client = createMatrixQaClient({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
fetchImpl: params.fetchImpl,
});
let lastError: unknown = null;
for (let attempt = 1; attempt <= 10; attempt += 1) {
try {
await client.joinRoom(params.roomId);
return;
} catch (error) {
lastError = error;
await new Promise((resolve) => setTimeout(resolve, 300 * attempt));
}
}
throw new Error(`Matrix join retry failed: ${formatErrorMessage(lastError)}`);
}
export async function provisionMatrixQaRoom(params: {
baseUrl: string;
fetchImpl?: FetchLike;
roomName: string;
driverLocalpart: string;
observerLocalpart: string;
registrationToken: string;
sutLocalpart: string;
}) {
const anonClient = createMatrixQaClient({
baseUrl: params.baseUrl,
fetchImpl: params.fetchImpl,
});
const driver = await anonClient.registerWithToken({
deviceName: "OpenClaw Matrix QA Driver",
localpart: params.driverLocalpart,
password: `driver-${randomUUID()}`,
registrationToken: params.registrationToken,
});
const sut = await anonClient.registerWithToken({
deviceName: "OpenClaw Matrix QA SUT",
localpart: params.sutLocalpart,
password: `sut-${randomUUID()}`,
registrationToken: params.registrationToken,
});
const observer = await anonClient.registerWithToken({
deviceName: "OpenClaw Matrix QA Observer",
localpart: params.observerLocalpart,
password: `observer-${randomUUID()}`,
registrationToken: params.registrationToken,
});
const driverClient = createMatrixQaClient({
accessToken: driver.accessToken,
baseUrl: params.baseUrl,
fetchImpl: params.fetchImpl,
});
const roomId = await driverClient.createPrivateRoom({
inviteUserIds: [sut.userId, observer.userId],
name: params.roomName,
});
await joinRoomWithRetry({
accessToken: sut.accessToken,
baseUrl: params.baseUrl,
fetchImpl: params.fetchImpl,
roomId,
});
await joinRoomWithRetry({
accessToken: observer.accessToken,
baseUrl: params.baseUrl,
fetchImpl: params.fetchImpl,
roomId,
});
return {
driver,
observer,
roomId,
sut,
} satisfies MatrixQaProvisionResult;
}
export const __testing = {
buildMatrixQaMessageContent,
buildMatrixReactionRelation,
buildMatrixThreadRelation,
normalizeMatrixQaObservedEvent,
resolveNextRegistrationAuth,
};

View File

@@ -1,271 +0,0 @@
import { mkdtemp, readFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
__testing,
startMatrixQaHarness,
writeMatrixQaHarnessFiles,
} from "./matrix-harness.runtime.js";
describe("matrix harness runtime", () => {
it("writes a pinned Tuwunel compose file and redacted manifest", async () => {
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await writeMatrixQaHarnessFiles({
outputDir,
homeserverPort: 28008,
registrationToken: "secret-token",
serverName: "matrix-qa.test",
});
const compose = await readFile(result.composeFile, "utf8");
const manifest = JSON.parse(await readFile(result.manifestPath, "utf8")) as {
image: string;
serverName: string;
homeserverPort: number;
composeFile: string;
};
expect(compose).toContain(`image: ${__testing.MATRIX_QA_DEFAULT_IMAGE}`);
expect(compose).toContain(' - "127.0.0.1:28008:8008"');
expect(compose).toContain('TUWUNEL_ALLOW_REGISTRATION: "true"');
expect(compose).toContain('TUWUNEL_REGISTRATION_TOKEN: "secret-token"');
expect(compose).toContain('TUWUNEL_SERVER_NAME: "matrix-qa.test"');
expect(manifest).toEqual({
image: __testing.MATRIX_QA_DEFAULT_IMAGE,
serverName: "matrix-qa.test",
homeserverPort: 28008,
composeFile: path.join(outputDir, "docker-compose.matrix-qa.yml"),
dataDir: path.join(outputDir, "data"),
});
expect(result.registrationToken).toBe("secret-token");
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
it("starts the harness, waits for versions, and exposes a stop command", async () => {
const calls: string[] = [];
const fetchCalls: string[] = [];
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await startMatrixQaHarness(
{
outputDir,
repoRoot: "/repo/openclaw",
homeserverPort: 28008,
},
{
async runCommand(command, args, cwd) {
calls.push([command, ...args, `@${cwd}`].join(" "));
if (args.join(" ").includes("ps --format json")) {
return { stdout: '[{"State":"running"}]\n', stderr: "" };
}
return { stdout: "", stderr: "" };
},
fetchImpl: vi.fn(async (input: string) => {
fetchCalls.push(input);
return { ok: true };
}),
sleepImpl: vi.fn(async () => {}),
resolveHostPortImpl: vi.fn(async (port: number) => port),
},
);
expect(calls).toEqual([
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml down --remove-orphans @/repo/openclaw`,
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml up -d @/repo/openclaw`,
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml ps --format json matrix-qa-homeserver @/repo/openclaw`,
]);
expect(fetchCalls).toEqual([
"http://127.0.0.1:28008/_matrix/client/versions",
"http://127.0.0.1:28008/_matrix/client/versions",
]);
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
expect(result.stopCommand).toBe(
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml down --remove-orphans`,
);
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
it("treats empty Docker health fields as a fallback to running state", async () => {
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await startMatrixQaHarness(
{
outputDir,
repoRoot: "/repo/openclaw",
homeserverPort: 28008,
},
{
async runCommand(_command, args) {
if (args.join(" ").includes("ps --format json")) {
return { stdout: '{"Health":"","State":"running"}\n', stderr: "" };
}
return { stdout: "", stderr: "" };
},
fetchImpl: vi.fn(async () => ({ ok: true })),
sleepImpl: vi.fn(async () => {}),
resolveHostPortImpl: vi.fn(async (port: number) => port),
},
);
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
it("falls back to the container IP when the host port is unreachable", async () => {
const calls: string[] = [];
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await startMatrixQaHarness(
{
outputDir,
repoRoot: "/repo/openclaw",
homeserverPort: 28008,
},
{
async runCommand(command, args, cwd) {
calls.push([command, ...args, `@${cwd}`].join(" "));
const rendered = args.join(" ");
if (rendered.includes("ps --format json")) {
return { stdout: '{"State":"running"}\n', stderr: "" };
}
if (rendered.includes("ps -q")) {
return { stdout: "container-123\n", stderr: "" };
}
if (rendered.includes("inspect --format")) {
return { stdout: "172.18.0.10\n", stderr: "" };
}
return { stdout: "", stderr: "" };
},
fetchImpl: vi.fn(async (input: string) => ({
ok: input.startsWith("http://172.18.0.10:8008/"),
})),
sleepImpl: vi.fn(async () => {}),
resolveHostPortImpl: vi.fn(async (port: number) => port),
},
);
expect(result.baseUrl).toBe("http://172.18.0.10:8008/");
expect(calls).toContain(
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml ps -q matrix-qa-homeserver @/repo/openclaw`,
);
expect(calls).toContain(
"docker inspect --format {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} container-123 @/repo/openclaw",
);
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
it("keeps the host URL when the container IP is also unreachable", async () => {
const fetchCalls: string[] = [];
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await startMatrixQaHarness(
{
outputDir,
repoRoot: "/repo/openclaw",
homeserverPort: 28008,
},
{
async runCommand(_command, args) {
const rendered = args.join(" ");
if (rendered.includes("ps --format json")) {
return { stdout: '{"State":"running"}\n', stderr: "" };
}
if (rendered.includes("ps -q")) {
return { stdout: "container-123\n", stderr: "" };
}
if (rendered.includes("inspect --format")) {
return { stdout: "172.18.0.10\n", stderr: "" };
}
return { stdout: "", stderr: "" };
},
fetchImpl: vi.fn(async (input: string) => {
fetchCalls.push(input);
return {
ok:
input === "http://127.0.0.1:28008/_matrix/client/versions" &&
fetchCalls.filter((url) => url === input).length > 1,
};
}),
sleepImpl: vi.fn(async () => {}),
resolveHostPortImpl: vi.fn(async (port: number) => port),
},
);
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
expect(fetchCalls).toEqual([
"http://127.0.0.1:28008/_matrix/client/versions",
"http://127.0.0.1:28008/_matrix/client/versions",
"http://127.0.0.1:28008/_matrix/client/versions",
]);
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
it("keeps probing the container URL until it becomes reachable", async () => {
const fetchCalls: string[] = [];
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
try {
const result = await startMatrixQaHarness(
{
outputDir,
repoRoot: "/repo/openclaw",
homeserverPort: 28008,
},
{
async runCommand(_command, args) {
const rendered = args.join(" ");
if (rendered.includes("ps --format json")) {
return { stdout: '{"State":"running"}\n', stderr: "" };
}
if (rendered.includes("ps -q")) {
return { stdout: "container-123\n", stderr: "" };
}
if (rendered.includes("inspect --format")) {
return { stdout: "172.18.0.10\n", stderr: "" };
}
return { stdout: "", stderr: "" };
},
fetchImpl: vi.fn(async (input: string) => {
fetchCalls.push(input);
return {
ok:
input === "http://172.18.0.10:8008/_matrix/client/versions" &&
fetchCalls.filter((url) => url === input).length > 1,
};
}),
sleepImpl: vi.fn(async () => {}),
resolveHostPortImpl: vi.fn(async (port: number) => port),
},
);
expect(result.baseUrl).toBe("http://172.18.0.10:8008/");
expect(fetchCalls).toEqual([
"http://127.0.0.1:28008/_matrix/client/versions",
"http://127.0.0.1:28008/_matrix/client/versions",
"http://172.18.0.10:8008/_matrix/client/versions",
"http://127.0.0.1:28008/_matrix/client/versions",
"http://172.18.0.10:8008/_matrix/client/versions",
"http://172.18.0.10:8008/_matrix/client/versions",
]);
} finally {
await rm(outputDir, { recursive: true, force: true });
}
});
});

View File

@@ -1,275 +0,0 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import {
execCommand,
fetchHealthUrl,
resolveComposeServiceUrl,
resolveHostPort,
waitForDockerServiceHealth,
waitForHealth,
type FetchLike,
type RunCommand,
} from "../../docker-runtime.js";
const MATRIX_QA_DEFAULT_IMAGE = "ghcr.io/matrix-construct/tuwunel:v1.5.1";
const MATRIX_QA_DEFAULT_SERVER_NAME = "matrix-qa.test";
const MATRIX_QA_DEFAULT_PORT = 28008;
const MATRIX_QA_INTERNAL_PORT = 8008;
const MATRIX_QA_SERVICE = "matrix-qa-homeserver";
type MatrixQaHarnessManifest = {
image: string;
serverName: string;
homeserverPort: number;
composeFile: string;
dataDir: string;
};
export type MatrixQaHarnessFiles = {
outputDir: string;
composeFile: string;
manifestPath: string;
image: string;
serverName: string;
homeserverPort: number;
registrationToken: string;
};
export type MatrixQaHarness = MatrixQaHarnessFiles & {
baseUrl: string;
stopCommand: string;
stop(): Promise<void>;
};
function buildVersionsUrl(baseUrl: string) {
return `${baseUrl}_matrix/client/versions`;
}
async function isMatrixVersionsReachable(baseUrl: string, fetchImpl: FetchLike) {
return await fetchImpl(buildVersionsUrl(baseUrl))
.then((response) => response.ok)
.catch(() => false);
}
async function waitForReachableMatrixBaseUrl(params: {
composeFile: string;
containerBaseUrl: string | null;
fetchImpl: FetchLike;
hostBaseUrl: string;
sleepImpl: (ms: number) => Promise<unknown>;
timeoutMs?: number;
pollMs?: number;
}) {
const timeoutMs = params.timeoutMs ?? 60_000;
const pollMs = params.pollMs ?? 1_000;
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
if (await isMatrixVersionsReachable(params.hostBaseUrl, params.fetchImpl)) {
return params.hostBaseUrl;
}
if (
params.containerBaseUrl &&
(await isMatrixVersionsReachable(params.containerBaseUrl, params.fetchImpl))
) {
return params.containerBaseUrl;
}
await params.sleepImpl(pollMs);
}
const candidateLabel = params.containerBaseUrl
? `${params.hostBaseUrl} or ${params.containerBaseUrl}`
: params.hostBaseUrl;
throw new Error(
[
`Matrix homeserver did not become healthy within ${Math.round(timeoutMs / 1000)}s.`,
`Last checked: ${candidateLabel}`,
`Hint: check container logs with \`docker compose -f ${params.composeFile} logs ${MATRIX_QA_SERVICE}\`.`,
].join("\n"),
);
}
function resolveMatrixQaHarnessImage(image?: string) {
return (
image?.trim() || process.env.OPENCLAW_QA_MATRIX_TUWUNEL_IMAGE?.trim() || MATRIX_QA_DEFAULT_IMAGE
);
}
function renderMatrixQaCompose(params: {
homeserverPort: number;
image: string;
registrationToken: string;
serverName: string;
}) {
return `services:
${MATRIX_QA_SERVICE}:
image: ${params.image}
ports:
- "127.0.0.1:${params.homeserverPort}:${MATRIX_QA_INTERNAL_PORT}"
environment:
TUWUNEL_ADDRESS: "0.0.0.0"
TUWUNEL_ALLOW_ENCRYPTION: "false"
TUWUNEL_ALLOW_FEDERATION: "false"
TUWUNEL_ALLOW_REGISTRATION: "true"
TUWUNEL_DATABASE_PATH: "/var/lib/tuwunel"
TUWUNEL_PORT: "${MATRIX_QA_INTERNAL_PORT}"
TUWUNEL_REGISTRATION_TOKEN: "${params.registrationToken}"
TUWUNEL_SERVER_NAME: "${params.serverName}"
volumes:
- ./data:/var/lib/tuwunel
`;
}
export async function writeMatrixQaHarnessFiles(params: {
outputDir: string;
image?: string;
homeserverPort: number;
registrationToken?: string;
serverName?: string;
}): Promise<MatrixQaHarnessFiles> {
const image = resolveMatrixQaHarnessImage(params.image);
const registrationToken = params.registrationToken?.trim() || `matrix-qa-${randomUUID()}`;
const serverName = params.serverName?.trim() || MATRIX_QA_DEFAULT_SERVER_NAME;
const composeFile = path.join(params.outputDir, "docker-compose.matrix-qa.yml");
const dataDir = path.join(params.outputDir, "data");
const manifestPath = path.join(params.outputDir, "matrix-qa-harness.json");
await fs.mkdir(dataDir, { recursive: true });
await fs.writeFile(
composeFile,
`${renderMatrixQaCompose({
homeserverPort: params.homeserverPort,
image,
registrationToken,
serverName,
})}\n`,
{ encoding: "utf8", mode: 0o600 },
);
const manifest: MatrixQaHarnessManifest = {
image,
serverName,
homeserverPort: params.homeserverPort,
composeFile,
dataDir,
};
await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
});
return {
outputDir: params.outputDir,
composeFile,
manifestPath,
image,
serverName,
homeserverPort: params.homeserverPort,
registrationToken,
};
}
export async function startMatrixQaHarness(
params: {
outputDir: string;
repoRoot?: string;
image?: string;
homeserverPort?: number;
serverName?: string;
},
deps?: {
fetchImpl?: FetchLike;
runCommand?: RunCommand;
sleepImpl?: (ms: number) => Promise<unknown>;
resolveHostPortImpl?: typeof resolveHostPort;
},
): Promise<MatrixQaHarness> {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const resolveHostPortImpl = deps?.resolveHostPortImpl ?? resolveHostPort;
const runCommand = deps?.runCommand ?? execCommand;
const fetchImpl = deps?.fetchImpl ?? fetchHealthUrl;
const sleepImpl = deps?.sleepImpl ?? sleep;
const homeserverPort = await resolveHostPortImpl(
params.homeserverPort ?? MATRIX_QA_DEFAULT_PORT,
params.homeserverPort != null,
);
const files = await writeMatrixQaHarnessFiles({
outputDir: path.resolve(params.outputDir),
image: params.image,
homeserverPort,
serverName: params.serverName,
});
try {
await runCommand(
"docker",
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
repoRoot,
);
} catch {
// First run or already stopped.
}
await runCommand("docker", ["compose", "-f", files.composeFile, "up", "-d"], repoRoot);
await sleepImpl(1_000);
await waitForDockerServiceHealth(
MATRIX_QA_SERVICE,
files.composeFile,
repoRoot,
runCommand,
sleepImpl,
);
const hostBaseUrl = `http://127.0.0.1:${homeserverPort}/`;
let baseUrl = hostBaseUrl;
const hostReachable = await isMatrixVersionsReachable(hostBaseUrl, fetchImpl);
if (!hostReachable) {
const containerBaseUrl = await resolveComposeServiceUrl(
MATRIX_QA_SERVICE,
MATRIX_QA_INTERNAL_PORT,
files.composeFile,
repoRoot,
runCommand,
);
baseUrl = await waitForReachableMatrixBaseUrl({
composeFile: files.composeFile,
containerBaseUrl,
fetchImpl,
hostBaseUrl,
sleepImpl,
});
}
await waitForHealth(buildVersionsUrl(baseUrl), {
label: "Matrix homeserver",
composeFile: files.composeFile,
fetchImpl,
sleepImpl,
});
return {
...files,
baseUrl,
stopCommand: `docker compose -f ${files.composeFile} down --remove-orphans`,
async stop() {
await runCommand(
"docker",
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
repoRoot,
);
},
};
}
export const __testing = {
MATRIX_QA_DEFAULT_IMAGE,
MATRIX_QA_DEFAULT_PORT,
MATRIX_QA_DEFAULT_SERVER_NAME,
MATRIX_QA_SERVICE,
buildVersionsUrl,
isMatrixVersionsReachable,
renderMatrixQaCompose,
resolveMatrixQaHarnessImage,
waitForReachableMatrixBaseUrl,
};

View File

@@ -1,153 +0,0 @@
import { describe, expect, it, beforeEach, vi } from "vitest";
const { createMatrixQaClient } = vi.hoisted(() => ({
createMatrixQaClient: vi.fn(),
}));
vi.mock("./matrix-driver-client.js", () => ({
createMatrixQaClient,
}));
import {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
findMissingLiveTransportStandardScenarios,
} from "../shared/live-transport-scenarios.js";
import {
__testing as scenarioTesting,
MATRIX_QA_SCENARIOS,
runMatrixQaScenario,
} from "./matrix-live-scenarios.js";
describe("matrix live qa scenarios", () => {
beforeEach(() => {
createMatrixQaClient.mockReset();
});
it("ships the Matrix live QA scenario set by default", () => {
expect(scenarioTesting.findMatrixQaScenarios().map((scenario) => scenario.id)).toEqual([
"matrix-thread-follow-up",
"matrix-thread-isolation",
"matrix-top-level-reply-shape",
"matrix-reaction-notification",
"matrix-restart-resume",
"matrix-mention-gating",
"matrix-allowlist-block",
]);
});
it("uses the repo-wide exact marker prompt shape for Matrix mentions", () => {
expect(
scenarioTesting.buildMentionPrompt("@sut:matrix-qa.test", "MATRIX_QA_CANARY_TOKEN"),
).toBe("@sut:matrix-qa.test reply with only this exact marker: MATRIX_QA_CANARY_TOKEN");
});
it("requires Matrix replies to match the exact marker body", () => {
expect(
scenarioTesting.buildMatrixReplyArtifact(
{
roomId: "!room:matrix-qa.test",
eventId: "$event",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "MATRIX_QA_TOKEN",
},
"MATRIX_QA_TOKEN",
).tokenMatched,
).toBe(true);
expect(
scenarioTesting.buildMatrixReplyArtifact(
{
roomId: "!room:matrix-qa.test",
eventId: "$event-2",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "prefix MATRIX_QA_TOKEN suffix",
},
"MATRIX_QA_TOKEN",
).tokenMatched,
).toBe(false);
});
it("fails when any requested Matrix scenario id is unknown", () => {
expect(() =>
scenarioTesting.findMatrixQaScenarios(["matrix-thread-follow-up", "typo-scenario"]),
).toThrow("unknown Matrix QA scenario id(s): typo-scenario");
});
it("covers the baseline live transport contract plus Matrix-specific extras", () => {
expect(scenarioTesting.MATRIX_QA_STANDARD_SCENARIO_IDS).toEqual([
"canary",
"thread-follow-up",
"thread-isolation",
"top-level-reply-shape",
"reaction-observation",
"restart-resume",
"mention-gating",
"allowlist-block",
]);
expect(
findMissingLiveTransportStandardScenarios({
coveredStandardScenarioIds: scenarioTesting.MATRIX_QA_STANDARD_SCENARIO_IDS,
expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
}),
).toEqual([]);
});
it("primes the observer sync cursor instead of reusing the driver's cursor", async () => {
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
const sendTextMessage = vi.fn().mockResolvedValue("$observer-trigger");
const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => {
expect(params.since).toBe("observer-sync-start");
return {
matched: false,
since: "observer-sync-next",
};
});
createMatrixQaClient.mockReturnValue({
primeRoom,
sendTextMessage,
waitForOptionalRoomEvent,
});
const scenario = MATRIX_QA_SCENARIOS.find((entry) => entry.id === "matrix-allowlist-block");
expect(scenario).toBeDefined();
const syncState = {
driver: "driver-sync-next",
};
await expect(
runMatrixQaScenario(scenario!, {
baseUrl: "http://127.0.0.1:28008/",
canary: undefined,
driverAccessToken: "driver-token",
driverUserId: "@driver:matrix-qa.test",
observedEvents: [],
observerAccessToken: "observer-token",
observerUserId: "@observer:matrix-qa.test",
roomId: "!room:matrix-qa.test",
restartGateway: undefined,
syncState,
sutUserId: "@sut:matrix-qa.test",
timeoutMs: 8_000,
}),
).resolves.toMatchObject({
artifacts: {
actorUserId: "@observer:matrix-qa.test",
expectedNoReplyWindowMs: 8_000,
},
});
expect(createMatrixQaClient).toHaveBeenCalledWith({
accessToken: "observer-token",
baseUrl: "http://127.0.0.1:28008/",
});
expect(primeRoom).toHaveBeenCalledTimes(1);
expect(sendTextMessage).toHaveBeenCalledTimes(1);
expect(waitForOptionalRoomEvent).toHaveBeenCalledTimes(1);
expect(syncState).toEqual({
driver: "driver-sync-next",
observer: "observer-sync-next",
});
});
});

View File

@@ -1,670 +0,0 @@
import { randomUUID } from "node:crypto";
import {
collectLiveTransportStandardScenarioCoverage,
selectLiveTransportScenarios,
type LiveTransportScenarioDefinition,
} from "../shared/live-transport-scenarios.js";
import { createMatrixQaClient, type MatrixQaObservedEvent } from "./matrix-driver-client.js";
export type MatrixQaScenarioId =
| "matrix-thread-follow-up"
| "matrix-thread-isolation"
| "matrix-top-level-reply-shape"
| "matrix-reaction-notification"
| "matrix-restart-resume"
| "matrix-mention-gating"
| "matrix-allowlist-block";
export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQaScenarioId>;
export type MatrixQaReplyArtifact = {
bodyPreview?: string;
eventId: string;
mentions?: MatrixQaObservedEvent["mentions"];
relatesTo?: MatrixQaObservedEvent["relatesTo"];
sender?: string;
tokenMatched?: boolean;
};
export type MatrixQaCanaryArtifact = {
driverEventId: string;
reply: MatrixQaReplyArtifact;
token: string;
};
export type MatrixQaScenarioArtifacts = {
actorUserId?: string;
driverEventId?: string;
expectedNoReplyWindowMs?: number;
reactionEmoji?: string;
reactionEventId?: string;
reactionTargetEventId?: string;
reply?: MatrixQaReplyArtifact;
restartSignal?: string;
rootEventId?: string;
threadDriverEventId?: string;
threadReply?: MatrixQaReplyArtifact;
threadRootEventId?: string;
threadToken?: string;
token?: string;
topLevelDriverEventId?: string;
topLevelReply?: MatrixQaReplyArtifact;
topLevelToken?: string;
triggerBody?: string;
};
export type MatrixQaScenarioExecution = {
artifacts?: MatrixQaScenarioArtifacts;
details: string;
};
type MatrixQaActorId = "driver" | "observer";
type MatrixQaSyncState = Partial<Record<MatrixQaActorId, string>>;
type MatrixQaScenarioContext = {
baseUrl: string;
canary?: MatrixQaCanaryArtifact;
driverAccessToken: string;
driverUserId: string;
observedEvents: MatrixQaObservedEvent[];
observerAccessToken: string;
observerUserId: string;
restartGateway?: () => Promise<void>;
roomId: string;
syncState: MatrixQaSyncState;
sutUserId: string;
timeoutMs: number;
};
const NO_REPLY_WINDOW_MS = 8_000;
export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
{
id: "matrix-thread-follow-up",
standardId: "thread-follow-up",
timeoutMs: 60_000,
title: "Matrix thread follow-up reply",
},
{
id: "matrix-thread-isolation",
standardId: "thread-isolation",
timeoutMs: 75_000,
title: "Matrix top-level reply stays out of prior thread",
},
{
id: "matrix-top-level-reply-shape",
standardId: "top-level-reply-shape",
timeoutMs: 45_000,
title: "Matrix top-level reply keeps replyToMode off",
},
{
id: "matrix-reaction-notification",
standardId: "reaction-observation",
timeoutMs: 45_000,
title: "Matrix reactions on bot replies are observed",
},
{
id: "matrix-restart-resume",
standardId: "restart-resume",
timeoutMs: 60_000,
title: "Matrix lane resumes cleanly after gateway restart",
},
{
id: "matrix-mention-gating",
standardId: "mention-gating",
timeoutMs: NO_REPLY_WINDOW_MS,
title: "Matrix room message without mention does not trigger",
},
{
id: "matrix-allowlist-block",
standardId: "allowlist-block",
timeoutMs: NO_REPLY_WINDOW_MS,
title: "Matrix allowlist blocks non-driver replies",
},
];
export const MATRIX_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
alwaysOnStandardScenarioIds: ["canary"],
scenarios: MATRIX_QA_SCENARIOS,
});
export function findMatrixQaScenarios(ids?: string[]) {
return selectLiveTransportScenarios({
ids,
laneLabel: "Matrix",
scenarios: MATRIX_QA_SCENARIOS,
});
}
export function buildMentionPrompt(sutUserId: string, token: string) {
return `${sutUserId} reply with only this exact marker: ${token}`;
}
function buildExactMarkerPrompt(token: string) {
return `reply with only this exact marker: ${token}`;
}
function buildMatrixReplyArtifact(
event: MatrixQaObservedEvent,
token?: string,
): MatrixQaReplyArtifact {
const replyBody = event.body?.trim();
return {
bodyPreview: replyBody?.slice(0, 200),
eventId: event.eventId,
mentions: event.mentions,
relatesTo: event.relatesTo,
sender: event.sender,
...(token ? { tokenMatched: replyBody === token } : {}),
};
}
export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) {
return [
`${label} event: ${artifact.eventId}`,
`${label} token matched: ${
artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no"
}`,
`${label} rel_type: ${artifact.relatesTo?.relType ?? "<none>"}`,
`${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? "<none>"}`,
`${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`,
];
}
function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) {
if (!artifact.tokenMatched) {
throw new Error(`${label} did not contain the expected token`);
}
if (artifact.relatesTo !== undefined) {
throw new Error(`${label} unexpectedly included relation metadata`);
}
}
function assertThreadReplyArtifact(
artifact: MatrixQaReplyArtifact,
params: {
expectedRootEventId: string;
label: string;
},
) {
if (!artifact.tokenMatched) {
throw new Error(`${params.label} did not contain the expected token`);
}
if (artifact.relatesTo?.relType !== "m.thread") {
throw new Error(`${params.label} did not use m.thread`);
}
if (artifact.relatesTo.eventId !== params.expectedRootEventId) {
throw new Error(
`${params.label} targeted ${artifact.relatesTo.eventId ?? "<none>"} instead of ${params.expectedRootEventId}`,
);
}
if (artifact.relatesTo.isFallingBack !== true) {
throw new Error(`${params.label} did not set is_falling_back`);
}
if (!artifact.relatesTo.inReplyToId) {
throw new Error(`${params.label} did not set m.in_reply_to`);
}
}
function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) {
return syncState[actorId];
}
function writeMatrixQaSyncCursor(
syncState: MatrixQaSyncState,
actorId: MatrixQaActorId,
since?: string,
) {
if (since) {
syncState[actorId] = since;
}
}
async function primeMatrixQaActorCursor(params: {
accessToken: string;
actorId: MatrixQaActorId;
baseUrl: string;
syncState: MatrixQaSyncState;
}) {
const client = createMatrixQaClient({
accessToken: params.accessToken,
baseUrl: params.baseUrl,
});
const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId);
if (existingSince) {
return { client, startSince: existingSince };
}
const startSince = await client.primeRoom();
if (!startSince) {
throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`);
}
return { client, startSince };
}
function advanceMatrixQaActorCursor(params: {
actorId: MatrixQaActorId;
syncState: MatrixQaSyncState;
nextSince?: string;
startSince: string;
}) {
writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince);
}
async function runTopLevelMentionScenario(params: {
accessToken: string;
actorId: MatrixQaActorId;
baseUrl: string;
observedEvents: MatrixQaObservedEvent[];
roomId: string;
syncState: MatrixQaSyncState;
sutUserId: string;
timeoutMs: number;
tokenPrefix: string;
withMention?: boolean;
}) {
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: params.accessToken,
actorId: params.actorId,
baseUrl: params.baseUrl,
syncState: params.syncState,
});
const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
const body =
params.withMention === false
? buildExactMarkerPrompt(token)
: buildMentionPrompt(params.sutUserId, token);
const driverEventId = await client.sendTextMessage({
body,
...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }),
roomId: params.roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
(event.body ?? "").includes(token) &&
event.relatesTo === undefined,
roomId: params.roomId,
since: startSince,
timeoutMs: params.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: params.actorId,
syncState: params.syncState,
nextSince: matched.since,
startSince,
});
return {
body,
driverEventId,
reply: buildMatrixReplyArtifact(matched.event, token),
since: matched.since,
token,
};
}
async function runThreadScenario(params: MatrixQaScenarioContext) {
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: params.driverAccessToken,
actorId: "driver",
baseUrl: params.baseUrl,
syncState: params.syncState,
});
const rootBody = `thread root ${randomUUID().slice(0, 8)}`;
const rootEventId = await client.sendTextMessage({
body: rootBody,
roomId: params.roomId,
});
const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`;
const driverEventId = await client.sendTextMessage({
body: buildMentionPrompt(params.sutUserId, token),
mentionUserIds: [params.sutUserId],
replyToEventId: rootEventId,
roomId: params.roomId,
threadRootEventId: rootEventId,
});
const matched = await client.waitForRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message" &&
(event.body ?? "").includes(token) &&
event.relatesTo?.relType === "m.thread" &&
event.relatesTo.eventId === rootEventId,
roomId: params.roomId,
since: startSince,
timeoutMs: params.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: params.syncState,
nextSince: matched.since,
startSince,
});
return {
driverEventId,
reply: buildMatrixReplyArtifact(matched.event, token),
rootEventId,
since: matched.since,
token,
};
}
async function runNoReplyExpectedScenario(params: {
accessToken: string;
actorId: MatrixQaActorId;
actorUserId: string;
baseUrl: string;
body: string;
mentionUserIds?: string[];
observedEvents: MatrixQaObservedEvent[];
roomId: string;
syncState: MatrixQaSyncState;
sutUserId: string;
timeoutMs: number;
token: string;
}) {
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: params.accessToken,
actorId: params.actorId,
baseUrl: params.baseUrl,
syncState: params.syncState,
});
const driverEventId = await client.sendTextMessage({
body: params.body,
...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}),
roomId: params.roomId,
});
const result = await client.waitForOptionalRoomEvent({
observedEvents: params.observedEvents,
predicate: (event) =>
event.roomId === params.roomId &&
event.sender === params.sutUserId &&
event.type === "m.room.message",
roomId: params.roomId,
since: startSince,
timeoutMs: params.timeoutMs,
});
if (result.matched) {
const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token);
throw new Error(
[
`unexpected SUT reply from ${params.sutUserId}`,
`trigger sender: ${params.actorUserId}`,
...buildMatrixReplyDetails("unexpected reply", unexpectedReply),
].join("\n"),
);
}
advanceMatrixQaActorCursor({
actorId: params.actorId,
syncState: params.syncState,
nextSince: result.since,
startSince,
});
return {
artifacts: {
actorUserId: params.actorUserId,
driverEventId,
expectedNoReplyWindowMs: params.timeoutMs,
token: params.token,
triggerBody: params.body,
},
details: [
`trigger event: ${driverEventId}`,
`trigger sender: ${params.actorUserId}`,
`waited ${params.timeoutMs}ms with no SUT reply`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
async function runReactionNotificationScenario(context: MatrixQaScenarioContext) {
const reactionTargetEventId = context.canary?.reply.eventId?.trim();
if (!reactionTargetEventId) {
throw new Error("Matrix reaction scenario requires a canary reply event id");
}
const { client, startSince } = await primeMatrixQaActorCursor({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
syncState: context.syncState,
});
const reactionEmoji = "👍";
const reactionEventId = await client.sendReaction({
emoji: reactionEmoji,
messageId: reactionTargetEventId,
roomId: context.roomId,
});
const matched = await client.waitForRoomEvent({
observedEvents: context.observedEvents,
predicate: (event) =>
event.roomId === context.roomId &&
event.sender === context.driverUserId &&
event.type === "m.reaction" &&
event.eventId === reactionEventId &&
event.reaction?.eventId === reactionTargetEventId &&
event.reaction?.key === reactionEmoji,
roomId: context.roomId,
since: startSince,
timeoutMs: context.timeoutMs,
});
advanceMatrixQaActorCursor({
actorId: "driver",
syncState: context.syncState,
nextSince: matched.since,
startSince,
});
return {
artifacts: {
reactionEmoji,
reactionEventId,
reactionTargetEventId,
},
details: [
`reaction event: ${reactionEventId}`,
`reaction target: ${reactionTargetEventId}`,
`reaction emoji: ${reactionEmoji}`,
`observed reaction key: ${matched.event.reaction?.key ?? "<none>"}`,
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
async function runRestartResumeScenario(context: MatrixQaScenarioContext) {
if (!context.restartGateway) {
throw new Error("Matrix restart scenario requires a gateway restart callback");
}
await context.restartGateway();
const result = await runTopLevelMentionScenario({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
roomId: context.roomId,
syncState: context.syncState,
sutUserId: context.sutUserId,
timeoutMs: context.timeoutMs,
tokenPrefix: "MATRIX_QA_RESTART",
});
assertTopLevelReplyArtifact("post-restart reply", result.reply);
return {
artifacts: {
driverEventId: result.driverEventId,
reply: result.reply,
restartSignal: "SIGUSR1",
token: result.token,
},
details: [
"restart signal: SIGUSR1",
`post-restart driver event: ${result.driverEventId}`,
...buildMatrixReplyDetails("reply", result.reply),
].join("\n"),
} satisfies MatrixQaScenarioExecution;
}
export async function runMatrixQaCanary(params: {
baseUrl: string;
driverAccessToken: string;
observedEvents: MatrixQaObservedEvent[];
roomId: string;
syncState: MatrixQaSyncState;
sutUserId: string;
timeoutMs: number;
}) {
const canary = await runTopLevelMentionScenario({
accessToken: params.driverAccessToken,
actorId: "driver",
baseUrl: params.baseUrl,
observedEvents: params.observedEvents,
roomId: params.roomId,
syncState: params.syncState,
sutUserId: params.sutUserId,
timeoutMs: params.timeoutMs,
tokenPrefix: "MATRIX_QA_CANARY",
});
assertTopLevelReplyArtifact("canary reply", canary.reply);
return canary;
}
export async function runMatrixQaScenario(
scenario: MatrixQaScenarioDefinition,
context: MatrixQaScenarioContext,
): Promise<MatrixQaScenarioExecution> {
switch (scenario.id) {
case "matrix-thread-follow-up": {
const result = await runThreadScenario(context);
assertThreadReplyArtifact(result.reply, {
expectedRootEventId: result.rootEventId,
label: "thread reply",
});
return {
artifacts: {
driverEventId: result.driverEventId,
reply: result.reply,
rootEventId: result.rootEventId,
token: result.token,
},
details: [
`root event: ${result.rootEventId}`,
`driver thread event: ${result.driverEventId}`,
...buildMatrixReplyDetails("reply", result.reply),
].join("\n"),
};
}
case "matrix-thread-isolation": {
const threadPhase = await runThreadScenario(context);
assertThreadReplyArtifact(threadPhase.reply, {
expectedRootEventId: threadPhase.rootEventId,
label: "thread isolation reply",
});
const topLevelPhase = await runTopLevelMentionScenario({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
roomId: context.roomId,
syncState: context.syncState,
sutUserId: context.sutUserId,
timeoutMs: context.timeoutMs,
tokenPrefix: "MATRIX_QA_TOPLEVEL",
});
assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply);
return {
artifacts: {
threadDriverEventId: threadPhase.driverEventId,
threadReply: threadPhase.reply,
threadRootEventId: threadPhase.rootEventId,
threadToken: threadPhase.token,
topLevelDriverEventId: topLevelPhase.driverEventId,
topLevelReply: topLevelPhase.reply,
topLevelToken: topLevelPhase.token,
},
details: [
`thread root event: ${threadPhase.rootEventId}`,
`thread driver event: ${threadPhase.driverEventId}`,
...buildMatrixReplyDetails("thread reply", threadPhase.reply),
`top-level driver event: ${topLevelPhase.driverEventId}`,
...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply),
].join("\n"),
};
}
case "matrix-top-level-reply-shape": {
const result = await runTopLevelMentionScenario({
accessToken: context.driverAccessToken,
actorId: "driver",
baseUrl: context.baseUrl,
observedEvents: context.observedEvents,
roomId: context.roomId,
syncState: context.syncState,
sutUserId: context.sutUserId,
timeoutMs: context.timeoutMs,
tokenPrefix: "MATRIX_QA_TOPLEVEL",
});
assertTopLevelReplyArtifact("top-level reply", result.reply);
return {
artifacts: {
driverEventId: result.driverEventId,
reply: result.reply,
token: result.token,
},
details: [
`driver event: ${result.driverEventId}`,
...buildMatrixReplyDetails("reply", result.reply),
].join("\n"),
};
}
case "matrix-reaction-notification":
return await runReactionNotificationScenario(context);
case "matrix-restart-resume":
return await runRestartResumeScenario(context);
case "matrix-mention-gating": {
const token = `MATRIX_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
return await runNoReplyExpectedScenario({
accessToken: context.driverAccessToken,
actorId: "driver",
actorUserId: context.driverUserId,
baseUrl: context.baseUrl,
body: buildExactMarkerPrompt(token),
observedEvents: context.observedEvents,
roomId: context.roomId,
syncState: context.syncState,
sutUserId: context.sutUserId,
timeoutMs: context.timeoutMs,
token,
});
}
case "matrix-allowlist-block": {
const token = `MATRIX_QA_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`;
return await runNoReplyExpectedScenario({
accessToken: context.observerAccessToken,
actorId: "observer",
actorUserId: context.observerUserId,
baseUrl: context.baseUrl,
body: buildMentionPrompt(context.sutUserId, token),
mentionUserIds: [context.sutUserId],
observedEvents: context.observedEvents,
roomId: context.roomId,
syncState: context.syncState,
sutUserId: context.sutUserId,
timeoutMs: context.timeoutMs,
token,
});
}
default: {
const exhaustiveScenarioId: never = scenario.id;
return exhaustiveScenarioId;
}
}
}
export const __testing = {
MATRIX_QA_STANDARD_SCENARIO_IDS,
buildMatrixReplyDetails,
buildMatrixReplyArtifact,
buildMentionPrompt,
findMatrixQaScenarios,
readMatrixQaSyncCursor,
writeMatrixQaSyncCursor,
};

View File

@@ -1,272 +0,0 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { afterEach, describe, expect, it, vi } from "vitest";
import { __testing as liveTesting } from "./matrix-live.runtime.js";
afterEach(() => {
vi.useRealTimers();
});
describe("matrix live qa runtime", () => {
it("injects a temporary Matrix account into the QA gateway config", () => {
const baseCfg: OpenClawConfig = {
plugins: {
allow: ["memory-core", "qa-channel"],
entries: {
"memory-core": { enabled: true },
"qa-channel": { enabled: true },
},
},
};
const next = liveTesting.buildMatrixQaConfig(baseCfg, {
driverUserId: "@driver:matrix-qa.test",
homeserver: "http://127.0.0.1:28008/",
roomId: "!room:matrix-qa.test",
sutAccessToken: "syt_sut",
sutAccountId: "sut",
sutDeviceId: "DEVICE123",
sutUserId: "@sut:matrix-qa.test",
});
expect(next.plugins?.allow).toContain("matrix");
expect(next.plugins?.entries?.matrix).toEqual({ enabled: true });
expect(next.channels?.matrix).toEqual({
enabled: true,
defaultAccount: "sut",
accounts: {
sut: {
accessToken: "syt_sut",
deviceId: "DEVICE123",
dm: { enabled: false },
enabled: true,
encryption: false,
groupAllowFrom: ["@driver:matrix-qa.test"],
groupPolicy: "allowlist",
groups: {
"!room:matrix-qa.test": {
enabled: true,
requireMention: true,
},
},
homeserver: "http://127.0.0.1:28008/",
network: {
dangerouslyAllowPrivateNetwork: true,
},
replyToMode: "off",
threadReplies: "inbound",
userId: "@sut:matrix-qa.test",
},
},
});
});
it("redacts Matrix observed event content by default in artifacts", () => {
expect(
liveTesting.buildObservedEventsArtifact({
includeContent: false,
observedEvents: [
{
roomId: "!room:matrix-qa.test",
eventId: "$event",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
body: "secret",
formattedBody: "<p>secret</p>",
msgtype: "m.text",
originServerTs: 1_700_000_000_000,
relatesTo: {
relType: "m.thread",
eventId: "$root",
inReplyToId: "$driver",
isFallingBack: true,
},
},
],
}),
).toEqual([
{
roomId: "!room:matrix-qa.test",
eventId: "$event",
sender: "@sut:matrix-qa.test",
type: "m.room.message",
msgtype: "m.text",
originServerTs: 1_700_000_000_000,
relatesTo: {
relType: "m.thread",
eventId: "$root",
inReplyToId: "$driver",
isFallingBack: true,
},
},
]);
});
it("keeps reaction metadata in redacted Matrix observed-event artifacts", () => {
expect(
liveTesting.buildObservedEventsArtifact({
includeContent: false,
observedEvents: [
{
roomId: "!room:matrix-qa.test",
eventId: "$reaction",
sender: "@driver:matrix-qa.test",
type: "m.reaction",
reaction: {
eventId: "$reply",
key: "👍",
},
relatesTo: {
relType: "m.annotation",
eventId: "$reply",
},
},
],
}),
).toEqual([
{
roomId: "!room:matrix-qa.test",
eventId: "$reaction",
sender: "@driver:matrix-qa.test",
type: "m.reaction",
originServerTs: undefined,
msgtype: undefined,
membership: undefined,
relatesTo: {
relType: "m.annotation",
eventId: "$reply",
},
mentions: undefined,
reaction: {
eventId: "$reply",
key: "👍",
},
},
]);
});
it("preserves negative-scenario artifacts in the Matrix summary", () => {
expect(
liveTesting.buildMatrixQaSummary({
artifactPaths: {
observedEvents: "/tmp/observed.json",
report: "/tmp/report.md",
summary: "/tmp/summary.json",
},
checks: [{ name: "Matrix harness ready", status: "pass" }],
finishedAt: "2026-04-10T10:05:00.000Z",
harness: {
baseUrl: "http://127.0.0.1:28008/",
composeFile: "/tmp/docker-compose.yml",
image: "ghcr.io/matrix-construct/tuwunel:v1.5.1",
roomId: "!room:matrix-qa.test",
serverName: "matrix-qa.test",
},
observedEventCount: 4,
scenarios: [
{
id: "matrix-mention-gating",
title: "Matrix room message without mention does not trigger",
status: "pass",
details: "no reply",
artifacts: {
actorUserId: "@driver:matrix-qa.test",
driverEventId: "$driver",
expectedNoReplyWindowMs: 8_000,
token: "MATRIX_QA_NOMENTION_TOKEN",
triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN",
},
},
],
startedAt: "2026-04-10T10:00:00.000Z",
sutAccountId: "sut",
userIds: {
driver: "@driver:matrix-qa.test",
observer: "@observer:matrix-qa.test",
sut: "@sut:matrix-qa.test",
},
}),
).toMatchObject({
counts: {
total: 2,
passed: 2,
failed: 0,
},
scenarios: [
{
id: "matrix-mention-gating",
artifacts: {
actorUserId: "@driver:matrix-qa.test",
expectedNoReplyWindowMs: 8_000,
triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN",
},
},
],
});
});
it("treats only connected, healthy Matrix accounts as ready", () => {
expect(liveTesting.isMatrixAccountReady({ running: true, connected: true })).toBe(true);
expect(liveTesting.isMatrixAccountReady({ running: true, connected: false })).toBe(false);
expect(
liveTesting.isMatrixAccountReady({
running: true,
connected: true,
restartPending: true,
}),
).toBe(false);
expect(
liveTesting.isMatrixAccountReady({
running: true,
connected: true,
healthState: "degraded",
}),
).toBe(false);
});
it("waits past not-ready Matrix status snapshots until the account is really ready", async () => {
vi.useFakeTimers();
const gateway = {
call: vi
.fn()
.mockResolvedValueOnce({
channelAccounts: {
matrix: [{ accountId: "sut", running: true, connected: false }],
},
})
.mockResolvedValueOnce({
channelAccounts: {
matrix: [{ accountId: "sut", running: true, connected: true }],
},
}),
};
const waitPromise = liveTesting.waitForMatrixChannelReady(gateway as never, "sut", {
timeoutMs: 1_000,
pollMs: 100,
});
await vi.advanceTimersByTimeAsync(100);
await expect(waitPromise).resolves.toBeUndefined();
expect(gateway.call).toHaveBeenCalledTimes(2);
});
it("fails readiness when the Matrix account never reaches a healthy connected state", async () => {
vi.useFakeTimers();
const gateway = {
call: vi.fn().mockResolvedValue({
channelAccounts: {
matrix: [{ accountId: "sut", running: true, connected: true, healthState: "degraded" }],
},
}),
};
const waitPromise = liveTesting.waitForMatrixChannelReady(gateway as never, "sut", {
timeoutMs: 250,
pollMs: 100,
});
const expectation = expect(waitPromise).rejects.toThrow(
'matrix account "sut" did not become ready',
);
await vi.advanceTimersByTimeAsync(300);
await expectation;
});
});

View File

@@ -1,559 +0,0 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { startQaGatewayChild } from "../../gateway-child.js";
import type { QaReportCheck } from "../../report.js";
import { renderQaMarkdownReport } from "../../report.js";
import {
defaultQaModelForMode,
normalizeQaProviderMode,
type QaProviderModeInput,
} from "../../run-config.js";
import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js";
import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js";
import {
provisionMatrixQaRoom,
type MatrixQaObservedEvent,
type MatrixQaProvisionResult,
} from "./matrix-driver-client.js";
import { startMatrixQaHarness } from "./matrix-harness.runtime.js";
import {
MATRIX_QA_SCENARIOS,
buildMatrixReplyDetails,
findMatrixQaScenarios,
runMatrixQaCanary,
runMatrixQaScenario,
type MatrixQaCanaryArtifact,
type MatrixQaScenarioArtifacts,
} from "./matrix-live-scenarios.js";
type MatrixQaScenarioResult = {
artifacts?: MatrixQaScenarioArtifacts;
details: string;
id: string;
status: "fail" | "pass";
title: string;
};
type MatrixQaSummary = {
checks: QaReportCheck[];
counts: {
failed: number;
passed: number;
total: number;
};
finishedAt: string;
harness: {
baseUrl: string;
composeFile: string;
image: string;
roomId: string;
serverName: string;
};
canary?: MatrixQaCanaryArtifact;
observedEventCount: number;
observedEventsPath: string;
reportPath: string;
scenarios: MatrixQaScenarioResult[];
startedAt: string;
summaryPath: string;
sutAccountId: string;
userIds: {
driver: string;
observer: string;
sut: string;
};
};
type MatrixQaArtifactPaths = {
observedEvents: string;
report: string;
summary: string;
};
export type MatrixQaRunResult = {
observedEventsPath: string;
outputDir: string;
reportPath: string;
scenarios: MatrixQaScenarioResult[];
summaryPath: string;
};
function buildMatrixQaSummary(params: {
artifactPaths: MatrixQaArtifactPaths;
canary?: MatrixQaCanaryArtifact;
checks: QaReportCheck[];
finishedAt: string;
harness: MatrixQaSummary["harness"];
observedEventCount: number;
scenarios: MatrixQaScenarioResult[];
startedAt: string;
sutAccountId: string;
userIds: MatrixQaSummary["userIds"];
}): MatrixQaSummary {
return {
checks: params.checks,
counts: {
total: params.checks.length + params.scenarios.length,
passed:
params.checks.filter((check) => check.status === "pass").length +
params.scenarios.filter((scenario) => scenario.status === "pass").length,
failed:
params.checks.filter((check) => check.status === "fail").length +
params.scenarios.filter((scenario) => scenario.status === "fail").length,
},
finishedAt: params.finishedAt,
harness: params.harness,
canary: params.canary,
observedEventCount: params.observedEventCount,
observedEventsPath: params.artifactPaths.observedEvents,
reportPath: params.artifactPaths.report,
scenarios: params.scenarios,
startedAt: params.startedAt,
summaryPath: params.artifactPaths.summary,
sutAccountId: params.sutAccountId,
userIds: params.userIds,
};
}
function buildMatrixQaConfig(
baseCfg: OpenClawConfig,
params: {
driverUserId: string;
homeserver: string;
roomId: string;
sutAccessToken: string;
sutAccountId: string;
sutDeviceId?: string;
sutUserId: string;
},
): OpenClawConfig {
const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "matrix"])];
return {
...baseCfg,
plugins: {
...baseCfg.plugins,
allow: pluginAllow,
entries: {
...baseCfg.plugins?.entries,
matrix: { enabled: true },
},
},
channels: {
...baseCfg.channels,
matrix: {
enabled: true,
defaultAccount: params.sutAccountId,
accounts: {
[params.sutAccountId]: {
accessToken: params.sutAccessToken,
...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}),
dm: { enabled: false },
enabled: true,
encryption: false,
groupAllowFrom: [params.driverUserId],
groupPolicy: "allowlist",
groups: {
[params.roomId]: {
enabled: true,
requireMention: true,
},
},
homeserver: params.homeserver,
network: {
dangerouslyAllowPrivateNetwork: true,
},
replyToMode: "off",
threadReplies: "inbound",
userId: params.sutUserId,
},
},
},
},
};
}
function buildObservedEventsArtifact(params: {
includeContent: boolean;
observedEvents: MatrixQaObservedEvent[];
}) {
return params.observedEvents.map((event) =>
params.includeContent
? event
: {
roomId: event.roomId,
eventId: event.eventId,
sender: event.sender,
stateKey: event.stateKey,
type: event.type,
originServerTs: event.originServerTs,
msgtype: event.msgtype,
membership: event.membership,
relatesTo: event.relatesTo,
mentions: event.mentions,
reaction: event.reaction,
},
);
}
function isMatrixAccountReady(entry?: {
connected?: boolean;
healthState?: string;
restartPending?: boolean;
running?: boolean;
}): boolean {
return (
entry?.running === true &&
entry.connected === true &&
entry.restartPending !== true &&
(entry.healthState === undefined || entry.healthState === "healthy")
);
}
async function waitForMatrixChannelReady(
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
accountId: string,
opts?: {
pollMs?: number;
timeoutMs?: number;
},
) {
const pollMs = opts?.pollMs ?? 500;
const timeoutMs = opts?.timeoutMs ?? 60_000;
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
try {
const payload = (await gateway.call(
"channels.status",
{ probe: false, timeoutMs: 2_000 },
{ timeoutMs: 5_000 },
)) as {
channelAccounts?: Record<
string,
Array<{
accountId?: string;
connected?: boolean;
healthState?: string;
restartPending?: boolean;
running?: boolean;
}>
>;
};
const accounts = payload.channelAccounts?.matrix ?? [];
const match = accounts.find((entry) => entry.accountId === accountId);
if (isMatrixAccountReady(match)) {
return;
}
} catch {
// retry
}
await sleep(pollMs);
}
throw new Error(`matrix account "${accountId}" did not become ready`);
}
export async function runMatrixQaLive(params: {
fastMode?: boolean;
outputDir?: string;
primaryModel?: string;
providerMode?: QaProviderModeInput;
repoRoot?: string;
scenarioIds?: string[];
sutAccountId?: string;
alternateModel?: string;
}): Promise<MatrixQaRunResult> {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const outputDir =
params.outputDir ??
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
await fs.mkdir(outputDir, { recursive: true });
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findMatrixQaScenarios(params.scenarioIds);
const observedEvents: MatrixQaObservedEvent[] = [];
const includeObservedEventContent = process.env.OPENCLAW_QA_MATRIX_CAPTURE_CONTENT === "1";
const startedAtDate = new Date();
const startedAt = startedAtDate.toISOString();
const runSuffix = randomUUID().slice(0, 8);
const harness = await startMatrixQaHarness({
outputDir: path.join(outputDir, "matrix-harness"),
repoRoot,
});
const provisioning: MatrixQaProvisionResult = await (async () => {
try {
return await provisionMatrixQaRoom({
baseUrl: harness.baseUrl,
driverLocalpart: `qa-driver-${runSuffix}`,
observerLocalpart: `qa-observer-${runSuffix}`,
registrationToken: harness.registrationToken,
roomName: `OpenClaw Matrix QA ${runSuffix}`,
sutLocalpart: `qa-sut-${runSuffix}`,
});
} catch (error) {
await harness.stop().catch(() => {});
throw error;
}
})();
const checks: QaReportCheck[] = [
{
name: "Matrix harness ready",
status: "pass",
details: [
`image: ${harness.image}`,
`baseUrl: ${harness.baseUrl}`,
`serverName: ${harness.serverName}`,
`roomId: ${provisioning.roomId}`,
].join("\n"),
},
];
const scenarioResults: MatrixQaScenarioResult[] = [];
const cleanupErrors: string[] = [];
let canaryArtifact: MatrixQaCanaryArtifact | undefined;
let gatewayHarness: Awaited<ReturnType<typeof startQaLiveLaneGateway>> | null = null;
let canaryFailed = false;
const syncState: { driver?: string; observer?: string } = {};
try {
gatewayHarness = await startQaLiveLaneGateway({
repoRoot,
transport: {
requiredPluginIds: [],
createGatewayConfig: () => ({}),
},
transportBaseUrl: "http://127.0.0.1:43123",
providerMode,
primaryModel,
alternateModel,
fastMode: params.fastMode,
controlUiEnabled: false,
mutateConfig: (cfg) =>
buildMatrixQaConfig(cfg, {
driverUserId: provisioning.driver.userId,
homeserver: harness.baseUrl,
roomId: provisioning.roomId,
sutAccessToken: provisioning.sut.accessToken,
sutAccountId,
sutDeviceId: provisioning.sut.deviceId,
sutUserId: provisioning.sut.userId,
}),
});
await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId);
checks.push({
name: "Matrix channel ready",
status: "pass",
details: `accountId: ${sutAccountId}\nuserId: ${provisioning.sut.userId}`,
});
try {
const canary = await runMatrixQaCanary({
baseUrl: harness.baseUrl,
driverAccessToken: provisioning.driver.accessToken,
observedEvents,
roomId: provisioning.roomId,
syncState,
sutUserId: provisioning.sut.userId,
timeoutMs: 45_000,
});
canaryArtifact = {
driverEventId: canary.driverEventId,
reply: canary.reply,
token: canary.token,
};
checks.push({
name: "Matrix canary",
status: "pass",
details: buildMatrixReplyDetails("reply", canary.reply).join("\n"),
});
} catch (error) {
canaryFailed = true;
checks.push({
name: "Matrix canary",
status: "fail",
details: formatErrorMessage(error),
});
}
if (!canaryFailed) {
for (const scenario of scenarios) {
try {
const result = await runMatrixQaScenario(scenario, {
baseUrl: harness.baseUrl,
canary: canaryArtifact,
driverAccessToken: provisioning.driver.accessToken,
driverUserId: provisioning.driver.userId,
observedEvents,
observerAccessToken: provisioning.observer.accessToken,
observerUserId: provisioning.observer.userId,
restartGateway: async () => {
if (!gatewayHarness) {
throw new Error("Matrix restart scenario requires a live gateway");
}
await gatewayHarness.gateway.restart();
await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId);
},
roomId: provisioning.roomId,
syncState,
sutUserId: provisioning.sut.userId,
timeoutMs: scenario.timeoutMs,
});
scenarioResults.push({
artifacts: result.artifacts,
id: scenario.id,
title: scenario.title,
status: "pass",
details: result.details,
});
} catch (error) {
scenarioResults.push({
id: scenario.id,
title: scenario.title,
status: "fail",
details: formatErrorMessage(error),
});
}
}
}
} finally {
if (gatewayHarness) {
try {
await gatewayHarness.stop();
} catch (error) {
appendLiveLaneIssue(cleanupErrors, "live gateway cleanup", error);
}
}
try {
await harness.stop();
} catch (error) {
appendLiveLaneIssue(cleanupErrors, "Matrix harness cleanup", error);
}
}
if (cleanupErrors.length > 0) {
checks.push({
name: "Matrix cleanup",
status: "fail",
details: cleanupErrors.join("\n"),
});
}
const finishedAtDate = new Date();
const finishedAt = finishedAtDate.toISOString();
const reportPath = path.join(outputDir, "matrix-qa-report.md");
const summaryPath = path.join(outputDir, "matrix-qa-summary.json");
const observedEventsPath = path.join(outputDir, "matrix-qa-observed-events.json");
const artifactPaths = {
observedEvents: observedEventsPath,
report: reportPath,
summary: summaryPath,
} satisfies MatrixQaArtifactPaths;
const report = renderQaMarkdownReport({
title: "Matrix QA Report",
startedAt: startedAtDate,
finishedAt: finishedAtDate,
checks,
scenarios: scenarioResults.map((scenario) => ({
details: scenario.details,
name: scenario.title,
status: scenario.status,
})),
notes: [
`roomId: ${provisioning.roomId}`,
`driver: ${provisioning.driver.userId}`,
`observer: ${provisioning.observer.userId}`,
`sut: ${provisioning.sut.userId}`,
`homeserver: ${harness.baseUrl}`,
`image: ${harness.image}`,
],
});
const summary: MatrixQaSummary = buildMatrixQaSummary({
artifactPaths,
canary: canaryArtifact,
checks,
finishedAt,
harness: {
baseUrl: harness.baseUrl,
composeFile: harness.composeFile,
image: harness.image,
roomId: provisioning.roomId,
serverName: harness.serverName,
},
observedEventCount: observedEvents.length,
scenarios: scenarioResults,
startedAt,
sutAccountId,
userIds: {
driver: provisioning.driver.userId,
observer: provisioning.observer.userId,
sut: provisioning.sut.userId,
},
});
await fs.writeFile(reportPath, `${report}\n`, { encoding: "utf8", mode: 0o600 });
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
});
await fs.writeFile(
observedEventsPath,
`${JSON.stringify(
buildObservedEventsArtifact({
includeContent: includeObservedEventContent,
observedEvents,
}),
null,
2,
)}\n`,
{ encoding: "utf8", mode: 0o600 },
);
const failedChecks = checks.filter(
(check) => check.status === "fail" && check.name !== "Matrix cleanup",
);
const failedScenarios = scenarioResults.filter((scenario) => scenario.status === "fail");
if (failedChecks.length > 0 || failedScenarios.length > 0) {
throw new Error(
buildLiveLaneArtifactsError({
heading: "Matrix QA failed.",
details: [
...failedChecks.map((check) => `check ${check.name}: ${check.details ?? "failed"}`),
...failedScenarios.map((scenario) => `scenario ${scenario.id}: ${scenario.details}`),
...cleanupErrors.map((error) => `cleanup: ${error}`),
],
artifacts: artifactPaths,
}),
);
}
if (cleanupErrors.length > 0) {
throw new Error(
buildLiveLaneArtifactsError({
heading: "Matrix QA cleanup failed after artifacts were written.",
details: cleanupErrors,
artifacts: artifactPaths,
}),
);
}
return {
observedEventsPath,
outputDir,
reportPath,
scenarios: scenarioResults,
summaryPath,
};
}
export const __testing = {
buildMatrixQaSummary,
MATRIX_QA_SCENARIOS,
buildMatrixQaConfig,
buildObservedEventsArtifact,
isMatrixAccountReady,
waitForMatrixChannelReady,
};

View File

@@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = {
register(qa: Command): void;
};
export type LiveTransportQaCredentialCliOptions = {
sourceDescription?: string;
roleDescription?: string;
};
export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
let promise: Promise<T> | null = null;
return async () => {
@@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions(
export function registerLiveTransportQaCli(params: {
qa: Command;
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
sutAccountHelp: string;
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
}) {
params.qa
const command = params.qa
.command(params.commandName)
.description(params.description)
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
@@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: {
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectString, [])
.option("--fast", "Enable provider fast mode where supported", false)
.option("--sut-account <id>", params.sutAccountHelp, "sut")
.option(
.option("--sut-account <id>", params.sutAccountHelp, "sut");
if (params.credentialOptions) {
command.option(
"--credential-source <source>",
"Credential source for live lanes: env or convex (default: env)",
)
.option(
"--credential-role <role>",
"Credential role for convex auth: maintainer or ci (default: maintainer)",
)
.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
params.credentialOptions.sourceDescription ??
"Credential source for live lanes: env or convex (default: env)",
);
if (params.credentialOptions.roleDescription) {
command.option("--credential-role <role>", params.credentialOptions.roleDescription);
}
}
command.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
}
export function createLiveTransportQaCliRegistration(params: {
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
@@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: {
registerLiveTransportQaCli({
qa,
commandName: params.commandName,
credentialOptions: params.credentialOptions,
description: params.description,
outputDirHelp: params.outputDirHelp,
scenarioHelp: params.scenarioHelp,

View File

@@ -20,6 +20,10 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
createLiveTransportQaCliRegistration({
commandName: "telegram",
credentialOptions: {
sourceDescription: "Credential source for Telegram QA: env or convex (default: env)",
roleDescription: "Credential role for convex auth: maintainer or ci (default: maintainer)",
},
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
outputDirHelp: "Telegram QA artifact directory",
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",

View File

@@ -3,6 +3,7 @@ export type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
export { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime";
export type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
export { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
export {
buildQaTarget,
createQaBusThread,

View File

@@ -81,7 +81,7 @@ export async function runQaSelfCheckAgainstState(params: {
timeline,
notes: params.notes ?? [
"Vertical slice: qa-channel + qa-lab bus + private debugger surface.",
"Docker orchestration, matrix runs, and auto-fix loops remain follow-up work.",
"Docker orchestration, additional QA runners, and auto-fix loops remain follow-up work.",
],
});