mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 21:00:44 +00:00
refactor(qa): split Matrix QA into optional plugin (#66723)
Merged via squash.
Prepared head SHA: 27241bd089
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
committed by
GitHub
parent
3425823dfb
commit
82a2db71e8
@@ -1 +1,2 @@
|
||||
export * from "./src/runtime-api.js";
|
||||
export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js";
|
||||
|
||||
@@ -8,7 +8,6 @@ const {
|
||||
runQaSuiteFromRuntime,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
runMatrixQaLive,
|
||||
runTelegramQaLive,
|
||||
startQaLabServer,
|
||||
writeQaDockerHarnessFiles,
|
||||
@@ -20,7 +19,6 @@ const {
|
||||
runQaSuiteFromRuntime: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
runMatrixQaLive: vi.fn(),
|
||||
runTelegramQaLive: vi.fn(),
|
||||
startQaLabServer: vi.fn(),
|
||||
writeQaDockerHarnessFiles: vi.fn(),
|
||||
@@ -52,10 +50,6 @@ vi.mock("./multipass.runtime.js", () => ({
|
||||
runQaMultipass,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/matrix/matrix-live.runtime.js", () => ({
|
||||
runMatrixQaLive,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
|
||||
runTelegramQaLive,
|
||||
}));
|
||||
@@ -88,7 +82,6 @@ import {
|
||||
runQaParityReportCommand,
|
||||
runQaSuiteCommand,
|
||||
} from "./cli.runtime.js";
|
||||
import { runQaMatrixCommand } from "./live-transports/matrix/cli.runtime.js";
|
||||
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
|
||||
|
||||
describe("qa cli runtime", () => {
|
||||
@@ -100,7 +93,6 @@ describe("qa cli runtime", () => {
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
runMatrixQaLive.mockReset();
|
||||
runTelegramQaLive.mockReset();
|
||||
startQaLabServer.mockReset();
|
||||
writeQaDockerHarnessFiles.mockReset();
|
||||
@@ -139,13 +131,6 @@ describe("qa cli runtime", () => {
|
||||
vmName: "openclaw-qa-test",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
runMatrixQaLive.mockResolvedValue({
|
||||
outputDir: "/tmp/matrix",
|
||||
reportPath: "/tmp/matrix/report.md",
|
||||
summaryPath: "/tmp/matrix/summary.json",
|
||||
observedEventsPath: "/tmp/matrix/observed.json",
|
||||
scenarios: [],
|
||||
});
|
||||
runTelegramQaLive.mockResolvedValue({
|
||||
outputDir: "/tmp/telegram",
|
||||
reportPath: "/tmp/telegram/report.md",
|
||||
@@ -226,30 +211,6 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves matrix qa repo-root-relative paths before dispatching", async () => {
|
||||
await runQaMatrixCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa/matrix",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
fastMode: true,
|
||||
scenarioIds: ["matrix-thread-follow-up"],
|
||||
sutAccountId: "sut-live",
|
||||
});
|
||||
|
||||
expect(runMatrixQaLive).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/matrix"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
fastMode: true,
|
||||
scenarioIds: ["matrix-thread-follow-up"],
|
||||
sutAccountId: "sut-live",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects output dirs that escape the repo root", () => {
|
||||
expect(() => resolveRepoRelativeOutputDir("/tmp/openclaw-repo", "../outside")).toThrow(
|
||||
"--output-dir must stay within the repo root.",
|
||||
@@ -273,20 +234,6 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("defaults matrix qa runs onto the live provider lane", async () => {
|
||||
await runQaMatrixCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
scenarioIds: ["matrix-thread-follow-up"],
|
||||
});
|
||||
|
||||
expect(runMatrixQaLive).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
|
||||
@@ -1,22 +1,76 @@
|
||||
import { Command } from "commander";
|
||||
import type { QaRunnerCliContribution } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const TEST_QA_RUNNER = {
|
||||
pluginId: "qa-runner-test",
|
||||
commandName: "runner-test",
|
||||
description: "Run the test live QA lane",
|
||||
npmSpec: "@openclaw/qa-runner-test",
|
||||
} as const;
|
||||
|
||||
function createAvailableQaRunnerContribution() {
|
||||
return {
|
||||
pluginId: TEST_QA_RUNNER.pluginId,
|
||||
commandName: TEST_QA_RUNNER.commandName,
|
||||
status: "available" as const,
|
||||
registration: {
|
||||
commandName: TEST_QA_RUNNER.commandName,
|
||||
register: vi.fn((qa: Command) => {
|
||||
qa.command(TEST_QA_RUNNER.commandName).action(() => undefined);
|
||||
}),
|
||||
},
|
||||
} satisfies QaRunnerCliContribution;
|
||||
}
|
||||
|
||||
function createMissingQaRunnerContribution(): QaRunnerCliContribution {
|
||||
return {
|
||||
pluginId: TEST_QA_RUNNER.pluginId,
|
||||
commandName: TEST_QA_RUNNER.commandName,
|
||||
description: TEST_QA_RUNNER.description,
|
||||
status: "missing",
|
||||
npmSpec: TEST_QA_RUNNER.npmSpec,
|
||||
};
|
||||
}
|
||||
|
||||
function createBlockedQaRunnerContribution(): QaRunnerCliContribution {
|
||||
return {
|
||||
pluginId: TEST_QA_RUNNER.pluginId,
|
||||
commandName: TEST_QA_RUNNER.commandName,
|
||||
description: TEST_QA_RUNNER.description,
|
||||
status: "blocked",
|
||||
};
|
||||
}
|
||||
|
||||
function createConflictingQaRunnerContribution(commandName: string): QaRunnerCliContribution {
|
||||
return {
|
||||
pluginId: TEST_QA_RUNNER.pluginId,
|
||||
commandName,
|
||||
description: TEST_QA_RUNNER.description,
|
||||
status: "blocked",
|
||||
};
|
||||
}
|
||||
|
||||
const {
|
||||
runQaCredentialsAddCommand,
|
||||
runQaCredentialsListCommand,
|
||||
runQaCredentialsRemoveCommand,
|
||||
runQaMatrixCommand,
|
||||
runQaTelegramCommand,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaCredentialsAddCommand: vi.fn(),
|
||||
runQaCredentialsListCommand: vi.fn(),
|
||||
runQaCredentialsRemoveCommand: vi.fn(),
|
||||
runQaMatrixCommand: vi.fn(),
|
||||
runQaTelegramCommand: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/matrix/cli.runtime.js", () => ({
|
||||
runQaMatrixCommand,
|
||||
const { listQaRunnerCliContributions } = vi.hoisted(() => ({
|
||||
listQaRunnerCliContributions: vi.fn<() => QaRunnerCliContribution[]>(() => [
|
||||
createAvailableQaRunnerContribution(),
|
||||
]),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({
|
||||
listQaRunnerCliContributions,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/cli.runtime.js", () => ({
|
||||
@@ -36,63 +90,71 @@ describe("qa cli registration", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
program = new Command();
|
||||
registerQaLabCli(program);
|
||||
runQaCredentialsAddCommand.mockReset();
|
||||
runQaCredentialsListCommand.mockReset();
|
||||
runQaCredentialsRemoveCommand.mockReset();
|
||||
runQaMatrixCommand.mockReset();
|
||||
runQaTelegramCommand.mockReset();
|
||||
listQaRunnerCliContributions
|
||||
.mockReset()
|
||||
.mockReturnValue([createAvailableQaRunnerContribution()]);
|
||||
registerQaLabCli(program);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("registers the matrix and telegram live transport subcommands", () => {
|
||||
it("registers discovered and built-in live transport subcommands", () => {
|
||||
const qa = program.commands.find((command) => command.name() === "qa");
|
||||
expect(qa).toBeDefined();
|
||||
expect(qa?.commands.map((command) => command.name())).toEqual(
|
||||
expect.arrayContaining(["matrix", "telegram", "credentials"]),
|
||||
expect.arrayContaining([TEST_QA_RUNNER.commandName, "telegram", "credentials"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes matrix CLI flags into the lane runtime", async () => {
|
||||
await program.parseAsync([
|
||||
"node",
|
||||
"openclaw",
|
||||
"qa",
|
||||
"matrix",
|
||||
"--repo-root",
|
||||
"/tmp/openclaw-repo",
|
||||
"--output-dir",
|
||||
".artifacts/qa/matrix",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--model",
|
||||
"mock-openai/gpt-5.4",
|
||||
"--alt-model",
|
||||
"mock-openai/gpt-5.4-alt",
|
||||
"--scenario",
|
||||
"matrix-thread-follow-up",
|
||||
"--scenario",
|
||||
"matrix-thread-isolation",
|
||||
"--fast",
|
||||
"--sut-account",
|
||||
"sut-live",
|
||||
]);
|
||||
it("delegates discovered qa runner registration through the generic host seam", () => {
|
||||
const [{ registration }] = listQaRunnerCliContributions.mock.results[0]?.value;
|
||||
expect(registration.register).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(runQaMatrixCommand).toHaveBeenCalledWith({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa/matrix",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.4",
|
||||
alternateModel: "mock-openai/gpt-5.4-alt",
|
||||
fastMode: true,
|
||||
scenarioIds: ["matrix-thread-follow-up", "matrix-thread-isolation"],
|
||||
sutAccountId: "sut-live",
|
||||
credentialSource: undefined,
|
||||
credentialRole: undefined,
|
||||
});
|
||||
it("keeps Telegram credential flags on the shared host CLI", () => {
|
||||
const qa = program.commands.find((command) => command.name() === "qa");
|
||||
const telegram = qa?.commands.find((command) => command.name() === "telegram");
|
||||
const optionNames = telegram?.options.map((option) => option.long) ?? [];
|
||||
|
||||
expect(optionNames).toEqual(
|
||||
expect.arrayContaining(["--credential-source", "--credential-role"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows an install hint when a discovered runner plugin is unavailable", async () => {
|
||||
listQaRunnerCliContributions.mockReset().mockReturnValue([createMissingQaRunnerContribution()]);
|
||||
const missingProgram = new Command();
|
||||
registerQaLabCli(missingProgram);
|
||||
|
||||
await expect(
|
||||
missingProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
|
||||
).rejects.toThrow(`openclaw plugins install ${TEST_QA_RUNNER.npmSpec}`);
|
||||
});
|
||||
|
||||
it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
|
||||
listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
|
||||
const blockedProgram = new Command();
|
||||
registerQaLabCli(blockedProgram);
|
||||
|
||||
await expect(
|
||||
blockedProgram.parseAsync(["node", "openclaw", "qa", TEST_QA_RUNNER.commandName]),
|
||||
).rejects.toThrow(`Enable or allow plugin "${TEST_QA_RUNNER.pluginId}"`);
|
||||
});
|
||||
|
||||
it("rejects discovered runners that collide with built-in qa subcommands", () => {
|
||||
listQaRunnerCliContributions
|
||||
.mockReset()
|
||||
.mockReturnValue([createConflictingQaRunnerContribution("manual")]);
|
||||
|
||||
expect(() => registerQaLabCli(new Command())).toThrow(
|
||||
'QA runner command "manual" conflicts with an existing qa subcommand',
|
||||
);
|
||||
});
|
||||
|
||||
it("routes telegram CLI defaults into the lane runtime", async () => {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Command } from "commander";
|
||||
import { collectString } from "./cli-options.js";
|
||||
import { LIVE_TRANSPORT_QA_CLI_REGISTRATIONS } from "./live-transports/cli.js";
|
||||
import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
import { hasQaScenarioPack } from "./scenario-catalog.js";
|
||||
|
||||
@@ -183,6 +183,12 @@ export function isQaLabCliAvailable(): boolean {
|
||||
return hasQaScenarioPack();
|
||||
}
|
||||
|
||||
function assertNoQaSubcommandCollision(qa: Command, commandName: string) {
|
||||
if (qa.commands.some((command) => command.name() === commandName)) {
|
||||
throw new Error(`QA runner command "${commandName}" conflicts with an existing qa subcommand`);
|
||||
}
|
||||
}
|
||||
|
||||
export function registerQaLabCli(program: Command) {
|
||||
const qa = program
|
||||
.command("qa")
|
||||
@@ -284,10 +290,6 @@ export function registerQaLabCli(program: Command) {
|
||||
},
|
||||
);
|
||||
|
||||
for (const lane of LIVE_TRANSPORT_QA_CLI_REGISTRATIONS) {
|
||||
lane.register(qa);
|
||||
}
|
||||
|
||||
qa.command("character-eval")
|
||||
.description("Run the character QA scenario across live models and write a judged report")
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
@@ -579,4 +581,9 @@ export function registerQaLabCli(program: Command) {
|
||||
.action(async (opts: { host?: string; port?: number }) => {
|
||||
await runQaMockOpenAi(opts);
|
||||
});
|
||||
|
||||
for (const lane of listLiveTransportQaCliRegistrations()) {
|
||||
assertNoQaSubcommandCollision(qa, lane.commandName);
|
||||
lane.register(qa);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,78 @@
|
||||
import { matrixQaCliRegistration } from "./matrix/cli.js";
|
||||
import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
|
||||
import { telegramQaCliRegistration } from "./telegram/cli.js";
|
||||
|
||||
function createMissingQaRunnerCliRegistration(params: {
|
||||
commandName: string;
|
||||
description: string;
|
||||
npmSpec: string;
|
||||
}): LiveTransportQaCliRegistration {
|
||||
return {
|
||||
commandName: params.commandName,
|
||||
register(qa) {
|
||||
qa.command(params.commandName)
|
||||
.description(params.description)
|
||||
.action(() => {
|
||||
throw new Error(
|
||||
`QA runner "${params.commandName}" not installed. Install it with "openclaw plugins install ${params.npmSpec}".`,
|
||||
);
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createBlockedQaRunnerCliRegistration(params: {
|
||||
commandName: string;
|
||||
description?: string;
|
||||
pluginId: string;
|
||||
}): LiveTransportQaCliRegistration {
|
||||
return {
|
||||
commandName: params.commandName,
|
||||
register(qa) {
|
||||
qa.command(params.commandName)
|
||||
.description(params.description ?? `Run the ${params.commandName} live QA lane`)
|
||||
.action(() => {
|
||||
throw new Error(
|
||||
`QA runner "${params.commandName}" is installed but not active. Enable or allow plugin "${params.pluginId}" in your OpenClaw config, then try again.`,
|
||||
);
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createQaRunnerCliRegistration(
|
||||
runner: ReturnType<typeof listQaRunnerCliContributions>[number],
|
||||
): LiveTransportQaCliRegistration {
|
||||
if (runner.status === "available") {
|
||||
return runner.registration;
|
||||
}
|
||||
if (runner.status === "blocked") {
|
||||
return createBlockedQaRunnerCliRegistration({
|
||||
commandName: runner.commandName,
|
||||
description: runner.description,
|
||||
pluginId: runner.pluginId,
|
||||
});
|
||||
}
|
||||
return createMissingQaRunnerCliRegistration({
|
||||
commandName: runner.commandName,
|
||||
description:
|
||||
runner.description ??
|
||||
`Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`,
|
||||
npmSpec: runner.npmSpec,
|
||||
});
|
||||
}
|
||||
|
||||
export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
|
||||
telegramQaCliRegistration,
|
||||
matrixQaCliRegistration,
|
||||
];
|
||||
|
||||
export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
|
||||
const liveRegistrations = [...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS];
|
||||
const discoveredRunners = listQaRunnerCliContributions();
|
||||
|
||||
for (const runner of discoveredRunners) {
|
||||
liveRegistrations.push(createQaRunnerCliRegistration(runner));
|
||||
}
|
||||
|
||||
return liveRegistrations;
|
||||
}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runMatrixQaLive = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./matrix-live.runtime.js", () => ({
|
||||
runMatrixQaLive,
|
||||
}));
|
||||
|
||||
import { runQaMatrixCommand } from "./cli.runtime.js";
|
||||
|
||||
describe("matrix qa cli runtime", () => {
|
||||
it("rejects non-env credential sources for the disposable Matrix lane", async () => {
|
||||
await expect(
|
||||
runQaMatrixCommand({
|
||||
credentialSource: "convex",
|
||||
}),
|
||||
).rejects.toThrow("Matrix QA currently supports only --credential-source env");
|
||||
});
|
||||
|
||||
it("passes through default env credential source options", async () => {
|
||||
runMatrixQaLive.mockResolvedValue({
|
||||
reportPath: "/tmp/matrix-report.md",
|
||||
summaryPath: "/tmp/matrix-summary.json",
|
||||
observedEventsPath: "/tmp/matrix-events.json",
|
||||
});
|
||||
|
||||
await runQaMatrixCommand({
|
||||
repoRoot: "/tmp/openclaw",
|
||||
outputDir: ".artifacts/qa-e2e/matrix",
|
||||
providerMode: "mock-openai",
|
||||
credentialSource: "env",
|
||||
});
|
||||
|
||||
expect(runMatrixQaLive).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: "/tmp/openclaw",
|
||||
outputDir: "/tmp/openclaw/.artifacts/qa-e2e/matrix",
|
||||
providerMode: "mock-openai",
|
||||
credentialSource: "env",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,23 +0,0 @@
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import {
|
||||
printLiveTransportQaArtifacts,
|
||||
resolveLiveTransportQaRunOptions,
|
||||
} from "../shared/live-transport-cli.runtime.js";
|
||||
import { runMatrixQaLive } from "./matrix-live.runtime.js";
|
||||
|
||||
export async function runQaMatrixCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
const credentialSource = runOptions.credentialSource?.toLowerCase();
|
||||
if (credentialSource && credentialSource !== "env") {
|
||||
throw new Error(
|
||||
"Matrix QA currently supports only --credential-source env (disposable local harness).",
|
||||
);
|
||||
}
|
||||
|
||||
const result = await runMatrixQaLive(runOptions);
|
||||
printLiveTransportQaArtifacts("Matrix QA", {
|
||||
report: result.reportPath,
|
||||
summary: result.summaryPath,
|
||||
"observed events": result.observedEventsPath,
|
||||
});
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import type { Command } from "commander";
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
createLiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCommandOptions,
|
||||
} from "../shared/live-transport-cli.js";
|
||||
|
||||
type MatrixQaCliRuntime = typeof import("./cli.runtime.js");
|
||||
|
||||
const loadMatrixQaCliRuntime = createLazyCliRuntimeLoader<MatrixQaCliRuntime>(
|
||||
() => import("./cli.runtime.js"),
|
||||
);
|
||||
|
||||
async function runQaMatrix(opts: LiveTransportQaCommandOptions) {
|
||||
const runtime = await loadMatrixQaCliRuntime();
|
||||
await runtime.runQaMatrixCommand(opts);
|
||||
}
|
||||
|
||||
export const matrixQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
createLiveTransportQaCliRegistration({
|
||||
commandName: "matrix",
|
||||
description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver",
|
||||
outputDirHelp: "Matrix QA artifact directory",
|
||||
scenarioHelp: "Run only the named Matrix QA scenario (repeatable)",
|
||||
sutAccountHelp: "Temporary Matrix account id inside the QA gateway config",
|
||||
run: runQaMatrix,
|
||||
});
|
||||
|
||||
export function registerMatrixQaCli(qa: Command) {
|
||||
matrixQaCliRegistration.register(qa);
|
||||
}
|
||||
@@ -1,416 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
__testing,
|
||||
createMatrixQaClient,
|
||||
provisionMatrixQaRoom,
|
||||
type MatrixQaObservedEvent,
|
||||
} from "./matrix-driver-client.js";
|
||||
|
||||
function resolveRequestUrl(input: RequestInfo | URL) {
|
||||
if (typeof input === "string") {
|
||||
return input;
|
||||
}
|
||||
if (input instanceof URL) {
|
||||
return input.toString();
|
||||
}
|
||||
return input.url;
|
||||
}
|
||||
|
||||
function parseJsonRequestBody(init?: RequestInit) {
|
||||
if (typeof init?.body !== "string") {
|
||||
return {};
|
||||
}
|
||||
return JSON.parse(init.body) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("matrix driver client", () => {
|
||||
it("builds Matrix HTML mentions for QA driver messages", () => {
|
||||
expect(
|
||||
__testing.buildMatrixQaMessageContent({
|
||||
body: "@sut:matrix-qa.test reply with exactly: TOKEN",
|
||||
mentionUserIds: ["@sut:matrix-qa.test"],
|
||||
}),
|
||||
).toEqual({
|
||||
body: "@sut:matrix-qa.test reply with exactly: TOKEN",
|
||||
msgtype: "m.text",
|
||||
format: "org.matrix.custom.html",
|
||||
formatted_body:
|
||||
'<a href="https://matrix.to/#/%40sut%3Amatrix-qa.test">@sut:matrix-qa.test</a> reply with exactly: TOKEN',
|
||||
"m.mentions": {
|
||||
user_ids: ["@sut:matrix-qa.test"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("omits Matrix HTML markup when the body has no visible mention token", () => {
|
||||
expect(
|
||||
__testing.buildMatrixQaMessageContent({
|
||||
body: "reply with exactly: TOKEN",
|
||||
mentionUserIds: ["@sut:matrix-qa.test"],
|
||||
}),
|
||||
).toEqual({
|
||||
body: "reply with exactly: TOKEN",
|
||||
msgtype: "m.text",
|
||||
"m.mentions": {
|
||||
user_ids: ["@sut:matrix-qa.test"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes message events with thread metadata", () => {
|
||||
expect(
|
||||
__testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
|
||||
event_id: "$event",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
origin_server_ts: 1_700_000_000_000,
|
||||
content: {
|
||||
body: "hello",
|
||||
msgtype: "m.text",
|
||||
"m.mentions": {
|
||||
user_ids: ["@sut:matrix-qa.test"],
|
||||
},
|
||||
"m.relates_to": {
|
||||
rel_type: "m.thread",
|
||||
event_id: "$root",
|
||||
is_falling_back: true,
|
||||
"m.in_reply_to": {
|
||||
event_id: "$driver",
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$event",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
originServerTs: 1_700_000_000_000,
|
||||
body: "hello",
|
||||
msgtype: "m.text",
|
||||
relatesTo: {
|
||||
relType: "m.thread",
|
||||
eventId: "$root",
|
||||
inReplyToId: "$driver",
|
||||
isFallingBack: true,
|
||||
},
|
||||
mentions: {
|
||||
userIds: ["@sut:matrix-qa.test"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("builds trimmed Matrix reaction relations for QA driver events", () => {
|
||||
expect(__testing.buildMatrixReactionRelation(" $msg-1 ", " 👍 ")).toEqual({
|
||||
"m.relates_to": {
|
||||
rel_type: "m.annotation",
|
||||
event_id: "$msg-1",
|
||||
key: "👍",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes Matrix reaction events with target metadata", () => {
|
||||
expect(
|
||||
__testing.normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
|
||||
event_id: "$reaction",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.reaction",
|
||||
origin_server_ts: 1_700_000_000_000,
|
||||
content: {
|
||||
"m.relates_to": {
|
||||
rel_type: "m.annotation",
|
||||
event_id: "$msg",
|
||||
key: "👍",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$reaction",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.reaction",
|
||||
originServerTs: 1_700_000_000_000,
|
||||
relatesTo: {
|
||||
eventId: "$msg",
|
||||
relType: "m.annotation",
|
||||
},
|
||||
reaction: {
|
||||
eventId: "$msg",
|
||||
key: "👍",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("advances Matrix registration through token then dummy auth stages", () => {
|
||||
const firstStage = __testing.resolveNextRegistrationAuth({
|
||||
registrationToken: "reg-token",
|
||||
response: {
|
||||
session: "uiaa-session",
|
||||
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(firstStage).toEqual({
|
||||
session: "uiaa-session",
|
||||
type: "m.login.registration_token",
|
||||
token: "reg-token",
|
||||
});
|
||||
|
||||
expect(
|
||||
__testing.resolveNextRegistrationAuth({
|
||||
registrationToken: "reg-token",
|
||||
response: {
|
||||
session: "uiaa-session",
|
||||
completed: ["m.login.registration_token"],
|
||||
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
session: "uiaa-session",
|
||||
type: "m.login.dummy",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects Matrix UIAA flows that require unsupported stages", () => {
|
||||
expect(() =>
|
||||
__testing.resolveNextRegistrationAuth({
|
||||
registrationToken: "reg-token",
|
||||
response: {
|
||||
session: "uiaa-session",
|
||||
flows: [{ stages: ["m.login.registration_token", "m.login.recaptcha", "m.login.dummy"] }],
|
||||
},
|
||||
}),
|
||||
).toThrow("Matrix registration requires unsupported auth stages:");
|
||||
});
|
||||
|
||||
it("returns a typed no-match result while preserving the latest sync token", async () => {
|
||||
const fetchImpl: typeof fetch = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
next_batch: "next-batch-2",
|
||||
rooms: {
|
||||
join: {
|
||||
"!room:matrix-qa.test": {
|
||||
timeline: {
|
||||
events: [
|
||||
{
|
||||
event_id: "$driver",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
content: { body: "hello", msgtype: "m.text" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
|
||||
const client = createMatrixQaClient({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
fetchImpl,
|
||||
});
|
||||
const observedEvents: MatrixQaObservedEvent[] = [];
|
||||
|
||||
const result = await client.waitForOptionalRoomEvent({
|
||||
observedEvents,
|
||||
predicate: (event) => event.sender === "@sut:matrix-qa.test",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
since: "start-batch",
|
||||
timeoutMs: 1,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
matched: false,
|
||||
since: "next-batch-2",
|
||||
});
|
||||
expect(observedEvents).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
body: "hello",
|
||||
eventId: "$driver",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps recording later same-batch events after the first match", async () => {
|
||||
const fetchImpl: typeof fetch = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
next_batch: "next-batch-2",
|
||||
rooms: {
|
||||
join: {
|
||||
"!room:matrix-qa.test": {
|
||||
timeline: {
|
||||
events: [
|
||||
{
|
||||
event_id: "$sut",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
content: { body: "target", msgtype: "m.text" },
|
||||
},
|
||||
{
|
||||
event_id: "$driver",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
content: { body: "trailing event", msgtype: "m.text" },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
|
||||
const client = createMatrixQaClient({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
fetchImpl,
|
||||
});
|
||||
const observedEvents: MatrixQaObservedEvent[] = [];
|
||||
|
||||
const result = await client.waitForOptionalRoomEvent({
|
||||
observedEvents,
|
||||
predicate: (event) => event.eventId === "$sut",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
since: "start-batch",
|
||||
timeoutMs: 1,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
event: expect.objectContaining({
|
||||
eventId: "$sut",
|
||||
}),
|
||||
matched: true,
|
||||
since: "next-batch-2",
|
||||
});
|
||||
expect(observedEvents).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
body: "target",
|
||||
eventId: "$sut",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
body: "trailing event",
|
||||
eventId: "$driver",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("sends Matrix reactions through the protocol send endpoint", async () => {
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
expect(resolveRequestUrl(input)).toContain(
|
||||
"/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.reaction/",
|
||||
);
|
||||
expect(parseJsonRequestBody(init)).toEqual({
|
||||
"m.relates_to": {
|
||||
rel_type: "m.annotation",
|
||||
event_id: "$msg-1",
|
||||
key: "👍",
|
||||
},
|
||||
});
|
||||
return new Response(JSON.stringify({ event_id: "$reaction-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
const client = createMatrixQaClient({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
await expect(
|
||||
client.sendReaction({
|
||||
emoji: "👍",
|
||||
messageId: "$msg-1",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
}),
|
||||
).resolves.toBe("$reaction-1");
|
||||
});
|
||||
|
||||
it("provisions a three-member room so Matrix QA runs in a group context", async () => {
|
||||
const createRoomBodies: Array<Record<string, unknown>> = [];
|
||||
const fetchImpl: typeof fetch = async (input, init) => {
|
||||
const url = resolveRequestUrl(input);
|
||||
const body = parseJsonRequestBody(init);
|
||||
if (url.endsWith("/_matrix/client/v3/register")) {
|
||||
const username = typeof body.username === "string" ? body.username : "";
|
||||
const auth = typeof body.auth === "object" && body.auth ? body.auth : undefined;
|
||||
if (!auth) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
session: `session-${username}`,
|
||||
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
|
||||
}),
|
||||
{ status: 401, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
}
|
||||
if ((auth as { type?: string }).type === "m.login.registration_token") {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
session: `session-${username}`,
|
||||
completed: ["m.login.registration_token"],
|
||||
flows: [{ stages: ["m.login.registration_token", "m.login.dummy"] }],
|
||||
}),
|
||||
{ status: 401, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
}
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
access_token: `token-${username}`,
|
||||
device_id: `device-${username}`,
|
||||
user_id: `@${username}:matrix-qa.test`,
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
}
|
||||
if (url.endsWith("/_matrix/client/v3/createRoom")) {
|
||||
createRoomBodies.push(body);
|
||||
return new Response(JSON.stringify({ room_id: "!room:matrix-qa.test" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
if (url.includes("/_matrix/client/v3/join/")) {
|
||||
return new Response(JSON.stringify({ room_id: "!room:matrix-qa.test" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected fetch ${url}`);
|
||||
};
|
||||
|
||||
const result = await provisionMatrixQaRoom({
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
driverLocalpart: "qa-driver",
|
||||
observerLocalpart: "qa-observer",
|
||||
registrationToken: "reg-token",
|
||||
roomName: "OpenClaw Matrix QA",
|
||||
sutLocalpart: "qa-sut",
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
expect(result.roomId).toBe("!room:matrix-qa.test");
|
||||
expect(result.observer.userId).toBe("@qa-observer:matrix-qa.test");
|
||||
expect(createRoomBodies).toEqual([
|
||||
expect.objectContaining({
|
||||
invite: ["@qa-sut:matrix-qa.test", "@qa-observer:matrix-qa.test"],
|
||||
is_direct: false,
|
||||
preset: "private_chat",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,728 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
type FetchLike = typeof fetch;
|
||||
|
||||
type MatrixQaAuthStage = "m.login.dummy" | "m.login.registration_token";
|
||||
|
||||
type MatrixQaRequestResult<T> = {
|
||||
status: number;
|
||||
body: T;
|
||||
};
|
||||
|
||||
type MatrixQaRegisterResponse = {
|
||||
access_token?: string;
|
||||
device_id?: string;
|
||||
user_id?: string;
|
||||
};
|
||||
|
||||
type MatrixQaRoomCreateResponse = {
|
||||
room_id?: string;
|
||||
};
|
||||
|
||||
type MatrixQaSendMessageContent = {
|
||||
body: string;
|
||||
format?: "org.matrix.custom.html";
|
||||
formatted_body?: string;
|
||||
"m.mentions"?: {
|
||||
user_ids?: string[];
|
||||
};
|
||||
"m.relates_to"?: {
|
||||
rel_type: "m.thread";
|
||||
event_id: string;
|
||||
is_falling_back: true;
|
||||
"m.in_reply_to": {
|
||||
event_id: string;
|
||||
};
|
||||
};
|
||||
msgtype: "m.text";
|
||||
};
|
||||
|
||||
type MatrixQaSendReactionContent = {
|
||||
"m.relates_to": {
|
||||
event_id: string;
|
||||
key: string;
|
||||
rel_type: "m.annotation";
|
||||
};
|
||||
};
|
||||
|
||||
type MatrixQaSyncResponse = {
|
||||
next_batch?: string;
|
||||
rooms?: {
|
||||
join?: Record<
|
||||
string,
|
||||
{
|
||||
timeline?: {
|
||||
events?: MatrixQaRoomEvent[];
|
||||
};
|
||||
}
|
||||
>;
|
||||
};
|
||||
};
|
||||
|
||||
type MatrixQaUiaaResponse = {
|
||||
completed?: string[];
|
||||
flows?: Array<{ stages?: string[] }>;
|
||||
session?: string;
|
||||
};
|
||||
|
||||
type MatrixQaRoomEvent = {
|
||||
content?: Record<string, unknown>;
|
||||
event_id?: string;
|
||||
origin_server_ts?: number;
|
||||
sender?: string;
|
||||
state_key?: string;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
export type MatrixQaObservedEvent = {
|
||||
roomId: string;
|
||||
eventId: string;
|
||||
sender?: string;
|
||||
stateKey?: string;
|
||||
type: string;
|
||||
originServerTs?: number;
|
||||
body?: string;
|
||||
formattedBody?: string;
|
||||
msgtype?: string;
|
||||
membership?: string;
|
||||
relatesTo?: {
|
||||
eventId?: string;
|
||||
inReplyToId?: string;
|
||||
isFallingBack?: boolean;
|
||||
relType?: string;
|
||||
};
|
||||
mentions?: {
|
||||
room?: boolean;
|
||||
userIds?: string[];
|
||||
};
|
||||
reaction?: {
|
||||
eventId?: string;
|
||||
key?: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type MatrixQaRegisteredAccount = {
|
||||
accessToken: string;
|
||||
deviceId?: string;
|
||||
localpart: string;
|
||||
password: string;
|
||||
userId: string;
|
||||
};
|
||||
|
||||
export type MatrixQaProvisionResult = {
|
||||
driver: MatrixQaRegisteredAccount;
|
||||
observer: MatrixQaRegisteredAccount;
|
||||
roomId: string;
|
||||
sut: MatrixQaRegisteredAccount;
|
||||
};
|
||||
|
||||
export type MatrixQaRoomEventWaitResult =
|
||||
| {
|
||||
event: MatrixQaObservedEvent;
|
||||
matched: true;
|
||||
since?: string;
|
||||
}
|
||||
| {
|
||||
matched: false;
|
||||
since?: string;
|
||||
};
|
||||
|
||||
function buildMatrixThreadRelation(threadRootEventId: string, replyToEventId?: string) {
|
||||
return {
|
||||
"m.relates_to": {
|
||||
rel_type: "m.thread" as const,
|
||||
event_id: threadRootEventId,
|
||||
is_falling_back: true as const,
|
||||
"m.in_reply_to": {
|
||||
event_id: replyToEventId?.trim() || threadRootEventId,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildMatrixReactionRelation(
|
||||
messageId: string,
|
||||
emoji: string,
|
||||
): MatrixQaSendReactionContent {
|
||||
const normalizedMessageId = messageId.trim();
|
||||
const normalizedEmoji = emoji.trim();
|
||||
if (!normalizedMessageId) {
|
||||
throw new Error("Matrix reaction requires a messageId");
|
||||
}
|
||||
if (!normalizedEmoji) {
|
||||
throw new Error("Matrix reaction requires an emoji");
|
||||
}
|
||||
return {
|
||||
"m.relates_to": {
|
||||
rel_type: "m.annotation",
|
||||
event_id: normalizedMessageId,
|
||||
key: normalizedEmoji,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function escapeMatrixHtml(value: string): string {
|
||||
return value.replace(/[&<>"']/g, (char) => {
|
||||
switch (char) {
|
||||
case "&":
|
||||
return "&";
|
||||
case "<":
|
||||
return "<";
|
||||
case ">":
|
||||
return ">";
|
||||
case '"':
|
||||
return """;
|
||||
case "'":
|
||||
return "'";
|
||||
default:
|
||||
return char;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function buildMatrixMentionLink(userId: string) {
|
||||
const href = `https://matrix.to/#/${encodeURIComponent(userId)}`;
|
||||
const label = escapeMatrixHtml(userId);
|
||||
return `<a href="${href}">${label}</a>`;
|
||||
}
|
||||
|
||||
function buildMatrixQaMessageContent(params: {
|
||||
body: string;
|
||||
mentionUserIds?: string[];
|
||||
replyToEventId?: string;
|
||||
threadRootEventId?: string;
|
||||
}): MatrixQaSendMessageContent {
|
||||
const body = params.body;
|
||||
const uniqueMentionUserIds = [...new Set(params.mentionUserIds?.filter(Boolean) ?? [])];
|
||||
const formattedParts: string[] = [];
|
||||
let cursor = 0;
|
||||
let usedFormattedMention = false;
|
||||
|
||||
while (cursor < body.length) {
|
||||
let matchedUserId: string | null = null;
|
||||
for (const userId of uniqueMentionUserIds) {
|
||||
if (body.startsWith(userId, cursor)) {
|
||||
matchedUserId = userId;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matchedUserId) {
|
||||
formattedParts.push(buildMatrixMentionLink(matchedUserId));
|
||||
cursor += matchedUserId.length;
|
||||
usedFormattedMention = true;
|
||||
continue;
|
||||
}
|
||||
formattedParts.push(escapeMatrixHtml(body[cursor] ?? ""));
|
||||
cursor += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
body,
|
||||
msgtype: "m.text",
|
||||
...(usedFormattedMention
|
||||
? {
|
||||
format: "org.matrix.custom.html" as const,
|
||||
formatted_body: formattedParts.join(""),
|
||||
}
|
||||
: {}),
|
||||
...(uniqueMentionUserIds.length > 0
|
||||
? { "m.mentions": { user_ids: uniqueMentionUserIds } }
|
||||
: {}),
|
||||
...(params.threadRootEventId
|
||||
? buildMatrixThreadRelation(params.threadRootEventId, params.replyToEventId)
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMentionUserIds(value: unknown) {
|
||||
return Array.isArray(value)
|
||||
? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function normalizeMatrixQaObservedEvent(
|
||||
roomId: string,
|
||||
event: MatrixQaRoomEvent,
|
||||
): MatrixQaObservedEvent | null {
|
||||
const eventId = event.event_id?.trim();
|
||||
const type = event.type?.trim();
|
||||
if (!eventId || !type) {
|
||||
return null;
|
||||
}
|
||||
const content = event.content ?? {};
|
||||
const relatesToRaw = content["m.relates_to"];
|
||||
const relatesTo =
|
||||
typeof relatesToRaw === "object" && relatesToRaw !== null
|
||||
? (relatesToRaw as Record<string, unknown>)
|
||||
: null;
|
||||
const inReplyToRaw = relatesTo?.["m.in_reply_to"];
|
||||
const inReplyTo =
|
||||
typeof inReplyToRaw === "object" && inReplyToRaw !== null
|
||||
? (inReplyToRaw as Record<string, unknown>)
|
||||
: null;
|
||||
const mentionsRaw = content["m.mentions"];
|
||||
const mentions =
|
||||
typeof mentionsRaw === "object" && mentionsRaw !== null
|
||||
? (mentionsRaw as Record<string, unknown>)
|
||||
: null;
|
||||
const mentionUserIds = normalizeMentionUserIds(mentions?.user_ids);
|
||||
const reactionKey =
|
||||
type === "m.reaction" && typeof relatesTo?.key === "string" ? relatesTo.key : undefined;
|
||||
const reactionEventId =
|
||||
type === "m.reaction" && typeof relatesTo?.event_id === "string"
|
||||
? relatesTo.event_id
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
roomId,
|
||||
eventId,
|
||||
sender: typeof event.sender === "string" ? event.sender : undefined,
|
||||
stateKey: typeof event.state_key === "string" ? event.state_key : undefined,
|
||||
type,
|
||||
originServerTs:
|
||||
typeof event.origin_server_ts === "number" ? Math.floor(event.origin_server_ts) : undefined,
|
||||
body: typeof content.body === "string" ? content.body : undefined,
|
||||
formattedBody: typeof content.formatted_body === "string" ? content.formatted_body : undefined,
|
||||
msgtype: typeof content.msgtype === "string" ? content.msgtype : undefined,
|
||||
membership: typeof content.membership === "string" ? content.membership : undefined,
|
||||
...(relatesTo
|
||||
? {
|
||||
relatesTo: {
|
||||
eventId: typeof relatesTo.event_id === "string" ? relatesTo.event_id : undefined,
|
||||
inReplyToId: typeof inReplyTo?.event_id === "string" ? inReplyTo.event_id : undefined,
|
||||
isFallingBack:
|
||||
typeof relatesTo.is_falling_back === "boolean"
|
||||
? relatesTo.is_falling_back
|
||||
: undefined,
|
||||
relType: typeof relatesTo.rel_type === "string" ? relatesTo.rel_type : undefined,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(mentions
|
||||
? {
|
||||
mentions: {
|
||||
...(mentions.room === true ? { room: true } : {}),
|
||||
...(mentionUserIds ? { userIds: mentionUserIds } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(reactionEventId || reactionKey
|
||||
? {
|
||||
reaction: {
|
||||
...(reactionEventId ? { eventId: reactionEventId } : {}),
|
||||
...(reactionKey ? { key: reactionKey } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveNextRegistrationAuth(params: {
|
||||
registrationToken: string;
|
||||
response: MatrixQaUiaaResponse;
|
||||
}) {
|
||||
const session = params.response.session?.trim();
|
||||
if (!session) {
|
||||
throw new Error("Matrix registration UIAA response did not include a session id.");
|
||||
}
|
||||
|
||||
const completed = new Set(
|
||||
(params.response.completed ?? []).filter(
|
||||
(stage): stage is MatrixQaAuthStage =>
|
||||
stage === "m.login.dummy" || stage === "m.login.registration_token",
|
||||
),
|
||||
);
|
||||
const supportedStages = new Set<MatrixQaAuthStage>([
|
||||
"m.login.registration_token",
|
||||
"m.login.dummy",
|
||||
]);
|
||||
|
||||
for (const flow of params.response.flows ?? []) {
|
||||
const flowStages = flow.stages ?? [];
|
||||
if (
|
||||
flowStages.length === 0 ||
|
||||
flowStages.some((stage) => !supportedStages.has(stage as MatrixQaAuthStage))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const stages = flowStages as MatrixQaAuthStage[];
|
||||
const nextStage = stages.find((stage) => !completed.has(stage));
|
||||
if (!nextStage) {
|
||||
continue;
|
||||
}
|
||||
if (nextStage === "m.login.registration_token") {
|
||||
return {
|
||||
session,
|
||||
type: nextStage,
|
||||
token: params.registrationToken,
|
||||
};
|
||||
}
|
||||
return {
|
||||
session,
|
||||
type: nextStage,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Matrix registration requires unsupported auth stages: ${JSON.stringify(params.response.flows ?? [])}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function requestMatrixJson<T>(params: {
|
||||
accessToken?: string;
|
||||
baseUrl: string;
|
||||
body?: unknown;
|
||||
endpoint: string;
|
||||
fetchImpl: FetchLike;
|
||||
method: "GET" | "POST" | "PUT";
|
||||
okStatuses?: number[];
|
||||
query?: Record<string, string | number | undefined>;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const url = new URL(params.endpoint, params.baseUrl);
|
||||
for (const [key, value] of Object.entries(params.query ?? {})) {
|
||||
if (value !== undefined) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
}
|
||||
const response = await params.fetchImpl(url, {
|
||||
method: params.method,
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
...(params.body !== undefined ? { "content-type": "application/json" } : {}),
|
||||
...(params.accessToken ? { authorization: `Bearer ${params.accessToken}` } : {}),
|
||||
},
|
||||
...(params.body !== undefined ? { body: JSON.stringify(params.body) } : {}),
|
||||
signal: AbortSignal.timeout(params.timeoutMs ?? 20_000),
|
||||
});
|
||||
let body: unknown = {};
|
||||
try {
|
||||
body = (await response.json()) as unknown;
|
||||
} catch {
|
||||
body = {};
|
||||
}
|
||||
const okStatuses = params.okStatuses ?? [200];
|
||||
if (!okStatuses.includes(response.status)) {
|
||||
const details =
|
||||
typeof body === "object" &&
|
||||
body !== null &&
|
||||
typeof (body as { error?: unknown }).error === "string"
|
||||
? (body as { error: string }).error
|
||||
: `${params.method} ${params.endpoint} failed with status ${response.status}`;
|
||||
throw new Error(details);
|
||||
}
|
||||
return {
|
||||
status: response.status,
|
||||
body: body as T,
|
||||
} satisfies MatrixQaRequestResult<T>;
|
||||
}
|
||||
|
||||
function buildRegisteredAccount(params: {
|
||||
localpart: string;
|
||||
password: string;
|
||||
response: MatrixQaRegisterResponse;
|
||||
}) {
|
||||
const userId = params.response.user_id?.trim();
|
||||
const accessToken = params.response.access_token?.trim();
|
||||
if (!userId || !accessToken) {
|
||||
throw new Error("Matrix registration did not return both user_id and access_token.");
|
||||
}
|
||||
return {
|
||||
accessToken,
|
||||
deviceId: params.response.device_id?.trim() || undefined,
|
||||
localpart: params.localpart,
|
||||
password: params.password,
|
||||
userId,
|
||||
} satisfies MatrixQaRegisteredAccount;
|
||||
}
|
||||
|
||||
export function createMatrixQaClient(params: {
|
||||
accessToken?: string;
|
||||
baseUrl: string;
|
||||
fetchImpl?: FetchLike;
|
||||
}) {
|
||||
const fetchImpl = params.fetchImpl ?? fetch;
|
||||
|
||||
async function waitForOptionalRoomEvent(opts: {
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
predicate: (event: MatrixQaObservedEvent) => boolean;
|
||||
roomId: string;
|
||||
since?: string;
|
||||
timeoutMs: number;
|
||||
}): Promise<MatrixQaRoomEventWaitResult> {
|
||||
const startedAt = Date.now();
|
||||
let since = opts.since;
|
||||
while (Date.now() - startedAt < opts.timeoutMs) {
|
||||
const remainingMs = Math.max(1_000, opts.timeoutMs - (Date.now() - startedAt));
|
||||
const response = await requestMatrixJson<MatrixQaSyncResponse>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
query: {
|
||||
...(since ? { since } : {}),
|
||||
timeout: Math.min(10_000, remainingMs),
|
||||
},
|
||||
timeoutMs: Math.min(15_000, remainingMs + 5_000),
|
||||
});
|
||||
since = response.body.next_batch?.trim() || since;
|
||||
const roomEvents = response.body.rooms?.join?.[opts.roomId]?.timeline?.events ?? [];
|
||||
let matchedEvent: MatrixQaObservedEvent | null = null;
|
||||
for (const event of roomEvents) {
|
||||
const normalized = normalizeMatrixQaObservedEvent(opts.roomId, event);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
opts.observedEvents.push(normalized);
|
||||
if (matchedEvent === null && opts.predicate(normalized)) {
|
||||
matchedEvent = normalized;
|
||||
}
|
||||
}
|
||||
if (matchedEvent) {
|
||||
return { event: matchedEvent, matched: true, since };
|
||||
}
|
||||
}
|
||||
return { matched: false, since };
|
||||
}
|
||||
|
||||
return {
|
||||
async createPrivateRoom(opts: { inviteUserIds: string[]; name: string }) {
|
||||
const result = await requestMatrixJson<MatrixQaRoomCreateResponse>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
body: {
|
||||
creation_content: { "m.federate": false },
|
||||
initial_state: [
|
||||
{
|
||||
type: "m.room.history_visibility",
|
||||
state_key: "",
|
||||
content: { history_visibility: "joined" },
|
||||
},
|
||||
],
|
||||
invite: opts.inviteUserIds,
|
||||
is_direct: false,
|
||||
name: opts.name,
|
||||
preset: "private_chat",
|
||||
},
|
||||
endpoint: "/_matrix/client/v3/createRoom",
|
||||
fetchImpl,
|
||||
method: "POST",
|
||||
});
|
||||
const roomId = result.body.room_id?.trim();
|
||||
if (!roomId) {
|
||||
throw new Error("Matrix createRoom did not return room_id.");
|
||||
}
|
||||
return roomId;
|
||||
},
|
||||
async primeRoom() {
|
||||
const response = await requestMatrixJson<MatrixQaSyncResponse>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
query: { timeout: 0 },
|
||||
});
|
||||
return response.body.next_batch?.trim() || undefined;
|
||||
},
|
||||
async registerWithToken(opts: {
|
||||
deviceName: string;
|
||||
localpart: string;
|
||||
password: string;
|
||||
registrationToken: string;
|
||||
}) {
|
||||
let auth: Record<string, unknown> | undefined;
|
||||
const baseBody = {
|
||||
inhibit_login: false,
|
||||
initial_device_display_name: opts.deviceName,
|
||||
password: opts.password,
|
||||
username: opts.localpart,
|
||||
};
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
const response = await requestMatrixJson<MatrixQaRegisterResponse | MatrixQaUiaaResponse>({
|
||||
baseUrl: params.baseUrl,
|
||||
body: {
|
||||
...baseBody,
|
||||
...(auth ? { auth } : {}),
|
||||
},
|
||||
endpoint: "/_matrix/client/v3/register",
|
||||
fetchImpl,
|
||||
method: "POST",
|
||||
okStatuses: [200, 401],
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
if (response.status === 200) {
|
||||
return buildRegisteredAccount({
|
||||
localpart: opts.localpart,
|
||||
password: opts.password,
|
||||
response: response.body as MatrixQaRegisterResponse,
|
||||
});
|
||||
}
|
||||
auth = resolveNextRegistrationAuth({
|
||||
registrationToken: opts.registrationToken,
|
||||
response: response.body as MatrixQaUiaaResponse,
|
||||
});
|
||||
}
|
||||
throw new Error(
|
||||
`Matrix registration for ${opts.localpart} did not complete after 4 attempts.`,
|
||||
);
|
||||
},
|
||||
async sendTextMessage(opts: {
|
||||
body: string;
|
||||
mentionUserIds?: string[];
|
||||
replyToEventId?: string;
|
||||
roomId: string;
|
||||
threadRootEventId?: string;
|
||||
}) {
|
||||
const txnId = randomUUID();
|
||||
const result = await requestMatrixJson<{ event_id?: string }>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
body: buildMatrixQaMessageContent(opts),
|
||||
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`,
|
||||
fetchImpl,
|
||||
method: "PUT",
|
||||
});
|
||||
const eventId = result.body.event_id?.trim();
|
||||
if (!eventId) {
|
||||
throw new Error("Matrix sendMessage did not return event_id.");
|
||||
}
|
||||
return eventId;
|
||||
},
|
||||
async sendReaction(opts: { emoji: string; messageId: string; roomId: string }) {
|
||||
const txnId = randomUUID();
|
||||
const result = await requestMatrixJson<{ event_id?: string }>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
body: buildMatrixReactionRelation(opts.messageId, opts.emoji),
|
||||
endpoint: `/_matrix/client/v3/rooms/${encodeURIComponent(opts.roomId)}/send/m.reaction/${encodeURIComponent(txnId)}`,
|
||||
fetchImpl,
|
||||
method: "PUT",
|
||||
});
|
||||
const eventId = result.body.event_id?.trim();
|
||||
if (!eventId) {
|
||||
throw new Error("Matrix sendReaction did not return event_id.");
|
||||
}
|
||||
return eventId;
|
||||
},
|
||||
async joinRoom(roomId: string) {
|
||||
const result = await requestMatrixJson<{ room_id?: string }>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
body: {},
|
||||
endpoint: `/_matrix/client/v3/join/${encodeURIComponent(roomId)}`,
|
||||
fetchImpl,
|
||||
method: "POST",
|
||||
});
|
||||
return result.body.room_id?.trim() || roomId;
|
||||
},
|
||||
waitForOptionalRoomEvent,
|
||||
async waitForRoomEvent(opts: {
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
predicate: (event: MatrixQaObservedEvent) => boolean;
|
||||
roomId: string;
|
||||
since?: string;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const result = await waitForOptionalRoomEvent(opts);
|
||||
if (result.matched) {
|
||||
return { event: result.event, since: result.since };
|
||||
}
|
||||
throw new Error(`timed out after ${opts.timeoutMs}ms waiting for Matrix room event`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function joinRoomWithRetry(params: {
|
||||
accessToken: string;
|
||||
baseUrl: string;
|
||||
fetchImpl?: FetchLike;
|
||||
roomId: string;
|
||||
}) {
|
||||
const client = createMatrixQaClient({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
fetchImpl: params.fetchImpl,
|
||||
});
|
||||
let lastError: unknown = null;
|
||||
for (let attempt = 1; attempt <= 10; attempt += 1) {
|
||||
try {
|
||||
await client.joinRoom(params.roomId);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 300 * attempt));
|
||||
}
|
||||
}
|
||||
throw new Error(`Matrix join retry failed: ${formatErrorMessage(lastError)}`);
|
||||
}
|
||||
|
||||
export async function provisionMatrixQaRoom(params: {
|
||||
baseUrl: string;
|
||||
fetchImpl?: FetchLike;
|
||||
roomName: string;
|
||||
driverLocalpart: string;
|
||||
observerLocalpart: string;
|
||||
registrationToken: string;
|
||||
sutLocalpart: string;
|
||||
}) {
|
||||
const anonClient = createMatrixQaClient({
|
||||
baseUrl: params.baseUrl,
|
||||
fetchImpl: params.fetchImpl,
|
||||
});
|
||||
const driver = await anonClient.registerWithToken({
|
||||
deviceName: "OpenClaw Matrix QA Driver",
|
||||
localpart: params.driverLocalpart,
|
||||
password: `driver-${randomUUID()}`,
|
||||
registrationToken: params.registrationToken,
|
||||
});
|
||||
const sut = await anonClient.registerWithToken({
|
||||
deviceName: "OpenClaw Matrix QA SUT",
|
||||
localpart: params.sutLocalpart,
|
||||
password: `sut-${randomUUID()}`,
|
||||
registrationToken: params.registrationToken,
|
||||
});
|
||||
const observer = await anonClient.registerWithToken({
|
||||
deviceName: "OpenClaw Matrix QA Observer",
|
||||
localpart: params.observerLocalpart,
|
||||
password: `observer-${randomUUID()}`,
|
||||
registrationToken: params.registrationToken,
|
||||
});
|
||||
const driverClient = createMatrixQaClient({
|
||||
accessToken: driver.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
fetchImpl: params.fetchImpl,
|
||||
});
|
||||
const roomId = await driverClient.createPrivateRoom({
|
||||
inviteUserIds: [sut.userId, observer.userId],
|
||||
name: params.roomName,
|
||||
});
|
||||
await joinRoomWithRetry({
|
||||
accessToken: sut.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
fetchImpl: params.fetchImpl,
|
||||
roomId,
|
||||
});
|
||||
await joinRoomWithRetry({
|
||||
accessToken: observer.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
fetchImpl: params.fetchImpl,
|
||||
roomId,
|
||||
});
|
||||
return {
|
||||
driver,
|
||||
observer,
|
||||
roomId,
|
||||
sut,
|
||||
} satisfies MatrixQaProvisionResult;
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
buildMatrixQaMessageContent,
|
||||
buildMatrixReactionRelation,
|
||||
buildMatrixThreadRelation,
|
||||
normalizeMatrixQaObservedEvent,
|
||||
resolveNextRegistrationAuth,
|
||||
};
|
||||
@@ -1,271 +0,0 @@
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
__testing,
|
||||
startMatrixQaHarness,
|
||||
writeMatrixQaHarnessFiles,
|
||||
} from "./matrix-harness.runtime.js";
|
||||
|
||||
describe("matrix harness runtime", () => {
|
||||
it("writes a pinned Tuwunel compose file and redacted manifest", async () => {
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await writeMatrixQaHarnessFiles({
|
||||
outputDir,
|
||||
homeserverPort: 28008,
|
||||
registrationToken: "secret-token",
|
||||
serverName: "matrix-qa.test",
|
||||
});
|
||||
|
||||
const compose = await readFile(result.composeFile, "utf8");
|
||||
const manifest = JSON.parse(await readFile(result.manifestPath, "utf8")) as {
|
||||
image: string;
|
||||
serverName: string;
|
||||
homeserverPort: number;
|
||||
composeFile: string;
|
||||
};
|
||||
|
||||
expect(compose).toContain(`image: ${__testing.MATRIX_QA_DEFAULT_IMAGE}`);
|
||||
expect(compose).toContain(' - "127.0.0.1:28008:8008"');
|
||||
expect(compose).toContain('TUWUNEL_ALLOW_REGISTRATION: "true"');
|
||||
expect(compose).toContain('TUWUNEL_REGISTRATION_TOKEN: "secret-token"');
|
||||
expect(compose).toContain('TUWUNEL_SERVER_NAME: "matrix-qa.test"');
|
||||
expect(manifest).toEqual({
|
||||
image: __testing.MATRIX_QA_DEFAULT_IMAGE,
|
||||
serverName: "matrix-qa.test",
|
||||
homeserverPort: 28008,
|
||||
composeFile: path.join(outputDir, "docker-compose.matrix-qa.yml"),
|
||||
dataDir: path.join(outputDir, "data"),
|
||||
});
|
||||
expect(result.registrationToken).toBe("secret-token");
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("starts the harness, waits for versions, and exposes a stop command", async () => {
|
||||
const calls: string[] = [];
|
||||
const fetchCalls: string[] = [];
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
{
|
||||
async runCommand(command, args, cwd) {
|
||||
calls.push([command, ...args, `@${cwd}`].join(" "));
|
||||
if (args.join(" ").includes("ps --format json")) {
|
||||
return { stdout: '[{"State":"running"}]\n', stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async (input: string) => {
|
||||
fetchCalls.push(input);
|
||||
return { ok: true };
|
||||
}),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
},
|
||||
);
|
||||
|
||||
expect(calls).toEqual([
|
||||
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml down --remove-orphans @/repo/openclaw`,
|
||||
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml up -d @/repo/openclaw`,
|
||||
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml ps --format json matrix-qa-homeserver @/repo/openclaw`,
|
||||
]);
|
||||
expect(fetchCalls).toEqual([
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
]);
|
||||
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
|
||||
expect(result.stopCommand).toBe(
|
||||
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml down --remove-orphans`,
|
||||
);
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("treats empty Docker health fields as a fallback to running state", async () => {
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
{
|
||||
async runCommand(_command, args) {
|
||||
if (args.join(" ").includes("ps --format json")) {
|
||||
return { stdout: '{"Health":"","State":"running"}\n', stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async () => ({ ok: true })),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to the container IP when the host port is unreachable", async () => {
|
||||
const calls: string[] = [];
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
{
|
||||
async runCommand(command, args, cwd) {
|
||||
calls.push([command, ...args, `@${cwd}`].join(" "));
|
||||
const rendered = args.join(" ");
|
||||
if (rendered.includes("ps --format json")) {
|
||||
return { stdout: '{"State":"running"}\n', stderr: "" };
|
||||
}
|
||||
if (rendered.includes("ps -q")) {
|
||||
return { stdout: "container-123\n", stderr: "" };
|
||||
}
|
||||
if (rendered.includes("inspect --format")) {
|
||||
return { stdout: "172.18.0.10\n", stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async (input: string) => ({
|
||||
ok: input.startsWith("http://172.18.0.10:8008/"),
|
||||
})),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.baseUrl).toBe("http://172.18.0.10:8008/");
|
||||
expect(calls).toContain(
|
||||
`docker compose -f ${outputDir}/docker-compose.matrix-qa.yml ps -q matrix-qa-homeserver @/repo/openclaw`,
|
||||
);
|
||||
expect(calls).toContain(
|
||||
"docker inspect --format {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} container-123 @/repo/openclaw",
|
||||
);
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the host URL when the container IP is also unreachable", async () => {
|
||||
const fetchCalls: string[] = [];
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
{
|
||||
async runCommand(_command, args) {
|
||||
const rendered = args.join(" ");
|
||||
if (rendered.includes("ps --format json")) {
|
||||
return { stdout: '{"State":"running"}\n', stderr: "" };
|
||||
}
|
||||
if (rendered.includes("ps -q")) {
|
||||
return { stdout: "container-123\n", stderr: "" };
|
||||
}
|
||||
if (rendered.includes("inspect --format")) {
|
||||
return { stdout: "172.18.0.10\n", stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async (input: string) => {
|
||||
fetchCalls.push(input);
|
||||
return {
|
||||
ok:
|
||||
input === "http://127.0.0.1:28008/_matrix/client/versions" &&
|
||||
fetchCalls.filter((url) => url === input).length > 1,
|
||||
};
|
||||
}),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.baseUrl).toBe("http://127.0.0.1:28008/");
|
||||
expect(fetchCalls).toEqual([
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
]);
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps probing the container URL until it becomes reachable", async () => {
|
||||
const fetchCalls: string[] = [];
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
{
|
||||
async runCommand(_command, args) {
|
||||
const rendered = args.join(" ");
|
||||
if (rendered.includes("ps --format json")) {
|
||||
return { stdout: '{"State":"running"}\n', stderr: "" };
|
||||
}
|
||||
if (rendered.includes("ps -q")) {
|
||||
return { stdout: "container-123\n", stderr: "" };
|
||||
}
|
||||
if (rendered.includes("inspect --format")) {
|
||||
return { stdout: "172.18.0.10\n", stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async (input: string) => {
|
||||
fetchCalls.push(input);
|
||||
return {
|
||||
ok:
|
||||
input === "http://172.18.0.10:8008/_matrix/client/versions" &&
|
||||
fetchCalls.filter((url) => url === input).length > 1,
|
||||
};
|
||||
}),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.baseUrl).toBe("http://172.18.0.10:8008/");
|
||||
expect(fetchCalls).toEqual([
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://172.18.0.10:8008/_matrix/client/versions",
|
||||
"http://127.0.0.1:28008/_matrix/client/versions",
|
||||
"http://172.18.0.10:8008/_matrix/client/versions",
|
||||
"http://172.18.0.10:8008/_matrix/client/versions",
|
||||
]);
|
||||
} finally {
|
||||
await rm(outputDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,275 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import {
|
||||
execCommand,
|
||||
fetchHealthUrl,
|
||||
resolveComposeServiceUrl,
|
||||
resolveHostPort,
|
||||
waitForDockerServiceHealth,
|
||||
waitForHealth,
|
||||
type FetchLike,
|
||||
type RunCommand,
|
||||
} from "../../docker-runtime.js";
|
||||
|
||||
const MATRIX_QA_DEFAULT_IMAGE = "ghcr.io/matrix-construct/tuwunel:v1.5.1";
|
||||
const MATRIX_QA_DEFAULT_SERVER_NAME = "matrix-qa.test";
|
||||
const MATRIX_QA_DEFAULT_PORT = 28008;
|
||||
const MATRIX_QA_INTERNAL_PORT = 8008;
|
||||
const MATRIX_QA_SERVICE = "matrix-qa-homeserver";
|
||||
|
||||
type MatrixQaHarnessManifest = {
|
||||
image: string;
|
||||
serverName: string;
|
||||
homeserverPort: number;
|
||||
composeFile: string;
|
||||
dataDir: string;
|
||||
};
|
||||
|
||||
export type MatrixQaHarnessFiles = {
|
||||
outputDir: string;
|
||||
composeFile: string;
|
||||
manifestPath: string;
|
||||
image: string;
|
||||
serverName: string;
|
||||
homeserverPort: number;
|
||||
registrationToken: string;
|
||||
};
|
||||
|
||||
export type MatrixQaHarness = MatrixQaHarnessFiles & {
|
||||
baseUrl: string;
|
||||
stopCommand: string;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
function buildVersionsUrl(baseUrl: string) {
|
||||
return `${baseUrl}_matrix/client/versions`;
|
||||
}
|
||||
|
||||
async function isMatrixVersionsReachable(baseUrl: string, fetchImpl: FetchLike) {
|
||||
return await fetchImpl(buildVersionsUrl(baseUrl))
|
||||
.then((response) => response.ok)
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
async function waitForReachableMatrixBaseUrl(params: {
|
||||
composeFile: string;
|
||||
containerBaseUrl: string | null;
|
||||
fetchImpl: FetchLike;
|
||||
hostBaseUrl: string;
|
||||
sleepImpl: (ms: number) => Promise<unknown>;
|
||||
timeoutMs?: number;
|
||||
pollMs?: number;
|
||||
}) {
|
||||
const timeoutMs = params.timeoutMs ?? 60_000;
|
||||
const pollMs = params.pollMs ?? 1_000;
|
||||
const startedAt = Date.now();
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (await isMatrixVersionsReachable(params.hostBaseUrl, params.fetchImpl)) {
|
||||
return params.hostBaseUrl;
|
||||
}
|
||||
if (
|
||||
params.containerBaseUrl &&
|
||||
(await isMatrixVersionsReachable(params.containerBaseUrl, params.fetchImpl))
|
||||
) {
|
||||
return params.containerBaseUrl;
|
||||
}
|
||||
await params.sleepImpl(pollMs);
|
||||
}
|
||||
|
||||
const candidateLabel = params.containerBaseUrl
|
||||
? `${params.hostBaseUrl} or ${params.containerBaseUrl}`
|
||||
: params.hostBaseUrl;
|
||||
throw new Error(
|
||||
[
|
||||
`Matrix homeserver did not become healthy within ${Math.round(timeoutMs / 1000)}s.`,
|
||||
`Last checked: ${candidateLabel}`,
|
||||
`Hint: check container logs with \`docker compose -f ${params.composeFile} logs ${MATRIX_QA_SERVICE}\`.`,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveMatrixQaHarnessImage(image?: string) {
|
||||
return (
|
||||
image?.trim() || process.env.OPENCLAW_QA_MATRIX_TUWUNEL_IMAGE?.trim() || MATRIX_QA_DEFAULT_IMAGE
|
||||
);
|
||||
}
|
||||
|
||||
function renderMatrixQaCompose(params: {
|
||||
homeserverPort: number;
|
||||
image: string;
|
||||
registrationToken: string;
|
||||
serverName: string;
|
||||
}) {
|
||||
return `services:
|
||||
${MATRIX_QA_SERVICE}:
|
||||
image: ${params.image}
|
||||
ports:
|
||||
- "127.0.0.1:${params.homeserverPort}:${MATRIX_QA_INTERNAL_PORT}"
|
||||
environment:
|
||||
TUWUNEL_ADDRESS: "0.0.0.0"
|
||||
TUWUNEL_ALLOW_ENCRYPTION: "false"
|
||||
TUWUNEL_ALLOW_FEDERATION: "false"
|
||||
TUWUNEL_ALLOW_REGISTRATION: "true"
|
||||
TUWUNEL_DATABASE_PATH: "/var/lib/tuwunel"
|
||||
TUWUNEL_PORT: "${MATRIX_QA_INTERNAL_PORT}"
|
||||
TUWUNEL_REGISTRATION_TOKEN: "${params.registrationToken}"
|
||||
TUWUNEL_SERVER_NAME: "${params.serverName}"
|
||||
volumes:
|
||||
- ./data:/var/lib/tuwunel
|
||||
`;
|
||||
}
|
||||
|
||||
export async function writeMatrixQaHarnessFiles(params: {
|
||||
outputDir: string;
|
||||
image?: string;
|
||||
homeserverPort: number;
|
||||
registrationToken?: string;
|
||||
serverName?: string;
|
||||
}): Promise<MatrixQaHarnessFiles> {
|
||||
const image = resolveMatrixQaHarnessImage(params.image);
|
||||
const registrationToken = params.registrationToken?.trim() || `matrix-qa-${randomUUID()}`;
|
||||
const serverName = params.serverName?.trim() || MATRIX_QA_DEFAULT_SERVER_NAME;
|
||||
const composeFile = path.join(params.outputDir, "docker-compose.matrix-qa.yml");
|
||||
const dataDir = path.join(params.outputDir, "data");
|
||||
const manifestPath = path.join(params.outputDir, "matrix-qa-harness.json");
|
||||
|
||||
await fs.mkdir(dataDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
composeFile,
|
||||
`${renderMatrixQaCompose({
|
||||
homeserverPort: params.homeserverPort,
|
||||
image,
|
||||
registrationToken,
|
||||
serverName,
|
||||
})}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
const manifest: MatrixQaHarnessManifest = {
|
||||
image,
|
||||
serverName,
|
||||
homeserverPort: params.homeserverPort,
|
||||
composeFile,
|
||||
dataDir,
|
||||
};
|
||||
await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
|
||||
return {
|
||||
outputDir: params.outputDir,
|
||||
composeFile,
|
||||
manifestPath,
|
||||
image,
|
||||
serverName,
|
||||
homeserverPort: params.homeserverPort,
|
||||
registrationToken,
|
||||
};
|
||||
}
|
||||
|
||||
export async function startMatrixQaHarness(
|
||||
params: {
|
||||
outputDir: string;
|
||||
repoRoot?: string;
|
||||
image?: string;
|
||||
homeserverPort?: number;
|
||||
serverName?: string;
|
||||
},
|
||||
deps?: {
|
||||
fetchImpl?: FetchLike;
|
||||
runCommand?: RunCommand;
|
||||
sleepImpl?: (ms: number) => Promise<unknown>;
|
||||
resolveHostPortImpl?: typeof resolveHostPort;
|
||||
},
|
||||
): Promise<MatrixQaHarness> {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
const resolveHostPortImpl = deps?.resolveHostPortImpl ?? resolveHostPort;
|
||||
const runCommand = deps?.runCommand ?? execCommand;
|
||||
const fetchImpl = deps?.fetchImpl ?? fetchHealthUrl;
|
||||
const sleepImpl = deps?.sleepImpl ?? sleep;
|
||||
const homeserverPort = await resolveHostPortImpl(
|
||||
params.homeserverPort ?? MATRIX_QA_DEFAULT_PORT,
|
||||
params.homeserverPort != null,
|
||||
);
|
||||
const files = await writeMatrixQaHarnessFiles({
|
||||
outputDir: path.resolve(params.outputDir),
|
||||
image: params.image,
|
||||
homeserverPort,
|
||||
serverName: params.serverName,
|
||||
});
|
||||
|
||||
try {
|
||||
await runCommand(
|
||||
"docker",
|
||||
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
|
||||
repoRoot,
|
||||
);
|
||||
} catch {
|
||||
// First run or already stopped.
|
||||
}
|
||||
|
||||
await runCommand("docker", ["compose", "-f", files.composeFile, "up", "-d"], repoRoot);
|
||||
await sleepImpl(1_000);
|
||||
await waitForDockerServiceHealth(
|
||||
MATRIX_QA_SERVICE,
|
||||
files.composeFile,
|
||||
repoRoot,
|
||||
runCommand,
|
||||
sleepImpl,
|
||||
);
|
||||
|
||||
const hostBaseUrl = `http://127.0.0.1:${homeserverPort}/`;
|
||||
let baseUrl = hostBaseUrl;
|
||||
const hostReachable = await isMatrixVersionsReachable(hostBaseUrl, fetchImpl);
|
||||
if (!hostReachable) {
|
||||
const containerBaseUrl = await resolveComposeServiceUrl(
|
||||
MATRIX_QA_SERVICE,
|
||||
MATRIX_QA_INTERNAL_PORT,
|
||||
files.composeFile,
|
||||
repoRoot,
|
||||
runCommand,
|
||||
);
|
||||
baseUrl = await waitForReachableMatrixBaseUrl({
|
||||
composeFile: files.composeFile,
|
||||
containerBaseUrl,
|
||||
fetchImpl,
|
||||
hostBaseUrl,
|
||||
sleepImpl,
|
||||
});
|
||||
}
|
||||
|
||||
await waitForHealth(buildVersionsUrl(baseUrl), {
|
||||
label: "Matrix homeserver",
|
||||
composeFile: files.composeFile,
|
||||
fetchImpl,
|
||||
sleepImpl,
|
||||
});
|
||||
|
||||
return {
|
||||
...files,
|
||||
baseUrl,
|
||||
stopCommand: `docker compose -f ${files.composeFile} down --remove-orphans`,
|
||||
async stop() {
|
||||
await runCommand(
|
||||
"docker",
|
||||
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
|
||||
repoRoot,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
MATRIX_QA_DEFAULT_IMAGE,
|
||||
MATRIX_QA_DEFAULT_PORT,
|
||||
MATRIX_QA_DEFAULT_SERVER_NAME,
|
||||
MATRIX_QA_SERVICE,
|
||||
buildVersionsUrl,
|
||||
isMatrixVersionsReachable,
|
||||
renderMatrixQaCompose,
|
||||
resolveMatrixQaHarnessImage,
|
||||
waitForReachableMatrixBaseUrl,
|
||||
};
|
||||
@@ -1,153 +0,0 @@
|
||||
import { describe, expect, it, beforeEach, vi } from "vitest";
|
||||
const { createMatrixQaClient } = vi.hoisted(() => ({
|
||||
createMatrixQaClient: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./matrix-driver-client.js", () => ({
|
||||
createMatrixQaClient,
|
||||
}));
|
||||
|
||||
import {
|
||||
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
|
||||
findMissingLiveTransportStandardScenarios,
|
||||
} from "../shared/live-transport-scenarios.js";
|
||||
import {
|
||||
__testing as scenarioTesting,
|
||||
MATRIX_QA_SCENARIOS,
|
||||
runMatrixQaScenario,
|
||||
} from "./matrix-live-scenarios.js";
|
||||
|
||||
describe("matrix live qa scenarios", () => {
|
||||
beforeEach(() => {
|
||||
createMatrixQaClient.mockReset();
|
||||
});
|
||||
|
||||
it("ships the Matrix live QA scenario set by default", () => {
|
||||
expect(scenarioTesting.findMatrixQaScenarios().map((scenario) => scenario.id)).toEqual([
|
||||
"matrix-thread-follow-up",
|
||||
"matrix-thread-isolation",
|
||||
"matrix-top-level-reply-shape",
|
||||
"matrix-reaction-notification",
|
||||
"matrix-restart-resume",
|
||||
"matrix-mention-gating",
|
||||
"matrix-allowlist-block",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the repo-wide exact marker prompt shape for Matrix mentions", () => {
|
||||
expect(
|
||||
scenarioTesting.buildMentionPrompt("@sut:matrix-qa.test", "MATRIX_QA_CANARY_TOKEN"),
|
||||
).toBe("@sut:matrix-qa.test reply with only this exact marker: MATRIX_QA_CANARY_TOKEN");
|
||||
});
|
||||
|
||||
it("requires Matrix replies to match the exact marker body", () => {
|
||||
expect(
|
||||
scenarioTesting.buildMatrixReplyArtifact(
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$event",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
body: "MATRIX_QA_TOKEN",
|
||||
},
|
||||
"MATRIX_QA_TOKEN",
|
||||
).tokenMatched,
|
||||
).toBe(true);
|
||||
expect(
|
||||
scenarioTesting.buildMatrixReplyArtifact(
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$event-2",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
body: "prefix MATRIX_QA_TOKEN suffix",
|
||||
},
|
||||
"MATRIX_QA_TOKEN",
|
||||
).tokenMatched,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("fails when any requested Matrix scenario id is unknown", () => {
|
||||
expect(() =>
|
||||
scenarioTesting.findMatrixQaScenarios(["matrix-thread-follow-up", "typo-scenario"]),
|
||||
).toThrow("unknown Matrix QA scenario id(s): typo-scenario");
|
||||
});
|
||||
|
||||
it("covers the baseline live transport contract plus Matrix-specific extras", () => {
|
||||
expect(scenarioTesting.MATRIX_QA_STANDARD_SCENARIO_IDS).toEqual([
|
||||
"canary",
|
||||
"thread-follow-up",
|
||||
"thread-isolation",
|
||||
"top-level-reply-shape",
|
||||
"reaction-observation",
|
||||
"restart-resume",
|
||||
"mention-gating",
|
||||
"allowlist-block",
|
||||
]);
|
||||
expect(
|
||||
findMissingLiveTransportStandardScenarios({
|
||||
coveredStandardScenarioIds: scenarioTesting.MATRIX_QA_STANDARD_SCENARIO_IDS,
|
||||
expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("primes the observer sync cursor instead of reusing the driver's cursor", async () => {
|
||||
const primeRoom = vi.fn().mockResolvedValue("observer-sync-start");
|
||||
const sendTextMessage = vi.fn().mockResolvedValue("$observer-trigger");
|
||||
const waitForOptionalRoomEvent = vi.fn().mockImplementation(async (params) => {
|
||||
expect(params.since).toBe("observer-sync-start");
|
||||
return {
|
||||
matched: false,
|
||||
since: "observer-sync-next",
|
||||
};
|
||||
});
|
||||
|
||||
createMatrixQaClient.mockReturnValue({
|
||||
primeRoom,
|
||||
sendTextMessage,
|
||||
waitForOptionalRoomEvent,
|
||||
});
|
||||
|
||||
const scenario = MATRIX_QA_SCENARIOS.find((entry) => entry.id === "matrix-allowlist-block");
|
||||
expect(scenario).toBeDefined();
|
||||
|
||||
const syncState = {
|
||||
driver: "driver-sync-next",
|
||||
};
|
||||
|
||||
await expect(
|
||||
runMatrixQaScenario(scenario!, {
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
canary: undefined,
|
||||
driverAccessToken: "driver-token",
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
observedEvents: [],
|
||||
observerAccessToken: "observer-token",
|
||||
observerUserId: "@observer:matrix-qa.test",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
restartGateway: undefined,
|
||||
syncState,
|
||||
sutUserId: "@sut:matrix-qa.test",
|
||||
timeoutMs: 8_000,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
artifacts: {
|
||||
actorUserId: "@observer:matrix-qa.test",
|
||||
expectedNoReplyWindowMs: 8_000,
|
||||
},
|
||||
});
|
||||
|
||||
expect(createMatrixQaClient).toHaveBeenCalledWith({
|
||||
accessToken: "observer-token",
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
});
|
||||
expect(primeRoom).toHaveBeenCalledTimes(1);
|
||||
expect(sendTextMessage).toHaveBeenCalledTimes(1);
|
||||
expect(waitForOptionalRoomEvent).toHaveBeenCalledTimes(1);
|
||||
expect(syncState).toEqual({
|
||||
driver: "driver-sync-next",
|
||||
observer: "observer-sync-next",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,670 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
collectLiveTransportStandardScenarioCoverage,
|
||||
selectLiveTransportScenarios,
|
||||
type LiveTransportScenarioDefinition,
|
||||
} from "../shared/live-transport-scenarios.js";
|
||||
import { createMatrixQaClient, type MatrixQaObservedEvent } from "./matrix-driver-client.js";
|
||||
|
||||
export type MatrixQaScenarioId =
|
||||
| "matrix-thread-follow-up"
|
||||
| "matrix-thread-isolation"
|
||||
| "matrix-top-level-reply-shape"
|
||||
| "matrix-reaction-notification"
|
||||
| "matrix-restart-resume"
|
||||
| "matrix-mention-gating"
|
||||
| "matrix-allowlist-block";
|
||||
|
||||
export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQaScenarioId>;
|
||||
|
||||
export type MatrixQaReplyArtifact = {
|
||||
bodyPreview?: string;
|
||||
eventId: string;
|
||||
mentions?: MatrixQaObservedEvent["mentions"];
|
||||
relatesTo?: MatrixQaObservedEvent["relatesTo"];
|
||||
sender?: string;
|
||||
tokenMatched?: boolean;
|
||||
};
|
||||
|
||||
export type MatrixQaCanaryArtifact = {
|
||||
driverEventId: string;
|
||||
reply: MatrixQaReplyArtifact;
|
||||
token: string;
|
||||
};
|
||||
|
||||
export type MatrixQaScenarioArtifacts = {
|
||||
actorUserId?: string;
|
||||
driverEventId?: string;
|
||||
expectedNoReplyWindowMs?: number;
|
||||
reactionEmoji?: string;
|
||||
reactionEventId?: string;
|
||||
reactionTargetEventId?: string;
|
||||
reply?: MatrixQaReplyArtifact;
|
||||
restartSignal?: string;
|
||||
rootEventId?: string;
|
||||
threadDriverEventId?: string;
|
||||
threadReply?: MatrixQaReplyArtifact;
|
||||
threadRootEventId?: string;
|
||||
threadToken?: string;
|
||||
token?: string;
|
||||
topLevelDriverEventId?: string;
|
||||
topLevelReply?: MatrixQaReplyArtifact;
|
||||
topLevelToken?: string;
|
||||
triggerBody?: string;
|
||||
};
|
||||
|
||||
export type MatrixQaScenarioExecution = {
|
||||
artifacts?: MatrixQaScenarioArtifacts;
|
||||
details: string;
|
||||
};
|
||||
|
||||
type MatrixQaActorId = "driver" | "observer";
|
||||
|
||||
type MatrixQaSyncState = Partial<Record<MatrixQaActorId, string>>;
|
||||
|
||||
type MatrixQaScenarioContext = {
|
||||
baseUrl: string;
|
||||
canary?: MatrixQaCanaryArtifact;
|
||||
driverAccessToken: string;
|
||||
driverUserId: string;
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
observerAccessToken: string;
|
||||
observerUserId: string;
|
||||
restartGateway?: () => Promise<void>;
|
||||
roomId: string;
|
||||
syncState: MatrixQaSyncState;
|
||||
sutUserId: string;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
const NO_REPLY_WINDOW_MS = 8_000;
|
||||
|
||||
export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
|
||||
{
|
||||
id: "matrix-thread-follow-up",
|
||||
standardId: "thread-follow-up",
|
||||
timeoutMs: 60_000,
|
||||
title: "Matrix thread follow-up reply",
|
||||
},
|
||||
{
|
||||
id: "matrix-thread-isolation",
|
||||
standardId: "thread-isolation",
|
||||
timeoutMs: 75_000,
|
||||
title: "Matrix top-level reply stays out of prior thread",
|
||||
},
|
||||
{
|
||||
id: "matrix-top-level-reply-shape",
|
||||
standardId: "top-level-reply-shape",
|
||||
timeoutMs: 45_000,
|
||||
title: "Matrix top-level reply keeps replyToMode off",
|
||||
},
|
||||
{
|
||||
id: "matrix-reaction-notification",
|
||||
standardId: "reaction-observation",
|
||||
timeoutMs: 45_000,
|
||||
title: "Matrix reactions on bot replies are observed",
|
||||
},
|
||||
{
|
||||
id: "matrix-restart-resume",
|
||||
standardId: "restart-resume",
|
||||
timeoutMs: 60_000,
|
||||
title: "Matrix lane resumes cleanly after gateway restart",
|
||||
},
|
||||
{
|
||||
id: "matrix-mention-gating",
|
||||
standardId: "mention-gating",
|
||||
timeoutMs: NO_REPLY_WINDOW_MS,
|
||||
title: "Matrix room message without mention does not trigger",
|
||||
},
|
||||
{
|
||||
id: "matrix-allowlist-block",
|
||||
standardId: "allowlist-block",
|
||||
timeoutMs: NO_REPLY_WINDOW_MS,
|
||||
title: "Matrix allowlist blocks non-driver replies",
|
||||
},
|
||||
];
|
||||
|
||||
export const MATRIX_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
alwaysOnStandardScenarioIds: ["canary"],
|
||||
scenarios: MATRIX_QA_SCENARIOS,
|
||||
});
|
||||
|
||||
export function findMatrixQaScenarios(ids?: string[]) {
|
||||
return selectLiveTransportScenarios({
|
||||
ids,
|
||||
laneLabel: "Matrix",
|
||||
scenarios: MATRIX_QA_SCENARIOS,
|
||||
});
|
||||
}
|
||||
|
||||
export function buildMentionPrompt(sutUserId: string, token: string) {
|
||||
return `${sutUserId} reply with only this exact marker: ${token}`;
|
||||
}
|
||||
|
||||
function buildExactMarkerPrompt(token: string) {
|
||||
return `reply with only this exact marker: ${token}`;
|
||||
}
|
||||
|
||||
function buildMatrixReplyArtifact(
|
||||
event: MatrixQaObservedEvent,
|
||||
token?: string,
|
||||
): MatrixQaReplyArtifact {
|
||||
const replyBody = event.body?.trim();
|
||||
return {
|
||||
bodyPreview: replyBody?.slice(0, 200),
|
||||
eventId: event.eventId,
|
||||
mentions: event.mentions,
|
||||
relatesTo: event.relatesTo,
|
||||
sender: event.sender,
|
||||
...(token ? { tokenMatched: replyBody === token } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildMatrixReplyDetails(label: string, artifact: MatrixQaReplyArtifact) {
|
||||
return [
|
||||
`${label} event: ${artifact.eventId}`,
|
||||
`${label} token matched: ${
|
||||
artifact.tokenMatched === undefined ? "n/a" : artifact.tokenMatched ? "yes" : "no"
|
||||
}`,
|
||||
`${label} rel_type: ${artifact.relatesTo?.relType ?? "<none>"}`,
|
||||
`${label} in_reply_to: ${artifact.relatesTo?.inReplyToId ?? "<none>"}`,
|
||||
`${label} is_falling_back: ${artifact.relatesTo?.isFallingBack === true ? "true" : "false"}`,
|
||||
];
|
||||
}
|
||||
|
||||
function assertTopLevelReplyArtifact(label: string, artifact: MatrixQaReplyArtifact) {
|
||||
if (!artifact.tokenMatched) {
|
||||
throw new Error(`${label} did not contain the expected token`);
|
||||
}
|
||||
if (artifact.relatesTo !== undefined) {
|
||||
throw new Error(`${label} unexpectedly included relation metadata`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertThreadReplyArtifact(
|
||||
artifact: MatrixQaReplyArtifact,
|
||||
params: {
|
||||
expectedRootEventId: string;
|
||||
label: string;
|
||||
},
|
||||
) {
|
||||
if (!artifact.tokenMatched) {
|
||||
throw new Error(`${params.label} did not contain the expected token`);
|
||||
}
|
||||
if (artifact.relatesTo?.relType !== "m.thread") {
|
||||
throw new Error(`${params.label} did not use m.thread`);
|
||||
}
|
||||
if (artifact.relatesTo.eventId !== params.expectedRootEventId) {
|
||||
throw new Error(
|
||||
`${params.label} targeted ${artifact.relatesTo.eventId ?? "<none>"} instead of ${params.expectedRootEventId}`,
|
||||
);
|
||||
}
|
||||
if (artifact.relatesTo.isFallingBack !== true) {
|
||||
throw new Error(`${params.label} did not set is_falling_back`);
|
||||
}
|
||||
if (!artifact.relatesTo.inReplyToId) {
|
||||
throw new Error(`${params.label} did not set m.in_reply_to`);
|
||||
}
|
||||
}
|
||||
|
||||
function readMatrixQaSyncCursor(syncState: MatrixQaSyncState, actorId: MatrixQaActorId) {
|
||||
return syncState[actorId];
|
||||
}
|
||||
|
||||
function writeMatrixQaSyncCursor(
|
||||
syncState: MatrixQaSyncState,
|
||||
actorId: MatrixQaActorId,
|
||||
since?: string,
|
||||
) {
|
||||
if (since) {
|
||||
syncState[actorId] = since;
|
||||
}
|
||||
}
|
||||
|
||||
async function primeMatrixQaActorCursor(params: {
|
||||
accessToken: string;
|
||||
actorId: MatrixQaActorId;
|
||||
baseUrl: string;
|
||||
syncState: MatrixQaSyncState;
|
||||
}) {
|
||||
const client = createMatrixQaClient({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
});
|
||||
const existingSince = readMatrixQaSyncCursor(params.syncState, params.actorId);
|
||||
if (existingSince) {
|
||||
return { client, startSince: existingSince };
|
||||
}
|
||||
const startSince = await client.primeRoom();
|
||||
if (!startSince) {
|
||||
throw new Error(`Matrix ${params.actorId} /sync prime did not return a next_batch cursor`);
|
||||
}
|
||||
return { client, startSince };
|
||||
}
|
||||
|
||||
function advanceMatrixQaActorCursor(params: {
|
||||
actorId: MatrixQaActorId;
|
||||
syncState: MatrixQaSyncState;
|
||||
nextSince?: string;
|
||||
startSince: string;
|
||||
}) {
|
||||
writeMatrixQaSyncCursor(params.syncState, params.actorId, params.nextSince ?? params.startSince);
|
||||
}
|
||||
|
||||
async function runTopLevelMentionScenario(params: {
|
||||
accessToken: string;
|
||||
actorId: MatrixQaActorId;
|
||||
baseUrl: string;
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
roomId: string;
|
||||
syncState: MatrixQaSyncState;
|
||||
sutUserId: string;
|
||||
timeoutMs: number;
|
||||
tokenPrefix: string;
|
||||
withMention?: boolean;
|
||||
}) {
|
||||
const { client, startSince } = await primeMatrixQaActorCursor({
|
||||
accessToken: params.accessToken,
|
||||
actorId: params.actorId,
|
||||
baseUrl: params.baseUrl,
|
||||
syncState: params.syncState,
|
||||
});
|
||||
const token = `${params.tokenPrefix}_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
const body =
|
||||
params.withMention === false
|
||||
? buildExactMarkerPrompt(token)
|
||||
: buildMentionPrompt(params.sutUserId, token);
|
||||
const driverEventId = await client.sendTextMessage({
|
||||
body,
|
||||
...(params.withMention === false ? {} : { mentionUserIds: [params.sutUserId] }),
|
||||
roomId: params.roomId,
|
||||
});
|
||||
const matched = await client.waitForRoomEvent({
|
||||
observedEvents: params.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === params.roomId &&
|
||||
event.sender === params.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
(event.body ?? "").includes(token) &&
|
||||
event.relatesTo === undefined,
|
||||
roomId: params.roomId,
|
||||
since: startSince,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: params.actorId,
|
||||
syncState: params.syncState,
|
||||
nextSince: matched.since,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
body,
|
||||
driverEventId,
|
||||
reply: buildMatrixReplyArtifact(matched.event, token),
|
||||
since: matched.since,
|
||||
token,
|
||||
};
|
||||
}
|
||||
|
||||
async function runThreadScenario(params: MatrixQaScenarioContext) {
|
||||
const { client, startSince } = await primeMatrixQaActorCursor({
|
||||
accessToken: params.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: params.baseUrl,
|
||||
syncState: params.syncState,
|
||||
});
|
||||
const rootBody = `thread root ${randomUUID().slice(0, 8)}`;
|
||||
const rootEventId = await client.sendTextMessage({
|
||||
body: rootBody,
|
||||
roomId: params.roomId,
|
||||
});
|
||||
const token = `MATRIX_QA_THREAD_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
const driverEventId = await client.sendTextMessage({
|
||||
body: buildMentionPrompt(params.sutUserId, token),
|
||||
mentionUserIds: [params.sutUserId],
|
||||
replyToEventId: rootEventId,
|
||||
roomId: params.roomId,
|
||||
threadRootEventId: rootEventId,
|
||||
});
|
||||
const matched = await client.waitForRoomEvent({
|
||||
observedEvents: params.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === params.roomId &&
|
||||
event.sender === params.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
(event.body ?? "").includes(token) &&
|
||||
event.relatesTo?.relType === "m.thread" &&
|
||||
event.relatesTo.eventId === rootEventId,
|
||||
roomId: params.roomId,
|
||||
since: startSince,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: "driver",
|
||||
syncState: params.syncState,
|
||||
nextSince: matched.since,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
driverEventId,
|
||||
reply: buildMatrixReplyArtifact(matched.event, token),
|
||||
rootEventId,
|
||||
since: matched.since,
|
||||
token,
|
||||
};
|
||||
}
|
||||
|
||||
async function runNoReplyExpectedScenario(params: {
|
||||
accessToken: string;
|
||||
actorId: MatrixQaActorId;
|
||||
actorUserId: string;
|
||||
baseUrl: string;
|
||||
body: string;
|
||||
mentionUserIds?: string[];
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
roomId: string;
|
||||
syncState: MatrixQaSyncState;
|
||||
sutUserId: string;
|
||||
timeoutMs: number;
|
||||
token: string;
|
||||
}) {
|
||||
const { client, startSince } = await primeMatrixQaActorCursor({
|
||||
accessToken: params.accessToken,
|
||||
actorId: params.actorId,
|
||||
baseUrl: params.baseUrl,
|
||||
syncState: params.syncState,
|
||||
});
|
||||
const driverEventId = await client.sendTextMessage({
|
||||
body: params.body,
|
||||
...(params.mentionUserIds ? { mentionUserIds: params.mentionUserIds } : {}),
|
||||
roomId: params.roomId,
|
||||
});
|
||||
const result = await client.waitForOptionalRoomEvent({
|
||||
observedEvents: params.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === params.roomId &&
|
||||
event.sender === params.sutUserId &&
|
||||
event.type === "m.room.message",
|
||||
roomId: params.roomId,
|
||||
since: startSince,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
if (result.matched) {
|
||||
const unexpectedReply = buildMatrixReplyArtifact(result.event, params.token);
|
||||
throw new Error(
|
||||
[
|
||||
`unexpected SUT reply from ${params.sutUserId}`,
|
||||
`trigger sender: ${params.actorUserId}`,
|
||||
...buildMatrixReplyDetails("unexpected reply", unexpectedReply),
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: params.actorId,
|
||||
syncState: params.syncState,
|
||||
nextSince: result.since,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
actorUserId: params.actorUserId,
|
||||
driverEventId,
|
||||
expectedNoReplyWindowMs: params.timeoutMs,
|
||||
token: params.token,
|
||||
triggerBody: params.body,
|
||||
},
|
||||
details: [
|
||||
`trigger event: ${driverEventId}`,
|
||||
`trigger sender: ${params.actorUserId}`,
|
||||
`waited ${params.timeoutMs}ms with no SUT reply`,
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
async function runReactionNotificationScenario(context: MatrixQaScenarioContext) {
|
||||
const reactionTargetEventId = context.canary?.reply.eventId?.trim();
|
||||
if (!reactionTargetEventId) {
|
||||
throw new Error("Matrix reaction scenario requires a canary reply event id");
|
||||
}
|
||||
const { client, startSince } = await primeMatrixQaActorCursor({
|
||||
accessToken: context.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: context.baseUrl,
|
||||
syncState: context.syncState,
|
||||
});
|
||||
const reactionEmoji = "👍";
|
||||
const reactionEventId = await client.sendReaction({
|
||||
emoji: reactionEmoji,
|
||||
messageId: reactionTargetEventId,
|
||||
roomId: context.roomId,
|
||||
});
|
||||
const matched = await client.waitForRoomEvent({
|
||||
observedEvents: context.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === context.roomId &&
|
||||
event.sender === context.driverUserId &&
|
||||
event.type === "m.reaction" &&
|
||||
event.eventId === reactionEventId &&
|
||||
event.reaction?.eventId === reactionTargetEventId &&
|
||||
event.reaction?.key === reactionEmoji,
|
||||
roomId: context.roomId,
|
||||
since: startSince,
|
||||
timeoutMs: context.timeoutMs,
|
||||
});
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: "driver",
|
||||
syncState: context.syncState,
|
||||
nextSince: matched.since,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
reactionEmoji,
|
||||
reactionEventId,
|
||||
reactionTargetEventId,
|
||||
},
|
||||
details: [
|
||||
`reaction event: ${reactionEventId}`,
|
||||
`reaction target: ${reactionTargetEventId}`,
|
||||
`reaction emoji: ${reactionEmoji}`,
|
||||
`observed reaction key: ${matched.event.reaction?.key ?? "<none>"}`,
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
async function runRestartResumeScenario(context: MatrixQaScenarioContext) {
|
||||
if (!context.restartGateway) {
|
||||
throw new Error("Matrix restart scenario requires a gateway restart callback");
|
||||
}
|
||||
await context.restartGateway();
|
||||
const result = await runTopLevelMentionScenario({
|
||||
accessToken: context.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: context.baseUrl,
|
||||
observedEvents: context.observedEvents,
|
||||
roomId: context.roomId,
|
||||
syncState: context.syncState,
|
||||
sutUserId: context.sutUserId,
|
||||
timeoutMs: context.timeoutMs,
|
||||
tokenPrefix: "MATRIX_QA_RESTART",
|
||||
});
|
||||
assertTopLevelReplyArtifact("post-restart reply", result.reply);
|
||||
return {
|
||||
artifacts: {
|
||||
driverEventId: result.driverEventId,
|
||||
reply: result.reply,
|
||||
restartSignal: "SIGUSR1",
|
||||
token: result.token,
|
||||
},
|
||||
details: [
|
||||
"restart signal: SIGUSR1",
|
||||
`post-restart driver event: ${result.driverEventId}`,
|
||||
...buildMatrixReplyDetails("reply", result.reply),
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
export async function runMatrixQaCanary(params: {
|
||||
baseUrl: string;
|
||||
driverAccessToken: string;
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
roomId: string;
|
||||
syncState: MatrixQaSyncState;
|
||||
sutUserId: string;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const canary = await runTopLevelMentionScenario({
|
||||
accessToken: params.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: params.baseUrl,
|
||||
observedEvents: params.observedEvents,
|
||||
roomId: params.roomId,
|
||||
syncState: params.syncState,
|
||||
sutUserId: params.sutUserId,
|
||||
timeoutMs: params.timeoutMs,
|
||||
tokenPrefix: "MATRIX_QA_CANARY",
|
||||
});
|
||||
assertTopLevelReplyArtifact("canary reply", canary.reply);
|
||||
return canary;
|
||||
}
|
||||
|
||||
export async function runMatrixQaScenario(
|
||||
scenario: MatrixQaScenarioDefinition,
|
||||
context: MatrixQaScenarioContext,
|
||||
): Promise<MatrixQaScenarioExecution> {
|
||||
switch (scenario.id) {
|
||||
case "matrix-thread-follow-up": {
|
||||
const result = await runThreadScenario(context);
|
||||
assertThreadReplyArtifact(result.reply, {
|
||||
expectedRootEventId: result.rootEventId,
|
||||
label: "thread reply",
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
driverEventId: result.driverEventId,
|
||||
reply: result.reply,
|
||||
rootEventId: result.rootEventId,
|
||||
token: result.token,
|
||||
},
|
||||
details: [
|
||||
`root event: ${result.rootEventId}`,
|
||||
`driver thread event: ${result.driverEventId}`,
|
||||
...buildMatrixReplyDetails("reply", result.reply),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
case "matrix-thread-isolation": {
|
||||
const threadPhase = await runThreadScenario(context);
|
||||
assertThreadReplyArtifact(threadPhase.reply, {
|
||||
expectedRootEventId: threadPhase.rootEventId,
|
||||
label: "thread isolation reply",
|
||||
});
|
||||
const topLevelPhase = await runTopLevelMentionScenario({
|
||||
accessToken: context.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: context.baseUrl,
|
||||
observedEvents: context.observedEvents,
|
||||
roomId: context.roomId,
|
||||
syncState: context.syncState,
|
||||
sutUserId: context.sutUserId,
|
||||
timeoutMs: context.timeoutMs,
|
||||
tokenPrefix: "MATRIX_QA_TOPLEVEL",
|
||||
});
|
||||
assertTopLevelReplyArtifact("top-level follow-up reply", topLevelPhase.reply);
|
||||
return {
|
||||
artifacts: {
|
||||
threadDriverEventId: threadPhase.driverEventId,
|
||||
threadReply: threadPhase.reply,
|
||||
threadRootEventId: threadPhase.rootEventId,
|
||||
threadToken: threadPhase.token,
|
||||
topLevelDriverEventId: topLevelPhase.driverEventId,
|
||||
topLevelReply: topLevelPhase.reply,
|
||||
topLevelToken: topLevelPhase.token,
|
||||
},
|
||||
details: [
|
||||
`thread root event: ${threadPhase.rootEventId}`,
|
||||
`thread driver event: ${threadPhase.driverEventId}`,
|
||||
...buildMatrixReplyDetails("thread reply", threadPhase.reply),
|
||||
`top-level driver event: ${topLevelPhase.driverEventId}`,
|
||||
...buildMatrixReplyDetails("top-level reply", topLevelPhase.reply),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
case "matrix-top-level-reply-shape": {
|
||||
const result = await runTopLevelMentionScenario({
|
||||
accessToken: context.driverAccessToken,
|
||||
actorId: "driver",
|
||||
baseUrl: context.baseUrl,
|
||||
observedEvents: context.observedEvents,
|
||||
roomId: context.roomId,
|
||||
syncState: context.syncState,
|
||||
sutUserId: context.sutUserId,
|
||||
timeoutMs: context.timeoutMs,
|
||||
tokenPrefix: "MATRIX_QA_TOPLEVEL",
|
||||
});
|
||||
assertTopLevelReplyArtifact("top-level reply", result.reply);
|
||||
return {
|
||||
artifacts: {
|
||||
driverEventId: result.driverEventId,
|
||||
reply: result.reply,
|
||||
token: result.token,
|
||||
},
|
||||
details: [
|
||||
`driver event: ${result.driverEventId}`,
|
||||
...buildMatrixReplyDetails("reply", result.reply),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
case "matrix-reaction-notification":
|
||||
return await runReactionNotificationScenario(context);
|
||||
case "matrix-restart-resume":
|
||||
return await runRestartResumeScenario(context);
|
||||
case "matrix-mention-gating": {
|
||||
const token = `MATRIX_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return await runNoReplyExpectedScenario({
|
||||
accessToken: context.driverAccessToken,
|
||||
actorId: "driver",
|
||||
actorUserId: context.driverUserId,
|
||||
baseUrl: context.baseUrl,
|
||||
body: buildExactMarkerPrompt(token),
|
||||
observedEvents: context.observedEvents,
|
||||
roomId: context.roomId,
|
||||
syncState: context.syncState,
|
||||
sutUserId: context.sutUserId,
|
||||
timeoutMs: context.timeoutMs,
|
||||
token,
|
||||
});
|
||||
}
|
||||
case "matrix-allowlist-block": {
|
||||
const token = `MATRIX_QA_ALLOWLIST_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return await runNoReplyExpectedScenario({
|
||||
accessToken: context.observerAccessToken,
|
||||
actorId: "observer",
|
||||
actorUserId: context.observerUserId,
|
||||
baseUrl: context.baseUrl,
|
||||
body: buildMentionPrompt(context.sutUserId, token),
|
||||
mentionUserIds: [context.sutUserId],
|
||||
observedEvents: context.observedEvents,
|
||||
roomId: context.roomId,
|
||||
syncState: context.syncState,
|
||||
sutUserId: context.sutUserId,
|
||||
timeoutMs: context.timeoutMs,
|
||||
token,
|
||||
});
|
||||
}
|
||||
default: {
|
||||
const exhaustiveScenarioId: never = scenario.id;
|
||||
return exhaustiveScenarioId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
MATRIX_QA_STANDARD_SCENARIO_IDS,
|
||||
buildMatrixReplyDetails,
|
||||
buildMatrixReplyArtifact,
|
||||
buildMentionPrompt,
|
||||
findMatrixQaScenarios,
|
||||
readMatrixQaSyncCursor,
|
||||
writeMatrixQaSyncCursor,
|
||||
};
|
||||
@@ -1,272 +0,0 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { __testing as liveTesting } from "./matrix-live.runtime.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("matrix live qa runtime", () => {
|
||||
it("injects a temporary Matrix account into the QA gateway config", () => {
|
||||
const baseCfg: OpenClawConfig = {
|
||||
plugins: {
|
||||
allow: ["memory-core", "qa-channel"],
|
||||
entries: {
|
||||
"memory-core": { enabled: true },
|
||||
"qa-channel": { enabled: true },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const next = liveTesting.buildMatrixQaConfig(baseCfg, {
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
homeserver: "http://127.0.0.1:28008/",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
sutAccessToken: "syt_sut",
|
||||
sutAccountId: "sut",
|
||||
sutDeviceId: "DEVICE123",
|
||||
sutUserId: "@sut:matrix-qa.test",
|
||||
});
|
||||
|
||||
expect(next.plugins?.allow).toContain("matrix");
|
||||
expect(next.plugins?.entries?.matrix).toEqual({ enabled: true });
|
||||
expect(next.channels?.matrix).toEqual({
|
||||
enabled: true,
|
||||
defaultAccount: "sut",
|
||||
accounts: {
|
||||
sut: {
|
||||
accessToken: "syt_sut",
|
||||
deviceId: "DEVICE123",
|
||||
dm: { enabled: false },
|
||||
enabled: true,
|
||||
encryption: false,
|
||||
groupAllowFrom: ["@driver:matrix-qa.test"],
|
||||
groupPolicy: "allowlist",
|
||||
groups: {
|
||||
"!room:matrix-qa.test": {
|
||||
enabled: true,
|
||||
requireMention: true,
|
||||
},
|
||||
},
|
||||
homeserver: "http://127.0.0.1:28008/",
|
||||
network: {
|
||||
dangerouslyAllowPrivateNetwork: true,
|
||||
},
|
||||
replyToMode: "off",
|
||||
threadReplies: "inbound",
|
||||
userId: "@sut:matrix-qa.test",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("redacts Matrix observed event content by default in artifacts", () => {
|
||||
expect(
|
||||
liveTesting.buildObservedEventsArtifact({
|
||||
includeContent: false,
|
||||
observedEvents: [
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$event",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
body: "secret",
|
||||
formattedBody: "<p>secret</p>",
|
||||
msgtype: "m.text",
|
||||
originServerTs: 1_700_000_000_000,
|
||||
relatesTo: {
|
||||
relType: "m.thread",
|
||||
eventId: "$root",
|
||||
inReplyToId: "$driver",
|
||||
isFallingBack: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$event",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
msgtype: "m.text",
|
||||
originServerTs: 1_700_000_000_000,
|
||||
relatesTo: {
|
||||
relType: "m.thread",
|
||||
eventId: "$root",
|
||||
inReplyToId: "$driver",
|
||||
isFallingBack: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps reaction metadata in redacted Matrix observed-event artifacts", () => {
|
||||
expect(
|
||||
liveTesting.buildObservedEventsArtifact({
|
||||
includeContent: false,
|
||||
observedEvents: [
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$reaction",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.reaction",
|
||||
reaction: {
|
||||
eventId: "$reply",
|
||||
key: "👍",
|
||||
},
|
||||
relatesTo: {
|
||||
relType: "m.annotation",
|
||||
eventId: "$reply",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$reaction",
|
||||
sender: "@driver:matrix-qa.test",
|
||||
type: "m.reaction",
|
||||
originServerTs: undefined,
|
||||
msgtype: undefined,
|
||||
membership: undefined,
|
||||
relatesTo: {
|
||||
relType: "m.annotation",
|
||||
eventId: "$reply",
|
||||
},
|
||||
mentions: undefined,
|
||||
reaction: {
|
||||
eventId: "$reply",
|
||||
key: "👍",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves negative-scenario artifacts in the Matrix summary", () => {
|
||||
expect(
|
||||
liveTesting.buildMatrixQaSummary({
|
||||
artifactPaths: {
|
||||
observedEvents: "/tmp/observed.json",
|
||||
report: "/tmp/report.md",
|
||||
summary: "/tmp/summary.json",
|
||||
},
|
||||
checks: [{ name: "Matrix harness ready", status: "pass" }],
|
||||
finishedAt: "2026-04-10T10:05:00.000Z",
|
||||
harness: {
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
composeFile: "/tmp/docker-compose.yml",
|
||||
image: "ghcr.io/matrix-construct/tuwunel:v1.5.1",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
serverName: "matrix-qa.test",
|
||||
},
|
||||
observedEventCount: 4,
|
||||
scenarios: [
|
||||
{
|
||||
id: "matrix-mention-gating",
|
||||
title: "Matrix room message without mention does not trigger",
|
||||
status: "pass",
|
||||
details: "no reply",
|
||||
artifacts: {
|
||||
actorUserId: "@driver:matrix-qa.test",
|
||||
driverEventId: "$driver",
|
||||
expectedNoReplyWindowMs: 8_000,
|
||||
token: "MATRIX_QA_NOMENTION_TOKEN",
|
||||
triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN",
|
||||
},
|
||||
},
|
||||
],
|
||||
startedAt: "2026-04-10T10:00:00.000Z",
|
||||
sutAccountId: "sut",
|
||||
userIds: {
|
||||
driver: "@driver:matrix-qa.test",
|
||||
observer: "@observer:matrix-qa.test",
|
||||
sut: "@sut:matrix-qa.test",
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
counts: {
|
||||
total: 2,
|
||||
passed: 2,
|
||||
failed: 0,
|
||||
},
|
||||
scenarios: [
|
||||
{
|
||||
id: "matrix-mention-gating",
|
||||
artifacts: {
|
||||
actorUserId: "@driver:matrix-qa.test",
|
||||
expectedNoReplyWindowMs: 8_000,
|
||||
triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("treats only connected, healthy Matrix accounts as ready", () => {
|
||||
expect(liveTesting.isMatrixAccountReady({ running: true, connected: true })).toBe(true);
|
||||
expect(liveTesting.isMatrixAccountReady({ running: true, connected: false })).toBe(false);
|
||||
expect(
|
||||
liveTesting.isMatrixAccountReady({
|
||||
running: true,
|
||||
connected: true,
|
||||
restartPending: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
liveTesting.isMatrixAccountReady({
|
||||
running: true,
|
||||
connected: true,
|
||||
healthState: "degraded",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("waits past not-ready Matrix status snapshots until the account is really ready", async () => {
|
||||
vi.useFakeTimers();
|
||||
const gateway = {
|
||||
call: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
matrix: [{ accountId: "sut", running: true, connected: false }],
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
matrix: [{ accountId: "sut", running: true, connected: true }],
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
const waitPromise = liveTesting.waitForMatrixChannelReady(gateway as never, "sut", {
|
||||
timeoutMs: 1_000,
|
||||
pollMs: 100,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await expect(waitPromise).resolves.toBeUndefined();
|
||||
expect(gateway.call).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("fails readiness when the Matrix account never reaches a healthy connected state", async () => {
|
||||
vi.useFakeTimers();
|
||||
const gateway = {
|
||||
call: vi.fn().mockResolvedValue({
|
||||
channelAccounts: {
|
||||
matrix: [{ accountId: "sut", running: true, connected: true, healthState: "degraded" }],
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
const waitPromise = liveTesting.waitForMatrixChannelReady(gateway as never, "sut", {
|
||||
timeoutMs: 250,
|
||||
pollMs: 100,
|
||||
});
|
||||
const expectation = expect(waitPromise).rejects.toThrow(
|
||||
'matrix account "sut" did not become ready',
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(300);
|
||||
await expectation;
|
||||
});
|
||||
});
|
||||
@@ -1,559 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import type { QaReportCheck } from "../../report.js";
|
||||
import { renderQaMarkdownReport } from "../../report.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "../../run-config.js";
|
||||
import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js";
|
||||
import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js";
|
||||
import {
|
||||
provisionMatrixQaRoom,
|
||||
type MatrixQaObservedEvent,
|
||||
type MatrixQaProvisionResult,
|
||||
} from "./matrix-driver-client.js";
|
||||
import { startMatrixQaHarness } from "./matrix-harness.runtime.js";
|
||||
import {
|
||||
MATRIX_QA_SCENARIOS,
|
||||
buildMatrixReplyDetails,
|
||||
findMatrixQaScenarios,
|
||||
runMatrixQaCanary,
|
||||
runMatrixQaScenario,
|
||||
type MatrixQaCanaryArtifact,
|
||||
type MatrixQaScenarioArtifacts,
|
||||
} from "./matrix-live-scenarios.js";
|
||||
|
||||
type MatrixQaScenarioResult = {
|
||||
artifacts?: MatrixQaScenarioArtifacts;
|
||||
details: string;
|
||||
id: string;
|
||||
status: "fail" | "pass";
|
||||
title: string;
|
||||
};
|
||||
|
||||
type MatrixQaSummary = {
|
||||
checks: QaReportCheck[];
|
||||
counts: {
|
||||
failed: number;
|
||||
passed: number;
|
||||
total: number;
|
||||
};
|
||||
finishedAt: string;
|
||||
harness: {
|
||||
baseUrl: string;
|
||||
composeFile: string;
|
||||
image: string;
|
||||
roomId: string;
|
||||
serverName: string;
|
||||
};
|
||||
canary?: MatrixQaCanaryArtifact;
|
||||
observedEventCount: number;
|
||||
observedEventsPath: string;
|
||||
reportPath: string;
|
||||
scenarios: MatrixQaScenarioResult[];
|
||||
startedAt: string;
|
||||
summaryPath: string;
|
||||
sutAccountId: string;
|
||||
userIds: {
|
||||
driver: string;
|
||||
observer: string;
|
||||
sut: string;
|
||||
};
|
||||
};
|
||||
|
||||
type MatrixQaArtifactPaths = {
|
||||
observedEvents: string;
|
||||
report: string;
|
||||
summary: string;
|
||||
};
|
||||
|
||||
export type MatrixQaRunResult = {
|
||||
observedEventsPath: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
scenarios: MatrixQaScenarioResult[];
|
||||
summaryPath: string;
|
||||
};
|
||||
|
||||
function buildMatrixQaSummary(params: {
|
||||
artifactPaths: MatrixQaArtifactPaths;
|
||||
canary?: MatrixQaCanaryArtifact;
|
||||
checks: QaReportCheck[];
|
||||
finishedAt: string;
|
||||
harness: MatrixQaSummary["harness"];
|
||||
observedEventCount: number;
|
||||
scenarios: MatrixQaScenarioResult[];
|
||||
startedAt: string;
|
||||
sutAccountId: string;
|
||||
userIds: MatrixQaSummary["userIds"];
|
||||
}): MatrixQaSummary {
|
||||
return {
|
||||
checks: params.checks,
|
||||
counts: {
|
||||
total: params.checks.length + params.scenarios.length,
|
||||
passed:
|
||||
params.checks.filter((check) => check.status === "pass").length +
|
||||
params.scenarios.filter((scenario) => scenario.status === "pass").length,
|
||||
failed:
|
||||
params.checks.filter((check) => check.status === "fail").length +
|
||||
params.scenarios.filter((scenario) => scenario.status === "fail").length,
|
||||
},
|
||||
finishedAt: params.finishedAt,
|
||||
harness: params.harness,
|
||||
canary: params.canary,
|
||||
observedEventCount: params.observedEventCount,
|
||||
observedEventsPath: params.artifactPaths.observedEvents,
|
||||
reportPath: params.artifactPaths.report,
|
||||
scenarios: params.scenarios,
|
||||
startedAt: params.startedAt,
|
||||
summaryPath: params.artifactPaths.summary,
|
||||
sutAccountId: params.sutAccountId,
|
||||
userIds: params.userIds,
|
||||
};
|
||||
}
|
||||
|
||||
function buildMatrixQaConfig(
|
||||
baseCfg: OpenClawConfig,
|
||||
params: {
|
||||
driverUserId: string;
|
||||
homeserver: string;
|
||||
roomId: string;
|
||||
sutAccessToken: string;
|
||||
sutAccountId: string;
|
||||
sutDeviceId?: string;
|
||||
sutUserId: string;
|
||||
},
|
||||
): OpenClawConfig {
|
||||
const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "matrix"])];
|
||||
return {
|
||||
...baseCfg,
|
||||
plugins: {
|
||||
...baseCfg.plugins,
|
||||
allow: pluginAllow,
|
||||
entries: {
|
||||
...baseCfg.plugins?.entries,
|
||||
matrix: { enabled: true },
|
||||
},
|
||||
},
|
||||
channels: {
|
||||
...baseCfg.channels,
|
||||
matrix: {
|
||||
enabled: true,
|
||||
defaultAccount: params.sutAccountId,
|
||||
accounts: {
|
||||
[params.sutAccountId]: {
|
||||
accessToken: params.sutAccessToken,
|
||||
...(params.sutDeviceId ? { deviceId: params.sutDeviceId } : {}),
|
||||
dm: { enabled: false },
|
||||
enabled: true,
|
||||
encryption: false,
|
||||
groupAllowFrom: [params.driverUserId],
|
||||
groupPolicy: "allowlist",
|
||||
groups: {
|
||||
[params.roomId]: {
|
||||
enabled: true,
|
||||
requireMention: true,
|
||||
},
|
||||
},
|
||||
homeserver: params.homeserver,
|
||||
network: {
|
||||
dangerouslyAllowPrivateNetwork: true,
|
||||
},
|
||||
replyToMode: "off",
|
||||
threadReplies: "inbound",
|
||||
userId: params.sutUserId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildObservedEventsArtifact(params: {
|
||||
includeContent: boolean;
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
}) {
|
||||
return params.observedEvents.map((event) =>
|
||||
params.includeContent
|
||||
? event
|
||||
: {
|
||||
roomId: event.roomId,
|
||||
eventId: event.eventId,
|
||||
sender: event.sender,
|
||||
stateKey: event.stateKey,
|
||||
type: event.type,
|
||||
originServerTs: event.originServerTs,
|
||||
msgtype: event.msgtype,
|
||||
membership: event.membership,
|
||||
relatesTo: event.relatesTo,
|
||||
mentions: event.mentions,
|
||||
reaction: event.reaction,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function isMatrixAccountReady(entry?: {
|
||||
connected?: boolean;
|
||||
healthState?: string;
|
||||
restartPending?: boolean;
|
||||
running?: boolean;
|
||||
}): boolean {
|
||||
return (
|
||||
entry?.running === true &&
|
||||
entry.connected === true &&
|
||||
entry.restartPending !== true &&
|
||||
(entry.healthState === undefined || entry.healthState === "healthy")
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForMatrixChannelReady(
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
|
||||
accountId: string,
|
||||
opts?: {
|
||||
pollMs?: number;
|
||||
timeoutMs?: number;
|
||||
},
|
||||
) {
|
||||
const pollMs = opts?.pollMs ?? 500;
|
||||
const timeoutMs = opts?.timeoutMs ?? 60_000;
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
try {
|
||||
const payload = (await gateway.call(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
)) as {
|
||||
channelAccounts?: Record<
|
||||
string,
|
||||
Array<{
|
||||
accountId?: string;
|
||||
connected?: boolean;
|
||||
healthState?: string;
|
||||
restartPending?: boolean;
|
||||
running?: boolean;
|
||||
}>
|
||||
>;
|
||||
};
|
||||
const accounts = payload.channelAccounts?.matrix ?? [];
|
||||
const match = accounts.find((entry) => entry.accountId === accountId);
|
||||
if (isMatrixAccountReady(match)) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// retry
|
||||
}
|
||||
await sleep(pollMs);
|
||||
}
|
||||
throw new Error(`matrix account "${accountId}" did not become ready`);
|
||||
}
|
||||
|
||||
export async function runMatrixQaLive(params: {
|
||||
fastMode?: boolean;
|
||||
outputDir?: string;
|
||||
primaryModel?: string;
|
||||
providerMode?: QaProviderModeInput;
|
||||
repoRoot?: string;
|
||||
scenarioIds?: string[];
|
||||
sutAccountId?: string;
|
||||
alternateModel?: string;
|
||||
}): Promise<MatrixQaRunResult> {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
const outputDir =
|
||||
params.outputDir ??
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
|
||||
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
const scenarios = findMatrixQaScenarios(params.scenarioIds);
|
||||
const observedEvents: MatrixQaObservedEvent[] = [];
|
||||
const includeObservedEventContent = process.env.OPENCLAW_QA_MATRIX_CAPTURE_CONTENT === "1";
|
||||
const startedAtDate = new Date();
|
||||
const startedAt = startedAtDate.toISOString();
|
||||
const runSuffix = randomUUID().slice(0, 8);
|
||||
|
||||
const harness = await startMatrixQaHarness({
|
||||
outputDir: path.join(outputDir, "matrix-harness"),
|
||||
repoRoot,
|
||||
});
|
||||
const provisioning: MatrixQaProvisionResult = await (async () => {
|
||||
try {
|
||||
return await provisionMatrixQaRoom({
|
||||
baseUrl: harness.baseUrl,
|
||||
driverLocalpart: `qa-driver-${runSuffix}`,
|
||||
observerLocalpart: `qa-observer-${runSuffix}`,
|
||||
registrationToken: harness.registrationToken,
|
||||
roomName: `OpenClaw Matrix QA ${runSuffix}`,
|
||||
sutLocalpart: `qa-sut-${runSuffix}`,
|
||||
});
|
||||
} catch (error) {
|
||||
await harness.stop().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
|
||||
const checks: QaReportCheck[] = [
|
||||
{
|
||||
name: "Matrix harness ready",
|
||||
status: "pass",
|
||||
details: [
|
||||
`image: ${harness.image}`,
|
||||
`baseUrl: ${harness.baseUrl}`,
|
||||
`serverName: ${harness.serverName}`,
|
||||
`roomId: ${provisioning.roomId}`,
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
const scenarioResults: MatrixQaScenarioResult[] = [];
|
||||
const cleanupErrors: string[] = [];
|
||||
let canaryArtifact: MatrixQaCanaryArtifact | undefined;
|
||||
let gatewayHarness: Awaited<ReturnType<typeof startQaLiveLaneGateway>> | null = null;
|
||||
let canaryFailed = false;
|
||||
const syncState: { driver?: string; observer?: string } = {};
|
||||
|
||||
try {
|
||||
gatewayHarness = await startQaLiveLaneGateway({
|
||||
repoRoot,
|
||||
transport: {
|
||||
requiredPluginIds: [],
|
||||
createGatewayConfig: () => ({}),
|
||||
},
|
||||
transportBaseUrl: "http://127.0.0.1:43123",
|
||||
providerMode,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: params.fastMode,
|
||||
controlUiEnabled: false,
|
||||
mutateConfig: (cfg) =>
|
||||
buildMatrixQaConfig(cfg, {
|
||||
driverUserId: provisioning.driver.userId,
|
||||
homeserver: harness.baseUrl,
|
||||
roomId: provisioning.roomId,
|
||||
sutAccessToken: provisioning.sut.accessToken,
|
||||
sutAccountId,
|
||||
sutDeviceId: provisioning.sut.deviceId,
|
||||
sutUserId: provisioning.sut.userId,
|
||||
}),
|
||||
});
|
||||
await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId);
|
||||
checks.push({
|
||||
name: "Matrix channel ready",
|
||||
status: "pass",
|
||||
details: `accountId: ${sutAccountId}\nuserId: ${provisioning.sut.userId}`,
|
||||
});
|
||||
|
||||
try {
|
||||
const canary = await runMatrixQaCanary({
|
||||
baseUrl: harness.baseUrl,
|
||||
driverAccessToken: provisioning.driver.accessToken,
|
||||
observedEvents,
|
||||
roomId: provisioning.roomId,
|
||||
syncState,
|
||||
sutUserId: provisioning.sut.userId,
|
||||
timeoutMs: 45_000,
|
||||
});
|
||||
canaryArtifact = {
|
||||
driverEventId: canary.driverEventId,
|
||||
reply: canary.reply,
|
||||
token: canary.token,
|
||||
};
|
||||
checks.push({
|
||||
name: "Matrix canary",
|
||||
status: "pass",
|
||||
details: buildMatrixReplyDetails("reply", canary.reply).join("\n"),
|
||||
});
|
||||
} catch (error) {
|
||||
canaryFailed = true;
|
||||
checks.push({
|
||||
name: "Matrix canary",
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
}
|
||||
|
||||
if (!canaryFailed) {
|
||||
for (const scenario of scenarios) {
|
||||
try {
|
||||
const result = await runMatrixQaScenario(scenario, {
|
||||
baseUrl: harness.baseUrl,
|
||||
canary: canaryArtifact,
|
||||
driverAccessToken: provisioning.driver.accessToken,
|
||||
driverUserId: provisioning.driver.userId,
|
||||
observedEvents,
|
||||
observerAccessToken: provisioning.observer.accessToken,
|
||||
observerUserId: provisioning.observer.userId,
|
||||
restartGateway: async () => {
|
||||
if (!gatewayHarness) {
|
||||
throw new Error("Matrix restart scenario requires a live gateway");
|
||||
}
|
||||
await gatewayHarness.gateway.restart();
|
||||
await waitForMatrixChannelReady(gatewayHarness.gateway, sutAccountId);
|
||||
},
|
||||
roomId: provisioning.roomId,
|
||||
syncState,
|
||||
sutUserId: provisioning.sut.userId,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
scenarioResults.push({
|
||||
artifacts: result.artifacts,
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: result.details,
|
||||
});
|
||||
} catch (error) {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (gatewayHarness) {
|
||||
try {
|
||||
await gatewayHarness.stop();
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupErrors, "live gateway cleanup", error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await harness.stop();
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupErrors, "Matrix harness cleanup", error);
|
||||
}
|
||||
}
|
||||
if (cleanupErrors.length > 0) {
|
||||
checks.push({
|
||||
name: "Matrix cleanup",
|
||||
status: "fail",
|
||||
details: cleanupErrors.join("\n"),
|
||||
});
|
||||
}
|
||||
|
||||
const finishedAtDate = new Date();
|
||||
const finishedAt = finishedAtDate.toISOString();
|
||||
const reportPath = path.join(outputDir, "matrix-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "matrix-qa-summary.json");
|
||||
const observedEventsPath = path.join(outputDir, "matrix-qa-observed-events.json");
|
||||
const artifactPaths = {
|
||||
observedEvents: observedEventsPath,
|
||||
report: reportPath,
|
||||
summary: summaryPath,
|
||||
} satisfies MatrixQaArtifactPaths;
|
||||
const report = renderQaMarkdownReport({
|
||||
title: "Matrix QA Report",
|
||||
startedAt: startedAtDate,
|
||||
finishedAt: finishedAtDate,
|
||||
checks,
|
||||
scenarios: scenarioResults.map((scenario) => ({
|
||||
details: scenario.details,
|
||||
name: scenario.title,
|
||||
status: scenario.status,
|
||||
})),
|
||||
notes: [
|
||||
`roomId: ${provisioning.roomId}`,
|
||||
`driver: ${provisioning.driver.userId}`,
|
||||
`observer: ${provisioning.observer.userId}`,
|
||||
`sut: ${provisioning.sut.userId}`,
|
||||
`homeserver: ${harness.baseUrl}`,
|
||||
`image: ${harness.image}`,
|
||||
],
|
||||
});
|
||||
const summary: MatrixQaSummary = buildMatrixQaSummary({
|
||||
artifactPaths,
|
||||
canary: canaryArtifact,
|
||||
checks,
|
||||
finishedAt,
|
||||
harness: {
|
||||
baseUrl: harness.baseUrl,
|
||||
composeFile: harness.composeFile,
|
||||
image: harness.image,
|
||||
roomId: provisioning.roomId,
|
||||
serverName: harness.serverName,
|
||||
},
|
||||
observedEventCount: observedEvents.length,
|
||||
scenarios: scenarioResults,
|
||||
startedAt,
|
||||
sutAccountId,
|
||||
userIds: {
|
||||
driver: provisioning.driver.userId,
|
||||
observer: provisioning.observer.userId,
|
||||
sut: provisioning.sut.userId,
|
||||
},
|
||||
});
|
||||
|
||||
await fs.writeFile(reportPath, `${report}\n`, { encoding: "utf8", mode: 0o600 });
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
await fs.writeFile(
|
||||
observedEventsPath,
|
||||
`${JSON.stringify(
|
||||
buildObservedEventsArtifact({
|
||||
includeContent: includeObservedEventContent,
|
||||
observedEvents,
|
||||
}),
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
|
||||
const failedChecks = checks.filter(
|
||||
(check) => check.status === "fail" && check.name !== "Matrix cleanup",
|
||||
);
|
||||
const failedScenarios = scenarioResults.filter((scenario) => scenario.status === "fail");
|
||||
if (failedChecks.length > 0 || failedScenarios.length > 0) {
|
||||
throw new Error(
|
||||
buildLiveLaneArtifactsError({
|
||||
heading: "Matrix QA failed.",
|
||||
details: [
|
||||
...failedChecks.map((check) => `check ${check.name}: ${check.details ?? "failed"}`),
|
||||
...failedScenarios.map((scenario) => `scenario ${scenario.id}: ${scenario.details}`),
|
||||
...cleanupErrors.map((error) => `cleanup: ${error}`),
|
||||
],
|
||||
artifacts: artifactPaths,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (cleanupErrors.length > 0) {
|
||||
throw new Error(
|
||||
buildLiveLaneArtifactsError({
|
||||
heading: "Matrix QA cleanup failed after artifacts were written.",
|
||||
details: cleanupErrors,
|
||||
artifacts: artifactPaths,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
observedEventsPath,
|
||||
outputDir,
|
||||
reportPath,
|
||||
scenarios: scenarioResults,
|
||||
summaryPath,
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
buildMatrixQaSummary,
|
||||
MATRIX_QA_SCENARIOS,
|
||||
buildMatrixQaConfig,
|
||||
buildObservedEventsArtifact,
|
||||
isMatrixAccountReady,
|
||||
waitForMatrixChannelReady,
|
||||
};
|
||||
@@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = {
|
||||
register(qa: Command): void;
|
||||
};
|
||||
|
||||
export type LiveTransportQaCredentialCliOptions = {
|
||||
sourceDescription?: string;
|
||||
roleDescription?: string;
|
||||
};
|
||||
|
||||
export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
|
||||
let promise: Promise<T> | null = null;
|
||||
return async () => {
|
||||
@@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions(
|
||||
export function registerLiveTransportQaCli(params: {
|
||||
qa: Command;
|
||||
commandName: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
description: string;
|
||||
outputDirHelp: string;
|
||||
scenarioHelp: string;
|
||||
sutAccountHelp: string;
|
||||
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
|
||||
}) {
|
||||
params.qa
|
||||
const command = params.qa
|
||||
.command(params.commandName)
|
||||
.description(params.description)
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
@@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: {
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--scenario <id>", params.scenarioHelp, collectString, [])
|
||||
.option("--fast", "Enable provider fast mode where supported", false)
|
||||
.option("--sut-account <id>", params.sutAccountHelp, "sut")
|
||||
.option(
|
||||
.option("--sut-account <id>", params.sutAccountHelp, "sut");
|
||||
|
||||
if (params.credentialOptions) {
|
||||
command.option(
|
||||
"--credential-source <source>",
|
||||
"Credential source for live lanes: env or convex (default: env)",
|
||||
)
|
||||
.option(
|
||||
"--credential-role <role>",
|
||||
"Credential role for convex auth: maintainer or ci (default: maintainer)",
|
||||
)
|
||||
.action(async (opts: LiveTransportQaCommanderOptions) => {
|
||||
await params.run(mapLiveTransportQaCommanderOptions(opts));
|
||||
});
|
||||
params.credentialOptions.sourceDescription ??
|
||||
"Credential source for live lanes: env or convex (default: env)",
|
||||
);
|
||||
if (params.credentialOptions.roleDescription) {
|
||||
command.option("--credential-role <role>", params.credentialOptions.roleDescription);
|
||||
}
|
||||
}
|
||||
|
||||
command.action(async (opts: LiveTransportQaCommanderOptions) => {
|
||||
await params.run(mapLiveTransportQaCommanderOptions(opts));
|
||||
});
|
||||
}
|
||||
|
||||
export function createLiveTransportQaCliRegistration(params: {
|
||||
commandName: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
description: string;
|
||||
outputDirHelp: string;
|
||||
scenarioHelp: string;
|
||||
@@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: {
|
||||
registerLiveTransportQaCli({
|
||||
qa,
|
||||
commandName: params.commandName,
|
||||
credentialOptions: params.credentialOptions,
|
||||
description: params.description,
|
||||
outputDirHelp: params.outputDirHelp,
|
||||
scenarioHelp: params.scenarioHelp,
|
||||
|
||||
@@ -20,6 +20,10 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
|
||||
export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
createLiveTransportQaCliRegistration({
|
||||
commandName: "telegram",
|
||||
credentialOptions: {
|
||||
sourceDescription: "Credential source for Telegram QA: env or convex (default: env)",
|
||||
roleDescription: "Credential role for convex auth: maintainer or ci (default: maintainer)",
|
||||
},
|
||||
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
|
||||
outputDirHelp: "Telegram QA artifact directory",
|
||||
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
|
||||
|
||||
@@ -3,6 +3,7 @@ export type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
export { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime";
|
||||
export type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
|
||||
export { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
export {
|
||||
buildQaTarget,
|
||||
createQaBusThread,
|
||||
|
||||
@@ -81,7 +81,7 @@ export async function runQaSelfCheckAgainstState(params: {
|
||||
timeline,
|
||||
notes: params.notes ?? [
|
||||
"Vertical slice: qa-channel + qa-lab bus + private debugger surface.",
|
||||
"Docker orchestration, matrix runs, and auto-fix loops remain follow-up work.",
|
||||
"Docker orchestration, additional QA runners, and auto-fix loops remain follow-up work.",
|
||||
],
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user