mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:50:43 +00:00
test(qa-matrix): cover approval metadata scenarios
This commit is contained in:
Gustavo Madeira Santana
@@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai
|
||||
### Changes
|
||||
|
||||
- Dependencies: refresh provider and tooling dependencies, including AWS SDK, PI runtime packages, AJV, Feishu SDK, Anthropic SDK, tokenjuice, and native TypeScript/oxlint tooling. Thanks @dependabot.
|
||||
- Matrix/QA: add live Matrix approval scenarios for exec metadata, chunked fallback, plugin approvals, deny reactions, thread targeting, and `target: "both"` delivery, with redacted artifacts preserving safe approval summaries. Thanks @gumadeiras.
|
||||
- Codex: add Computer Use setup for Codex-mode agents, including `/codex computer-use status/install`, marketplace discovery, optional auto-install, and fail-closed MCP server checks before Codex-mode turns start. Fixes #72094. (#71842) Thanks @pash-openai.
|
||||
- Apps: consume Peekaboo 3.0.0-beta4 and ElevenLabsKit 0.1.1, align Swabble on Commander 0.2.2, and refresh macOS/iOS SwiftPM resolutions against the released dependency graph. Thanks @Blaizzy.
|
||||
- Plugin SDK: expose shared channel route normalization, parser-driven target resolution, raw-target compact keys, parsed-target types, and route comparison helpers through `openclaw/plugin-sdk/channel-route`, switch native approval origin matching onto that route contract with optional delivery and match-only target normalization, and retire the internal channel-route shim behind dated compatibility aliases for legacy key/comparable-target helpers. Thanks @vincentkoc.
|
||||
|
||||
@@ -64,15 +64,15 @@ Matrix QA does not accept `--credential-source` or `--credential-role`. The lane
|
||||
|
||||
The selected profile decides which scenarios run.
|
||||
|
||||
| Profile | Use it for |
|
||||
| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `all` (default) | Full catalog. Slow but exhaustive. |
|
||||
| `fast` | Release-gate subset that exercises the live transport contract: canary, mention gating, allowlist block, reply shape, restart resume, thread follow-up, thread isolation, reaction observation. |
|
||||
| `transport` | Transport-level threading, DM, room, autojoin, mention/allowlist scenarios. |
|
||||
| `media` | Image, audio, video, PDF, EPUB attachment coverage. |
|
||||
| `e2ee-smoke` | Minimum E2EE coverage — basic encrypted reply, thread follow-up, bootstrap success. |
|
||||
| `e2ee-deep` | Exhaustive E2EE state-loss, backup, key, and recovery scenarios. |
|
||||
| `e2ee-cli` | `openclaw matrix encryption setup` and `verify *` CLI scenarios driven through the QA harness. |
|
||||
| Profile | Use it for |
|
||||
| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `all` (default) | Full catalog. Slow but exhaustive. |
|
||||
| `fast` | Release-gate subset that exercises the live transport contract: canary, mention gating, allowlist block, reply shape, restart resume, thread follow-up, thread isolation, reaction observation, and exec approval metadata delivery. |
|
||||
| `transport` | Transport-level threading, DM, room, autojoin, mention/allowlist, approval, and reaction scenarios. |
|
||||
| `media` | Image, audio, video, PDF, EPUB attachment coverage. |
|
||||
| `e2ee-smoke` | Minimum E2EE coverage — basic encrypted reply, thread follow-up, bootstrap success. |
|
||||
| `e2ee-deep` | Exhaustive E2EE state-loss, backup, key, and recovery scenarios. |
|
||||
| `e2ee-cli` | `openclaw matrix encryption setup` and `verify *` CLI scenarios driven through the QA harness. |
|
||||
|
||||
The exact mapping lives in `extensions/qa-matrix/src/runners/contract/scenario-catalog.ts`.
|
||||
|
||||
@@ -86,6 +86,7 @@ The full scenario id list is the `MatrixQaScenarioId` union in `extensions/qa-ma
|
||||
- media — `matrix-media-type-coverage`, `matrix-room-image-understanding-attachment`, `matrix-attachment-only-ignored`, `matrix-unsupported-media-safe`
|
||||
- routing — `matrix-room-autojoin-invite`, `matrix-secondary-room-*`
|
||||
- reactions — `matrix-reaction-*`
|
||||
- approvals — `matrix-approval-*` (exec/plugin metadata, chunked fallback, deny reactions, threads, and `target: "both"` routing)
|
||||
- restart and replay — `matrix-restart-*`, `matrix-stale-sync-replay-dedupe`, `matrix-room-membership-loss`, `matrix-homeserver-restart-resume`, `matrix-initial-catchup-then-incremental`
|
||||
- mention gating and allowlists — `matrix-mention-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override`
|
||||
- E2EE — `matrix-e2ee-*` (basic reply, thread follow-up, bootstrap, recovery key lifecycle, state-loss variants, server backup behavior, device hygiene, SAS / QR / DM verification, restart, artifact redaction)
|
||||
@@ -112,7 +113,7 @@ Written to `--output-dir`:
|
||||
|
||||
- `matrix-qa-report.md` — Markdown protocol report (what passed, failed, was skipped, and why).
|
||||
- `matrix-qa-summary.json` — Structured summary suitable for CI parsing and dashboards.
|
||||
- `matrix-qa-observed-events.json` — Observed Matrix events from the driver and observer clients. Bodies are redacted unless `OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1`.
|
||||
- `matrix-qa-observed-events.json` — Observed Matrix events from the driver and observer clients. Bodies are redacted unless `OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1`; approval metadata is summarized with selected safe fields and truncated command preview.
|
||||
- `matrix-qa-output.log` — Combined stdout/stderr from the run. If `OPENCLAW_RUN_NODE_OUTPUT_LOG` is set, the outer launcher's log is reused instead.
|
||||
|
||||
The default output dir is `<repo>/.artifacts/qa-e2e/matrix-<timestamp>` so successive runs do not overwrite each other.
|
||||
|
||||
@@ -40,7 +40,7 @@ type MatrixQaGatewayChild = {
|
||||
call(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
options?: { timeoutMs?: number },
|
||||
options?: { expectFinal?: boolean; timeoutMs?: number },
|
||||
): Promise<unknown>;
|
||||
restartAfterStateMutation?: (
|
||||
mutateState: (context: { stateDir: string }) => Promise<void>,
|
||||
@@ -789,6 +789,8 @@ export async function runMatrixQaLive(params: {
|
||||
observerUserId: provisioning.observer.userId,
|
||||
gatewayRuntimeEnv: scenarioGateway.harness.gateway.runtimeEnv,
|
||||
gatewayStateDir: scenarioGateway.harness.gateway.runtimeEnv?.OPENCLAW_STATE_DIR,
|
||||
gatewayCall: async (method, params, opts) =>
|
||||
await scenarioGateway.harness.gateway.call(method, params ?? {}, opts),
|
||||
outputDir,
|
||||
registrationToken: harness.registrationToken,
|
||||
restartGateway: async () => {
|
||||
|
||||
@@ -43,6 +43,12 @@ export type MatrixQaScenarioId =
|
||||
| "matrix-reaction-threaded"
|
||||
| "matrix-reaction-not-a-reply"
|
||||
| "matrix-reaction-redaction-observed"
|
||||
| "matrix-approval-exec-metadata-single-event"
|
||||
| "matrix-approval-exec-metadata-chunked"
|
||||
| "matrix-approval-plugin-metadata-single-event"
|
||||
| "matrix-approval-deny-reaction"
|
||||
| "matrix-approval-thread-target"
|
||||
| "matrix-approval-channel-target-both"
|
||||
| "matrix-restart-resume"
|
||||
| "matrix-post-restart-room-continue"
|
||||
| "matrix-initial-catchup-then-incremental"
|
||||
@@ -266,6 +272,51 @@ const MATRIX_QA_E2EE_CLI_SETUP_CONFIG = {
|
||||
startupVerification: "off",
|
||||
} satisfies MatrixQaConfigOverrides;
|
||||
|
||||
const MATRIX_QA_APPROVAL_CHANNEL_CONFIG = {
|
||||
approvalForwarding: {
|
||||
exec: true,
|
||||
},
|
||||
dm: {
|
||||
enabled: true,
|
||||
},
|
||||
execApprovals: {
|
||||
enabled: true,
|
||||
target: "channel",
|
||||
},
|
||||
} satisfies MatrixQaConfigOverrides;
|
||||
|
||||
const MATRIX_QA_APPROVAL_CHUNKED_CONFIG = {
|
||||
...MATRIX_QA_APPROVAL_CHANNEL_CONFIG,
|
||||
chunkMode: "length",
|
||||
textChunkLimit: 280,
|
||||
} satisfies MatrixQaConfigOverrides;
|
||||
|
||||
const MATRIX_QA_APPROVAL_PLUGIN_CONFIG = {
|
||||
approvalForwarding: {
|
||||
plugin: true,
|
||||
},
|
||||
dm: {
|
||||
enabled: true,
|
||||
},
|
||||
execApprovals: {
|
||||
enabled: true,
|
||||
target: "channel",
|
||||
},
|
||||
} satisfies MatrixQaConfigOverrides;
|
||||
|
||||
const MATRIX_QA_APPROVAL_BOTH_CONFIG = {
|
||||
approvalForwarding: {
|
||||
exec: true,
|
||||
},
|
||||
dm: {
|
||||
enabled: true,
|
||||
},
|
||||
execApprovals: {
|
||||
enabled: true,
|
||||
target: "both",
|
||||
},
|
||||
} satisfies MatrixQaConfigOverrides;
|
||||
|
||||
export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
|
||||
{
|
||||
id: "matrix-thread-follow-up",
|
||||
@@ -518,6 +569,43 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
|
||||
timeoutMs: 45_000,
|
||||
title: "Matrix reaction removals are observed as redactions",
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-exec-metadata-single-event",
|
||||
timeoutMs: 75_000,
|
||||
title: "Matrix exec approval prompt carries structured metadata on one event",
|
||||
configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-exec-metadata-chunked",
|
||||
timeoutMs: 90_000,
|
||||
title: "Matrix exec approval prompt fallback keeps metadata on the first chunk",
|
||||
configOverrides: MATRIX_QA_APPROVAL_CHUNKED_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-plugin-metadata-single-event",
|
||||
timeoutMs: 75_000,
|
||||
title: "Matrix plugin approval prompt carries plugin metadata",
|
||||
configOverrides: MATRIX_QA_APPROVAL_PLUGIN_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-deny-reaction",
|
||||
timeoutMs: 75_000,
|
||||
title: "Matrix approval deny reaction resolves the metadata-bearing event",
|
||||
configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-thread-target",
|
||||
timeoutMs: 75_000,
|
||||
title: "Matrix approval prompt preserves thread targeting metadata",
|
||||
configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-approval-channel-target-both",
|
||||
timeoutMs: 90_000,
|
||||
title: "Matrix approval target=both delivers channel and DM metadata once",
|
||||
topology: MATRIX_QA_DRIVER_DM_TOPOLOGY,
|
||||
configOverrides: MATRIX_QA_APPROVAL_BOTH_CONFIG,
|
||||
},
|
||||
{
|
||||
id: "matrix-restart-resume",
|
||||
standardId: "restart-resume",
|
||||
@@ -985,6 +1073,8 @@ const MATRIX_QA_FAST_PROFILE_SCENARIO_IDS = [
|
||||
"matrix-thread-isolation",
|
||||
"matrix-top-level-reply-shape",
|
||||
"matrix-reaction-notification",
|
||||
"matrix-approval-exec-metadata-single-event",
|
||||
"matrix-approval-exec-metadata-chunked",
|
||||
"matrix-restart-resume",
|
||||
"matrix-mention-gating",
|
||||
"matrix-allowlist-block",
|
||||
|
||||
@@ -0,0 +1,599 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
|
||||
import { MATRIX_QA_DRIVER_DM_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js";
|
||||
import {
|
||||
advanceMatrixQaActorCursor,
|
||||
buildMatrixQaToken,
|
||||
createMatrixQaScenarioClient,
|
||||
primeMatrixQaDriverScenarioClient,
|
||||
type MatrixQaScenarioContext,
|
||||
} from "./scenario-runtime-shared.js";
|
||||
import type { MatrixQaScenarioExecution } from "./scenario-types.js";
|
||||
|
||||
const MATRIX_QA_APPROVAL_ALLOW_ONCE_REACTION = "✅";
|
||||
const MATRIX_QA_APPROVAL_DENY_REACTION = "❌";
|
||||
const MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS = 30_000;
|
||||
const MATRIX_QA_APPROVAL_SHORT_WINDOW_MS = 4_000;
|
||||
const MATRIX_QA_APPROVAL_LONG_COMMAND_TEXT = "matrix approval chunk fallback ".repeat(40);
|
||||
|
||||
type MatrixQaApprovalDecision = "allow-once" | "deny";
|
||||
type MatrixQaApprovalKind = "exec" | "plugin";
|
||||
type MatrixQaApprovalOptionReactionParams = {
|
||||
context: MatrixQaScenarioContext;
|
||||
emoji: string;
|
||||
roomId: string;
|
||||
targetEventId: string;
|
||||
};
|
||||
|
||||
function requireMatrixQaGatewayCall(context: MatrixQaScenarioContext) {
|
||||
if (!context.gatewayCall) {
|
||||
throw new Error("Matrix approval QA scenario requires a live gateway RPC client");
|
||||
}
|
||||
return context.gatewayCall;
|
||||
}
|
||||
|
||||
function buildMatrixApprovalArtifact(event: MatrixQaObservedEvent) {
|
||||
if (!event.approval) {
|
||||
throw new Error(`Matrix event ${event.eventId} did not include approval metadata`);
|
||||
}
|
||||
return {
|
||||
...event.approval,
|
||||
eventId: event.eventId,
|
||||
roomId: event.roomId,
|
||||
};
|
||||
}
|
||||
|
||||
function isApprovalOptionReaction(
|
||||
event: MatrixQaObservedEvent,
|
||||
params: MatrixQaApprovalOptionReactionParams,
|
||||
) {
|
||||
return (
|
||||
event.roomId === params.roomId &&
|
||||
event.sender === params.context.sutUserId &&
|
||||
event.type === "m.reaction" &&
|
||||
event.reaction?.eventId === params.targetEventId &&
|
||||
event.reaction.key === params.emoji
|
||||
);
|
||||
}
|
||||
|
||||
function hasObservedApprovalOptionReaction(params: MatrixQaApprovalOptionReactionParams) {
|
||||
return params.context.observedEvents.some((event) => isApprovalOptionReaction(event, params));
|
||||
}
|
||||
|
||||
function assertApprovalMetadata(params: {
|
||||
event: { approval?: unknown; eventId: string };
|
||||
expectedKind: MatrixQaApprovalKind;
|
||||
}) {
|
||||
const approval =
|
||||
typeof params.event.approval === "object" && params.event.approval !== null
|
||||
? (params.event.approval as {
|
||||
allowedDecisions?: string[];
|
||||
hasCommandText?: boolean;
|
||||
id?: string;
|
||||
kind?: string;
|
||||
state?: string;
|
||||
type?: string;
|
||||
version?: number;
|
||||
})
|
||||
: null;
|
||||
if (!approval) {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose metadata`);
|
||||
}
|
||||
if (approval.kind !== params.expectedKind) {
|
||||
throw new Error(
|
||||
`approval event ${params.event.eventId} kind was ${approval.kind ?? "<missing>"} instead of ${params.expectedKind}`,
|
||||
);
|
||||
}
|
||||
if (!approval.id) {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose an approval id`);
|
||||
}
|
||||
if (approval.version !== 1) {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose version=1`);
|
||||
}
|
||||
if (approval.type !== "approval.request") {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose type=approval.request`);
|
||||
}
|
||||
if (approval.state !== "pending") {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose state=pending`);
|
||||
}
|
||||
if (!approval.allowedDecisions?.includes("deny")) {
|
||||
throw new Error(`approval event ${params.event.eventId} did not include deny`);
|
||||
}
|
||||
if (
|
||||
params.expectedKind === "exec" &&
|
||||
(!approval.allowedDecisions.includes("allow-once") || approval.hasCommandText !== true)
|
||||
) {
|
||||
throw new Error(`approval event ${params.event.eventId} did not expose exec approval fields`);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForApprovalEvent(params: {
|
||||
context: MatrixQaScenarioContext;
|
||||
expectedApprovalId: string;
|
||||
expectedKind: MatrixQaApprovalKind;
|
||||
roomId: string;
|
||||
since?: string;
|
||||
threadRootEventId?: string;
|
||||
}) {
|
||||
const client = createMatrixQaScenarioClient({
|
||||
accessToken: params.context.driverAccessToken,
|
||||
baseUrl: params.context.baseUrl,
|
||||
});
|
||||
const matched = await client.waitForRoomEvent({
|
||||
observedEvents: params.context.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === params.roomId &&
|
||||
event.sender === params.context.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
event.approval?.kind === params.expectedKind &&
|
||||
event.approval.id === params.expectedApprovalId &&
|
||||
(!params.threadRootEventId || event.relatesTo?.eventId === params.threadRootEventId),
|
||||
roomId: params.roomId,
|
||||
since: params.since,
|
||||
timeoutMs: params.context.timeoutMs,
|
||||
});
|
||||
assertApprovalMetadata({
|
||||
event: matched.event,
|
||||
expectedKind: params.expectedKind,
|
||||
});
|
||||
return matched;
|
||||
}
|
||||
|
||||
async function reactToApproval(params: {
|
||||
context: MatrixQaScenarioContext;
|
||||
decision: MatrixQaApprovalDecision;
|
||||
roomId: string;
|
||||
targetEventId: string;
|
||||
}) {
|
||||
const client = createMatrixQaScenarioClient({
|
||||
accessToken: params.context.driverAccessToken,
|
||||
baseUrl: params.context.baseUrl,
|
||||
});
|
||||
const emoji =
|
||||
params.decision === "allow-once"
|
||||
? MATRIX_QA_APPROVAL_ALLOW_ONCE_REACTION
|
||||
: MATRIX_QA_APPROVAL_DENY_REACTION;
|
||||
if (
|
||||
!hasObservedApprovalOptionReaction({
|
||||
context: params.context,
|
||||
emoji,
|
||||
roomId: params.roomId,
|
||||
targetEventId: params.targetEventId,
|
||||
})
|
||||
) {
|
||||
await client.waitForRoomEvent({
|
||||
observedEvents: params.context.observedEvents,
|
||||
predicate: (event) =>
|
||||
isApprovalOptionReaction(event, {
|
||||
context: params.context,
|
||||
emoji,
|
||||
roomId: params.roomId,
|
||||
targetEventId: params.targetEventId,
|
||||
}),
|
||||
roomId: params.roomId,
|
||||
timeoutMs: params.context.timeoutMs,
|
||||
});
|
||||
}
|
||||
const eventId = await client.sendReaction({
|
||||
emoji,
|
||||
messageId: params.targetEventId,
|
||||
roomId: params.roomId,
|
||||
});
|
||||
return {
|
||||
eventId,
|
||||
reaction: {
|
||||
eventId: params.targetEventId,
|
||||
key: emoji,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertApprovalDecisionResult(params: {
|
||||
approvalId: string;
|
||||
decision: MatrixQaApprovalDecision;
|
||||
result: unknown;
|
||||
}) {
|
||||
const result =
|
||||
typeof params.result === "object" && params.result !== null
|
||||
? (params.result as { decision?: unknown; id?: unknown })
|
||||
: null;
|
||||
if (result?.id !== params.approvalId) {
|
||||
throw new Error(
|
||||
`approval decision result id was ${formatApprovalResultValue(result?.id)} instead of ${params.approvalId}`,
|
||||
);
|
||||
}
|
||||
if (result?.decision !== params.decision) {
|
||||
throw new Error(
|
||||
`approval decision was ${formatApprovalResultValue(result?.decision)} instead of ${params.decision}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function formatApprovalResultValue(value: unknown) {
|
||||
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
if (value == null) {
|
||||
return "<missing>";
|
||||
}
|
||||
return JSON.stringify(value) ?? "<unserializable>";
|
||||
}
|
||||
|
||||
async function requestExecApproval(params: {
|
||||
context: MatrixQaScenarioContext;
|
||||
command: string;
|
||||
id?: string;
|
||||
threadRootEventId?: string;
|
||||
}) {
|
||||
const gatewayCall = requireMatrixQaGatewayCall(params.context);
|
||||
return await gatewayCall(
|
||||
"exec.approval.request",
|
||||
{
|
||||
...(params.id ? { id: params.id } : {}),
|
||||
ask: "always",
|
||||
command: params.command,
|
||||
host: "gateway",
|
||||
security: "full",
|
||||
timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS,
|
||||
twoPhase: true,
|
||||
turnSourceAccountId: params.context.sutAccountId,
|
||||
turnSourceChannel: "matrix",
|
||||
turnSourceTo: `room:${params.context.roomId}`,
|
||||
...(params.threadRootEventId ? { turnSourceThreadId: params.threadRootEventId } : {}),
|
||||
},
|
||||
{
|
||||
expectFinal: false,
|
||||
timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function requestPluginApproval(params: { context: MatrixQaScenarioContext; token: string }) {
|
||||
const gatewayCall = requireMatrixQaGatewayCall(params.context);
|
||||
return await gatewayCall(
|
||||
"plugin.approval.request",
|
||||
{
|
||||
agentId: "qa",
|
||||
description: `Matrix plugin approval QA request ${params.token}`,
|
||||
pluginId: "qa-matrix-plugin",
|
||||
severity: "warning",
|
||||
timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS,
|
||||
title: "Matrix plugin approval QA",
|
||||
toolName: "matrix_qa_tool",
|
||||
twoPhase: true,
|
||||
turnSourceAccountId: params.context.sutAccountId,
|
||||
turnSourceChannel: "matrix",
|
||||
turnSourceTo: `room:${params.context.roomId}`,
|
||||
},
|
||||
{
|
||||
expectFinal: false,
|
||||
timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForApprovalDecision(params: {
|
||||
approvalId: string;
|
||||
context: MatrixQaScenarioContext;
|
||||
kind: MatrixQaApprovalKind;
|
||||
}) {
|
||||
const gatewayCall = requireMatrixQaGatewayCall(params.context);
|
||||
const method =
|
||||
params.kind === "exec" ? "exec.approval.waitDecision" : "plugin.approval.waitDecision";
|
||||
return await gatewayCall(
|
||||
method,
|
||||
{ id: params.approvalId },
|
||||
{
|
||||
expectFinal: true,
|
||||
timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function readAcceptedApprovalRequest(result: unknown) {
|
||||
const accepted =
|
||||
typeof result === "object" && result !== null
|
||||
? (result as { id?: unknown; status?: unknown })
|
||||
: null;
|
||||
if (accepted?.status !== "accepted") {
|
||||
throw new Error(
|
||||
`approval request status was ${formatApprovalResultValue(accepted?.status)} instead of accepted`,
|
||||
);
|
||||
}
|
||||
return accepted;
|
||||
}
|
||||
|
||||
function assertAcceptedApprovalRequest(params: { approvalId: string; result: unknown }) {
|
||||
const id = readAcceptedApprovalRequest(params.result).id;
|
||||
if (id !== params.approvalId) {
|
||||
throw new Error(
|
||||
`accepted approval id was ${formatApprovalResultValue(id)} instead of ${params.approvalId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function readAcceptedApprovalRequestId(result: unknown) {
|
||||
const id = readAcceptedApprovalRequest(result).id;
|
||||
if (typeof id !== "string" || !id.trim()) {
|
||||
throw new Error("approval request did not return an accepted approval id");
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
function buildExecApprovalCommand(params: { expectChunk?: boolean; token: string }) {
|
||||
if (!params.expectChunk) {
|
||||
return `printf ${params.token}`;
|
||||
}
|
||||
return `printf '${params.token} ${MATRIX_QA_APPROVAL_LONG_COMMAND_TEXT}'`;
|
||||
}
|
||||
|
||||
async function runExecApprovalScenario(params: {
|
||||
context: MatrixQaScenarioContext;
|
||||
decision: MatrixQaApprovalDecision;
|
||||
expectChunk?: boolean;
|
||||
tokenPrefix: string;
|
||||
threadRootEventId?: string;
|
||||
}) {
|
||||
const { client, startSince } = await primeMatrixQaDriverScenarioClient(params.context);
|
||||
const token = buildMatrixQaToken(params.tokenPrefix);
|
||||
const command = buildExecApprovalCommand({ expectChunk: params.expectChunk, token });
|
||||
const approvalId = `qa-${token.toLowerCase()}-${randomUUID().slice(0, 8)}`;
|
||||
const accepted = await requestExecApproval({
|
||||
context: params.context,
|
||||
command,
|
||||
id: approvalId,
|
||||
threadRootEventId: params.threadRootEventId,
|
||||
});
|
||||
assertAcceptedApprovalRequest({ approvalId, result: accepted });
|
||||
const approval = await waitForApprovalEvent({
|
||||
context: params.context,
|
||||
expectedApprovalId: approvalId,
|
||||
expectedKind: "exec",
|
||||
roomId: params.context.roomId,
|
||||
since: startSince,
|
||||
threadRootEventId: params.threadRootEventId,
|
||||
});
|
||||
if (params.expectChunk) {
|
||||
const chunk = await client.waitForRoomEvent({
|
||||
observedEvents: params.context.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.roomId === params.context.roomId &&
|
||||
event.sender === params.context.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
event.body?.includes(token) === true &&
|
||||
event.eventId !== approval.event.eventId &&
|
||||
event.approval === undefined,
|
||||
roomId: params.context.roomId,
|
||||
timeoutMs: params.context.timeoutMs,
|
||||
});
|
||||
if (chunk.event.approval) {
|
||||
throw new Error(`chunk event ${chunk.event.eventId} unexpectedly duplicated metadata`);
|
||||
}
|
||||
}
|
||||
const reaction = await reactToApproval({
|
||||
context: params.context,
|
||||
decision: params.decision,
|
||||
roomId: params.context.roomId,
|
||||
targetEventId: approval.event.eventId,
|
||||
});
|
||||
const result = await waitForApprovalDecision({
|
||||
approvalId,
|
||||
context: params.context,
|
||||
kind: "exec",
|
||||
});
|
||||
assertApprovalDecisionResult({
|
||||
approvalId,
|
||||
decision: params.decision,
|
||||
result,
|
||||
});
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: "driver",
|
||||
syncState: params.context.syncState,
|
||||
nextSince: approval.since,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
approval: buildMatrixApprovalArtifact(approval.event),
|
||||
reactionEmoji: reaction.reaction?.key,
|
||||
reactionEventId: reaction.eventId,
|
||||
reactionTargetEventId: reaction.reaction?.eventId,
|
||||
token,
|
||||
},
|
||||
details: [
|
||||
`approval event: ${approval.event.eventId}`,
|
||||
`approval id: ${approvalId}`,
|
||||
`approval kind: ${approval.event.approval?.kind ?? "<missing>"}`,
|
||||
`decision: ${params.decision}`,
|
||||
`reaction event: ${reaction.eventId}`,
|
||||
`reaction target: ${reaction.reaction?.eventId ?? "<missing>"}`,
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
export async function runApprovalExecMetadataSingleEventScenario(context: MatrixQaScenarioContext) {
|
||||
return await runExecApprovalScenario({
|
||||
context,
|
||||
decision: "allow-once",
|
||||
tokenPrefix: "MATRIX_QA_APPROVAL_EXEC",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runApprovalExecMetadataChunkedScenario(context: MatrixQaScenarioContext) {
|
||||
return await runExecApprovalScenario({
|
||||
context,
|
||||
decision: "allow-once",
|
||||
expectChunk: true,
|
||||
tokenPrefix: "MATRIX_QA_APPROVAL_CHUNKED",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runApprovalDenyReactionScenario(context: MatrixQaScenarioContext) {
|
||||
return await runExecApprovalScenario({
|
||||
context,
|
||||
decision: "deny",
|
||||
tokenPrefix: "MATRIX_QA_APPROVAL_DENY",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runApprovalThreadTargetScenario(context: MatrixQaScenarioContext) {
|
||||
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
|
||||
const token = buildMatrixQaToken("MATRIX_QA_APPROVAL_THREAD_ROOT");
|
||||
const rootEventId = await client.sendTextMessage({
|
||||
body: `Matrix approval thread root ${token}`,
|
||||
roomId: context.roomId,
|
||||
});
|
||||
const result = await runExecApprovalScenario({
|
||||
context,
|
||||
decision: "allow-once",
|
||||
threadRootEventId: rootEventId,
|
||||
tokenPrefix: "MATRIX_QA_APPROVAL_THREAD",
|
||||
});
|
||||
advanceMatrixQaActorCursor({
|
||||
actorId: "driver",
|
||||
syncState: context.syncState,
|
||||
startSince,
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
...result.artifacts,
|
||||
rootEventId,
|
||||
},
|
||||
details: [result.details, `thread root event: ${rootEventId}`].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
export async function runApprovalPluginMetadataSingleEventScenario(
|
||||
context: MatrixQaScenarioContext,
|
||||
) {
|
||||
const { startSince } = await primeMatrixQaDriverScenarioClient(context);
|
||||
const token = buildMatrixQaToken("MATRIX_QA_PLUGIN_APPROVAL");
|
||||
const accepted = await requestPluginApproval({ context, token });
|
||||
const approvalId = readAcceptedApprovalRequestId(accepted);
|
||||
const approval = await waitForApprovalEvent({
|
||||
context,
|
||||
expectedApprovalId: approvalId,
|
||||
expectedKind: "plugin",
|
||||
roomId: context.roomId,
|
||||
since: startSince,
|
||||
});
|
||||
const approvalMetadata = approval.event.approval;
|
||||
if (
|
||||
approvalMetadata?.pluginId !== "qa-matrix-plugin" ||
|
||||
approvalMetadata.toolName !== "matrix_qa_tool" ||
|
||||
approvalMetadata.severity !== "warning" ||
|
||||
approvalMetadata.agentId !== "qa"
|
||||
) {
|
||||
throw new Error(`plugin approval event ${approval.event.eventId} did not expose plugin fields`);
|
||||
}
|
||||
const reaction = await reactToApproval({
|
||||
context,
|
||||
decision: "allow-once",
|
||||
roomId: context.roomId,
|
||||
targetEventId: approval.event.eventId,
|
||||
});
|
||||
const result = await waitForApprovalDecision({
|
||||
approvalId,
|
||||
context,
|
||||
kind: "plugin",
|
||||
});
|
||||
assertApprovalDecisionResult({
|
||||
approvalId,
|
||||
decision: "allow-once",
|
||||
result,
|
||||
});
|
||||
return {
|
||||
artifacts: {
|
||||
approval: buildMatrixApprovalArtifact(approval.event),
|
||||
reactionEmoji: reaction.reaction?.key,
|
||||
reactionEventId: reaction.eventId,
|
||||
reactionTargetEventId: reaction.reaction?.eventId,
|
||||
token,
|
||||
},
|
||||
details: [
|
||||
`approval event: ${approval.event.eventId}`,
|
||||
`approval id: ${approvalMetadata.id}`,
|
||||
`plugin id: ${approvalMetadata.pluginId ?? "<missing>"}`,
|
||||
`tool name: ${approvalMetadata.toolName ?? "<missing>"}`,
|
||||
`decision: allow-once`,
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
|
||||
export async function runApprovalChannelTargetBothScenario(context: MatrixQaScenarioContext) {
|
||||
const { client, startSince } = await primeMatrixQaDriverScenarioClient(context);
|
||||
const dmRoomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY);
|
||||
const token = buildMatrixQaToken("MATRIX_QA_APPROVAL_BOTH");
|
||||
const approvalId = `qa-${token.toLowerCase()}-${randomUUID().slice(0, 8)}`;
|
||||
const accepted = await requestExecApproval({
|
||||
context,
|
||||
command: `printf ${token}`,
|
||||
id: approvalId,
|
||||
});
|
||||
assertAcceptedApprovalRequest({ approvalId, result: accepted });
|
||||
const channelApproval = await waitForApprovalEvent({
|
||||
context,
|
||||
expectedApprovalId: approvalId,
|
||||
expectedKind: "exec",
|
||||
roomId: context.roomId,
|
||||
since: startSince,
|
||||
});
|
||||
const dmApproval = await waitForApprovalEvent({
|
||||
context,
|
||||
expectedApprovalId: approvalId,
|
||||
expectedKind: "exec",
|
||||
roomId: dmRoomId,
|
||||
since: startSince,
|
||||
});
|
||||
if (channelApproval.event.approval?.id !== dmApproval.event.approval?.id) {
|
||||
throw new Error("target=both delivered different approval ids to channel and DM");
|
||||
}
|
||||
const reaction = await reactToApproval({
|
||||
context,
|
||||
decision: "allow-once",
|
||||
roomId: context.roomId,
|
||||
targetEventId: channelApproval.event.eventId,
|
||||
});
|
||||
const result = await waitForApprovalDecision({
|
||||
approvalId,
|
||||
context,
|
||||
kind: "exec",
|
||||
});
|
||||
assertApprovalDecisionResult({
|
||||
approvalId,
|
||||
decision: "allow-once",
|
||||
result,
|
||||
});
|
||||
const lateDuplicate = await client.waitForOptionalRoomEvent({
|
||||
observedEvents: context.observedEvents,
|
||||
predicate: (event) =>
|
||||
event.sender === context.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
event.approval?.id === approvalId &&
|
||||
event.eventId !== channelApproval.event.eventId &&
|
||||
event.eventId !== dmApproval.event.eventId,
|
||||
roomId: context.roomId,
|
||||
timeoutMs: MATRIX_QA_APPROVAL_SHORT_WINDOW_MS,
|
||||
});
|
||||
if (lateDuplicate.matched) {
|
||||
throw new Error(`approval ${approvalId} was re-delivered after resolution`);
|
||||
}
|
||||
return {
|
||||
artifacts: {
|
||||
approvals: [
|
||||
buildMatrixApprovalArtifact(channelApproval.event),
|
||||
buildMatrixApprovalArtifact(dmApproval.event),
|
||||
],
|
||||
reactionEmoji: reaction.reaction?.key,
|
||||
reactionEventId: reaction.eventId,
|
||||
reactionTargetEventId: reaction.reaction?.eventId,
|
||||
token,
|
||||
},
|
||||
details: [
|
||||
`channel approval event: ${channelApproval.event.eventId}`,
|
||||
`dm approval event: ${dmApproval.event.eventId}`,
|
||||
`approval id: ${approvalId}`,
|
||||
`decision: allow-once`,
|
||||
].join("\n"),
|
||||
} satisfies MatrixQaScenarioExecution;
|
||||
}
|
||||
@@ -29,6 +29,11 @@ export type MatrixQaScenarioContext = {
|
||||
observerUserId: string;
|
||||
gatewayRuntimeEnv?: NodeJS.ProcessEnv;
|
||||
gatewayStateDir?: string;
|
||||
gatewayCall?: (
|
||||
method: string,
|
||||
params?: Record<string, unknown>,
|
||||
opts?: { expectFinal?: boolean; timeoutMs?: number },
|
||||
) => Promise<unknown>;
|
||||
outputDir?: string;
|
||||
registrationToken?: string;
|
||||
restartGateway?: () => Promise<void>;
|
||||
|
||||
@@ -3,6 +3,14 @@ import {
|
||||
MATRIX_QA_SECONDARY_ROOM_KEY,
|
||||
type MatrixQaScenarioDefinition,
|
||||
} from "./scenario-catalog.js";
|
||||
import {
|
||||
runApprovalChannelTargetBothScenario,
|
||||
runApprovalDenyReactionScenario,
|
||||
runApprovalExecMetadataChunkedScenario,
|
||||
runApprovalExecMetadataSingleEventScenario,
|
||||
runApprovalPluginMetadataSingleEventScenario,
|
||||
runApprovalThreadTargetScenario,
|
||||
} from "./scenario-runtime-approval.js";
|
||||
import {
|
||||
runDmPerRoomSessionOverrideScenario,
|
||||
runDmSharedSessionNoticeScenario,
|
||||
@@ -268,6 +276,18 @@ export async function runMatrixQaScenario(
|
||||
return await runReactionNotAReplyScenario(context);
|
||||
case "matrix-reaction-redaction-observed":
|
||||
return await runReactionRedactionObservedScenario(context);
|
||||
case "matrix-approval-exec-metadata-single-event":
|
||||
return await runApprovalExecMetadataSingleEventScenario(context);
|
||||
case "matrix-approval-exec-metadata-chunked":
|
||||
return await runApprovalExecMetadataChunkedScenario(context);
|
||||
case "matrix-approval-plugin-metadata-single-event":
|
||||
return await runApprovalPluginMetadataSingleEventScenario(context);
|
||||
case "matrix-approval-deny-reaction":
|
||||
return await runApprovalDenyReactionScenario(context);
|
||||
case "matrix-approval-thread-target":
|
||||
return await runApprovalThreadTargetScenario(context);
|
||||
case "matrix-approval-channel-target-both":
|
||||
return await runApprovalChannelTargetBothScenario(context);
|
||||
case "matrix-restart-resume":
|
||||
return await runRestartResumeScenario(context);
|
||||
case "matrix-post-restart-room-continue":
|
||||
|
||||
@@ -17,6 +17,16 @@ export type MatrixQaCanaryArtifact = {
|
||||
|
||||
export type MatrixQaScenarioArtifacts = {
|
||||
accepted?: MatrixQaScenarioArtifacts;
|
||||
approval?: MatrixQaObservedEvent["approval"] & {
|
||||
eventId: string;
|
||||
roomId: string;
|
||||
};
|
||||
approvals?: Array<
|
||||
MatrixQaObservedEvent["approval"] & {
|
||||
eventId: string;
|
||||
roomId: string;
|
||||
}
|
||||
>;
|
||||
attachments?: Array<{
|
||||
eventId: string;
|
||||
filename?: string;
|
||||
|
||||
@@ -230,6 +230,12 @@ describe("matrix live qa scenarios", () => {
|
||||
"matrix-reaction-threaded",
|
||||
"matrix-reaction-not-a-reply",
|
||||
"matrix-reaction-redaction-observed",
|
||||
"matrix-approval-exec-metadata-single-event",
|
||||
"matrix-approval-exec-metadata-chunked",
|
||||
"matrix-approval-plugin-metadata-single-event",
|
||||
"matrix-approval-deny-reaction",
|
||||
"matrix-approval-thread-target",
|
||||
"matrix-approval-channel-target-both",
|
||||
"matrix-restart-resume",
|
||||
"matrix-post-restart-room-continue",
|
||||
"matrix-initial-catchup-then-incremental",
|
||||
@@ -319,6 +325,8 @@ describe("matrix live qa scenarios", () => {
|
||||
"matrix-thread-isolation",
|
||||
"matrix-top-level-reply-shape",
|
||||
"matrix-reaction-notification",
|
||||
"matrix-approval-exec-metadata-single-event",
|
||||
"matrix-approval-exec-metadata-chunked",
|
||||
"matrix-restart-resume",
|
||||
"matrix-mention-gating",
|
||||
"matrix-allowlist-block",
|
||||
|
||||
@@ -99,6 +99,58 @@ describe("matrix observed event artifacts", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps approval summaries in redacted Matrix observed-event artifacts", () => {
|
||||
expect(
|
||||
buildMatrixQaObservedEventsArtifact({
|
||||
includeContent: false,
|
||||
observedEvents: [
|
||||
{
|
||||
kind: "message",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$approval",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
body: "secret command body",
|
||||
approval: {
|
||||
id: "approval-1",
|
||||
kind: "exec",
|
||||
state: "pending",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
hasCommandText: true,
|
||||
commandTextPreview: "printf MATRIX_QA",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
kind: "message",
|
||||
roomId: "!room:matrix-qa.test",
|
||||
eventId: "$approval",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
originServerTs: undefined,
|
||||
msgtype: undefined,
|
||||
membership: undefined,
|
||||
relatesTo: undefined,
|
||||
mentions: undefined,
|
||||
reaction: undefined,
|
||||
approval: {
|
||||
id: "approval-1",
|
||||
kind: "exec",
|
||||
state: "pending",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
hasCommandText: true,
|
||||
commandTextPreview: "printf MATRIX_QA",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps redaction metadata while still stripping Matrix event content", () => {
|
||||
expect(
|
||||
buildMatrixQaObservedEventsArtifact({
|
||||
|
||||
@@ -20,6 +20,7 @@ export function buildMatrixQaObservedEventsArtifact(params: {
|
||||
relatesTo: event.relatesTo,
|
||||
mentions: event.mentions,
|
||||
reaction: event.reaction,
|
||||
...(event.approval ? { approval: event.approval } : {}),
|
||||
attachment: event.attachment
|
||||
? {
|
||||
kind: event.attachment.kind,
|
||||
|
||||
@@ -227,9 +227,14 @@ describe("matrix qa config", () => {
|
||||
});
|
||||
|
||||
expect(snapshot).toEqual({
|
||||
approvalForwarding: {
|
||||
exec: false,
|
||||
plugin: false,
|
||||
},
|
||||
autoJoin: "allowlist",
|
||||
autoJoinAllowlist: ["!ops:matrix-qa.test"],
|
||||
blockStreaming: true,
|
||||
chunkMode: undefined,
|
||||
dm: {
|
||||
allowFrom: ["@driver:matrix-qa.test"],
|
||||
enabled: true,
|
||||
@@ -238,6 +243,7 @@ describe("matrix qa config", () => {
|
||||
threadReplies: "inbound",
|
||||
},
|
||||
encryption: false,
|
||||
execApprovals: undefined,
|
||||
groupAllowFrom: ["@driver:matrix-qa.test"],
|
||||
groupPolicy: "open",
|
||||
groupsByKey: {
|
||||
@@ -255,6 +261,7 @@ describe("matrix qa config", () => {
|
||||
replyToMode: "off",
|
||||
streaming: "partial",
|
||||
streamingPreviewToolProgress: true,
|
||||
textChunkLimit: undefined,
|
||||
threadBindings: {},
|
||||
threadReplies: "inbound",
|
||||
});
|
||||
@@ -289,6 +296,50 @@ describe("matrix qa config", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("applies Matrix approval delivery overrides with gateway forwarding enabled", () => {
|
||||
const next = buildMatrixQaConfig({} as OpenClawConfig, {
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
homeserver: "http://127.0.0.1:28008/",
|
||||
observerUserId: "@observer:matrix-qa.test",
|
||||
overrides: {
|
||||
approvalForwarding: {
|
||||
exec: true,
|
||||
plugin: true,
|
||||
},
|
||||
chunkMode: "length",
|
||||
dm: {
|
||||
enabled: true,
|
||||
},
|
||||
execApprovals: {
|
||||
enabled: true,
|
||||
target: "both",
|
||||
},
|
||||
textChunkLimit: 280,
|
||||
},
|
||||
sutAccessToken: "sut-token",
|
||||
sutAccountId: "sut",
|
||||
sutUserId: "@sut:matrix-qa.test",
|
||||
topology,
|
||||
});
|
||||
|
||||
expect(next.approvals).toMatchObject({
|
||||
exec: { enabled: true, mode: "session" },
|
||||
plugin: { enabled: true, mode: "session" },
|
||||
});
|
||||
expect(next.channels?.matrix?.accounts?.sut).toMatchObject({
|
||||
chunkMode: "length",
|
||||
dm: {
|
||||
allowFrom: ["@driver:matrix-qa.test"],
|
||||
enabled: true,
|
||||
},
|
||||
execApprovals: {
|
||||
enabled: true,
|
||||
target: "both",
|
||||
},
|
||||
textChunkLimit: 280,
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves role-based Matrix sender allowlist overrides", () => {
|
||||
const snapshot = buildMatrixQaConfigSnapshot({
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
|
||||
@@ -8,6 +8,9 @@ export type MatrixQaGroupPolicy = "allowlist" | "disabled" | "open";
|
||||
export type MatrixQaAutoJoinMode = "allowlist" | "always" | "off";
|
||||
export type MatrixQaStreamingMode = "off" | "partial" | "quiet";
|
||||
export type MatrixQaActorRole = "driver" | "observer" | "sut";
|
||||
export type MatrixQaChunkMode = "length" | "newline";
|
||||
export type MatrixQaExecApprovalTarget = "both" | "channel" | "dm";
|
||||
export type MatrixQaExecApprovalsEnabled = boolean | "auto";
|
||||
|
||||
export type MatrixQaStreamingConfig = {
|
||||
mode?: MatrixQaStreamingMode;
|
||||
@@ -56,13 +59,27 @@ export type MatrixQaThreadBindingsConfigOverrides = {
|
||||
spawnSubagentSessions?: boolean;
|
||||
};
|
||||
|
||||
export type MatrixQaExecApprovalsConfigOverrides = {
|
||||
agentFilter?: string[];
|
||||
approvers?: string[];
|
||||
enabled?: MatrixQaExecApprovalsEnabled;
|
||||
sessionFilter?: string[];
|
||||
target?: MatrixQaExecApprovalTarget;
|
||||
};
|
||||
|
||||
export type MatrixQaConfigOverrides = {
|
||||
approvalForwarding?: {
|
||||
exec?: boolean;
|
||||
plugin?: boolean;
|
||||
};
|
||||
agentDefaults?: MatrixQaAgentDefaultsOverrides;
|
||||
autoJoin?: MatrixQaAutoJoinMode;
|
||||
autoJoinAllowlist?: string[];
|
||||
blockStreaming?: boolean;
|
||||
chunkMode?: MatrixQaChunkMode;
|
||||
dm?: MatrixQaDmConfigOverrides;
|
||||
encryption?: boolean;
|
||||
execApprovals?: MatrixQaExecApprovalsConfigOverrides;
|
||||
groupAllowFrom?: string[];
|
||||
groupAllowRoles?: MatrixQaActorRole[];
|
||||
groupPolicy?: MatrixQaGroupPolicy;
|
||||
@@ -70,15 +87,21 @@ export type MatrixQaConfigOverrides = {
|
||||
replyToMode?: MatrixQaReplyToMode;
|
||||
startupVerification?: "if-unverified" | "off";
|
||||
streaming?: MatrixQaStreamingMode | MatrixQaStreamingConfig | boolean;
|
||||
textChunkLimit?: number;
|
||||
threadBindings?: MatrixQaThreadBindingsConfigOverrides;
|
||||
threadReplies?: MatrixQaThreadRepliesMode;
|
||||
toolProfile?: "coding" | "messaging" | "minimal";
|
||||
};
|
||||
|
||||
export type MatrixQaConfigSnapshot = {
|
||||
approvalForwarding: {
|
||||
exec: boolean;
|
||||
plugin: boolean;
|
||||
};
|
||||
autoJoin: MatrixQaAutoJoinMode;
|
||||
autoJoinAllowlist: string[];
|
||||
blockStreaming: boolean;
|
||||
chunkMode?: MatrixQaChunkMode;
|
||||
dm: {
|
||||
allowFrom: string[];
|
||||
enabled: boolean;
|
||||
@@ -87,6 +110,7 @@ export type MatrixQaConfigSnapshot = {
|
||||
threadReplies: MatrixQaThreadRepliesMode;
|
||||
};
|
||||
encryption: boolean;
|
||||
execApprovals?: MatrixQaExecApprovalsConfigOverrides;
|
||||
groupAllowFrom: string[];
|
||||
groupPolicy: MatrixQaGroupPolicy;
|
||||
groupsByKey: Record<string, MatrixQaGroupSnapshot>;
|
||||
@@ -94,6 +118,7 @@ export type MatrixQaConfigSnapshot = {
|
||||
startupVerification?: "if-unverified" | "off";
|
||||
streaming: MatrixQaStreamingMode;
|
||||
streamingPreviewToolProgress: boolean;
|
||||
textChunkLimit?: number;
|
||||
threadBindings: MatrixQaThreadBindingsConfigOverrides;
|
||||
threadReplies: MatrixQaThreadRepliesMode;
|
||||
};
|
||||
@@ -121,6 +146,14 @@ type MatrixQaAccountDmConfig =
|
||||
threadReplies?: MatrixQaThreadRepliesMode;
|
||||
};
|
||||
|
||||
type MatrixQaAccountExecApprovalsConfig = {
|
||||
agentFilter?: string[];
|
||||
approvers?: string[];
|
||||
enabled?: MatrixQaExecApprovalsEnabled;
|
||||
sessionFilter?: string[];
|
||||
target?: MatrixQaExecApprovalTarget;
|
||||
};
|
||||
|
||||
function normalizeMatrixQaAllowlist(entries?: string[]) {
|
||||
return [...new Set((entries ?? []).map((entry) => entry.trim()).filter(Boolean))];
|
||||
}
|
||||
@@ -299,6 +332,21 @@ function buildMatrixQaAccountDmConfig(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function buildMatrixQaAccountExecApprovalsConfig(
|
||||
overrides?: MatrixQaExecApprovalsConfigOverrides,
|
||||
): MatrixQaAccountExecApprovalsConfig | undefined {
|
||||
if (!overrides) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...(overrides.agentFilter ? { agentFilter: overrides.agentFilter } : {}),
|
||||
...(overrides.approvers ? { approvers: normalizeMatrixQaAllowlist(overrides.approvers) } : {}),
|
||||
...(overrides.enabled !== undefined ? { enabled: overrides.enabled } : {}),
|
||||
...(overrides.sessionFilter ? { sessionFilter: overrides.sessionFilter } : {}),
|
||||
...(overrides.target ? { target: overrides.target } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildMatrixQaChannelAccountConfig(params: {
|
||||
groups: Record<string, MatrixQaGroupEntry>;
|
||||
homeserver: string;
|
||||
@@ -319,6 +367,11 @@ function buildMatrixQaChannelAccountConfig(params: {
|
||||
params.overrides?.blockStreaming !== undefined
|
||||
? { blockStreaming: params.snapshot.blockStreaming }
|
||||
: {};
|
||||
const chunkModeConfig =
|
||||
params.snapshot.chunkMode !== undefined ? { chunkMode: params.snapshot.chunkMode } : {};
|
||||
const execApprovalsConfig = buildMatrixQaAccountExecApprovalsConfig(
|
||||
params.snapshot.execApprovals,
|
||||
);
|
||||
const streamingConfig =
|
||||
params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming } : {};
|
||||
const startupVerificationConfig =
|
||||
@@ -329,6 +382,10 @@ function buildMatrixQaChannelAccountConfig(params: {
|
||||
params.overrides?.threadBindings !== undefined
|
||||
? { threadBindings: params.snapshot.threadBindings }
|
||||
: {};
|
||||
const textChunkLimitConfig =
|
||||
params.snapshot.textChunkLimit !== undefined
|
||||
? { textChunkLimit: params.snapshot.textChunkLimit }
|
||||
: {};
|
||||
|
||||
return {
|
||||
accessToken: params.sutAccessToken,
|
||||
@@ -354,7 +411,10 @@ function buildMatrixQaChannelAccountConfig(params: {
|
||||
...autoJoinConfig,
|
||||
...autoJoinAllowlistConfig,
|
||||
...blockStreamingConfig,
|
||||
...chunkModeConfig,
|
||||
...(execApprovalsConfig ? { execApprovals: execApprovalsConfig } : {}),
|
||||
...streamingConfig,
|
||||
...textChunkLimitConfig,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -369,8 +429,10 @@ export function buildMatrixQaConfigSnapshot(params: {
|
||||
autoJoin: params.overrides?.autoJoin ?? "off",
|
||||
autoJoinAllowlist: resolveMatrixQaAutoJoinAllowlist(params),
|
||||
blockStreaming: params.overrides?.blockStreaming ?? false,
|
||||
chunkMode: params.overrides?.chunkMode,
|
||||
dm: resolveMatrixQaDmConfigSnapshot(params),
|
||||
encryption: params.overrides?.encryption ?? false,
|
||||
execApprovals: params.overrides?.execApprovals,
|
||||
groupAllowFrom: resolveMatrixQaGroupAllowFrom(params),
|
||||
groupPolicy: params.overrides?.groupPolicy ?? "allowlist",
|
||||
groupsByKey: resolveMatrixQaGroupSnapshots({
|
||||
@@ -384,7 +446,13 @@ export function buildMatrixQaConfigSnapshot(params: {
|
||||
params.overrides?.streaming,
|
||||
),
|
||||
threadBindings: { ...params.overrides?.threadBindings },
|
||||
textChunkLimit: params.overrides?.textChunkLimit,
|
||||
threadReplies: params.overrides?.threadReplies ?? "inbound",
|
||||
approvalForwarding: {
|
||||
exec:
|
||||
params.overrides?.approvalForwarding?.exec ?? params.overrides?.execApprovals !== undefined,
|
||||
plugin: params.overrides?.approvalForwarding?.plugin ?? false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -398,12 +466,18 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot
|
||||
`dm.threadReplies=${snapshot.dm.threadReplies}`,
|
||||
`streaming=${snapshot.streaming}`,
|
||||
`streaming.preview.toolProgress=${formatMatrixQaBoolean(snapshot.streamingPreviewToolProgress)}`,
|
||||
`textChunkLimit=${snapshot.textChunkLimit ?? "<default>"}`,
|
||||
`chunkMode=${snapshot.chunkMode ?? "<default>"}`,
|
||||
`execApprovals.enabled=${snapshot.execApprovals?.enabled ?? "<default>"}`,
|
||||
`execApprovals.target=${snapshot.execApprovals?.target ?? "<default>"}`,
|
||||
`blockStreaming=${formatMatrixQaBoolean(snapshot.blockStreaming)}`,
|
||||
`autoJoin=${snapshot.autoJoin}`,
|
||||
`encryption=${formatMatrixQaBoolean(snapshot.encryption)}`,
|
||||
`startupVerification=${snapshot.startupVerification ?? "<default>"}`,
|
||||
`threadBindings.enabled=${snapshot.threadBindings.enabled ?? "<default>"}`,
|
||||
`threadBindings.spawnSubagentSessions=${snapshot.threadBindings.spawnSubagentSessions ?? "<default>"}`,
|
||||
`approvals.exec.enabled=${formatMatrixQaBoolean(snapshot.approvalForwarding.exec)}`,
|
||||
`approvals.plugin.enabled=${formatMatrixQaBoolean(snapshot.approvalForwarding.plugin)}`,
|
||||
].join(", ");
|
||||
}
|
||||
|
||||
@@ -430,9 +504,36 @@ export function buildMatrixQaConfig(
|
||||
topology: params.topology,
|
||||
});
|
||||
const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey);
|
||||
const approvalForwardingConfig =
|
||||
snapshot.approvalForwarding.exec || snapshot.approvalForwarding.plugin
|
||||
? {
|
||||
approvals: {
|
||||
...baseCfg.approvals,
|
||||
...(snapshot.approvalForwarding.exec
|
||||
? {
|
||||
exec: {
|
||||
...baseCfg.approvals?.exec,
|
||||
enabled: true,
|
||||
mode: "session" as const,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(snapshot.approvalForwarding.plugin
|
||||
? {
|
||||
plugin: {
|
||||
...baseCfg.approvals?.plugin,
|
||||
enabled: true,
|
||||
mode: "session" as const,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
||||
return {
|
||||
...baseCfg,
|
||||
...approvalForwardingConfig,
|
||||
...(params.overrides?.toolProfile
|
||||
? {
|
||||
tools: {
|
||||
|
||||
@@ -133,6 +133,84 @@ describe("matrix observed event normalization", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("summarizes Matrix approval metadata without dumping full command text", () => {
|
||||
const commandText = `printf ${"A".repeat(300)}`;
|
||||
expect(
|
||||
normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
|
||||
event_id: "$approval",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
content: {
|
||||
body: "React here: ✅ Allow once, ❌ Deny",
|
||||
msgtype: "m.text",
|
||||
"com.openclaw.approval": {
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
commandText,
|
||||
id: "approval-1",
|
||||
kind: "exec",
|
||||
state: "pending",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
approval: {
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
commandTextPreview: commandText.slice(0, 160),
|
||||
hasCommandText: true,
|
||||
id: "approval-1",
|
||||
kind: "exec",
|
||||
state: "pending",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("summarizes Matrix plugin approval metadata fields", () => {
|
||||
expect(
|
||||
normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
|
||||
event_id: "$plugin-approval",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
content: {
|
||||
body: "Plugin approval required",
|
||||
msgtype: "m.text",
|
||||
"com.openclaw.approval": {
|
||||
agentId: "qa",
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
id: "plugin:approval-1",
|
||||
kind: "plugin",
|
||||
pluginId: "qa-plugin",
|
||||
severity: "medium",
|
||||
state: "pending",
|
||||
toolName: "qa_tool",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual(
|
||||
expect.objectContaining({
|
||||
approval: {
|
||||
agentId: "qa",
|
||||
allowedDecisions: ["allow-once", "deny"],
|
||||
id: "plugin:approval-1",
|
||||
kind: "plugin",
|
||||
pluginId: "qa-plugin",
|
||||
severity: "medium",
|
||||
state: "pending",
|
||||
toolName: "qa_tool",
|
||||
type: "approval.request",
|
||||
version: 1,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes Matrix image messages with attachment metadata", () => {
|
||||
expect(
|
||||
normalizeMatrixQaObservedEvent("!room:matrix-qa.test", {
|
||||
|
||||
@@ -21,6 +21,21 @@ export type MatrixQaObservedEventAttachment = {
|
||||
kind: "audio" | "file" | "image" | "sticker" | "video";
|
||||
};
|
||||
|
||||
export type MatrixQaObservedApproval = {
|
||||
agentId?: string;
|
||||
allowedDecisions?: string[];
|
||||
commandTextPreview?: string;
|
||||
hasCommandText?: boolean;
|
||||
id: string;
|
||||
kind: "exec" | "plugin";
|
||||
pluginId?: string;
|
||||
severity?: string;
|
||||
state?: string;
|
||||
toolName?: string;
|
||||
type?: string;
|
||||
version?: number;
|
||||
};
|
||||
|
||||
export type MatrixQaObservedEvent = {
|
||||
kind: MatrixQaObservedEventKind;
|
||||
roomId: string;
|
||||
@@ -48,8 +63,12 @@ export type MatrixQaObservedEvent = {
|
||||
key?: string;
|
||||
};
|
||||
attachment?: MatrixQaObservedEventAttachment;
|
||||
approval?: MatrixQaObservedApproval;
|
||||
};
|
||||
|
||||
const MATRIX_QA_APPROVAL_METADATA_KEY = "com.openclaw.approval";
|
||||
const MATRIX_QA_APPROVAL_COMMAND_PREVIEW_CHARS = 160;
|
||||
|
||||
function normalizeMentionUserIds(value: unknown) {
|
||||
return Array.isArray(value)
|
||||
? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
||||
@@ -130,6 +149,54 @@ function resolveMatrixQaAttachmentSummary(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMatrixQaApprovalAllowedDecisions(value: unknown) {
|
||||
return Array.isArray(value)
|
||||
? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function normalizeMatrixQaApprovalMetadata(value: unknown): MatrixQaObservedApproval | undefined {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const metadata = value as Record<string, unknown>;
|
||||
const id = typeof metadata.id === "string" ? metadata.id.trim() : "";
|
||||
const kind = metadata.kind;
|
||||
if (!id || (kind !== "exec" && kind !== "plugin")) {
|
||||
return undefined;
|
||||
}
|
||||
const commandText =
|
||||
typeof metadata.commandText === "string" ? metadata.commandText.trim() : undefined;
|
||||
const commandPreview =
|
||||
typeof metadata.commandPreview === "string" ? metadata.commandPreview.trim() : undefined;
|
||||
const commandTextPreview = (commandPreview || commandText)?.slice(
|
||||
0,
|
||||
MATRIX_QA_APPROVAL_COMMAND_PREVIEW_CHARS,
|
||||
);
|
||||
return {
|
||||
id,
|
||||
kind,
|
||||
...(typeof metadata.agentId === "string" ? { agentId: metadata.agentId } : {}),
|
||||
...(typeof metadata.state === "string" ? { state: metadata.state } : {}),
|
||||
...(typeof metadata.type === "string" ? { type: metadata.type } : {}),
|
||||
...(typeof metadata.version === "number" ? { version: metadata.version } : {}),
|
||||
...(metadata.allowedDecisions
|
||||
? { allowedDecisions: normalizeMatrixQaApprovalAllowedDecisions(metadata.allowedDecisions) }
|
||||
: {}),
|
||||
...(commandText ? { hasCommandText: true } : {}),
|
||||
...(commandTextPreview ? { commandTextPreview } : {}),
|
||||
...(kind === "plugin" && typeof metadata.pluginId === "string"
|
||||
? { pluginId: metadata.pluginId }
|
||||
: {}),
|
||||
...(kind === "plugin" && typeof metadata.severity === "string"
|
||||
? { severity: metadata.severity }
|
||||
: {}),
|
||||
...(kind === "plugin" && typeof metadata.toolName === "string"
|
||||
? { toolName: metadata.toolName }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeMatrixQaObservedEvent(
|
||||
roomId: string,
|
||||
event: MatrixQaRoomEvent,
|
||||
@@ -177,6 +244,9 @@ export function normalizeMatrixQaObservedEvent(
|
||||
filename: normalizedFilename,
|
||||
msgtype: normalizedMsgtype,
|
||||
});
|
||||
const approval = normalizeMatrixQaApprovalMetadata(
|
||||
messageContent[MATRIX_QA_APPROVAL_METADATA_KEY] ?? content[MATRIX_QA_APPROVAL_METADATA_KEY],
|
||||
);
|
||||
|
||||
return {
|
||||
kind: resolveMatrixQaObservedEventKind({ msgtype: normalizedMsgtype, type }),
|
||||
@@ -222,6 +292,7 @@ export function normalizeMatrixQaObservedEvent(
|
||||
}
|
||||
: {}),
|
||||
...(attachment ? { attachment } : {}),
|
||||
...(approval ? { approval } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user