From ae616777f3b185d7e12da4e1dac999bf0ace8bb5 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Mon, 27 Apr 2026 22:32:55 -0400 Subject: [PATCH] test(qa-matrix): cover approval metadata scenarios --- CHANGELOG.md | 1 + docs/concepts/qa-matrix.md | 21 +- .../qa-matrix/src/runners/contract/runtime.ts | 4 +- .../src/runners/contract/scenario-catalog.ts | 90 +++ .../contract/scenario-runtime-approval.ts | 599 ++++++++++++++++++ .../contract/scenario-runtime-shared.ts | 5 + .../src/runners/contract/scenario-runtime.ts | 20 + .../src/runners/contract/scenario-types.ts | 10 + .../src/runners/contract/scenarios.test.ts | 8 + .../qa-matrix/src/substrate/artifacts.test.ts | 52 ++ .../qa-matrix/src/substrate/artifacts.ts | 1 + .../qa-matrix/src/substrate/config.test.ts | 51 ++ extensions/qa-matrix/src/substrate/config.ts | 101 +++ .../qa-matrix/src/substrate/events.test.ts | 78 +++ extensions/qa-matrix/src/substrate/events.ts | 71 +++ 15 files changed, 1101 insertions(+), 11 deletions(-) create mode 100644 extensions/qa-matrix/src/runners/contract/scenario-runtime-approval.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c796b9a02..1b53f10e3e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai ### Changes - Dependencies: refresh provider and tooling dependencies, including AWS SDK, PI runtime packages, AJV, Feishu SDK, Anthropic SDK, tokenjuice, and native TypeScript/oxlint tooling. Thanks @dependabot. +- Matrix/QA: add live Matrix approval scenarios for exec metadata, chunked fallback, plugin approvals, deny reactions, thread targeting, and `target: "both"` delivery, with redacted artifacts preserving safe approval summaries. Thanks @gumadeiras. - Codex: add Computer Use setup for Codex-mode agents, including `/codex computer-use status/install`, marketplace discovery, optional auto-install, and fail-closed MCP server checks before Codex-mode turns start. Fixes #72094. (#71842) Thanks @pash-openai. - Apps: consume Peekaboo 3.0.0-beta4 and ElevenLabsKit 0.1.1, align Swabble on Commander 0.2.2, and refresh macOS/iOS SwiftPM resolutions against the released dependency graph. Thanks @Blaizzy. - Plugin SDK: expose shared channel route normalization, parser-driven target resolution, raw-target compact keys, parsed-target types, and route comparison helpers through `openclaw/plugin-sdk/channel-route`, switch native approval origin matching onto that route contract with optional delivery and match-only target normalization, and retire the internal channel-route shim behind dated compatibility aliases for legacy key/comparable-target helpers. Thanks @vincentkoc. diff --git a/docs/concepts/qa-matrix.md b/docs/concepts/qa-matrix.md index b2899fa1033..13cf49da864 100644 --- a/docs/concepts/qa-matrix.md +++ b/docs/concepts/qa-matrix.md @@ -64,15 +64,15 @@ Matrix QA does not accept `--credential-source` or `--credential-role`. The lane The selected profile decides which scenarios run. -| Profile | Use it for | -| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `all` (default) | Full catalog. Slow but exhaustive. | -| `fast` | Release-gate subset that exercises the live transport contract: canary, mention gating, allowlist block, reply shape, restart resume, thread follow-up, thread isolation, reaction observation. | -| `transport` | Transport-level threading, DM, room, autojoin, mention/allowlist scenarios. | -| `media` | Image, audio, video, PDF, EPUB attachment coverage. | -| `e2ee-smoke` | Minimum E2EE coverage — basic encrypted reply, thread follow-up, bootstrap success. | -| `e2ee-deep` | Exhaustive E2EE state-loss, backup, key, and recovery scenarios. | -| `e2ee-cli` | `openclaw matrix encryption setup` and `verify *` CLI scenarios driven through the QA harness. | +| Profile | Use it for | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `all` (default) | Full catalog. Slow but exhaustive. | +| `fast` | Release-gate subset that exercises the live transport contract: canary, mention gating, allowlist block, reply shape, restart resume, thread follow-up, thread isolation, reaction observation, and exec approval metadata delivery. | +| `transport` | Transport-level threading, DM, room, autojoin, mention/allowlist, approval, and reaction scenarios. | +| `media` | Image, audio, video, PDF, EPUB attachment coverage. | +| `e2ee-smoke` | Minimum E2EE coverage — basic encrypted reply, thread follow-up, bootstrap success. | +| `e2ee-deep` | Exhaustive E2EE state-loss, backup, key, and recovery scenarios. | +| `e2ee-cli` | `openclaw matrix encryption setup` and `verify *` CLI scenarios driven through the QA harness. | The exact mapping lives in `extensions/qa-matrix/src/runners/contract/scenario-catalog.ts`. @@ -86,6 +86,7 @@ The full scenario id list is the `MatrixQaScenarioId` union in `extensions/qa-ma - media — `matrix-media-type-coverage`, `matrix-room-image-understanding-attachment`, `matrix-attachment-only-ignored`, `matrix-unsupported-media-safe` - routing — `matrix-room-autojoin-invite`, `matrix-secondary-room-*` - reactions — `matrix-reaction-*` +- approvals — `matrix-approval-*` (exec/plugin metadata, chunked fallback, deny reactions, threads, and `target: "both"` routing) - restart and replay — `matrix-restart-*`, `matrix-stale-sync-replay-dedupe`, `matrix-room-membership-loss`, `matrix-homeserver-restart-resume`, `matrix-initial-catchup-then-incremental` - mention gating and allowlists — `matrix-mention-*`, `matrix-allowlist-*`, `matrix-multi-actor-ordering`, `matrix-inbound-edit-*`, `matrix-mxid-prefixed-command-block`, `matrix-observer-allowlist-override` - E2EE — `matrix-e2ee-*` (basic reply, thread follow-up, bootstrap, recovery key lifecycle, state-loss variants, server backup behavior, device hygiene, SAS / QR / DM verification, restart, artifact redaction) @@ -112,7 +113,7 @@ Written to `--output-dir`: - `matrix-qa-report.md` — Markdown protocol report (what passed, failed, was skipped, and why). - `matrix-qa-summary.json` — Structured summary suitable for CI parsing and dashboards. -- `matrix-qa-observed-events.json` — Observed Matrix events from the driver and observer clients. Bodies are redacted unless `OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1`. +- `matrix-qa-observed-events.json` — Observed Matrix events from the driver and observer clients. Bodies are redacted unless `OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1`; approval metadata is summarized with selected safe fields and truncated command preview. - `matrix-qa-output.log` — Combined stdout/stderr from the run. If `OPENCLAW_RUN_NODE_OUTPUT_LOG` is set, the outer launcher's log is reused instead. The default output dir is `/.artifacts/qa-e2e/matrix-` so successive runs do not overwrite each other. diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 765e0dba82e..87cf542aab4 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -40,7 +40,7 @@ type MatrixQaGatewayChild = { call( method: string, params: Record, - options?: { timeoutMs?: number }, + options?: { expectFinal?: boolean; timeoutMs?: number }, ): Promise; restartAfterStateMutation?: ( mutateState: (context: { stateDir: string }) => Promise, @@ -789,6 +789,8 @@ export async function runMatrixQaLive(params: { observerUserId: provisioning.observer.userId, gatewayRuntimeEnv: scenarioGateway.harness.gateway.runtimeEnv, gatewayStateDir: scenarioGateway.harness.gateway.runtimeEnv?.OPENCLAW_STATE_DIR, + gatewayCall: async (method, params, opts) => + await scenarioGateway.harness.gateway.call(method, params ?? {}, opts), outputDir, registrationToken: harness.registrationToken, restartGateway: async () => { diff --git a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts index 195935a424a..9c73b074c07 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-catalog.ts @@ -43,6 +43,12 @@ export type MatrixQaScenarioId = | "matrix-reaction-threaded" | "matrix-reaction-not-a-reply" | "matrix-reaction-redaction-observed" + | "matrix-approval-exec-metadata-single-event" + | "matrix-approval-exec-metadata-chunked" + | "matrix-approval-plugin-metadata-single-event" + | "matrix-approval-deny-reaction" + | "matrix-approval-thread-target" + | "matrix-approval-channel-target-both" | "matrix-restart-resume" | "matrix-post-restart-room-continue" | "matrix-initial-catchup-then-incremental" @@ -266,6 +272,51 @@ const MATRIX_QA_E2EE_CLI_SETUP_CONFIG = { startupVerification: "off", } satisfies MatrixQaConfigOverrides; +const MATRIX_QA_APPROVAL_CHANNEL_CONFIG = { + approvalForwarding: { + exec: true, + }, + dm: { + enabled: true, + }, + execApprovals: { + enabled: true, + target: "channel", + }, +} satisfies MatrixQaConfigOverrides; + +const MATRIX_QA_APPROVAL_CHUNKED_CONFIG = { + ...MATRIX_QA_APPROVAL_CHANNEL_CONFIG, + chunkMode: "length", + textChunkLimit: 280, +} satisfies MatrixQaConfigOverrides; + +const MATRIX_QA_APPROVAL_PLUGIN_CONFIG = { + approvalForwarding: { + plugin: true, + }, + dm: { + enabled: true, + }, + execApprovals: { + enabled: true, + target: "channel", + }, +} satisfies MatrixQaConfigOverrides; + +const MATRIX_QA_APPROVAL_BOTH_CONFIG = { + approvalForwarding: { + exec: true, + }, + dm: { + enabled: true, + }, + execApprovals: { + enabled: true, + target: "both", + }, +} satisfies MatrixQaConfigOverrides; + export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ { id: "matrix-thread-follow-up", @@ -518,6 +569,43 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [ timeoutMs: 45_000, title: "Matrix reaction removals are observed as redactions", }, + { + id: "matrix-approval-exec-metadata-single-event", + timeoutMs: 75_000, + title: "Matrix exec approval prompt carries structured metadata on one event", + configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG, + }, + { + id: "matrix-approval-exec-metadata-chunked", + timeoutMs: 90_000, + title: "Matrix exec approval prompt fallback keeps metadata on the first chunk", + configOverrides: MATRIX_QA_APPROVAL_CHUNKED_CONFIG, + }, + { + id: "matrix-approval-plugin-metadata-single-event", + timeoutMs: 75_000, + title: "Matrix plugin approval prompt carries plugin metadata", + configOverrides: MATRIX_QA_APPROVAL_PLUGIN_CONFIG, + }, + { + id: "matrix-approval-deny-reaction", + timeoutMs: 75_000, + title: "Matrix approval deny reaction resolves the metadata-bearing event", + configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG, + }, + { + id: "matrix-approval-thread-target", + timeoutMs: 75_000, + title: "Matrix approval prompt preserves thread targeting metadata", + configOverrides: MATRIX_QA_APPROVAL_CHANNEL_CONFIG, + }, + { + id: "matrix-approval-channel-target-both", + timeoutMs: 90_000, + title: "Matrix approval target=both delivers channel and DM metadata once", + topology: MATRIX_QA_DRIVER_DM_TOPOLOGY, + configOverrides: MATRIX_QA_APPROVAL_BOTH_CONFIG, + }, { id: "matrix-restart-resume", standardId: "restart-resume", @@ -985,6 +1073,8 @@ const MATRIX_QA_FAST_PROFILE_SCENARIO_IDS = [ "matrix-thread-isolation", "matrix-top-level-reply-shape", "matrix-reaction-notification", + "matrix-approval-exec-metadata-single-event", + "matrix-approval-exec-metadata-chunked", "matrix-restart-resume", "matrix-mention-gating", "matrix-allowlist-block", diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-approval.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-approval.ts new file mode 100644 index 00000000000..55e0978c914 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-approval.ts @@ -0,0 +1,599 @@ +import { randomUUID } from "node:crypto"; +import type { MatrixQaObservedEvent } from "../../substrate/events.js"; +import { MATRIX_QA_DRIVER_DM_ROOM_KEY, resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js"; +import { + advanceMatrixQaActorCursor, + buildMatrixQaToken, + createMatrixQaScenarioClient, + primeMatrixQaDriverScenarioClient, + type MatrixQaScenarioContext, +} from "./scenario-runtime-shared.js"; +import type { MatrixQaScenarioExecution } from "./scenario-types.js"; + +const MATRIX_QA_APPROVAL_ALLOW_ONCE_REACTION = "✅"; +const MATRIX_QA_APPROVAL_DENY_REACTION = "❌"; +const MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS = 30_000; +const MATRIX_QA_APPROVAL_SHORT_WINDOW_MS = 4_000; +const MATRIX_QA_APPROVAL_LONG_COMMAND_TEXT = "matrix approval chunk fallback ".repeat(40); + +type MatrixQaApprovalDecision = "allow-once" | "deny"; +type MatrixQaApprovalKind = "exec" | "plugin"; +type MatrixQaApprovalOptionReactionParams = { + context: MatrixQaScenarioContext; + emoji: string; + roomId: string; + targetEventId: string; +}; + +function requireMatrixQaGatewayCall(context: MatrixQaScenarioContext) { + if (!context.gatewayCall) { + throw new Error("Matrix approval QA scenario requires a live gateway RPC client"); + } + return context.gatewayCall; +} + +function buildMatrixApprovalArtifact(event: MatrixQaObservedEvent) { + if (!event.approval) { + throw new Error(`Matrix event ${event.eventId} did not include approval metadata`); + } + return { + ...event.approval, + eventId: event.eventId, + roomId: event.roomId, + }; +} + +function isApprovalOptionReaction( + event: MatrixQaObservedEvent, + params: MatrixQaApprovalOptionReactionParams, +) { + return ( + event.roomId === params.roomId && + event.sender === params.context.sutUserId && + event.type === "m.reaction" && + event.reaction?.eventId === params.targetEventId && + event.reaction.key === params.emoji + ); +} + +function hasObservedApprovalOptionReaction(params: MatrixQaApprovalOptionReactionParams) { + return params.context.observedEvents.some((event) => isApprovalOptionReaction(event, params)); +} + +function assertApprovalMetadata(params: { + event: { approval?: unknown; eventId: string }; + expectedKind: MatrixQaApprovalKind; +}) { + const approval = + typeof params.event.approval === "object" && params.event.approval !== null + ? (params.event.approval as { + allowedDecisions?: string[]; + hasCommandText?: boolean; + id?: string; + kind?: string; + state?: string; + type?: string; + version?: number; + }) + : null; + if (!approval) { + throw new Error(`approval event ${params.event.eventId} did not expose metadata`); + } + if (approval.kind !== params.expectedKind) { + throw new Error( + `approval event ${params.event.eventId} kind was ${approval.kind ?? ""} instead of ${params.expectedKind}`, + ); + } + if (!approval.id) { + throw new Error(`approval event ${params.event.eventId} did not expose an approval id`); + } + if (approval.version !== 1) { + throw new Error(`approval event ${params.event.eventId} did not expose version=1`); + } + if (approval.type !== "approval.request") { + throw new Error(`approval event ${params.event.eventId} did not expose type=approval.request`); + } + if (approval.state !== "pending") { + throw new Error(`approval event ${params.event.eventId} did not expose state=pending`); + } + if (!approval.allowedDecisions?.includes("deny")) { + throw new Error(`approval event ${params.event.eventId} did not include deny`); + } + if ( + params.expectedKind === "exec" && + (!approval.allowedDecisions.includes("allow-once") || approval.hasCommandText !== true) + ) { + throw new Error(`approval event ${params.event.eventId} did not expose exec approval fields`); + } +} + +async function waitForApprovalEvent(params: { + context: MatrixQaScenarioContext; + expectedApprovalId: string; + expectedKind: MatrixQaApprovalKind; + roomId: string; + since?: string; + threadRootEventId?: string; +}) { + const client = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const matched = await client.waitForRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === params.roomId && + event.sender === params.context.sutUserId && + event.type === "m.room.message" && + event.approval?.kind === params.expectedKind && + event.approval.id === params.expectedApprovalId && + (!params.threadRootEventId || event.relatesTo?.eventId === params.threadRootEventId), + roomId: params.roomId, + since: params.since, + timeoutMs: params.context.timeoutMs, + }); + assertApprovalMetadata({ + event: matched.event, + expectedKind: params.expectedKind, + }); + return matched; +} + +async function reactToApproval(params: { + context: MatrixQaScenarioContext; + decision: MatrixQaApprovalDecision; + roomId: string; + targetEventId: string; +}) { + const client = createMatrixQaScenarioClient({ + accessToken: params.context.driverAccessToken, + baseUrl: params.context.baseUrl, + }); + const emoji = + params.decision === "allow-once" + ? MATRIX_QA_APPROVAL_ALLOW_ONCE_REACTION + : MATRIX_QA_APPROVAL_DENY_REACTION; + if ( + !hasObservedApprovalOptionReaction({ + context: params.context, + emoji, + roomId: params.roomId, + targetEventId: params.targetEventId, + }) + ) { + await client.waitForRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + isApprovalOptionReaction(event, { + context: params.context, + emoji, + roomId: params.roomId, + targetEventId: params.targetEventId, + }), + roomId: params.roomId, + timeoutMs: params.context.timeoutMs, + }); + } + const eventId = await client.sendReaction({ + emoji, + messageId: params.targetEventId, + roomId: params.roomId, + }); + return { + eventId, + reaction: { + eventId: params.targetEventId, + key: emoji, + }, + }; +} + +function assertApprovalDecisionResult(params: { + approvalId: string; + decision: MatrixQaApprovalDecision; + result: unknown; +}) { + const result = + typeof params.result === "object" && params.result !== null + ? (params.result as { decision?: unknown; id?: unknown }) + : null; + if (result?.id !== params.approvalId) { + throw new Error( + `approval decision result id was ${formatApprovalResultValue(result?.id)} instead of ${params.approvalId}`, + ); + } + if (result?.decision !== params.decision) { + throw new Error( + `approval decision was ${formatApprovalResultValue(result?.decision)} instead of ${params.decision}`, + ); + } +} + +function formatApprovalResultValue(value: unknown) { + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return String(value); + } + if (value == null) { + return ""; + } + return JSON.stringify(value) ?? ""; +} + +async function requestExecApproval(params: { + context: MatrixQaScenarioContext; + command: string; + id?: string; + threadRootEventId?: string; +}) { + const gatewayCall = requireMatrixQaGatewayCall(params.context); + return await gatewayCall( + "exec.approval.request", + { + ...(params.id ? { id: params.id } : {}), + ask: "always", + command: params.command, + host: "gateway", + security: "full", + timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS, + twoPhase: true, + turnSourceAccountId: params.context.sutAccountId, + turnSourceChannel: "matrix", + turnSourceTo: `room:${params.context.roomId}`, + ...(params.threadRootEventId ? { turnSourceThreadId: params.threadRootEventId } : {}), + }, + { + expectFinal: false, + timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); +} + +async function requestPluginApproval(params: { context: MatrixQaScenarioContext; token: string }) { + const gatewayCall = requireMatrixQaGatewayCall(params.context); + return await gatewayCall( + "plugin.approval.request", + { + agentId: "qa", + description: `Matrix plugin approval QA request ${params.token}`, + pluginId: "qa-matrix-plugin", + severity: "warning", + timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS, + title: "Matrix plugin approval QA", + toolName: "matrix_qa_tool", + twoPhase: true, + turnSourceAccountId: params.context.sutAccountId, + turnSourceChannel: "matrix", + turnSourceTo: `room:${params.context.roomId}`, + }, + { + expectFinal: false, + timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); +} + +async function waitForApprovalDecision(params: { + approvalId: string; + context: MatrixQaScenarioContext; + kind: MatrixQaApprovalKind; +}) { + const gatewayCall = requireMatrixQaGatewayCall(params.context); + const method = + params.kind === "exec" ? "exec.approval.waitDecision" : "plugin.approval.waitDecision"; + return await gatewayCall( + method, + { id: params.approvalId }, + { + expectFinal: true, + timeoutMs: MATRIX_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); +} + +function readAcceptedApprovalRequest(result: unknown) { + const accepted = + typeof result === "object" && result !== null + ? (result as { id?: unknown; status?: unknown }) + : null; + if (accepted?.status !== "accepted") { + throw new Error( + `approval request status was ${formatApprovalResultValue(accepted?.status)} instead of accepted`, + ); + } + return accepted; +} + +function assertAcceptedApprovalRequest(params: { approvalId: string; result: unknown }) { + const id = readAcceptedApprovalRequest(params.result).id; + if (id !== params.approvalId) { + throw new Error( + `accepted approval id was ${formatApprovalResultValue(id)} instead of ${params.approvalId}`, + ); + } +} + +function readAcceptedApprovalRequestId(result: unknown) { + const id = readAcceptedApprovalRequest(result).id; + if (typeof id !== "string" || !id.trim()) { + throw new Error("approval request did not return an accepted approval id"); + } + return id; +} + +function buildExecApprovalCommand(params: { expectChunk?: boolean; token: string }) { + if (!params.expectChunk) { + return `printf ${params.token}`; + } + return `printf '${params.token} ${MATRIX_QA_APPROVAL_LONG_COMMAND_TEXT}'`; +} + +async function runExecApprovalScenario(params: { + context: MatrixQaScenarioContext; + decision: MatrixQaApprovalDecision; + expectChunk?: boolean; + tokenPrefix: string; + threadRootEventId?: string; +}) { + const { client, startSince } = await primeMatrixQaDriverScenarioClient(params.context); + const token = buildMatrixQaToken(params.tokenPrefix); + const command = buildExecApprovalCommand({ expectChunk: params.expectChunk, token }); + const approvalId = `qa-${token.toLowerCase()}-${randomUUID().slice(0, 8)}`; + const accepted = await requestExecApproval({ + context: params.context, + command, + id: approvalId, + threadRootEventId: params.threadRootEventId, + }); + assertAcceptedApprovalRequest({ approvalId, result: accepted }); + const approval = await waitForApprovalEvent({ + context: params.context, + expectedApprovalId: approvalId, + expectedKind: "exec", + roomId: params.context.roomId, + since: startSince, + threadRootEventId: params.threadRootEventId, + }); + if (params.expectChunk) { + const chunk = await client.waitForRoomEvent({ + observedEvents: params.context.observedEvents, + predicate: (event) => + event.roomId === params.context.roomId && + event.sender === params.context.sutUserId && + event.type === "m.room.message" && + event.body?.includes(token) === true && + event.eventId !== approval.event.eventId && + event.approval === undefined, + roomId: params.context.roomId, + timeoutMs: params.context.timeoutMs, + }); + if (chunk.event.approval) { + throw new Error(`chunk event ${chunk.event.eventId} unexpectedly duplicated metadata`); + } + } + const reaction = await reactToApproval({ + context: params.context, + decision: params.decision, + roomId: params.context.roomId, + targetEventId: approval.event.eventId, + }); + const result = await waitForApprovalDecision({ + approvalId, + context: params.context, + kind: "exec", + }); + assertApprovalDecisionResult({ + approvalId, + decision: params.decision, + result, + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: params.context.syncState, + nextSince: approval.since, + startSince, + }); + return { + artifacts: { + approval: buildMatrixApprovalArtifact(approval.event), + reactionEmoji: reaction.reaction?.key, + reactionEventId: reaction.eventId, + reactionTargetEventId: reaction.reaction?.eventId, + token, + }, + details: [ + `approval event: ${approval.event.eventId}`, + `approval id: ${approvalId}`, + `approval kind: ${approval.event.approval?.kind ?? ""}`, + `decision: ${params.decision}`, + `reaction event: ${reaction.eventId}`, + `reaction target: ${reaction.reaction?.eventId ?? ""}`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runApprovalExecMetadataSingleEventScenario(context: MatrixQaScenarioContext) { + return await runExecApprovalScenario({ + context, + decision: "allow-once", + tokenPrefix: "MATRIX_QA_APPROVAL_EXEC", + }); +} + +export async function runApprovalExecMetadataChunkedScenario(context: MatrixQaScenarioContext) { + return await runExecApprovalScenario({ + context, + decision: "allow-once", + expectChunk: true, + tokenPrefix: "MATRIX_QA_APPROVAL_CHUNKED", + }); +} + +export async function runApprovalDenyReactionScenario(context: MatrixQaScenarioContext) { + return await runExecApprovalScenario({ + context, + decision: "deny", + tokenPrefix: "MATRIX_QA_APPROVAL_DENY", + }); +} + +export async function runApprovalThreadTargetScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const token = buildMatrixQaToken("MATRIX_QA_APPROVAL_THREAD_ROOT"); + const rootEventId = await client.sendTextMessage({ + body: `Matrix approval thread root ${token}`, + roomId: context.roomId, + }); + const result = await runExecApprovalScenario({ + context, + decision: "allow-once", + threadRootEventId: rootEventId, + tokenPrefix: "MATRIX_QA_APPROVAL_THREAD", + }); + advanceMatrixQaActorCursor({ + actorId: "driver", + syncState: context.syncState, + startSince, + }); + return { + artifacts: { + ...result.artifacts, + rootEventId, + }, + details: [result.details, `thread root event: ${rootEventId}`].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runApprovalPluginMetadataSingleEventScenario( + context: MatrixQaScenarioContext, +) { + const { startSince } = await primeMatrixQaDriverScenarioClient(context); + const token = buildMatrixQaToken("MATRIX_QA_PLUGIN_APPROVAL"); + const accepted = await requestPluginApproval({ context, token }); + const approvalId = readAcceptedApprovalRequestId(accepted); + const approval = await waitForApprovalEvent({ + context, + expectedApprovalId: approvalId, + expectedKind: "plugin", + roomId: context.roomId, + since: startSince, + }); + const approvalMetadata = approval.event.approval; + if ( + approvalMetadata?.pluginId !== "qa-matrix-plugin" || + approvalMetadata.toolName !== "matrix_qa_tool" || + approvalMetadata.severity !== "warning" || + approvalMetadata.agentId !== "qa" + ) { + throw new Error(`plugin approval event ${approval.event.eventId} did not expose plugin fields`); + } + const reaction = await reactToApproval({ + context, + decision: "allow-once", + roomId: context.roomId, + targetEventId: approval.event.eventId, + }); + const result = await waitForApprovalDecision({ + approvalId, + context, + kind: "plugin", + }); + assertApprovalDecisionResult({ + approvalId, + decision: "allow-once", + result, + }); + return { + artifacts: { + approval: buildMatrixApprovalArtifact(approval.event), + reactionEmoji: reaction.reaction?.key, + reactionEventId: reaction.eventId, + reactionTargetEventId: reaction.reaction?.eventId, + token, + }, + details: [ + `approval event: ${approval.event.eventId}`, + `approval id: ${approvalMetadata.id}`, + `plugin id: ${approvalMetadata.pluginId ?? ""}`, + `tool name: ${approvalMetadata.toolName ?? ""}`, + `decision: allow-once`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} + +export async function runApprovalChannelTargetBothScenario(context: MatrixQaScenarioContext) { + const { client, startSince } = await primeMatrixQaDriverScenarioClient(context); + const dmRoomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_DRIVER_DM_ROOM_KEY); + const token = buildMatrixQaToken("MATRIX_QA_APPROVAL_BOTH"); + const approvalId = `qa-${token.toLowerCase()}-${randomUUID().slice(0, 8)}`; + const accepted = await requestExecApproval({ + context, + command: `printf ${token}`, + id: approvalId, + }); + assertAcceptedApprovalRequest({ approvalId, result: accepted }); + const channelApproval = await waitForApprovalEvent({ + context, + expectedApprovalId: approvalId, + expectedKind: "exec", + roomId: context.roomId, + since: startSince, + }); + const dmApproval = await waitForApprovalEvent({ + context, + expectedApprovalId: approvalId, + expectedKind: "exec", + roomId: dmRoomId, + since: startSince, + }); + if (channelApproval.event.approval?.id !== dmApproval.event.approval?.id) { + throw new Error("target=both delivered different approval ids to channel and DM"); + } + const reaction = await reactToApproval({ + context, + decision: "allow-once", + roomId: context.roomId, + targetEventId: channelApproval.event.eventId, + }); + const result = await waitForApprovalDecision({ + approvalId, + context, + kind: "exec", + }); + assertApprovalDecisionResult({ + approvalId, + decision: "allow-once", + result, + }); + const lateDuplicate = await client.waitForOptionalRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.sender === context.sutUserId && + event.type === "m.room.message" && + event.approval?.id === approvalId && + event.eventId !== channelApproval.event.eventId && + event.eventId !== dmApproval.event.eventId, + roomId: context.roomId, + timeoutMs: MATRIX_QA_APPROVAL_SHORT_WINDOW_MS, + }); + if (lateDuplicate.matched) { + throw new Error(`approval ${approvalId} was re-delivered after resolution`); + } + return { + artifacts: { + approvals: [ + buildMatrixApprovalArtifact(channelApproval.event), + buildMatrixApprovalArtifact(dmApproval.event), + ], + reactionEmoji: reaction.reaction?.key, + reactionEventId: reaction.eventId, + reactionTargetEventId: reaction.reaction?.eventId, + token, + }, + details: [ + `channel approval event: ${channelApproval.event.eventId}`, + `dm approval event: ${dmApproval.event.eventId}`, + `approval id: ${approvalId}`, + `decision: allow-once`, + ].join("\n"), + } satisfies MatrixQaScenarioExecution; +} diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts index 0a595203c36..d5dad267d89 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts @@ -29,6 +29,11 @@ export type MatrixQaScenarioContext = { observerUserId: string; gatewayRuntimeEnv?: NodeJS.ProcessEnv; gatewayStateDir?: string; + gatewayCall?: ( + method: string, + params?: Record, + opts?: { expectFinal?: boolean; timeoutMs?: number }, + ) => Promise; outputDir?: string; registrationToken?: string; restartGateway?: () => Promise; diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts index a67d21b56ce..e82fe2a2001 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime.ts @@ -3,6 +3,14 @@ import { MATRIX_QA_SECONDARY_ROOM_KEY, type MatrixQaScenarioDefinition, } from "./scenario-catalog.js"; +import { + runApprovalChannelTargetBothScenario, + runApprovalDenyReactionScenario, + runApprovalExecMetadataChunkedScenario, + runApprovalExecMetadataSingleEventScenario, + runApprovalPluginMetadataSingleEventScenario, + runApprovalThreadTargetScenario, +} from "./scenario-runtime-approval.js"; import { runDmPerRoomSessionOverrideScenario, runDmSharedSessionNoticeScenario, @@ -268,6 +276,18 @@ export async function runMatrixQaScenario( return await runReactionNotAReplyScenario(context); case "matrix-reaction-redaction-observed": return await runReactionRedactionObservedScenario(context); + case "matrix-approval-exec-metadata-single-event": + return await runApprovalExecMetadataSingleEventScenario(context); + case "matrix-approval-exec-metadata-chunked": + return await runApprovalExecMetadataChunkedScenario(context); + case "matrix-approval-plugin-metadata-single-event": + return await runApprovalPluginMetadataSingleEventScenario(context); + case "matrix-approval-deny-reaction": + return await runApprovalDenyReactionScenario(context); + case "matrix-approval-thread-target": + return await runApprovalThreadTargetScenario(context); + case "matrix-approval-channel-target-both": + return await runApprovalChannelTargetBothScenario(context); case "matrix-restart-resume": return await runRestartResumeScenario(context); case "matrix-post-restart-room-continue": diff --git a/extensions/qa-matrix/src/runners/contract/scenario-types.ts b/extensions/qa-matrix/src/runners/contract/scenario-types.ts index a9176ede86b..7c44ff628a6 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-types.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-types.ts @@ -17,6 +17,16 @@ export type MatrixQaCanaryArtifact = { export type MatrixQaScenarioArtifacts = { accepted?: MatrixQaScenarioArtifacts; + approval?: MatrixQaObservedEvent["approval"] & { + eventId: string; + roomId: string; + }; + approvals?: Array< + MatrixQaObservedEvent["approval"] & { + eventId: string; + roomId: string; + } + >; attachments?: Array<{ eventId: string; filename?: string; diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 5e1caa825a4..4dc9b2624fc 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -230,6 +230,12 @@ describe("matrix live qa scenarios", () => { "matrix-reaction-threaded", "matrix-reaction-not-a-reply", "matrix-reaction-redaction-observed", + "matrix-approval-exec-metadata-single-event", + "matrix-approval-exec-metadata-chunked", + "matrix-approval-plugin-metadata-single-event", + "matrix-approval-deny-reaction", + "matrix-approval-thread-target", + "matrix-approval-channel-target-both", "matrix-restart-resume", "matrix-post-restart-room-continue", "matrix-initial-catchup-then-incremental", @@ -319,6 +325,8 @@ describe("matrix live qa scenarios", () => { "matrix-thread-isolation", "matrix-top-level-reply-shape", "matrix-reaction-notification", + "matrix-approval-exec-metadata-single-event", + "matrix-approval-exec-metadata-chunked", "matrix-restart-resume", "matrix-mention-gating", "matrix-allowlist-block", diff --git a/extensions/qa-matrix/src/substrate/artifacts.test.ts b/extensions/qa-matrix/src/substrate/artifacts.test.ts index 951d5676e6c..423f17252c0 100644 --- a/extensions/qa-matrix/src/substrate/artifacts.test.ts +++ b/extensions/qa-matrix/src/substrate/artifacts.test.ts @@ -99,6 +99,58 @@ describe("matrix observed event artifacts", () => { ]); }); + it("keeps approval summaries in redacted Matrix observed-event artifacts", () => { + expect( + buildMatrixQaObservedEventsArtifact({ + includeContent: false, + observedEvents: [ + { + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$approval", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + body: "secret command body", + approval: { + id: "approval-1", + kind: "exec", + state: "pending", + type: "approval.request", + version: 1, + allowedDecisions: ["allow-once", "deny"], + hasCommandText: true, + commandTextPreview: "printf MATRIX_QA", + }, + }, + ], + }), + ).toEqual([ + { + kind: "message", + roomId: "!room:matrix-qa.test", + eventId: "$approval", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + originServerTs: undefined, + msgtype: undefined, + membership: undefined, + relatesTo: undefined, + mentions: undefined, + reaction: undefined, + approval: { + id: "approval-1", + kind: "exec", + state: "pending", + type: "approval.request", + version: 1, + allowedDecisions: ["allow-once", "deny"], + hasCommandText: true, + commandTextPreview: "printf MATRIX_QA", + }, + }, + ]); + }); + it("keeps redaction metadata while still stripping Matrix event content", () => { expect( buildMatrixQaObservedEventsArtifact({ diff --git a/extensions/qa-matrix/src/substrate/artifacts.ts b/extensions/qa-matrix/src/substrate/artifacts.ts index 250fbb83527..e4176db3b4f 100644 --- a/extensions/qa-matrix/src/substrate/artifacts.ts +++ b/extensions/qa-matrix/src/substrate/artifacts.ts @@ -20,6 +20,7 @@ export function buildMatrixQaObservedEventsArtifact(params: { relatesTo: event.relatesTo, mentions: event.mentions, reaction: event.reaction, + ...(event.approval ? { approval: event.approval } : {}), attachment: event.attachment ? { kind: event.attachment.kind, diff --git a/extensions/qa-matrix/src/substrate/config.test.ts b/extensions/qa-matrix/src/substrate/config.test.ts index 34f06089469..0537072642e 100644 --- a/extensions/qa-matrix/src/substrate/config.test.ts +++ b/extensions/qa-matrix/src/substrate/config.test.ts @@ -227,9 +227,14 @@ describe("matrix qa config", () => { }); expect(snapshot).toEqual({ + approvalForwarding: { + exec: false, + plugin: false, + }, autoJoin: "allowlist", autoJoinAllowlist: ["!ops:matrix-qa.test"], blockStreaming: true, + chunkMode: undefined, dm: { allowFrom: ["@driver:matrix-qa.test"], enabled: true, @@ -238,6 +243,7 @@ describe("matrix qa config", () => { threadReplies: "inbound", }, encryption: false, + execApprovals: undefined, groupAllowFrom: ["@driver:matrix-qa.test"], groupPolicy: "open", groupsByKey: { @@ -255,6 +261,7 @@ describe("matrix qa config", () => { replyToMode: "off", streaming: "partial", streamingPreviewToolProgress: true, + textChunkLimit: undefined, threadBindings: {}, threadReplies: "inbound", }); @@ -289,6 +296,50 @@ describe("matrix qa config", () => { ); }); + it("applies Matrix approval delivery overrides with gateway forwarding enabled", () => { + const next = buildMatrixQaConfig({} as OpenClawConfig, { + driverUserId: "@driver:matrix-qa.test", + homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", + overrides: { + approvalForwarding: { + exec: true, + plugin: true, + }, + chunkMode: "length", + dm: { + enabled: true, + }, + execApprovals: { + enabled: true, + target: "both", + }, + textChunkLimit: 280, + }, + sutAccessToken: "sut-token", + sutAccountId: "sut", + sutUserId: "@sut:matrix-qa.test", + topology, + }); + + expect(next.approvals).toMatchObject({ + exec: { enabled: true, mode: "session" }, + plugin: { enabled: true, mode: "session" }, + }); + expect(next.channels?.matrix?.accounts?.sut).toMatchObject({ + chunkMode: "length", + dm: { + allowFrom: ["@driver:matrix-qa.test"], + enabled: true, + }, + execApprovals: { + enabled: true, + target: "both", + }, + textChunkLimit: 280, + }); + }); + it("resolves role-based Matrix sender allowlist overrides", () => { const snapshot = buildMatrixQaConfigSnapshot({ driverUserId: "@driver:matrix-qa.test", diff --git a/extensions/qa-matrix/src/substrate/config.ts b/extensions/qa-matrix/src/substrate/config.ts index 7fb5eff47e4..ee6c292d9b4 100644 --- a/extensions/qa-matrix/src/substrate/config.ts +++ b/extensions/qa-matrix/src/substrate/config.ts @@ -8,6 +8,9 @@ export type MatrixQaGroupPolicy = "allowlist" | "disabled" | "open"; export type MatrixQaAutoJoinMode = "allowlist" | "always" | "off"; export type MatrixQaStreamingMode = "off" | "partial" | "quiet"; export type MatrixQaActorRole = "driver" | "observer" | "sut"; +export type MatrixQaChunkMode = "length" | "newline"; +export type MatrixQaExecApprovalTarget = "both" | "channel" | "dm"; +export type MatrixQaExecApprovalsEnabled = boolean | "auto"; export type MatrixQaStreamingConfig = { mode?: MatrixQaStreamingMode; @@ -56,13 +59,27 @@ export type MatrixQaThreadBindingsConfigOverrides = { spawnSubagentSessions?: boolean; }; +export type MatrixQaExecApprovalsConfigOverrides = { + agentFilter?: string[]; + approvers?: string[]; + enabled?: MatrixQaExecApprovalsEnabled; + sessionFilter?: string[]; + target?: MatrixQaExecApprovalTarget; +}; + export type MatrixQaConfigOverrides = { + approvalForwarding?: { + exec?: boolean; + plugin?: boolean; + }; agentDefaults?: MatrixQaAgentDefaultsOverrides; autoJoin?: MatrixQaAutoJoinMode; autoJoinAllowlist?: string[]; blockStreaming?: boolean; + chunkMode?: MatrixQaChunkMode; dm?: MatrixQaDmConfigOverrides; encryption?: boolean; + execApprovals?: MatrixQaExecApprovalsConfigOverrides; groupAllowFrom?: string[]; groupAllowRoles?: MatrixQaActorRole[]; groupPolicy?: MatrixQaGroupPolicy; @@ -70,15 +87,21 @@ export type MatrixQaConfigOverrides = { replyToMode?: MatrixQaReplyToMode; startupVerification?: "if-unverified" | "off"; streaming?: MatrixQaStreamingMode | MatrixQaStreamingConfig | boolean; + textChunkLimit?: number; threadBindings?: MatrixQaThreadBindingsConfigOverrides; threadReplies?: MatrixQaThreadRepliesMode; toolProfile?: "coding" | "messaging" | "minimal"; }; export type MatrixQaConfigSnapshot = { + approvalForwarding: { + exec: boolean; + plugin: boolean; + }; autoJoin: MatrixQaAutoJoinMode; autoJoinAllowlist: string[]; blockStreaming: boolean; + chunkMode?: MatrixQaChunkMode; dm: { allowFrom: string[]; enabled: boolean; @@ -87,6 +110,7 @@ export type MatrixQaConfigSnapshot = { threadReplies: MatrixQaThreadRepliesMode; }; encryption: boolean; + execApprovals?: MatrixQaExecApprovalsConfigOverrides; groupAllowFrom: string[]; groupPolicy: MatrixQaGroupPolicy; groupsByKey: Record; @@ -94,6 +118,7 @@ export type MatrixQaConfigSnapshot = { startupVerification?: "if-unverified" | "off"; streaming: MatrixQaStreamingMode; streamingPreviewToolProgress: boolean; + textChunkLimit?: number; threadBindings: MatrixQaThreadBindingsConfigOverrides; threadReplies: MatrixQaThreadRepliesMode; }; @@ -121,6 +146,14 @@ type MatrixQaAccountDmConfig = threadReplies?: MatrixQaThreadRepliesMode; }; +type MatrixQaAccountExecApprovalsConfig = { + agentFilter?: string[]; + approvers?: string[]; + enabled?: MatrixQaExecApprovalsEnabled; + sessionFilter?: string[]; + target?: MatrixQaExecApprovalTarget; +}; + function normalizeMatrixQaAllowlist(entries?: string[]) { return [...new Set((entries ?? []).map((entry) => entry.trim()).filter(Boolean))]; } @@ -299,6 +332,21 @@ function buildMatrixQaAccountDmConfig(params: { }; } +function buildMatrixQaAccountExecApprovalsConfig( + overrides?: MatrixQaExecApprovalsConfigOverrides, +): MatrixQaAccountExecApprovalsConfig | undefined { + if (!overrides) { + return undefined; + } + return { + ...(overrides.agentFilter ? { agentFilter: overrides.agentFilter } : {}), + ...(overrides.approvers ? { approvers: normalizeMatrixQaAllowlist(overrides.approvers) } : {}), + ...(overrides.enabled !== undefined ? { enabled: overrides.enabled } : {}), + ...(overrides.sessionFilter ? { sessionFilter: overrides.sessionFilter } : {}), + ...(overrides.target ? { target: overrides.target } : {}), + }; +} + function buildMatrixQaChannelAccountConfig(params: { groups: Record; homeserver: string; @@ -319,6 +367,11 @@ function buildMatrixQaChannelAccountConfig(params: { params.overrides?.blockStreaming !== undefined ? { blockStreaming: params.snapshot.blockStreaming } : {}; + const chunkModeConfig = + params.snapshot.chunkMode !== undefined ? { chunkMode: params.snapshot.chunkMode } : {}; + const execApprovalsConfig = buildMatrixQaAccountExecApprovalsConfig( + params.snapshot.execApprovals, + ); const streamingConfig = params.overrides?.streaming !== undefined ? { streaming: params.overrides.streaming } : {}; const startupVerificationConfig = @@ -329,6 +382,10 @@ function buildMatrixQaChannelAccountConfig(params: { params.overrides?.threadBindings !== undefined ? { threadBindings: params.snapshot.threadBindings } : {}; + const textChunkLimitConfig = + params.snapshot.textChunkLimit !== undefined + ? { textChunkLimit: params.snapshot.textChunkLimit } + : {}; return { accessToken: params.sutAccessToken, @@ -354,7 +411,10 @@ function buildMatrixQaChannelAccountConfig(params: { ...autoJoinConfig, ...autoJoinAllowlistConfig, ...blockStreamingConfig, + ...chunkModeConfig, + ...(execApprovalsConfig ? { execApprovals: execApprovalsConfig } : {}), ...streamingConfig, + ...textChunkLimitConfig, }; } @@ -369,8 +429,10 @@ export function buildMatrixQaConfigSnapshot(params: { autoJoin: params.overrides?.autoJoin ?? "off", autoJoinAllowlist: resolveMatrixQaAutoJoinAllowlist(params), blockStreaming: params.overrides?.blockStreaming ?? false, + chunkMode: params.overrides?.chunkMode, dm: resolveMatrixQaDmConfigSnapshot(params), encryption: params.overrides?.encryption ?? false, + execApprovals: params.overrides?.execApprovals, groupAllowFrom: resolveMatrixQaGroupAllowFrom(params), groupPolicy: params.overrides?.groupPolicy ?? "allowlist", groupsByKey: resolveMatrixQaGroupSnapshots({ @@ -384,7 +446,13 @@ export function buildMatrixQaConfigSnapshot(params: { params.overrides?.streaming, ), threadBindings: { ...params.overrides?.threadBindings }, + textChunkLimit: params.overrides?.textChunkLimit, threadReplies: params.overrides?.threadReplies ?? "inbound", + approvalForwarding: { + exec: + params.overrides?.approvalForwarding?.exec ?? params.overrides?.execApprovals !== undefined, + plugin: params.overrides?.approvalForwarding?.plugin ?? false, + }, }; } @@ -398,12 +466,18 @@ export function summarizeMatrixQaConfigSnapshot(snapshot: MatrixQaConfigSnapshot `dm.threadReplies=${snapshot.dm.threadReplies}`, `streaming=${snapshot.streaming}`, `streaming.preview.toolProgress=${formatMatrixQaBoolean(snapshot.streamingPreviewToolProgress)}`, + `textChunkLimit=${snapshot.textChunkLimit ?? ""}`, + `chunkMode=${snapshot.chunkMode ?? ""}`, + `execApprovals.enabled=${snapshot.execApprovals?.enabled ?? ""}`, + `execApprovals.target=${snapshot.execApprovals?.target ?? ""}`, `blockStreaming=${formatMatrixQaBoolean(snapshot.blockStreaming)}`, `autoJoin=${snapshot.autoJoin}`, `encryption=${formatMatrixQaBoolean(snapshot.encryption)}`, `startupVerification=${snapshot.startupVerification ?? ""}`, `threadBindings.enabled=${snapshot.threadBindings.enabled ?? ""}`, `threadBindings.spawnSubagentSessions=${snapshot.threadBindings.spawnSubagentSessions ?? ""}`, + `approvals.exec.enabled=${formatMatrixQaBoolean(snapshot.approvalForwarding.exec)}`, + `approvals.plugin.enabled=${formatMatrixQaBoolean(snapshot.approvalForwarding.plugin)}`, ].join(", "); } @@ -430,9 +504,36 @@ export function buildMatrixQaConfig( topology: params.topology, }); const groups = buildMatrixQaGroupEntries(snapshot.groupsByKey); + const approvalForwardingConfig = + snapshot.approvalForwarding.exec || snapshot.approvalForwarding.plugin + ? { + approvals: { + ...baseCfg.approvals, + ...(snapshot.approvalForwarding.exec + ? { + exec: { + ...baseCfg.approvals?.exec, + enabled: true, + mode: "session" as const, + }, + } + : {}), + ...(snapshot.approvalForwarding.plugin + ? { + plugin: { + ...baseCfg.approvals?.plugin, + enabled: true, + mode: "session" as const, + }, + } + : {}), + }, + } + : {}; return { ...baseCfg, + ...approvalForwardingConfig, ...(params.overrides?.toolProfile ? { tools: { diff --git a/extensions/qa-matrix/src/substrate/events.test.ts b/extensions/qa-matrix/src/substrate/events.test.ts index fcbb2513f36..5b30acbb478 100644 --- a/extensions/qa-matrix/src/substrate/events.test.ts +++ b/extensions/qa-matrix/src/substrate/events.test.ts @@ -133,6 +133,84 @@ describe("matrix observed event normalization", () => { }); }); + it("summarizes Matrix approval metadata without dumping full command text", () => { + const commandText = `printf ${"A".repeat(300)}`; + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$approval", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + content: { + body: "React here: ✅ Allow once, ❌ Deny", + msgtype: "m.text", + "com.openclaw.approval": { + allowedDecisions: ["allow-once", "deny"], + commandText, + id: "approval-1", + kind: "exec", + state: "pending", + type: "approval.request", + version: 1, + }, + }, + }), + ).toEqual( + expect.objectContaining({ + approval: { + allowedDecisions: ["allow-once", "deny"], + commandTextPreview: commandText.slice(0, 160), + hasCommandText: true, + id: "approval-1", + kind: "exec", + state: "pending", + type: "approval.request", + version: 1, + }, + }), + ); + }); + + it("summarizes Matrix plugin approval metadata fields", () => { + expect( + normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { + event_id: "$plugin-approval", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + content: { + body: "Plugin approval required", + msgtype: "m.text", + "com.openclaw.approval": { + agentId: "qa", + allowedDecisions: ["allow-once", "deny"], + id: "plugin:approval-1", + kind: "plugin", + pluginId: "qa-plugin", + severity: "medium", + state: "pending", + toolName: "qa_tool", + type: "approval.request", + version: 1, + }, + }, + }), + ).toEqual( + expect.objectContaining({ + approval: { + agentId: "qa", + allowedDecisions: ["allow-once", "deny"], + id: "plugin:approval-1", + kind: "plugin", + pluginId: "qa-plugin", + severity: "medium", + state: "pending", + toolName: "qa_tool", + type: "approval.request", + version: 1, + }, + }), + ); + }); + it("normalizes Matrix image messages with attachment metadata", () => { expect( normalizeMatrixQaObservedEvent("!room:matrix-qa.test", { diff --git a/extensions/qa-matrix/src/substrate/events.ts b/extensions/qa-matrix/src/substrate/events.ts index e5434a368d5..3769cf1e492 100644 --- a/extensions/qa-matrix/src/substrate/events.ts +++ b/extensions/qa-matrix/src/substrate/events.ts @@ -21,6 +21,21 @@ export type MatrixQaObservedEventAttachment = { kind: "audio" | "file" | "image" | "sticker" | "video"; }; +export type MatrixQaObservedApproval = { + agentId?: string; + allowedDecisions?: string[]; + commandTextPreview?: string; + hasCommandText?: boolean; + id: string; + kind: "exec" | "plugin"; + pluginId?: string; + severity?: string; + state?: string; + toolName?: string; + type?: string; + version?: number; +}; + export type MatrixQaObservedEvent = { kind: MatrixQaObservedEventKind; roomId: string; @@ -48,8 +63,12 @@ export type MatrixQaObservedEvent = { key?: string; }; attachment?: MatrixQaObservedEventAttachment; + approval?: MatrixQaObservedApproval; }; +const MATRIX_QA_APPROVAL_METADATA_KEY = "com.openclaw.approval"; +const MATRIX_QA_APPROVAL_COMMAND_PREVIEW_CHARS = 160; + function normalizeMentionUserIds(value: unknown) { return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0) @@ -130,6 +149,54 @@ function resolveMatrixQaAttachmentSummary(params: { }; } +function normalizeMatrixQaApprovalAllowedDecisions(value: unknown) { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0) + : undefined; +} + +function normalizeMatrixQaApprovalMetadata(value: unknown): MatrixQaObservedApproval | undefined { + if (typeof value !== "object" || value === null) { + return undefined; + } + const metadata = value as Record; + const id = typeof metadata.id === "string" ? metadata.id.trim() : ""; + const kind = metadata.kind; + if (!id || (kind !== "exec" && kind !== "plugin")) { + return undefined; + } + const commandText = + typeof metadata.commandText === "string" ? metadata.commandText.trim() : undefined; + const commandPreview = + typeof metadata.commandPreview === "string" ? metadata.commandPreview.trim() : undefined; + const commandTextPreview = (commandPreview || commandText)?.slice( + 0, + MATRIX_QA_APPROVAL_COMMAND_PREVIEW_CHARS, + ); + return { + id, + kind, + ...(typeof metadata.agentId === "string" ? { agentId: metadata.agentId } : {}), + ...(typeof metadata.state === "string" ? { state: metadata.state } : {}), + ...(typeof metadata.type === "string" ? { type: metadata.type } : {}), + ...(typeof metadata.version === "number" ? { version: metadata.version } : {}), + ...(metadata.allowedDecisions + ? { allowedDecisions: normalizeMatrixQaApprovalAllowedDecisions(metadata.allowedDecisions) } + : {}), + ...(commandText ? { hasCommandText: true } : {}), + ...(commandTextPreview ? { commandTextPreview } : {}), + ...(kind === "plugin" && typeof metadata.pluginId === "string" + ? { pluginId: metadata.pluginId } + : {}), + ...(kind === "plugin" && typeof metadata.severity === "string" + ? { severity: metadata.severity } + : {}), + ...(kind === "plugin" && typeof metadata.toolName === "string" + ? { toolName: metadata.toolName } + : {}), + }; +} + export function normalizeMatrixQaObservedEvent( roomId: string, event: MatrixQaRoomEvent, @@ -177,6 +244,9 @@ export function normalizeMatrixQaObservedEvent( filename: normalizedFilename, msgtype: normalizedMsgtype, }); + const approval = normalizeMatrixQaApprovalMetadata( + messageContent[MATRIX_QA_APPROVAL_METADATA_KEY] ?? content[MATRIX_QA_APPROVAL_METADATA_KEY], + ); return { kind: resolveMatrixQaObservedEventKind({ msgtype: normalizedMsgtype, type }), @@ -222,6 +292,7 @@ export function normalizeMatrixQaObservedEvent( } : {}), ...(attachment ? { attachment } : {}), + ...(approval ? { approval } : {}), }; }