From e582cebf2d4c17f37d678b1b8ecfdf8502a3f6dc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 9 May 2026 09:15:08 +0100 Subject: [PATCH] fix(matrix): wire presentation metadata delivery --- CHANGELOG.md | 1 + docs/.i18n/glossary.zh-CN.json | 4 ++ docs/docs.json | 1 + .../src/channel.message-adapter.test.ts | 66 +++++++++++++++++++ extensions/matrix/src/channel.ts | 15 +++++ extensions/matrix/src/outbound.test.ts | 52 +++++++++++++-- extensions/matrix/src/outbound.ts | 20 +++++- .../plugins/runtime-forwarders.test.ts | 28 +++++++- src/channels/plugins/runtime-forwarders.ts | 34 +++++++++- 9 files changed, 211 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab2c735798c..b0dc0ed0373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ Docs: https://docs.openclaw.ai - Plugins/install: add `npm-pack:` installs so local npm pack artifacts run through the same managed npm-root install, lockfile verification, dependency scan, and install-record path as registry npm plugins. - Channels/plugins: show configured official external channels as missing-plugin status rows and send errors with exact install/doctor repair commands after raw package-manager upgrades leave Feishu or WhatsApp uninstalled. Fixes #78702 and #78593. Thanks @MarkMa84 and @mkupiainen. - Matrix: move the Matrix channel back to an official external ClawHub/npm plugin so core installs no longer need Matrix SDK runtime dependencies. +- Matrix: attach `com.openclaw.presentation` metadata to semantic presentation replies so OpenClaw-aware Matrix clients can render rich buttons, selects, context rows, and dividers while stock clients keep the plain text fallback. (#73312) Thanks @kakahu2015. - Codex app-server: disarm the short post-tool completion watchdog after current-turn activity, expose `appServer.turnCompletionIdleTimeoutMs`, and include raw assistant item context in idle-timeout diagnostics so status-only post-tool stalls stop failing as idle. Fixes #77984. Thanks @roseware-dev and @rubencu. - Plugin skills/Windows: publish plugin-provided skill directories as junctions on Windows so standard users without Developer Mode can register plugin skills without symlink EPERM failures. Fixes #77958. (#77971) Thanks @hclsys and @jarro. - Shell env/Windows: hide the login-shell environment probe child window so gateway startup and shell-env refreshes do not flash a console on Windows. Fixes #78159. (#78266) Thanks @BradGroux. diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 42105dded98..d968ed7a847 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -767,6 +767,10 @@ "source": "Matrix QA", "target": "Matrix QA" }, + { + "source": "Matrix presentation metadata", + "target": "Matrix 呈现元数据" + }, { "source": "QA overview", "target": "QA overview" diff --git a/docs/docs.json b/docs/docs.json index e69ec858b6f..4a13c2d59eb 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1066,6 +1066,7 @@ "channels/imessage-from-bluebubbles", "channels/matrix", "channels/matrix-migration", + "channels/matrix-presentation", "channels/matrix-push-rules" ] }, diff --git a/extensions/matrix/src/channel.message-adapter.test.ts b/extensions/matrix/src/channel.message-adapter.test.ts index 604eb978a96..80f455855d0 100644 --- a/extensions/matrix/src/channel.message-adapter.test.ts +++ b/extensions/matrix/src/channel.message-adapter.test.ts @@ -130,6 +130,72 @@ describe("matrix channel message adapter", () => { }); }); + it("forwards presentation payload hooks through the registered outbound adapter", async () => { + const outbound = matrixPlugin.outbound; + expect(outbound?.presentationCapabilities).toMatchObject({ + supported: true, + buttons: true, + selects: true, + context: true, + divider: true, + }); + if (!outbound?.renderPresentation || !outbound.sendPayload) { + throw new Error("Expected Matrix outbound presentation payload hooks."); + } + + const presentation = { + title: "Select thinking level", + tone: "info" as const, + blocks: [ + { + type: "buttons" as const, + buttons: [{ label: "Low", value: "/think low" }], + }, + ], + }; + const rendered = await outbound.renderPresentation({ + payload: { text: "fallback", presentation }, + presentation, + ctx: {} as never, + }); + + expect(rendered?.channelData?.matrix).toMatchObject({ + extraContent: { + "com.openclaw.presentation": { + ...presentation, + version: 1, + type: "message.presentation", + }, + }, + }); + + await outbound.sendPayload({ + cfg, + to: "room:!room:example", + text: rendered?.text ?? "", + payload: rendered!, + accountId: "default", + threadId: "$thread", + }); + + expect(mocks.sendMessageMatrix).toHaveBeenLastCalledWith( + "room:!room:example", + rendered?.text, + expect.objectContaining({ + cfg, + accountId: "default", + threadId: "$thread", + extraContent: { + "com.openclaw.presentation": { + ...presentation, + version: 1, + type: "message.presentation", + }, + }, + }), + ); + }); + it("backs declared live preview finalizer capabilities with adapter proofs", async () => { const adapter = matrixPlugin.message; diff --git a/extensions/matrix/src/channel.ts b/extensions/matrix/src/channel.ts index 6497122ca6e..e3b9bc9cfed 100644 --- a/extensions/matrix/src/channel.ts +++ b/extensions/matrix/src/channel.ts @@ -335,6 +335,13 @@ const matrixChannelOutbound: ChannelOutboundAdapter = { messageSendingHooks: true, }, }, + presentationCapabilities: { + supported: true, + buttons: true, + selects: true, + context: true, + divider: true, + }, shouldSuppressLocalPayloadPrompt: ({ cfg, accountId, payload }) => shouldSuppressLocalMatrixExecApprovalPrompt({ cfg, @@ -343,6 +350,14 @@ const matrixChannelOutbound: ChannelOutboundAdapter = { }), ...createRuntimeOutboundDelegates({ getRuntime: loadMatrixChannelRuntime, + renderPresentation: { + resolve: (runtime) => runtime.matrixOutbound.renderPresentation, + unavailableMessage: "Matrix outbound presentation rendering is unavailable", + }, + sendPayload: { + resolve: (runtime) => runtime.matrixOutbound.sendPayload, + unavailableMessage: "Matrix outbound payload delivery is unavailable", + }, sendText: { resolve: (runtime) => runtime.matrixOutbound.sendText, unavailableMessage: "Matrix outbound text delivery is unavailable", diff --git a/extensions/matrix/src/outbound.test.ts b/extensions/matrix/src/outbound.test.ts index d2fe20e8d1b..97a5951836f 100644 --- a/extensions/matrix/src/outbound.test.ts +++ b/extensions/matrix/src/outbound.test.ts @@ -259,6 +259,53 @@ describe("matrixOutbound cfg threading", () => { ); }); + it("only forwards presentation metadata from Matrix extraContent", async () => { + const cfg = { + channels: { + matrix: { + accessToken: "resolved-token", + }, + }, + } as OpenClawConfig; + + const presentationContent = { + version: 1, + type: "message.presentation", + title: "Select model", + blocks: [{ type: "divider" }], + }; + + await matrixOutbound.sendPayload!({ + cfg, + to: "room:!room:example", + text: "Select model", + payload: { + text: "Select model", + channelData: { + matrix: { + extraContent: { + body: "spoofed", + msgtype: "m.notice", + "m.relates_to": { "m.in_reply_to": { event_id: "$spoof" } }, + "com.openclaw.presentation": presentationContent, + }, + }, + }, + }, + accountId: "default", + }); + + expect(mocks.sendMessageMatrix).toHaveBeenCalledWith( + "room:!room:example", + "Select model", + expect.objectContaining({ + extraContent: { + "com.openclaw.presentation": presentationContent, + }, + }), + ); + }); + it("sends all media URLs via sendPayload", async () => { const cfg = { channels: { @@ -281,7 +328,6 @@ describe("matrixOutbound cfg threading", () => { }); expect(mocks.sendMessageMatrix).toHaveBeenCalledTimes(2); - // First call: caption + media expect(mocks.sendMessageMatrix).toHaveBeenNthCalledWith( 1, "room:!room:example", @@ -291,7 +337,6 @@ describe("matrixOutbound cfg threading", () => { threadId: "$thread", }), ); - // Second call: no text, just media expect(mocks.sendMessageMatrix).toHaveBeenNthCalledWith( 2, "room:!room:example", @@ -335,7 +380,6 @@ describe("matrixOutbound cfg threading", () => { }); expect(mocks.sendMessageMatrix).toHaveBeenCalledTimes(2); - // First call gets extraContent expect(mocks.sendMessageMatrix).toHaveBeenNthCalledWith( 1, "room:!room:example", @@ -349,7 +393,6 @@ describe("matrixOutbound cfg threading", () => { }, }), ); - // Second call does NOT get extraContent expect(mocks.sendMessageMatrix).toHaveBeenNthCalledWith( 2, "room:!room:example", @@ -380,7 +423,6 @@ describe("matrixOutbound cfg threading", () => { accountId: "default", }); - // Every URL must be sent — none may be silently dropped expect(mocks.sendMessageMatrix).toHaveBeenCalledTimes(3); expect(mocks.sendMessageMatrix).toHaveBeenNthCalledWith( 1, diff --git a/extensions/matrix/src/outbound.ts b/extensions/matrix/src/outbound.ts index 5b86c9b1536..60961a21b26 100644 --- a/extensions/matrix/src/outbound.ts +++ b/extensions/matrix/src/outbound.ts @@ -39,6 +39,21 @@ function buildMatrixPresentationContent(presentation: MessagePresentation) { }; } +function resolveMatrixPresentationContent( + payload: ReplyPayload, +): Record | undefined { + const extraContent = toRecord(resolveMatrixChannelData(payload).extraContent); + const presentation = toRecord(extraContent?.[MATRIX_OPENCLAW_PRESENTATION_KEY]); + if ( + !presentation || + presentation.version !== 1 || + presentation.type !== MATRIX_OPENCLAW_PRESENTATION_TYPE + ) { + return undefined; + } + return presentation; +} + function renderMatrixPresentationPayload(params: { payload: ReplyPayload; presentation: MessagePresentation; @@ -55,7 +70,6 @@ function renderMatrixPresentationPayload(params: { matrix: { ...matrixData, extraContent: { - ...matrixData.extraContent, [MATRIX_OPENCLAW_PRESENTATION_KEY]: buildMatrixPresentationContent(params.presentation), }, }, @@ -64,8 +78,8 @@ function renderMatrixPresentationPayload(params: { } function resolveMatrixExtraContent(payload: ReplyPayload): MatrixExtraContentFields | undefined { - const extraContent = resolveMatrixChannelData(payload).extraContent; - return extraContent && Object.keys(extraContent).length > 0 ? extraContent : undefined; + const presentation = resolveMatrixPresentationContent(payload); + return presentation ? { [MATRIX_OPENCLAW_PRESENTATION_KEY]: presentation } : undefined; } export const matrixOutbound: ChannelOutboundAdapter = { diff --git a/src/channels/plugins/runtime-forwarders.test.ts b/src/channels/plugins/runtime-forwarders.test.ts index e780f20407f..319740721de 100644 --- a/src/channels/plugins/runtime-forwarders.test.ts +++ b/src/channels/plugins/runtime-forwarders.test.ts @@ -3,6 +3,11 @@ import { createRuntimeDirectoryLiveAdapter, createRuntimeOutboundDelegates, } from "./runtime-forwarders.js"; +import type { ChannelOutboundAdapter } from "./types.adapters.js"; + +type RenderPresentationParams = Parameters< + NonNullable +>[0]; describe("createRuntimeDirectoryLiveAdapter", () => { it("forwards live directory calls through the runtime getter", async () => { @@ -28,16 +33,37 @@ describe("createRuntimeDirectoryLiveAdapter", () => { describe("createRuntimeOutboundDelegates", () => { it("forwards outbound methods through the runtime getter", async () => { + const renderPresentation = vi.fn(async (ctx: RenderPresentationParams) => ({ + ...ctx.payload, + text: "rendered", + })); + const sendPayload = vi.fn(async () => ({ channel: "x", messageId: "payload-1" })); const sendText = vi.fn(async () => ({ channel: "x", messageId: "1" })); const outbound = createRuntimeOutboundDelegates({ - getRuntime: async () => ({ outbound: { sendText } }), + getRuntime: async () => ({ outbound: { renderPresentation, sendPayload, sendText } }), + renderPresentation: { resolve: (runtime) => runtime.outbound.renderPresentation }, + sendPayload: { resolve: (runtime) => runtime.outbound.sendPayload }, sendText: { resolve: (runtime) => runtime.outbound.sendText }, }); + await expect( + outbound.renderPresentation?.({ + payload: { text: "raw" }, + presentation: { blocks: [{ type: "text", text: "shown" }] }, + ctx: {} as never, + }), + ).resolves.toEqual({ + text: "rendered", + }); + await expect( + outbound.sendPayload?.({ cfg: {} as never, to: "a", text: "hi", payload: { text: "hi" } }), + ).resolves.toEqual({ channel: "x", messageId: "payload-1" }); await expect(outbound.sendText?.({ cfg: {} as never, to: "a", text: "hi" })).resolves.toEqual({ channel: "x", messageId: "1", }); + expect(renderPresentation).toHaveBeenCalled(); + expect(sendPayload).toHaveBeenCalled(); expect(sendText).toHaveBeenCalled(); }); diff --git a/src/channels/plugins/runtime-forwarders.ts b/src/channels/plugins/runtime-forwarders.ts index ab4322ee973..5bb96c8571b 100644 --- a/src/channels/plugins/runtime-forwarders.ts +++ b/src/channels/plugins/runtime-forwarders.ts @@ -3,7 +3,7 @@ import type { ChannelDirectoryAdapter, ChannelOutboundAdapter } from "./types.ad type MaybePromise = T | Promise; type DirectoryMethod = "self" | "listPeersLive" | "listGroupsLive" | "listGroupMembers"; -type OutboundMethod = "sendText" | "sendMedia" | "sendPoll"; +type OutboundMethod = "renderPresentation" | "sendPayload" | "sendText" | "sendMedia" | "sendPoll"; type DirectorySelfParams = Parameters>[0]; type DirectoryListParams = Parameters>[0]; @@ -13,6 +13,10 @@ type DirectoryGroupMembersParams = Parameters< type SendTextParams = Parameters>[0]; type SendMediaParams = Parameters>[0]; type SendPollParams = Parameters>[0]; +type RenderPresentationParams = Parameters< + NonNullable +>[0]; +type SendPayloadParams = Parameters>[0]; async function resolveForwardedMethod(params: { getRuntime: () => MaybePromise; @@ -80,6 +84,14 @@ export function createRuntimeDirectoryLiveAdapter(params: { export function createRuntimeOutboundDelegates(params: { getRuntime: () => MaybePromise; + renderPresentation?: { + resolve: (runtime: Runtime) => ChannelOutboundAdapter["renderPresentation"] | null | undefined; + unavailableMessage?: string; + }; + sendPayload?: { + resolve: (runtime: Runtime) => ChannelOutboundAdapter["sendPayload"] | null | undefined; + unavailableMessage?: string; + }; sendText?: { resolve: (runtime: Runtime) => ChannelOutboundAdapter["sendText"] | null | undefined; unavailableMessage?: string; @@ -94,6 +106,26 @@ export function createRuntimeOutboundDelegates(params: { }; }): Pick { return { + renderPresentation: params.renderPresentation + ? async (ctx: RenderPresentationParams) => + await ( + await resolveForwardedMethod({ + getRuntime: params.getRuntime, + resolve: params.renderPresentation!.resolve, + unavailableMessage: params.renderPresentation!.unavailableMessage, + }) + )(ctx) + : undefined, + sendPayload: params.sendPayload + ? async (ctx: SendPayloadParams) => + await ( + await resolveForwardedMethod({ + getRuntime: params.getRuntime, + resolve: params.sendPayload!.resolve, + unavailableMessage: params.sendPayload!.unavailableMessage, + }) + )(ctx) + : undefined, sendText: params.sendText ? async (ctx: SendTextParams) => await (