Refactor message tool media-source discovery

This commit is contained in:
Gustavo Madeira Santana
2026-04-14 11:10:55 -04:00
parent 5d3d50cba5
commit fe29ec5157
15 changed files with 316 additions and 52 deletions

View File

@@ -1,2 +1,2 @@
a1e765cf426077085975f1f00847026b71f301cad35cb9168713e2b6249c4a47 plugin-sdk-api-baseline.json
9f1cdbe8d9bfbd582edb671729c4c09e578fb1940e787cfd6aa82dee0bdf5de7 plugin-sdk-api-baseline.jsonl
b446e9695b5f5c61d9a404d88fab9200752fffbc320dde9eac6d7f47027f75a4 plugin-sdk-api-baseline.json
f3d19cbae5a5e77cdfb8ebd20c0fc7f14471cba9ca08c0b539d8b18c614750b7 plugin-sdk-api-baseline.jsonl

View File

@@ -173,6 +173,12 @@ For channel plugins, the SDK surface is
call lets a plugin return its visible actions, capabilities, and schema
contributions together so those pieces do not drift apart.
When a channel-specific message-tool param carries a media source such as a
local path or remote media URL, the plugin should also return
`mediaSourceParams` from `describeMessageTool(...)`. Core uses that explicit
list to apply sandbox path normalization and outbound media-access hints
without hardcoding plugin-owned param names.
Core passes runtime scope into that discovery step. Important fields include:
- `accountId`

View File

@@ -35,6 +35,12 @@ shared `message` tool in core. Your plugin owns:
Core owns the shared message tool, prompt wiring, the outer session-key shape,
generic `:thread:` bookkeeping, and dispatch.
If your channel adds message-tool params that carry media sources, expose those
param names through `describeMessageTool(...).mediaSourceParams`. Core uses
that explicit list for sandbox path normalization and outbound media-access
policy, so plugins do not need shared-core special cases for provider-specific
avatar, attachment, or cover-image params.
If your platform stores extra scope inside conversation ids, keep that parsing
in the plugin with `messaging.resolveSessionConversation(...)`. That is the
canonical hook for mapping `rawId` to the base conversation id, optional thread

View File

@@ -92,6 +92,12 @@ describe("matrixMessageActions", () => {
expect(actions).toContain(profileAction);
expect(supportsAction({ action: profileAction } as never)).toBe(true);
expect(discovery.mediaSourceParams).toEqual([
"avatarUrl",
"avatar_url",
"avatarPath",
"avatar_path",
]);
expect(properties.displayName).toBeDefined();
expect(properties.avatarUrl).toBeDefined();
expect(properties.avatarPath).toBeDefined();

View File

@@ -31,6 +31,35 @@ const MATRIX_PLUGIN_HANDLED_ACTIONS = new Set<ChannelMessageActionName>([
"channel-info",
"permissions",
]);
const MATRIX_PROFILE_MEDIA_PROPERTIES = {
avatarUrl: Type.Optional(
Type.String({
description:
"Profile avatar URL for Matrix self-profile update actions. Matrix accepts mxc:// and http(s) URLs.",
}),
),
avatar_url: Type.Optional(
Type.String({
description:
"snake_case alias of avatarUrl for Matrix self-profile update actions. Matrix accepts mxc:// and http(s) URLs.",
}),
),
avatarPath: Type.Optional(
Type.String({
description:
"Local avatar file path for Matrix self-profile update actions. Matrix uploads this file and sets the resulting MXC URI.",
}),
),
avatar_path: Type.Optional(
Type.String({
description:
"snake_case alias of avatarPath for Matrix self-profile update actions. Matrix uploads this file and sets the resulting MXC URI.",
}),
),
} as const;
const MATRIX_PROFILE_MEDIA_SOURCE_PARAMS = Object.freeze(
Object.keys(MATRIX_PROFILE_MEDIA_PROPERTIES),
);
function createMatrixExposedActions(params: {
gate: ReturnType<typeof createActionGate>;
@@ -81,30 +110,7 @@ function buildMatrixProfileToolSchema(): NonNullable<ChannelMessageToolDiscovery
description: "snake_case alias of displayName for Matrix self-profile update actions.",
}),
),
avatarUrl: Type.Optional(
Type.String({
description:
"Profile avatar URL for Matrix self-profile update actions. Matrix accepts mxc:// and http(s) URLs.",
}),
),
avatar_url: Type.Optional(
Type.String({
description:
"snake_case alias of avatarUrl for Matrix self-profile update actions. Matrix accepts mxc:// and http(s) URLs.",
}),
),
avatarPath: Type.Optional(
Type.String({
description:
"Local avatar file path for Matrix self-profile update actions. Matrix uploads this file and sets the resulting MXC URI.",
}),
),
avatar_path: Type.Optional(
Type.String({
description:
"snake_case alias of avatarPath for Matrix self-profile update actions. Matrix uploads this file and sets the resulting MXC URI.",
}),
),
...MATRIX_PROFILE_MEDIA_PROPERTIES,
},
};
}
@@ -133,6 +139,9 @@ export const matrixMessageActions: ChannelMessageActionAdapter = {
actions: listedActions,
capabilities: [],
schema: listedActions.includes("set-profile") ? buildMatrixProfileToolSchema() : null,
mediaSourceParams: listedActions.includes("set-profile")
? MATRIX_PROFILE_MEDIA_SOURCE_PARAMS
: [],
};
},
supportsAction: ({ action }) => MATRIX_PLUGIN_HANDLED_ACTIONS.has(action),

View File

@@ -9,11 +9,11 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { logVerbose } from "../../globals.js";
import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js";
import { resolveConfiguredMediaMaxBytes } from "../../media/configured-max-bytes.js";
import { isPassThroughRemoteMediaSource } from "../../media/media-source-url.js";
import { saveMediaSource } from "../../media/store.js";
import { resolveConfigDir } from "../../utils.js";
import type { ReplyPayload } from "../types.js";
const HTTP_URL_RE = /^https?:\/\//i;
const FILE_URL_RE = /^file:\/\//i;
const WINDOWS_DRIVE_RE = /^[a-zA-Z]:[\\/]/;
const SCHEME_RE = /^[a-zA-Z][a-zA-Z0-9+.-]*:/;
@@ -156,7 +156,7 @@ export function createReplyMediaPathNormalizer(params: {
return media;
}
assertMediaNotDataUrl(media);
if (HTTP_URL_RE.test(media)) {
if (isPassThroughRemoteMediaSource(media)) {
return media;
}
const sandboxRoot = await resolveSandboxRoot();

View File

@@ -108,6 +108,7 @@ type ResolvedChannelMessageActionDiscovery = {
actions: ChannelMessageActionName[];
capabilities: readonly ChannelMessageCapability[];
schemaContributions: ChannelMessageToolSchemaContribution[];
mediaSourceParams: readonly string[];
};
export function resolveMessageActionDiscoveryForPlugin(params: {
@@ -124,6 +125,7 @@ export function resolveMessageActionDiscoveryForPlugin(params: {
actions: [],
capabilities: [],
schemaContributions: [],
mediaSourceParams: [],
};
}
@@ -142,6 +144,9 @@ export function resolveMessageActionDiscoveryForPlugin(params: {
schemaContributions: params.includeSchema
? normalizeToolSchemaContributions(described?.schema)
: [],
mediaSourceParams: Array.isArray(described?.mediaSourceParams)
? described.mediaSourceParams
: [],
};
}
@@ -260,6 +265,37 @@ export function resolveChannelMessageToolSchemaProperties(params: {
return properties;
}
export function resolveChannelMessageToolMediaSourceParamKeys(params: {
cfg: OpenClawConfig;
channel?: string;
currentChannelId?: string | null;
currentThreadTs?: string | null;
currentMessageId?: string | number | null;
accountId?: string | null;
sessionKey?: string | null;
sessionId?: string | null;
agentId?: string | null;
requesterSenderId?: string | null;
senderIsOwner?: boolean;
}): string[] {
const channelId = resolveMessageActionDiscoveryChannelId(params.channel);
if (!channelId) {
return [];
}
const plugin = getChannelPlugin(channelId as Parameters<typeof getChannelPlugin>[0]);
if (!plugin?.actions) {
return [];
}
const described = resolveMessageActionDiscoveryForPlugin({
pluginId: plugin.id,
actions: plugin.actions,
context: createMessageActionDiscoveryContext(params),
includeSchema: false,
});
return Array.from(new Set(described.mediaSourceParams));
}
export function channelSupportsMessageCapability(
cfg: OpenClawConfig,
capability: ChannelMessageCapability,

View File

@@ -14,6 +14,7 @@ import {
listChannelMessageActions,
listChannelMessageCapabilities,
listChannelMessageCapabilitiesForChannel,
resolveChannelMessageToolMediaSourceParamKeys,
resolveChannelMessageToolSchemaProperties,
} from "./message-action-discovery.js";
import type { ChannelMessageCapability } from "./message-capabilities.js";
@@ -199,6 +200,42 @@ describe("message action capability checks", () => {
).toHaveProperty("components");
});
it("derives plugin-owned media-source params from message-tool discovery", () => {
const mediaPlugin: ChannelPlugin = {
...createChannelTestPluginBase({
id: "demo-media",
label: "Demo Media",
capabilities: { chatTypes: ["direct", "group"] },
config: {
listAccountIds: () => ["default"],
},
}),
actions: {
describeMessageTool: () => ({
actions: ["set-profile"],
mediaSourceParams: ["avatarUrl", "avatarPath"],
schema: {
properties: {
avatarUrl: Type.Optional(Type.String({ description: "Remote avatar URL" })),
avatarPath: Type.Optional(Type.String({ description: "Local avatar path" })),
displayName: Type.Optional(Type.String()),
},
},
}),
},
};
setActivePluginRegistry(
createTestRegistry([{ pluginId: "demo-media", source: "test", plugin: mediaPlugin }]),
);
expect(
resolveChannelMessageToolMediaSourceParamKeys({
cfg: {} as OpenClawConfig,
channel: "demo-media",
}),
).toEqual(["avatarUrl", "avatarPath"]);
});
it("skips crashing action/capability discovery paths and logs once", () => {
const crashingPlugin: ChannelPlugin = {
...createChannelTestPluginBase({

View File

@@ -66,6 +66,12 @@ export type ChannelMessageToolDiscovery = {
actions?: readonly ChannelMessageActionName[] | null;
capabilities?: readonly ChannelMessageCapability[] | null;
schema?: ChannelMessageToolSchemaContribution | ChannelMessageToolSchemaContribution[] | null;
/**
* Plugin-owned message-tool params that carry media sources.
* Core uses this to derive sandbox path normalization and host media-access
* hints without hardcoding plugin-specific param names.
*/
mediaSourceParams?: readonly string[] | null;
};
/** Shared setup input bag used by CLI, onboarding, and setup adapters. */
@@ -630,7 +636,8 @@ export type ChannelMessageActionAdapter = {
/**
* Unified discovery surface for the shared `message` tool.
* This returns the scoped actions,
* capabilities, and schema fragments together so they cannot drift.
* capabilities, schema fragments, and any plugin-owned media-source params
* together so they cannot drift.
*/
describeMessageTool: (
params: ChannelMessageActionDiscoveryContext,

View File

@@ -13,6 +13,12 @@ import {
const cfg = {} as OpenClawConfig;
const maybeIt = process.platform === "win32" ? it.skip : it;
const matrixMediaSourceParamKeys = [
"avatarPath",
"avatar_path",
"avatarUrl",
"avatar_url",
] as const;
describe("message action media helpers", () => {
it("prefers sandbox media policy when sandbox roots are non-blank", () => {
@@ -119,6 +125,7 @@ describe("message action media helpers", () => {
mode: "sandbox",
sandboxRoot,
},
extraParamKeys: matrixMediaSourceParamKeys,
});
expect(args).toMatchObject({
@@ -132,14 +139,17 @@ describe("message action media helpers", () => {
it("collects host media source hints from the shared media-source key set", () => {
expect(
collectActionMediaSourceHints({
media: " /workspace/uploads/photo.png ",
filePath: "",
image: "file:///workspace/assets/event-cover.png",
avatarPath: "/workspace/avatars/profile.png",
avatar_url: "mxc://matrix.org/abc123def456",
ignored: "/workspace/not-included.png",
}),
collectActionMediaSourceHints(
{
media: " /workspace/uploads/photo.png ",
filePath: "",
image: "file:///workspace/assets/event-cover.png",
avatarPath: "/workspace/avatars/profile.png",
avatar_url: "mxc://matrix.org/abc123def456",
ignored: "/workspace/not-included.png",
},
matrixMediaSourceParamKeys,
),
).toEqual([
" /workspace/uploads/photo.png ",
"file:///workspace/assets/event-cover.png",
@@ -162,6 +172,7 @@ describe("message action media helpers", () => {
mode: "sandbox",
sandboxRoot,
},
extraParamKeys: matrixMediaSourceParamKeys,
});
expect(args).toMatchObject({
@@ -187,6 +198,7 @@ describe("message action media helpers", () => {
mode: "sandbox",
sandboxRoot,
},
extraParamKeys: matrixMediaSourceParamKeys,
});
expect(args).toMatchObject({
@@ -212,6 +224,7 @@ describe("message action media helpers", () => {
mode: "sandbox",
sandboxRoot,
},
extraParamKeys: matrixMediaSourceParamKeys,
});
expect(args).toMatchObject({

View File

@@ -1,5 +1,6 @@
import { assertMediaNotDataUrl, resolveSandboxedMediaSource } from "../../agents/sandbox-paths.js";
import { readStringParam } from "../../agents/tools/common.js";
import { resolveChannelMessageToolMediaSourceParamKeys } from "../../channels/plugins/message-action-discovery.js";
import type { ChannelId, ChannelMessageActionName } from "../../channels/plugins/types.public.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { createRootScopedReadFile } from "../../infra/fs-safe.js";
@@ -17,29 +18,53 @@ import { normalizeOptionalString } from "../../shared/string-coerce.js";
export const readBooleanParam = readBooleanParamShared;
export const ACTION_MEDIA_SOURCE_PARAM_KEYS = [
export const BASE_ACTION_MEDIA_SOURCE_PARAM_KEYS = [
"media",
"path",
"filePath",
"mediaUrl",
"fileUrl",
"image",
"avatarPath",
"avatar_path",
"avatarUrl",
"avatar_url",
] as const;
function readMediaParam(
args: Record<string, unknown>,
key: (typeof ACTION_MEDIA_SOURCE_PARAM_KEYS)[number],
): string | undefined {
function readMediaParam(args: Record<string, unknown>, key: string): string | undefined {
return readStringParam(args, key, { trim: false });
}
export function collectActionMediaSourceHints(args: Record<string, unknown>): string[] {
function buildActionMediaSourceParamKeys(extraParamKeys?: readonly string[]): string[] {
const keys = new Set<string>(BASE_ACTION_MEDIA_SOURCE_PARAM_KEYS);
extraParamKeys?.forEach((key) => keys.add(key));
return Array.from(keys);
}
export function resolveExtraActionMediaSourceParamKeys(params: {
cfg: OpenClawConfig;
channel?: string;
accountId?: string | null;
sessionKey?: string | null;
sessionId?: string | null;
agentId?: string | null;
requesterSenderId?: string | null;
senderIsOwner?: boolean;
}): string[] {
return resolveChannelMessageToolMediaSourceParamKeys({
cfg: params.cfg,
channel: params.channel,
accountId: params.accountId,
sessionKey: params.sessionKey,
sessionId: params.sessionId,
agentId: params.agentId,
requesterSenderId: params.requesterSenderId,
senderIsOwner: params.senderIsOwner,
});
}
export function collectActionMediaSourceHints(
args: Record<string, unknown>,
extraParamKeys?: readonly string[],
): string[] {
const sources: string[] = [];
for (const key of ACTION_MEDIA_SOURCE_PARAM_KEYS) {
for (const key of buildActionMediaSourceParamKeys(extraParamKeys)) {
const source = typeof args[key] === "string" ? args[key] : undefined;
if (source && normalizeOptionalString(source)) {
sources.push(source);
@@ -245,10 +270,11 @@ async function hydrateAttachmentPayload(params: {
export async function normalizeSandboxMediaParams(params: {
args: Record<string, unknown>;
mediaPolicy: AttachmentMediaPolicy;
extraParamKeys?: readonly string[];
}): Promise<void> {
const sandboxRoot =
params.mediaPolicy.mode === "sandbox" ? params.mediaPolicy.sandboxRoot.trim() : undefined;
for (const key of ACTION_MEDIA_SOURCE_PARAM_KEYS) {
for (const key of buildActionMediaSourceParamKeys(params.extraParamKeys)) {
const raw = readMediaParam(params.args, key);
if (!raw) {
continue;

View File

@@ -1,6 +1,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Type } from "@sinclair/typebox";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { jsonResult } from "../../agents/tools/common.js";
import type { ChannelPlugin } from "../../channels/plugins/types.js";
@@ -524,6 +525,105 @@ describe("runMessageAction media behavior", () => {
});
});
describe("plugin-owned media-source discovery routing", () => {
const profilePlugin: ChannelPlugin = {
...createChannelTestPluginBase({
id: "profile-demo",
label: "Profile Demo",
capabilities: { chatTypes: ["direct"] },
config: {
listAccountIds: () => ["default"],
isConfigured: () => true,
},
}),
actions: {
describeMessageTool: () => ({
actions: ["set-profile"],
mediaSourceParams: ["avatarPath", "avatarUrl"],
schema: {
properties: {
avatarPath: Type.Optional(Type.String({ description: "Local avatar path" })),
avatarUrl: Type.Optional(Type.String({ description: "Remote avatar URL" })),
displayName: Type.Optional(Type.String()),
},
},
}),
supportsAction: ({ action }) => action === "set-profile",
handleAction: async ({ params, mediaLocalRoots }) =>
jsonResult({
ok: true,
avatarPath: params.avatarPath,
avatarUrl: params.avatarUrl,
mediaLocalRoots,
}),
},
};
beforeEach(() => {
setActivePluginRegistry(
createTestRegistry([
{
pluginId: "profile-demo",
source: "test",
plugin: profilePlugin,
},
]),
);
});
afterEach(() => {
setActivePluginRegistry(createTestRegistry([]));
});
it("rewrites plugin-owned sandbox media params and preserves mxc URLs", async () => {
await withSandbox(async (sandboxDir) => {
const result = await runMessageAction({
cfg: {} as OpenClawConfig,
action: "set-profile",
params: {
channel: "profile-demo",
avatarPath: "/workspace/avatars/profile.png",
avatarUrl: "mxc://matrix.org/abc123def456",
},
sandboxRoot: sandboxDir,
});
expect(result.kind).toBe("action");
expect(result.payload).toMatchObject({
ok: true,
avatarPath: path.join(sandboxDir, "avatars", "profile.png"),
avatarUrl: "mxc://matrix.org/abc123def456",
});
});
});
it("routes plugin-owned host media hints into local-root expansion", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "msg-profile-media-"));
try {
const avatarPath = path.join(tempDir, "profile.png");
await fs.writeFile(avatarPath, onePixelPng);
const result = await runMessageAction({
cfg: {
tools: { fs: { workspaceOnly: false } },
} as OpenClawConfig,
action: "set-profile",
params: {
channel: "profile-demo",
avatarPath,
},
});
expect(result.kind).toBe("action");
expect((result.payload as { mediaLocalRoots?: string[] }).mediaLocalRoots).toEqual(
expect.arrayContaining([tempDir]),
);
} finally {
await fs.rm(tempDir, { recursive: true, force: true });
}
});
});
describe("sandboxed media validation", () => {
beforeEach(() => {
setActivePluginRegistry(

View File

@@ -52,6 +52,7 @@ import {
parseInteractiveParam,
readBooleanParam,
resolveAttachmentMediaPolicy,
resolveExtraActionMediaSourceParamKeys,
} from "./message-action-params.js";
import {
prepareOutboundMirrorRoute,
@@ -849,16 +850,27 @@ export async function runMessageAction(
sandboxRoot: input.sandboxRoot,
mediaLocalRoots: getAgentScopedMediaLocalRoots(cfg, resolvedAgentId),
});
const extraActionMediaSourceParamKeys = resolveExtraActionMediaSourceParamKeys({
cfg,
channel,
accountId,
sessionKey: input.sessionKey,
sessionId: input.sessionId,
agentId: resolvedAgentId,
requesterSenderId: input.requesterSenderId,
senderIsOwner: input.senderIsOwner,
});
await normalizeSandboxMediaParams({
args: params,
mediaPolicy: normalizationPolicy,
extraParamKeys: extraActionMediaSourceParamKeys,
});
const mediaAccess = resolveAgentScopedOutboundMediaAccess({
cfg,
agentId: resolvedAgentId,
mediaSources: collectActionMediaSourceHints(params),
mediaSources: collectActionMediaSourceHints(params, extraActionMediaSourceParamKeys),
sessionKey: input.sessionKey,
messageProvider: input.sessionKey ? undefined : channel,
accountId: input.sessionKey ? (input.requesterAccountId ?? accountId) : accountId,

View File

@@ -128,6 +128,12 @@ describe("local media roots", () => {
expect(roots.map(normalizeHostPath)).not.toContain(normalizeHostPath("/"));
});
it("does not widen local roots for pass-through remote media schemes", () => {
const roots = appendLocalMediaParentRoots(["/tmp/base"], ["mxc://matrix.org/abc123def456"]);
expect(roots.map(normalizeHostPath)).toEqual([normalizeHostPath("/tmp/base")]);
});
it.each([
{
name: "widens agent media roots for concrete local sources when workspaceOnly is disabled",

View File

@@ -10,13 +10,13 @@ import { safeFileURLToPath } from "../infra/local-file-access.js";
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
import { normalizeOptionalString } from "../shared/string-coerce.js";
import { resolveConfigDir, resolveUserPath } from "../utils.js";
import { isPassThroughRemoteMediaSource } from "./media-source-url.js";
type BuildMediaLocalRootsOptions = {
preferredTmpDir?: string;
};
let cachedPreferredTmpDir: string | undefined;
const HTTP_URL_RE = /^https?:\/\//i;
const DATA_URL_RE = /^data:/i;
const WINDOWS_DRIVE_RE = /^[A-Za-z]:[\\/]/;
@@ -73,7 +73,7 @@ export function getAgentScopedMediaLocalRoots(
function resolveLocalMediaPath(source: string): string | undefined {
const trimmed = source.trim();
if (!trimmed || HTTP_URL_RE.test(trimmed) || DATA_URL_RE.test(trimmed)) {
if (!trimmed || isPassThroughRemoteMediaSource(trimmed) || DATA_URL_RE.test(trimmed)) {
return undefined;
}
if (trimmed.startsWith("file://")) {