vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-03-19 14:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: deduplicate reply payload handling

Browse Source
This commit is contained in:
Peter Steinberger
2026-03-18 18:14:36 +00:00
parent 152d179302
commit 62edfdffbd
58 changed files with 704 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/agents/pi-embedded-runner/run/payloads.ts
View File

@@ -4,6 +4,7 @@ import type { ReasoningLevel, VerboseLevel } from "../../../auto-reply/thinking.
import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js";
import { formatToolAggregate } from "../../../auto-reply/tool-meta.js";
import type { OpenClawConfig } from "../../../config/config.js";
import { hasOutboundReplyContent } from "../../../plugin-sdk/reply-payload.js";
import {
BILLING_ERROR_USER_MESSAGE,
formatAssistantErrorText,
@@ -336,7 +337,7 @@ export function buildEmbeddedRunPayloads(params: {
audioAsVoice: item.audioAsVoice || Boolean(hasAudioAsVoiceTag && item.media?.length),
}))
.filter((p) => {
if (!p.text && !p.mediaUrl && (!p.mediaUrls || p.mediaUrls.length === 0)) {
if (!hasOutboundReplyContent(p)) {
return false;
}
if (p.text && isSilentReplyText(p.text, SILENT_REPLY_TOKEN)) {

34
src/agents/pi-embedded-subscribe.handlers.messages.test.ts
View File

@@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
import { resolveSilentReplyFallbackText } from "./pi-embedded-subscribe.handlers.messages.js";
import {
buildAssistantStreamData,
hasAssistantVisibleReply,
resolveSilentReplyFallbackText,
} from "./pi-embedded-subscribe.handlers.messages.js";
describe("resolveSilentReplyFallbackText", () => {
it("replaces NO_REPLY with latest messaging tool text when available", () => {
@@ -29,3 +33,31 @@ describe("resolveSilentReplyFallbackText", () => {
).toBe("NO_REPLY");
});
});
describe("hasAssistantVisibleReply", () => {
it("treats audio-only payloads as visible", () => {
expect(hasAssistantVisibleReply({ audioAsVoice: true })).toBe(true);
});
it("detects text or media visibility", () => {
expect(hasAssistantVisibleReply({ text: "hello" })).toBe(true);
expect(hasAssistantVisibleReply({ mediaUrls: ["https://example.com/a.png"] })).toBe(true);
expect(hasAssistantVisibleReply({})).toBe(false);
});
});
describe("buildAssistantStreamData", () => {
it("normalizes media payloads for assistant stream events", () => {
expect(
buildAssistantStreamData({
text: "hello",
delta: "he",
mediaUrl: "https://example.com/a.png",
}),
).toEqual({
text: "hello",
delta: "he",
mediaUrls: ["https://example.com/a.png"],
});
});
});

76
src/agents/pi-embedded-subscribe.handlers.messages.ts
View File

@@ -3,6 +3,7 @@ import { parseReplyDirectives } from "../auto-reply/reply/reply-directives.js";
import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import { emitAgentEvent } from "../infra/agent-events.js";
import { createInlineCodeState } from "../markdown/code-spans.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
import {
isMessagingToolDuplicateNormalized,
normalizeTextForComparison,
@@ -56,6 +57,29 @@ export function resolveSilentReplyFallbackText(params: {
return fallback;
}
export function hasAssistantVisibleReply(params: {
text?: string;
mediaUrls?: string[];
mediaUrl?: string;
audioAsVoice?: boolean;
}): boolean {
return resolveSendableOutboundReplyParts(params).hasContent || Boolean(params.audioAsVoice);
}
export function buildAssistantStreamData(params: {
text?: string;
delta?: string;
mediaUrls?: string[];
mediaUrl?: string;
}): { text: string; delta: string; mediaUrls?: string[] } {
const mediaUrls = resolveSendableOutboundReplyParts(params).mediaUrls;
return {
text: params.text ?? "",
delta: params.delta ?? "",
mediaUrls: mediaUrls.length ? mediaUrls : undefined,
};
}
export function handleMessageStart(
ctx: EmbeddedPiSubscribeContext,
evt: AgentEvent & { message: AgentMessage },
@@ -196,14 +220,13 @@ export function handleMessageUpdate(
const parsedDelta = visibleDelta ? ctx.consumePartialReplyDirectives(visibleDelta) : null;
const parsedFull = parseReplyDirectives(stripTrailingDirective(next));
const cleanedText = parsedFull.text;
const mediaUrls = parsedDelta?.mediaUrls;
const hasMedia = Boolean(mediaUrls && mediaUrls.length > 0);
const { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedDelta ?? {});
const hasAudio = Boolean(parsedDelta?.audioAsVoice);
const previousCleaned = ctx.state.lastStreamedAssistantCleaned ?? "";
let shouldEmit = false;
let deltaText = "";
if (!cleanedText && !hasMedia && !hasAudio) {
if (!hasAssistantVisibleReply({ text: cleanedText, mediaUrls, audioAsVoice: hasAudio })) {
shouldEmit = false;
} else if (previousCleaned && !cleanedText.startsWith(previousCleaned)) {
shouldEmit = false;
@@ -216,29 +239,23 @@ export function handleMessageUpdate(
ctx.state.lastStreamedAssistantCleaned = cleanedText;
if (shouldEmit) {
const data = buildAssistantStreamData({
text: cleanedText,
delta: deltaText,
mediaUrls,
});
emitAgentEvent({
runId: ctx.params.runId,
stream: "assistant",
data: {
text: cleanedText,
delta: deltaText,
mediaUrls: hasMedia ? mediaUrls : undefined,
},
data,
});
void ctx.params.onAgentEvent?.({
stream: "assistant",
data: {
text: cleanedText,
delta: deltaText,
mediaUrls: hasMedia ? mediaUrls : undefined,
},
data,
});
ctx.state.emittedAssistantUpdate = true;
if (ctx.params.onPartialReply && ctx.state.shouldEmitPartialReplies) {
void ctx.params.onPartialReply({
text: cleanedText,
mediaUrls: hasMedia ? mediaUrls : undefined,
});
void ctx.params.onPartialReply(data);
}
}
}
@@ -291,8 +308,7 @@ export function handleMessageEnd(
const trimmedText = text.trim();
const parsedText = trimmedText ? parseReplyDirectives(stripTrailingDirective(trimmedText)) : null;
let cleanedText = parsedText?.text ?? "";
let mediaUrls = parsedText?.mediaUrls;
let hasMedia = Boolean(mediaUrls && mediaUrls.length > 0);
let { mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedText ?? {});
if (!cleanedText && !hasMedia && !ctx.params.enforceFinalTag) {
const rawTrimmed = rawText.trim();
@@ -301,28 +317,24 @@ export function handleMessageEnd(
if (rawCandidate) {
const parsedFallback = parseReplyDirectives(stripTrailingDirective(rawCandidate));
cleanedText = parsedFallback.text ?? rawCandidate;
mediaUrls = parsedFallback.mediaUrls;
hasMedia = Boolean(mediaUrls && mediaUrls.length > 0);
({ mediaUrls, hasMedia } = resolveSendableOutboundReplyParts(parsedFallback));
}
}
if (!ctx.state.emittedAssistantUpdate && (cleanedText || hasMedia)) {
const data = buildAssistantStreamData({
text: cleanedText,
delta: cleanedText,
mediaUrls,
});
emitAgentEvent({
runId: ctx.params.runId,
stream: "assistant",
data: {
text: cleanedText,
delta: cleanedText,
mediaUrls: hasMedia ? mediaUrls : undefined,
},
data,
});
void ctx.params.onAgentEvent?.({
stream: "assistant",
data: {
text: cleanedText,
delta: cleanedText,
mediaUrls: hasMedia ? mediaUrls : undefined,
},
data,
});
ctx.state.emittedAssistantUpdate = true;
}
@@ -377,7 +389,7 @@ export function handleMessageEnd(
replyToCurrent,
} = splitResult;
// Emit if there's content OR audioAsVoice flag (to propagate the flag).
if (cleanedText || (mediaUrls && mediaUrls.length > 0) || audioAsVoice) {
if (hasAssistantVisibleReply({ text: cleanedText, mediaUrls, audioAsVoice })) {
emitBlockReplySafely({
text: cleanedText,
mediaUrls: mediaUrls?.length ? mediaUrls : undefined,

3
src/auto-reply/heartbeat-reply-payload.ts
View File

@@ -1,3 +1,4 @@
import { hasOutboundReplyContent } from "../plugin-sdk/reply-payload.js";
import type { ReplyPayload } from "./types.js";
export function resolveHeartbeatReplyPayload(
@@ -14,7 +15,7 @@ export function resolveHeartbeatReplyPayload(
if (!payload) {
continue;
}
if (payload.text || payload.mediaUrl || (payload.mediaUrls && payload.mediaUrls.length > 0)) {
if (hasOutboundReplyContent(payload)) {
return payload;
}
}

6
src/auto-reply/reply/agent-runner-execution.ts
View File

@@ -23,6 +23,7 @@ import {
} from "../../config/sessions.js";
import { logVerbose } from "../../globals.js";
import { emitAgentEvent, registerAgentRunContext } from "../../infra/agent-events.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import { defaultRuntime } from "../../runtime.js";
import {
isMarkdownCapableMessageChannel,
@@ -148,6 +149,7 @@ export async function runAgentTurnWithFallback(params: {
try {
const normalizeStreamingText = (payload: ReplyPayload): { text?: string; skip: boolean } => {
let text = payload.text;
const reply = resolveSendableOutboundReplyParts(payload);
if (!params.isHeartbeat && text?.includes("HEARTBEAT_OK")) {
const stripped = stripHeartbeatToken(text, {
mode: "message",
@@ -156,7 +158,7 @@ export async function runAgentTurnWithFallback(params: {
didLogHeartbeatStrip = true;
logVerbose("Stripped stray HEARTBEAT_OK token from reply");
}
if (stripped.shouldSkip && (payload.mediaUrls?.length ?? 0) === 0) {
if (stripped.shouldSkip && !reply.hasMedia) {
return { skip: true };
}
text = stripped.text;
@@ -172,7 +174,7 @@ export async function runAgentTurnWithFallback(params: {
}
if (!text) {
// Allow media-only payloads (e.g. tool result screenshots) through.
if ((payload.mediaUrls?.length ?? 0) > 0) {
if (reply.hasMedia) {
return { text: undefined, skip: false };
}
return { skip: true };

22
src/auto-reply/reply/agent-runner-helpers.ts
View File

@@ -1,5 +1,9 @@
import { loadSessionStore } from "../../config/sessions.js";
import { isAudioFileName } from "../../media/mime.js";
import {
hasOutboundReplyContent,
resolveSendableOutboundReplyParts,
} from "../../plugin-sdk/reply-payload.js";
import { normalizeVerboseLevel, type VerboseLevel } from "../thinking.js";
import type { ReplyPayload } from "../types.js";
import { scheduleFollowupDrain } from "./queue.js";
@@ -9,7 +13,7 @@ const hasAudioMedia = (urls?: string[]): boolean =>
Boolean(urls?.some((url) => isAudioFileName(url)));
export const isAudioPayload = (payload: ReplyPayload): boolean =>
hasAudioMedia(payload.mediaUrls ?? (payload.mediaUrl ? [payload.mediaUrl] : undefined));
hasAudioMedia(resolveSendableOutboundReplyParts(payload).mediaUrls);
type VerboseGateParams = {
sessionKey?: string;
@@ -63,19 +67,9 @@ export const signalTypingIfNeeded = async (
payloads: ReplyPayload[],
typingSignals: TypingSignaler,
): Promise<void> => {
const shouldSignalTyping = payloads.some((payload) => {
const trimmed = payload.text?.trim();
if (trimmed) {
return true;
}
if (payload.mediaUrl) {
return true;
}
if (payload.mediaUrls && payload.mediaUrls.length > 0) {
return true;
}
return false;
});
const shouldSignalTyping = payloads.some((payload) =>
hasOutboundReplyContent(payload, { trimText: true }),
);
if (shouldSignalTyping) {
await typingSignals.signalRunStart();
}

15
src/auto-reply/reply/agent-runner-payloads.ts
View File

@@ -1,5 +1,6 @@
import type { ReplyToMode } from "../../config/types.js";
import { logVerbose } from "../../globals.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import { stripHeartbeatToken } from "../heartbeat.js";
import type { OriginatingChannelType } from "../templating.js";
import { SILENT_REPLY_TOKEN } from "../tokens.js";
@@ -20,15 +21,11 @@ import {
shouldSuppressMessagingToolReplies,
} from "./reply-payloads.js";
function hasPayloadMedia(payload: ReplyPayload): boolean {
return Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
}
async function normalizeReplyPayloadMedia(params: {
payload: ReplyPayload;
normalizeMediaPaths?: (payload: ReplyPayload) => Promise<ReplyPayload>;
}): Promise<ReplyPayload> {
if (!params.normalizeMediaPaths || !hasPayloadMedia(params.payload)) {
if (!params.normalizeMediaPaths || !resolveSendableOutboundReplyParts(params.payload).hasMedia) {
return params.payload;
}
@@ -69,11 +66,7 @@ async function normalizeSentMediaUrlsForDedupe(params: {
mediaUrl: trimmed,
mediaUrls: [trimmed],
});
const normalizedMediaUrls = normalized.mediaUrls?.length
? normalized.mediaUrls
: normalized.mediaUrl
? [normalized.mediaUrl]
: [];
const normalizedMediaUrls = resolveSendableOutboundReplyParts(normalized).mediaUrls;
for (const mediaUrl of normalizedMediaUrls) {
const candidate = mediaUrl.trim();
if (!candidate || seen.has(candidate)) {
@@ -130,7 +123,7 @@ export async function buildReplyPayloads(params: {
didLogHeartbeatStrip = true;
logVerbose("Stripped stray HEARTBEAT_OK token from reply");
}
const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
const hasMedia = resolveSendableOutboundReplyParts(payload).hasMedia;
if (stripped.shouldSkip && !hasMedia) {
return [];
}

8
src/auto-reply/reply/block-reply-coalescer.ts
View File

@@ -1,3 +1,4 @@
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import type { ReplyPayload } from "../types.js";
import type { BlockStreamingCoalescing } from "./block-streaming.js";
@@ -75,9 +76,10 @@ export function createBlockReplyCoalescer(params: {
if (shouldAbort()) {
return;
}
const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
const text = payload.text ?? "";
const hasText = text.trim().length > 0;
const reply = resolveSendableOutboundReplyParts(payload);
const hasMedia = reply.hasMedia;
const text = reply.text;
const hasText = reply.hasText;
if (hasMedia) {
void flush({ force: true });
void onFlush(payload);

23
src/auto-reply/reply/block-reply-pipeline.ts
View File

@@ -1,4 +1,5 @@
import { logVerbose } from "../../globals.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import type { ReplyPayload } from "../types.js";
import { createBlockReplyCoalescer } from "./block-reply-coalescer.js";
import type { BlockStreamingCoalescing } from "./block-streaming.js";
@@ -35,30 +36,20 @@ export function createAudioAsVoiceBuffer(params: {
}
export function createBlockReplyPayloadKey(payload: ReplyPayload): string {
const text = payload.text?.trim() ?? "";
const mediaList = payload.mediaUrls?.length
? payload.mediaUrls
: payload.mediaUrl
? [payload.mediaUrl]
: [];
const reply = resolveSendableOutboundReplyParts(payload);
return JSON.stringify({
text,
mediaList,
text: reply.trimmedText,
mediaList: reply.mediaUrls,
replyToId: payload.replyToId ?? null,
});
}
export function createBlockReplyContentKey(payload: ReplyPayload): string {
const text = payload.text?.trim() ?? "";
const mediaList = payload.mediaUrls?.length
? payload.mediaUrls
: payload.mediaUrl
? [payload.mediaUrl]
: [];
const reply = resolveSendableOutboundReplyParts(payload);
// Content-only key used for final-payload suppression after block streaming.
// This intentionally ignores replyToId so a streamed threaded payload and the
// later final payload still collapse when they carry the same content.
return JSON.stringify({ text, mediaList });
return JSON.stringify({ text: reply.trimmedText, mediaList: reply.mediaUrls });
}
const withTimeout = async <T>(
@@ -217,7 +208,7 @@ export function createBlockReplyPipeline(params: {
if (bufferPayload(payload)) {
return;
}
const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
const hasMedia = resolveSendableOutboundReplyParts(payload).hasMedia;
if (hasMedia) {
void coalescer?.flush({ force: true });
sendPayload(payload, /* bypassSeenCheck */ false);

3
src/auto-reply/reply/dispatch-acp-delivery.ts
View File

@@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../../config/config.js";
import type { TtsAutoMode } from "../../config/types.tts.js";
import { logVerbose } from "../../globals.js";
import { runMessageAction } from "../../infra/outbound/message-action-runner.js";
import { hasOutboundReplyContent } from "../../plugin-sdk/reply-payload.js";
import { maybeApplyTtsToPayload } from "../../tts/tts.js";
import type { FinalizedMsgContext } from "../templating.js";
import type { ReplyPayload } from "../types.js";
@@ -127,7 +128,7 @@ export function createAcpDispatchDeliveryCoordinator(params: {
state.blockCount += 1;
}
if ((payload.text?.trim() ?? "").length > 0 || payload.mediaUrl || payload.mediaUrls?.length) {
if (hasOutboundReplyContent(payload, { trimText: true })) {
await startReplyLifecycleOnce();
}

3
src/auto-reply/reply/dispatch-from-config.ts
View File

@@ -29,6 +29,7 @@ import {
logMessageQueued,
logSessionStateChange,
} from "../../logging/diagnostic.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import {
buildPluginBindingDeclinedText,
buildPluginBindingErrorText,
@@ -532,7 +533,7 @@ export async function dispatchReplyFromConfig(params: {
}
// Group/native flows intentionally suppress tool summary text, but media-only
// tool results (for example TTS audio) must still be delivered.
const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
const hasMedia = resolveSendableOutboundReplyParts(payload).hasMedia;
if (!hasMedia) {
return null;
}

11
src/auto-reply/reply/followup-runner.ts
View File

@@ -9,6 +9,10 @@ import type { SessionEntry } from "../../config/sessions.js";
import type { TypingMode } from "../../config/types.js";
import { logVerbose } from "../../globals.js";
import { registerAgentRunContext } from "../../infra/agent-events.js";
import {
hasOutboundReplyContent,
resolveSendableOutboundReplyParts,
} from "../../plugin-sdk/reply-payload.js";
import { defaultRuntime } from "../../runtime.js";
import { isInternalMessageChannel } from "../../utils/message-channel.js";
import { stripHeartbeatToken } from "../heartbeat.js";
@@ -81,13 +85,12 @@ export function createFollowupRunner(params: {
}
for (const payload of payloads) {
if (!payload?.text && !payload?.mediaUrl && !payload?.mediaUrls?.length) {
if (!payload || !hasOutboundReplyContent(payload)) {
continue;
}
if (
isSilentReplyText(payload.text, SILENT_REPLY_TOKEN) &&
!payload.mediaUrl &&
!payload.mediaUrls?.length
!resolveSendableOutboundReplyParts(payload).hasMedia
) {
continue;
}
@@ -289,7 +292,7 @@ export function createFollowupRunner(params: {
return [payload];
}
const stripped = stripHeartbeatToken(text, { mode: "message" });
const hasMedia = Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
const hasMedia = resolveSendableOutboundReplyParts(payload).hasMedia;
if (stripped.shouldSkip && !hasMedia) {
return [];
}

63
src/auto-reply/reply/normalize-reply.ts
View File

@@ -1,5 +1,5 @@
import { sanitizeUserFacingText } from "../../agents/pi-embedded-helpers.js";
import { hasReplyChannelData, hasReplyContent } from "../../interactive/payload.js";
import { hasReplyPayloadContent } from "../../interactive/payload.js";
import { stripHeartbeatToken } from "../heartbeat.js";
import {
HEARTBEAT_TOKEN,
@@ -32,17 +32,18 @@ export function normalizeReplyPayload(
payload: ReplyPayload,
opts: NormalizeReplyOptions = {},
): ReplyPayload | null {
const hasChannelData = hasReplyChannelData(payload.channelData);
const hasContent = (text: string | undefined) =>
hasReplyPayloadContent(
{
...payload,
text,
},
{
trimText: true,
},
);
const trimmed = payload.text?.trim() ?? "";
if (
!hasReplyContent({
text: trimmed,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (!hasContent(trimmed)) {
opts.onSkip?.("empty");
return null;
}
@@ -50,14 +51,7 @@ export function normalizeReplyPayload(
const silentToken = opts.silentToken ?? SILENT_REPLY_TOKEN;
let text = payload.text ?? undefined;
if (text && isSilentReplyText(text, silentToken)) {
if (
!hasReplyContent({
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (!hasContent("")) {
opts.onSkip?.("silent");
return null;
}
@@ -68,15 +62,7 @@ export function normalizeReplyPayload(
// silent just like the exact-match path above. (#30916, #30955)
if (text && text.includes(silentToken) && !isSilentReplyText(text, silentToken)) {
text = stripSilentToken(text, silentToken);
if (
!hasReplyContent({
text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (!hasContent(text)) {
opts.onSkip?.("silent");
return null;
}
@@ -92,16 +78,7 @@ export function normalizeReplyPayload(
if (stripped.didStrip) {
opts.onHeartbeatStrip?.();
}
if (
stripped.shouldSkip &&
!hasReplyContent({
text: stripped.text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (stripped.shouldSkip && !hasContent(stripped.text)) {
opts.onSkip?.("heartbeat");
return null;
}
@@ -111,15 +88,7 @@ export function normalizeReplyPayload(
if (text) {
text = sanitizeUserFacingText(text, { errorContext: Boolean(payload.isError) });
}
if (
!hasReplyContent({
text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (!hasContent(text)) {
opts.onSkip?.("empty");
return null;
}

8
src/auto-reply/reply/reply-delivery.ts
View File

@@ -1,4 +1,5 @@
import { logVerbose } from "../../globals.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import { SILENT_REPLY_TOKEN } from "../tokens.js";
import type { BlockReplyContext, ReplyPayload } from "../types.js";
import type { BlockReplyPipeline } from "./block-reply-pipeline.js";
@@ -57,9 +58,6 @@ export function normalizeReplyPayloadDirectives(params: {
};
}
const hasRenderableMedia = (payload: ReplyPayload): boolean =>
Boolean(payload.mediaUrl) || (payload.mediaUrls?.length ?? 0) > 0;
export function createBlockReplyDeliveryHandler(params: {
onBlockReply: (payload: ReplyPayload, context?: BlockReplyContext) => Promise<void> | void;
currentMessageId?: string;
@@ -73,7 +71,7 @@ export function createBlockReplyDeliveryHandler(params: {
}): (payload: ReplyPayload) => Promise<void> {
return async (payload) => {
const { text, skip } = params.normalizeStreamingText(payload);
if (skip && !hasRenderableMedia(payload)) {
if (skip && !resolveSendableOutboundReplyParts(payload).hasMedia) {
return;
}
@@ -106,7 +104,7 @@ export function createBlockReplyDeliveryHandler(params: {
? await params.normalizeMediaPaths(normalized.payload)
: normalized.payload;
const blockPayload = params.applyReplyToMode(mediaNormalizedPayload);
const blockHasMedia = hasRenderableMedia(blockPayload);
const blockHasMedia = resolveSendableOutboundReplyParts(blockPayload).hasMedia;
// Skip empty payloads unless they have audioAsVoice flag (need to track it).
if (!blockPayload.text && !blockHasMedia && !blockPayload.audioAsVoice) {

3
src/auto-reply/reply/reply-media-paths.ts
View File

@@ -2,6 +2,7 @@ import { resolvePathFromInput } from "../../agents/path-policy.js";
import { assertMediaNotDataUrl, resolveSandboxedMediaSource } from "../../agents/sandbox-paths.js";
import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox.js";
import type { OpenClawConfig } from "../../config/config.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import type { ReplyPayload } from "../types.js";
const HTTP_URL_RE = /^https?:\/\//i;
@@ -25,7 +26,7 @@ function isLikelyLocalMediaSource(media: string): boolean {
}
function getPayloadMediaList(payload: ReplyPayload): string[] {
return payload.mediaUrls?.length ? payload.mediaUrls : payload.mediaUrl ? [payload.mediaUrl] : [];
return resolveSendableOutboundReplyParts(payload).mediaUrls;
}
export function createReplyMediaPathNormalizer(params: {

11
src/auto-reply/reply/reply-payloads.ts
View File

@@ -4,7 +4,7 @@ import { normalizeChannelId } from "../../channels/plugins/index.js";
import { parseExplicitTargetForChannel } from "../../channels/plugins/target-parsing.js";
import type { ReplyToMode } from "../../config/types.js";
import { normalizeTargetForProvider } from "../../infra/outbound/target-normalization.js";
import { hasReplyChannelData, hasReplyContent } from "../../interactive/payload.js";
import { hasReplyPayloadContent } from "../../interactive/payload.js";
import { normalizeOptionalAccountId } from "../../routing/account-id.js";
import type { OriginatingChannelType } from "../templating.js";
import type { ReplyPayload } from "../types.js";
@@ -75,14 +75,7 @@ export function applyReplyTagsToPayload(
}
export function isRenderablePayload(payload: ReplyPayload): boolean {
return hasReplyContent({
text: payload.text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData: hasReplyChannelData(payload.channelData),
extraContent: payload.audioAsVoice,
});
return hasReplyPayloadContent(payload, { extraContent: payload.audioAsVoice });
}
export function shouldSuppressReasoningPayload(payload: ReplyPayload): boolean {

18
src/auto-reply/reply/route-reply.ts
View File

@@ -12,7 +12,7 @@ import { resolveEffectiveMessagesConfig } from "../../agents/identity.js";
import { getChannelPlugin, normalizeChannelId } from "../../channels/plugins/index.js";
import type { OpenClawConfig } from "../../config/config.js";
import { buildOutboundSessionContext } from "../../infra/outbound/session-context.js";
import { hasReplyContent } from "../../interactive/payload.js";
import { hasReplyPayloadContent } from "../../interactive/payload.js";
import { INTERNAL_MESSAGE_CHANNEL, normalizeMessageChannel } from "../../utils/message-channel.js";
import type { OriginatingChannelType } from "../templating.js";
import type { ReplyPayload } from "../types.js";
@@ -126,12 +126,16 @@ export async function routeReply(params: RouteReplyParams): Promise<RouteReplyRe
// Skip empty replies.
if (
!hasReplyContent({
text,
mediaUrls,
interactive: externalPayload.interactive,
hasChannelData,
})
!hasReplyPayloadContent(
{
...externalPayload,
text,
mediaUrls,
},
{
hasChannelData,
},
)
) {
return { ok: true };
}

6
src/auto-reply/reply/streaming-directives.ts
View File

@@ -1,4 +1,5 @@
import { splitMediaFromOutput } from "../../media/parse.js";
import { hasOutboundReplyContent } from "../../plugin-sdk/reply-payload.js";
import { parseInlineDirectives } from "../../utils/directive-tags.js";
import { isSilentReplyPrefixText, isSilentReplyText, SILENT_REPLY_TOKEN } from "../tokens.js";
import type { ReplyDirectiveParseResult } from "./reply-directives.js";
@@ -67,10 +68,7 @@ const parseChunk = (raw: string, options?: { silentToken?: string }): ParsedChun
};
const hasRenderableContent = (parsed: ReplyDirectiveParseResult): boolean =>
Boolean(parsed.text) ||
Boolean(parsed.mediaUrl) ||
(parsed.mediaUrls?.length ?? 0) > 0 ||
Boolean(parsed.audioAsVoice);
hasOutboundReplyContent(parsed) || Boolean(parsed.audioAsVoice);
export function createStreamingDirectiveAccumulator() {
let pendingTail = "";

3
src/channels/plugins/outbound/direct-text-media.ts
View File

@@ -1,6 +1,7 @@
import { chunkText } from "../../../auto-reply/chunk.js";
import type { OpenClawConfig } from "../../../config/config.js";
import type { OutboundSendDeps } from "../../../infra/outbound/deliver.js";
import { resolveOutboundMediaUrls } from "../../../plugin-sdk/reply-payload.js";
import { resolveChannelMediaMaxBytes } from "../media-limits.js";
import type { ChannelOutboundAdapter } from "../types.js";
@@ -29,7 +30,7 @@ type SendPayloadAdapter = Pick<
>;
export function resolvePayloadMediaUrls(payload: SendPayloadContext["payload"]): string[] {
return payload.mediaUrls?.length ? payload.mediaUrls : payload.mediaUrl ? [payload.mediaUrl] : [];
return resolveOutboundMediaUrls(payload);
}
export async function sendPayloadMediaSequence<TResult>(params: {

17
src/commands/agent-via-gateway.ts
View File

@@ -4,6 +4,7 @@ import type { CliDeps } from "../cli/deps.js";
import { withProgress } from "../cli/progress.js";
import { loadConfig } from "../config/config.js";
import { callGateway, randomIdempotencyKey } from "../gateway/call.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
import { normalizeAgentId } from "../routing/session-key.js";
import type { RuntimeEnv } from "../runtime.js";
import {
@@ -69,16 +70,16 @@ function formatPayloadForLog(payload: {
mediaUrls?: string[];
mediaUrl?: string | null;
}) {
const parts = resolveSendableOutboundReplyParts({
text: payload.text,
mediaUrls: payload.mediaUrls,
mediaUrl: typeof payload.mediaUrl === "string" ? payload.mediaUrl : undefined,
});
const lines: string[] = [];
if (payload.text) {
lines.push(payload.text.trimEnd());
if (parts.text) {
lines.push(parts.text.trimEnd());
}
const mediaUrl =
typeof payload.mediaUrl === "string" && payload.mediaUrl.trim()
? payload.mediaUrl.trim()
: undefined;
const media = payload.mediaUrls ?? (mediaUrl ? [mediaUrl] : []);
for (const url of media) {
for (const url of parts.mediaUrls) {
lines.push(`MEDIA:${url}`);
}
return lines.join("\n").trimEnd();

3
src/cron/heartbeat-policy.ts
View File

@@ -1,4 +1,5 @@
import { stripHeartbeatToken } from "../auto-reply/heartbeat.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
export type HeartbeatDeliveryPayload = {
text?: string;
@@ -14,7 +15,7 @@ export function shouldSkipHeartbeatOnlyDelivery(
return true;
}
const hasAnyMedia = payloads.some(
(payload) => (payload.mediaUrls?.length ?? 0) > 0 || Boolean(payload.mediaUrl),
(payload) => resolveSendableOutboundReplyParts(payload).hasMedia,
);
if (hasAnyMedia) {
return false;

5
src/cron/isolated-agent/helpers.ts
View File

@@ -1,5 +1,6 @@
import { DEFAULT_HEARTBEAT_ACK_MAX_CHARS } from "../../auto-reply/heartbeat.js";
import type { ReplyPayload } from "../../auto-reply/types.js";
import { hasOutboundReplyContent } from "../../plugin-sdk/reply-payload.js";
import { truncateUtf16Safe } from "../../utils.js";
import { shouldSkipHeartbeatOnlyDelivery } from "../heartbeat-policy.js";
@@ -61,11 +62,9 @@ export function pickLastNonEmptyTextFromPayloads(
export function pickLastDeliverablePayload(payloads: DeliveryPayload[]) {
const isDeliverable = (p: DeliveryPayload) => {
const text = (p?.text ?? "").trim();
const hasMedia = Boolean(p?.mediaUrl) || (p?.mediaUrls?.length ?? 0) > 0;
const hasInteractive = (p?.interactive?.blocks?.length ?? 0) > 0;
const hasChannelData = Object.keys(p?.channelData ?? {}).length > 0;
return text || hasMedia || hasInteractive || hasChannelData;
return hasOutboundReplyContent(p, { trimText: true }) || hasInteractive || hasChannelData;
};
for (let i = payloads.length - 1; i >= 0; i--) {
if (payloads[i]?.isError) {

10
src/cron/isolated-agent/run.ts
View File

@@ -48,6 +48,7 @@ import {
import type { AgentDefaultsConfig } from "../../config/types.js";
import { registerAgentRunContext } from "../../infra/agent-events.js";
import { logWarn } from "../../logger.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import { normalizeAgentId } from "../../routing/session-key.js";
import {
buildSafeExternalPrompt,
@@ -687,9 +688,9 @@ export async function runCronIsolatedAgentTurn(params: {
const interimPayloads = interimRunResult.payloads ?? [];
const interimDeliveryPayload = pickLastDeliverablePayload(interimPayloads);
const interimPayloadHasStructuredContent =
Boolean(interimDeliveryPayload?.mediaUrl) ||
(interimDeliveryPayload?.mediaUrls?.length ?? 0) > 0 ||
Object.keys(interimDeliveryPayload?.channelData ?? {}).length > 0;
(interimDeliveryPayload
? resolveSendableOutboundReplyParts(interimDeliveryPayload).hasMedia
: false) || Object.keys(interimDeliveryPayload?.channelData ?? {}).length > 0;
const interimText = pickLastNonEmptyTextFromPayloads(interimPayloads)?.trim() ?? "";
const hasDescendantsSinceRunStart = listDescendantRunsForRequester(agentSessionKey).some(
(entry) => {
@@ -809,8 +810,7 @@ export async function runCronIsolatedAgentTurn(params: {
? [{ text: synthesizedText }]
: [];
const deliveryPayloadHasStructuredContent =
Boolean(deliveryPayload?.mediaUrl) ||
(deliveryPayload?.mediaUrls?.length ?? 0) > 0 ||
(deliveryPayload ? resolveSendableOutboundReplyParts(deliveryPayload).hasMedia : false) ||
Object.keys(deliveryPayload?.channelData ?? {}).length > 0;
const deliveryBestEffort = resolveCronDeliveryBestEffort(params.job);
const hasErrorPayload = payloads.some((payload) => payload?.isError === true);

6
src/gateway/server-methods/send.ts
View File

@@ -13,7 +13,7 @@ import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads
import { buildOutboundSessionContext } from "../../infra/outbound/session-context.js";
import { maybeResolveIdLikeTarget } from "../../infra/outbound/target-resolver.js";
import { resolveOutboundTarget } from "../../infra/outbound/targets.js";
import { resolveOutboundMediaUrls } from "../../plugin-sdk/reply-payload.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import { normalizePollInput } from "../../polls.js";
import {
ErrorCodes,
@@ -211,8 +211,8 @@ export const sendHandlers: GatewayRequestHandlers = {
.map((payload) => payload.text)
.filter(Boolean)
.join("\n");
const mirrorMediaUrls = mirrorPayloads.flatMap((payload) =>
resolveOutboundMediaUrls(payload),
const mirrorMediaUrls = mirrorPayloads.flatMap(
(payload) => resolveSendableOutboundReplyParts(payload).mediaUrls,
);
const providedSessionKey =
typeof request.sessionKey === "string" && request.sessionKey.trim()

9
src/gateway/ws-log.ts
View File

@@ -3,6 +3,7 @@ import { isVerbose } from "../globals.js";
import { shouldLogSubsystemToConsole } from "../logging/console.js";
import { getDefaultRedactPatterns, redactSensitiveText } from "../logging/redact.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
import { parseAgentSessionKey } from "../routing/session-key.js";
import { DEFAULT_WS_SLOW_MS, getGatewayWsLogStyle } from "./ws-logging.js";
@@ -204,9 +205,11 @@ export function summarizeAgentEventForWsLog(payload: unknown): Record<string, un
if (text?.trim()) {
extra.text = compactPreview(text);
}
const mediaUrls = Array.isArray(data.mediaUrls) ? data.mediaUrls : undefined;
if (mediaUrls && mediaUrls.length > 0) {
extra.media = mediaUrls.length;
const mediaCount = resolveSendableOutboundReplyParts({
mediaUrls: Array.isArray(data.mediaUrls) ? data.mediaUrls : undefined,
}).mediaCount;
if (mediaCount > 0) {
extra.media = mediaCount;
}
return extra;
}

14
src/infra/heartbeat-runner.ts
View File

@@ -35,6 +35,10 @@ import {
import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js";
import { resolveCronSession } from "../cron/isolated-agent/session.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
hasOutboundReplyContent,
resolveSendableOutboundReplyParts,
} from "../plugin-sdk/reply-payload.js";
import { getQueueSize } from "../process/command-queue.js";
import { CommandLane } from "../process/lanes.js";
import {
@@ -368,7 +372,7 @@ function normalizeHeartbeatReply(
mode: "heartbeat",
maxAckChars: ackMaxChars,
});
const hasMedia = Boolean(payload.mediaUrl || (payload.mediaUrls?.length ?? 0) > 0);
const hasMedia = resolveSendableOutboundReplyParts(payload).hasMedia;
if (stripped.shouldSkip && !hasMedia) {
return {
shouldSkip: true,
@@ -720,10 +724,7 @@ export async function runHeartbeatOnce(opts: {
? resolveHeartbeatReasoningPayloads(replyResult).filter((payload) => payload !== replyPayload)
: [];
if (
!replyPayload ||
(!replyPayload.text && !replyPayload.mediaUrl && !replyPayload.mediaUrls?.length)
) {
if (!replyPayload || !hasOutboundReplyContent(replyPayload)) {
await restoreHeartbeatUpdatedAt({
storePath,
sessionKey,
@@ -780,8 +781,7 @@ export async function runHeartbeatOnce(opts: {
return { status: "ran", durationMs: Date.now() - startedAt };
}
const mediaUrls =
replyPayload.mediaUrls ?? (replyPayload.mediaUrl ? [replyPayload.mediaUrl] : []);
const mediaUrls = resolveSendableOutboundReplyParts(replyPayload).mediaUrls;
// Suppress duplicate heartbeats (same payload) within a short window.
// This prevents "nagging" when nothing changed but the model repeats the same items.

28
src/infra/outbound/deliver.ts
View File

@@ -23,11 +23,11 @@ import {
toPluginMessageContext,
toPluginMessageSentEvent,
} from "../../hooks/message-hook-mappers.js";
import { hasReplyChannelData, hasReplyContent } from "../../interactive/payload.js";
import { hasReplyPayloadContent } from "../../interactive/payload.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { getAgentScopedMediaLocalRoots } from "../../media/local-roots.js";
import {
resolveOutboundMediaUrls,
resolveSendableOutboundReplyParts,
sendMediaWithLeadingCaption,
} from "../../plugin-sdk/reply-payload.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
@@ -284,17 +284,8 @@ type MessageSentEvent = {
function normalizeEmptyPayloadForDelivery(payload: ReplyPayload): ReplyPayload | null {
const text = typeof payload.text === "string" ? payload.text : "";
const hasChannelData = hasReplyChannelData(payload.channelData);
if (!text.trim()) {
if (
!hasReplyContent({
text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData,
})
) {
if (!hasReplyPayloadContent({ ...payload, text })) {
return null;
}
if (text) {
@@ -340,9 +331,10 @@ function normalizePayloadsForChannelDelivery(
}
function buildPayloadSummary(payload: ReplyPayload): NormalizedOutboundPayload {
const parts = resolveSendableOutboundReplyParts(payload);
return {
text: payload.text ?? "",
mediaUrls: resolveOutboundMediaUrls(payload),
text: parts.text,
mediaUrls: parts.mediaUrls,
interactive: payload.interactive,
channelData: payload.channelData,
};
@@ -669,10 +661,10 @@ async function deliverOutboundPayloadsCore(
};
if (
handler.sendPayload &&
(effectivePayload.channelData ||
hasReplyContent({
interactive: effectivePayload.interactive,
}))
hasReplyPayloadContent({
interactive: effectivePayload.interactive,
channelData: effectivePayload.channelData,
})
) {
const delivery = await handler.sendPayload(effectivePayload, sendOverrides);
results.push(delivery);

20
src/infra/outbound/message-action-runner.ts
View File

@@ -14,7 +14,7 @@ import type {
ChannelThreadingToolContext,
} from "../../channels/plugins/types.js";
import type { OpenClawConfig } from "../../config/config.js";
import { hasInteractiveReplyBlocks, hasReplyContent } from "../../interactive/payload.js";
import { hasInteractiveReplyBlocks, hasReplyPayloadContent } from "../../interactive/payload.js";
import { getAgentScopedMediaLocalRoots } from "../../media/local-roots.js";
import { hasPollCreationParams, resolveTelegramPollVisibility } from "../../poll-params.js";
import { resolvePollMaxSelections } from "../../polls.js";
@@ -484,13 +484,17 @@ async function handleSendAction(ctx: ResolvedActionContext): Promise<MessageActi
}
}
if (
!hasReplyContent({
text: message,
mediaUrl,
mediaUrls: mergedMediaUrls,
interactive: params.interactive,
extraContent: hasButtons || hasCard || hasComponents || hasBlocks,
})
!hasReplyPayloadContent(
{
text: message,
mediaUrl,
mediaUrls: mergedMediaUrls,
interactive: params.interactive,
},
{
extraContent: hasButtons || hasCard || hasComponents || hasBlocks,
},
)
) {
throw new Error("send requires text or media");
}

6
src/infra/outbound/message.ts
View File

@@ -1,7 +1,7 @@
import type { OpenClawConfig } from "../../config/config.js";
import { loadConfig } from "../../config/config.js";
import { callGatewayLeastPrivilege, randomIdempotencyKey } from "../../gateway/call.js";
import { resolveOutboundMediaUrls } from "../../plugin-sdk/reply-payload.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
import type { PollInput } from "../../polls.js";
import { normalizePollInput } from "../../polls.js";
import {
@@ -203,8 +203,8 @@ export async function sendMessage(params: MessageSendParams): Promise<MessageSen
.map((payload) => payload.text)
.filter(Boolean)
.join("\n");
const mirrorMediaUrls = normalizedPayloads.flatMap((payload) =>
resolveOutboundMediaUrls(payload),
const mirrorMediaUrls = normalizedPayloads.flatMap(
(payload) => resolveSendableOutboundReplyParts(payload).mediaUrls,
);
const primaryMediaUrl = mirrorMediaUrls[0] ?? params.mediaUrl ?? null;

23
src/infra/outbound/payloads.ts
View File

@@ -8,10 +8,10 @@ import type { ReplyPayload } from "../../auto-reply/types.js";
import {
hasInteractiveReplyBlocks,
hasReplyChannelData,
hasReplyContent,
hasReplyPayloadContent,
type InteractiveReply,
} from "../../interactive/payload.js";
import { resolveOutboundMediaUrls } from "../../plugin-sdk/reply-payload.js";
import { resolveSendableOutboundReplyParts } from "../../plugin-sdk/reply-payload.js";
export type NormalizedOutboundPayload = {
text: string;
@@ -97,25 +97,20 @@ export function normalizeOutboundPayloads(
): NormalizedOutboundPayload[] {
const normalizedPayloads: NormalizedOutboundPayload[] = [];
for (const payload of normalizeReplyPayloadsForDelivery(payloads)) {
const mediaUrls = resolveOutboundMediaUrls(payload);
const parts = resolveSendableOutboundReplyParts(payload);
const interactive = payload.interactive;
const channelData = payload.channelData;
const hasChannelData = hasReplyChannelData(channelData);
const hasInteractive = hasInteractiveReplyBlocks(interactive);
const text = payload.text ?? "";
const text = parts.text;
if (
!hasReplyContent({
text,
mediaUrls,
interactive,
hasChannelData,
})
!hasReplyPayloadContent({ ...payload, text, mediaUrls: parts.mediaUrls }, { hasChannelData })
) {
continue;
}
normalizedPayloads.push({
text,
mediaUrls,
mediaUrls: parts.mediaUrls,
...(hasInteractive ? { interactive } : {}),
...(hasChannelData ? { channelData } : {}),
});
@@ -128,11 +123,11 @@ export function normalizeOutboundPayloadsForJson(
): OutboundPayloadJson[] {
const normalized: OutboundPayloadJson[] = [];
for (const payload of normalizeReplyPayloadsForDelivery(payloads)) {
const mediaUrls = resolveOutboundMediaUrls(payload);
const parts = resolveSendableOutboundReplyParts(payload);
normalized.push({
text: payload.text ?? "",
text: parts.text,
mediaUrl: payload.mediaUrl ?? null,
mediaUrls: mediaUrls.length ? mediaUrls : undefined,
mediaUrls: parts.mediaUrls.length ? parts.mediaUrls : undefined,
interactive: payload.interactive,
channelData: payload.channelData,
});

36
src/interactive/payload.test.ts
View File

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
hasReplyChannelData,
hasReplyContent,
hasReplyPayloadContent,
normalizeInteractiveReply,
resolveInteractiveTextFallback,
} from "./payload.js";
@@ -44,6 +45,41 @@ describe("hasReplyContent", () => {
});
});
describe("hasReplyPayloadContent", () => {
it("trims text and falls back to channel data by default", () => {
expect(
hasReplyPayloadContent({
text: " ",
channelData: { slack: { blocks: [] } },
}),
).toBe(true);
});
it("accepts explicit channel-data overrides and extra content", () => {
expect(
hasReplyPayloadContent(
{
text: " ",
channelData: {},
},
{
hasChannelData: true,
},
),
).toBe(true);
expect(
hasReplyPayloadContent(
{
text: " ",
},
{
extraContent: true,
},
),
).toBe(true);
});
});
describe("interactive payload helpers", () => {
it("normalizes interactive replies and resolves text fallbacks", () => {
const interactive = normalizeInteractiveReply({

24
src/interactive/payload.ts
View File

@@ -160,6 +160,30 @@ export function hasReplyContent(params: {
);
}
export function hasReplyPayloadContent(
payload: {
text?: string | null;
mediaUrl?: string | null;
mediaUrls?: ReadonlyArray<string | null | undefined>;
interactive?: unknown;
channelData?: unknown;
},
options?: {
trimText?: boolean;
hasChannelData?: boolean;
extraContent?: boolean;
},
): boolean {
return hasReplyContent({
text: options?.trimText ? payload.text?.trim() : payload.text,
mediaUrl: payload.mediaUrl,
mediaUrls: payload.mediaUrls,
interactive: payload.interactive,
hasChannelData: options?.hasChannelData ?? hasReplyChannelData(payload.channelData),
extraContent: options?.extraContent,
});
}
export function resolveInteractiveTextFallback(params: {
text?: string;
interactive?: InteractiveReply;

4
src/line/auto-reply-delivery.ts
View File

@@ -1,6 +1,6 @@
import type { messagingApi } from "@line/bot-sdk";
import type { ReplyPayload } from "../auto-reply/types.js";
import { resolveOutboundMediaUrls } from "../plugin-sdk/reply-payload.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
import type { FlexContainer } from "./flex-templates.js";
import type { ProcessedLineMessage } from "./markdown-to-line.js";
import type { SendLineReplyChunksParams } from "./reply-chunks.js";
@@ -124,7 +124,7 @@ export async function deliverLineAutoReply(params: {
const chunks = processed.text ? deps.chunkMarkdownText(processed.text, textLimit) : [];
const mediaUrls = resolveOutboundMediaUrls(payload);
const mediaUrls = resolveSendableOutboundReplyParts(payload).mediaUrls;
const mediaMessages = mediaUrls
.map((url) => url?.trim())
.filter((url): url is string => Boolean(url))

2
src/plugin-sdk/msteams.ts
View File

@@ -46,7 +46,7 @@ export {
splitSetupEntries,
} from "../channels/plugins/setup-wizard-helpers.js";
export { PAIRING_APPROVED_MESSAGE } from "../channels/plugins/pairing-message.js";
export { resolveOutboundMediaUrls } from "./reply-payload.js";
export { resolveOutboundMediaUrls, resolveSendableOutboundReplyParts } from "./reply-payload.js";
export type {
BaseProbeResult,
ChannelDirectoryEntry,

121
src/plugin-sdk/reply-payload.test.ts
View File

@@ -1,9 +1,14 @@
import { describe, expect, it, vi } from "vitest";
import {
countOutboundMedia,
deliverFormattedTextWithAttachments,
deliverTextOrMediaReply,
hasOutboundMedia,
hasOutboundReplyContent,
hasOutboundText,
isNumericTargetId,
resolveOutboundMediaUrls,
resolveSendableOutboundReplyParts,
resolveTextChunksWithFallback,
sendMediaWithLeadingCaption,
sendPayloadWithChunkedTextAndMedia,
@@ -84,6 +89,102 @@ describe("resolveOutboundMediaUrls", () => {
});
});
describe("countOutboundMedia", () => {
it("counts normalized media entries", () => {
expect(
countOutboundMedia({
mediaUrls: ["https://example.com/a.png", "https://example.com/b.png"],
}),
).toBe(2);
});
it("counts legacy single-media payloads", () => {
expect(
countOutboundMedia({
mediaUrl: "https://example.com/legacy.png",
}),
).toBe(1);
});
});
describe("hasOutboundMedia", () => {
it("reports whether normalized payloads include media", () => {
expect(hasOutboundMedia({ mediaUrls: ["https://example.com/a.png"] })).toBe(true);
expect(hasOutboundMedia({ mediaUrl: "https://example.com/legacy.png" })).toBe(true);
expect(hasOutboundMedia({})).toBe(false);
});
});
describe("hasOutboundText", () => {
it("checks raw text presence by default", () => {
expect(hasOutboundText({ text: "hello" })).toBe(true);
expect(hasOutboundText({ text: " " })).toBe(true);
expect(hasOutboundText({})).toBe(false);
});
it("can trim whitespace-only text", () => {
expect(hasOutboundText({ text: " " }, { trim: true })).toBe(false);
expect(hasOutboundText({ text: " hi " }, { trim: true })).toBe(true);
});
});
describe("hasOutboundReplyContent", () => {
it("detects text or media content", () => {
expect(hasOutboundReplyContent({ text: "hello" })).toBe(true);
expect(hasOutboundReplyContent({ mediaUrl: "https://example.com/a.png" })).toBe(true);
expect(hasOutboundReplyContent({})).toBe(false);
});
it("can ignore whitespace-only text unless media exists", () => {
expect(hasOutboundReplyContent({ text: " " }, { trimText: true })).toBe(false);
expect(
hasOutboundReplyContent(
{ text: " ", mediaUrls: ["https://example.com/a.png"] },
{ trimText: true },
),
).toBe(true);
});
});
describe("resolveSendableOutboundReplyParts", () => {
it("normalizes missing text and trims media urls", () => {
expect(
resolveSendableOutboundReplyParts({
mediaUrls: [" https://example.com/a.png ", " "],
}),
).toEqual({
text: "",
trimmedText: "",
mediaUrls: ["https://example.com/a.png"],
mediaCount: 1,
hasText: false,
hasMedia: true,
hasContent: true,
});
});
it("accepts transformed text overrides", () => {
expect(
resolveSendableOutboundReplyParts(
{
text: "ignored",
},
{
text: " hello ",
},
),
).toEqual({
text: " hello ",
trimmedText: "hello",
mediaUrls: [],
mediaCount: 0,
hasText: true,
hasMedia: false,
hasContent: true,
});
});
});
describe("resolveTextChunksWithFallback", () => {
it("returns existing chunks unchanged", () => {
expect(resolveTextChunksWithFallback("hello", ["a", "b"])).toEqual(["a", "b"]);
@@ -161,6 +262,26 @@ describe("deliverTextOrMediaReply", () => {
expect(sendText).not.toHaveBeenCalled();
expect(sendMedia).not.toHaveBeenCalled();
});
it("ignores blank media urls before sending", async () => {
const sendMedia = vi.fn(async () => undefined);
const sendText = vi.fn(async () => undefined);
await expect(
deliverTextOrMediaReply({
payload: { text: "hello", mediaUrls: [" ", " https://a "] },
text: "hello",
sendText,
sendMedia,
}),
).resolves.toBe("media");
expect(sendMedia).toHaveBeenCalledTimes(1);
expect(sendMedia).toHaveBeenCalledWith({
mediaUrl: "https://a",
caption: "hello",
});
});
});
describe("sendMediaWithLeadingCaption", () => {

62
src/plugin-sdk/reply-payload.ts
View File

@@ -5,6 +5,16 @@ export type OutboundReplyPayload = {
replyToId?: string;
};
export type SendableOutboundReplyParts = {
text: string;
trimmedText: string;
mediaUrls: string[];
mediaCount: number;
hasText: boolean;
hasMedia: boolean;
hasContent: boolean;
};
/** Extract the supported outbound reply fields from loose tool or agent payload objects. */
export function normalizeOutboundReplyPayload(
payload: Record<string, unknown>,
@@ -52,6 +62,54 @@ export function resolveOutboundMediaUrls(payload: {
return [];
}
/** Count outbound media items after legacy single-media fallback normalization. */
export function countOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): number {
return resolveOutboundMediaUrls(payload).length;
}
/** Check whether an outbound payload includes any media after normalization. */
export function hasOutboundMedia(payload: { mediaUrls?: string[]; mediaUrl?: string }): boolean {
return countOutboundMedia(payload) > 0;
}
/** Check whether an outbound payload includes text, optionally trimming whitespace first. */
export function hasOutboundText(payload: { text?: string }, options?: { trim?: boolean }): boolean {
const text = options?.trim ? payload.text?.trim() : payload.text;
return Boolean(text);
}
/** Check whether an outbound payload includes any sendable text or media. */
export function hasOutboundReplyContent(
payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string },
options?: { trimText?: boolean },
): boolean {
return hasOutboundText(payload, { trim: options?.trimText }) || hasOutboundMedia(payload);
}
/** Normalize reply payload text/media into a trimmed, sendable shape for delivery paths. */
export function resolveSendableOutboundReplyParts(
payload: { text?: string; mediaUrls?: string[]; mediaUrl?: string },
options?: { text?: string },
): SendableOutboundReplyParts {
const text = options?.text ?? payload.text ?? "";
const trimmedText = text.trim();
const mediaUrls = resolveOutboundMediaUrls(payload)
.map((entry) => entry.trim())
.filter(Boolean);
const mediaCount = mediaUrls.length;
const hasText = Boolean(trimmedText);
const hasMedia = mediaCount > 0;
return {
text,
trimmedText,
mediaUrls,
mediaCount,
hasText,
hasMedia,
hasContent: hasText || hasMedia,
};
}
/** Preserve caller-provided chunking, but fall back to the full text when chunkers return nothing. */
export function resolveTextChunksWithFallback(text: string, chunks: readonly string[]): string[] {
if (chunks.length > 0) {
@@ -188,7 +246,9 @@ export async function deliverTextOrMediaReply(params: {
isFirst: boolean;
}) => Promise<void> | void;
}): Promise<"empty" | "text" | "media"> {
const mediaUrls = resolveOutboundMediaUrls(params.payload);
const { mediaUrls } = resolveSendableOutboundReplyParts(params.payload, {
text: params.text,
});
const sentMedia = await sendMediaWithLeadingCaption({
mediaUrls,
caption: params.text,

4
src/plugin-sdk/subpaths.test.ts
View File

@@ -98,9 +98,13 @@ describe("plugin-sdk subpath exports", () => {
});
it("exports reply payload helpers from the dedicated subpath", () => {
expect(typeof replyPayloadSdk.countOutboundMedia).toBe("function");
expect(typeof replyPayloadSdk.deliverFormattedTextWithAttachments).toBe("function");
expect(typeof replyPayloadSdk.deliverTextOrMediaReply).toBe("function");
expect(typeof replyPayloadSdk.formatTextWithAttachmentLinks).toBe("function");
expect(typeof replyPayloadSdk.hasOutboundMedia).toBe("function");
expect(typeof replyPayloadSdk.hasOutboundReplyContent).toBe("function");
expect(typeof replyPayloadSdk.hasOutboundText).toBe("function");
expect(typeof replyPayloadSdk.resolveOutboundMediaUrls).toBe("function");
expect(typeof replyPayloadSdk.resolveTextChunksWithFallback).toBe("function");
expect(typeof replyPayloadSdk.sendMediaWithLeadingCaption).toBe("function");

1
src/plugin-sdk/zalouser.ts
View File

@@ -71,6 +71,7 @@ export {
deliverTextOrMediaReply,
isNumericTargetId,
resolveOutboundMediaUrls,
resolveSendableOutboundReplyParts,
sendMediaWithLeadingCaption,
sendPayloadWithChunkedTextAndMedia,
} from "./reply-payload.js";

6
src/tts/tts.ts
View File

@@ -24,6 +24,7 @@ import type {
import { logVerbose } from "../globals.js";
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
import { stripMarkdown } from "../line/markdown-to-line.js";
import { resolveSendableOutboundReplyParts } from "../plugin-sdk/reply-payload.js";
import { CONFIG_DIR, resolveUserPath } from "../utils.js";
import {
getSpeechProvider,
@@ -793,7 +794,8 @@ export async function maybeApplyTtsToPayload(params: {
return params.payload;
}
const text = params.payload.text ?? "";
const reply = resolveSendableOutboundReplyParts(params.payload);
const text = reply.text;
const directives = parseTtsDirectives(text, config.modelOverrides, config.openai.baseUrl);
if (directives.warnings.length > 0) {
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
@@ -827,7 +829,7 @@ export async function maybeApplyTtsToPayload(params: {
if (!ttsText.trim()) {
return nextPayload;
}
if (params.payload.mediaUrl || (params.payload.mediaUrls?.length ?? 0) > 0) {
if (reply.hasMedia) {
return nextPayload;
}
if (text.includes("MEDIA:")) {