openclaw/src/auto-reply/reply-payload.ts

import type {
  InteractiveReply,
  MessagePresentation,
  ReplyPayloadDelivery,
} from "../interactive/payload.js";

export type ReplyPayload = {
  text?: string;
  mediaUrl?: string;
  mediaUrls?: string[];
  /** Internal-only trust signal for gateway webchat local media embedding. */
  trustedLocalMedia?: boolean;
  /** Treat media as live-only content and avoid persisting the underlying media reference. */
  sensitiveMedia?: boolean;
  /** Channel-agnostic rich presentation. Core degrades or asks the channel renderer to map it. */
  presentation?: MessagePresentation;
  /** Channel-agnostic delivery preferences, e.g. pin the sent message when supported. */
  delivery?: ReplyPayloadDelivery;
  /**
   * @deprecated Use presentation.
   *
   * Internal legacy representation used by existing approval/reply helpers during migration.
   */
  interactive?: InteractiveReply;
  btw?: {
    question: string;
  };
  replyToId?: string;
  replyToTag?: boolean;
  /** True when [[reply_to_current]] was present but not yet mapped to a message id. */
  replyToCurrent?: boolean;
  /** Send audio as voice message (bubble) instead of audio file. Defaults to false. */
  audioAsVoice?: boolean;
  /**
   * Text synthesized into an audio-only TTS payload. Exposed to hooks for
   * archival/search use when no visible channel text is sent.
   */
  spokenText?: string;
  /**
   * Marks a TTS media payload as supplemental audio for assistant text that is
   * already visible through streaming or transcript projection.
   */
  ttsSupplement?: ReplyPayloadTtsSupplement;
  isError?: boolean;
  /** Marks this payload as a reasoning/thinking block. Channels that do not
   *  have a dedicated reasoning lane (e.g. WhatsApp, web) should suppress it. */
  isReasoning?: boolean;
  /** Marks this payload as a compaction status notice (start/end).
   *  Should be excluded from TTS transcript accumulation so compaction
   *  status lines are not synthesised into the spoken assistant reply. */
  isCompactionNotice?: boolean;
  /** Marks this payload as a model-fallback transition/recovery notice. */
  isFallbackNotice?: boolean;
  /** Marks this payload as transient status, not assistant answer content. */
  isStatusNotice?: boolean;
  /** Channel-specific payload data (per-channel envelope). */
  channelData?: Record<string, unknown>;
};

export type ReplyPayloadTtsSupplement = {
  spokenText: string;
  visibleTextAlreadyDelivered?: boolean;
};

export const REPLY_MEDIA_FAILURE_WARNING = "⚠️ Media failed.";

export function appendReplyMediaFailureWarning(text: string | undefined): string {
  if (!text?.trim()) {
    return REPLY_MEDIA_FAILURE_WARNING;
  }
  if (text.includes(REPLY_MEDIA_FAILURE_WARNING)) {
    return text;
  }
  return `${text}\n${REPLY_MEDIA_FAILURE_WARNING}`;
}

function normalizeTtsSupplementSpokenText(value: unknown): string | undefined {
  return typeof value === "string" && value.trim() ? value : undefined;
}

function hasReplyPayloadMedia(payload: Pick<ReplyPayload, "mediaUrl" | "mediaUrls">): boolean {
  return Boolean(payload.mediaUrl?.trim() || payload.mediaUrls?.some((url) => url.trim()));
}

export function getReplyPayloadTtsSupplement(
  payload: Pick<ReplyPayload, "mediaUrl" | "mediaUrls" | "ttsSupplement">,
): ReplyPayloadTtsSupplement | undefined {
  const spokenText = normalizeTtsSupplementSpokenText(payload.ttsSupplement?.spokenText);
  if (!spokenText || !hasReplyPayloadMedia(payload)) {
    return undefined;
  }
  return {
    spokenText,
    ...(payload.ttsSupplement?.visibleTextAlreadyDelivered === true
      ? { visibleTextAlreadyDelivered: true }
      : {}),
  };
}

export function isReplyPayloadTtsSupplement(
  payload: Pick<ReplyPayload, "mediaUrl" | "mediaUrls" | "ttsSupplement">,
): boolean {
  return Boolean(getReplyPayloadTtsSupplement(payload));
}

export function markReplyPayloadAsTtsSupplement<T extends ReplyPayload>(
  payload: T,
  spokenText: string = payload.spokenText ?? payload.text ?? "",
  options?: { visibleTextAlreadyDelivered?: boolean },
): T {
  const normalizedSpokenText = normalizeTtsSupplementSpokenText(spokenText);
  if (!normalizedSpokenText) {
    return payload;
  }
  return {
    ...payload,
    spokenText: normalizedSpokenText,
    ttsSupplement: {
      spokenText: normalizedSpokenText,
      ...(options?.visibleTextAlreadyDelivered === true
        ? { visibleTextAlreadyDelivered: true }
        : {}),
    },
  };
}

export function buildTtsSupplementMediaPayload(payload: ReplyPayload): ReplyPayload {
  const supplement = getReplyPayloadTtsSupplement(payload);
  if (!supplement) {
    return payload;
  }
  const {
    text: _text,
    presentation: _presentation,
    interactive: _interactive,
    btw: _btw,
    ...mediaPayload
  } = payload;
  return {
    ...mediaPayload,
    spokenText: supplement.spokenText,
    ttsSupplement: supplement,
  };
}

export type ReplyPayloadMetadata = {
  assistantMessageIndex?: number;
  /**
   * Internal OpenClaw notices generated after a runtime/provider failure are
   * not assistant source replies. Dispatch may deliver them even when normal
   * assistant source replies are message-tool-only; sendPolicy deny still wins.
   */
  deliverDespiteSourceReplySuppression?: boolean;
  /**
   * A message-tool reply to the active internal UI source. The final payload is
   * still the live delivery vehicle; this mirror makes the reply durable for
   * chat.history and page reloads without turning the internal UI into an
   * outbound channel.
   */
  sourceReplyTranscriptMirror?: {
    sessionKey: string;
    agentId?: string;
    text?: string;
    mediaUrls?: string[];
    idempotencyKey?: string;
  };
  beforeAgentRunBlocked?: boolean;
  /** Warning synthesized from an observed tool error after the run produced assistant output. */
  nonTerminalToolErrorWarning?: boolean;
};

const replyPayloadMetadata = new WeakMap<object, ReplyPayloadMetadata>();

export function setReplyPayloadMetadata<T extends object>(
  payload: T,
  metadata: ReplyPayloadMetadata,
): T {
  const previous = replyPayloadMetadata.get(payload);
  replyPayloadMetadata.set(payload, { ...previous, ...metadata });
  return payload;
}

export function getReplyPayloadMetadata(payload: object): ReplyPayloadMetadata | undefined {
  return replyPayloadMetadata.get(payload);
}

export function isReplyPayloadNonTerminalToolErrorWarning(payload: object): boolean {
  return getReplyPayloadMetadata(payload)?.nonTerminalToolErrorWarning === true;
}

export function copyReplyPayloadMetadata<T extends object>(source: object, payload: T): T {
  const metadata = getReplyPayloadMetadata(source);
  return metadata ? setReplyPayloadMetadata(payload, metadata) : payload;
}

export function markReplyPayloadForSourceSuppressionDelivery<T extends object>(payload: T): T {
  return setReplyPayloadMetadata(payload, {
    deliverDespiteSourceReplySuppression: true,
  });
}

export function isReplyPayloadStatusNotice(
  payload: Pick<ReplyPayload, "isCompactionNotice" | "isFallbackNotice" | "isStatusNotice">,
): boolean {
  return Boolean(payload.isCompactionNotice || payload.isFallbackNotice || payload.isStatusNotice);
}