diff --git a/CHANGELOG.md b/CHANGELOG.md index d6a37ddba6f..05806adb4f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Control UI/WebChat: keep large attachment payloads out of Lit state and optimistic chat messages, using object URL previews plus send-time payload serialization so PDF/image uploads no longer trigger `RangeError: Maximum call stack size exceeded`. Fixes #73360; refs #54378 and #63432. Thanks @hejunhui-73, @Ansub, and @christianhernandez3-afk. - Agents/models: keep per-agent primary models strict when `fallbacks` is omitted, so probe-only custom providers are not tried as hidden fallback candidates unless the agent explicitly opts in. Fixes #73332. Thanks @haumanto. - Cron/Telegram: preserve explicit `:topic:` delivery targets over stale session-derived thread IDs when isolated cron announces to Telegram forum topics. Carries forward #59069; refs #49704 and #43808. Thanks @roytong9. - Build/runtime: write the runtime-postbuild stamp after `pnpm build` writes the build stamp, so the next CLI invocation does not re-sync runtime artifacts after a successful build. Fixes #73151. Thanks @bittoby. diff --git a/src/commands/channels.list.auth-profiles.test.ts b/src/commands/channels.list.auth-profiles.test.ts index 6fe62d8a747..de51794994d 100644 --- a/src/commands/channels.list.auth-profiles.test.ts +++ b/src/commands/channels.list.auth-profiles.test.ts @@ -40,6 +40,24 @@ vi.mock("../channels/plugins/status.js", () => ({ import { channelsListCommand } from "./channels/list.js"; +function createMockChannelPlugin(accountIds: string[]): ChannelPlugin { + return { + id: "telegram", + meta: { + id: "telegram", + label: "Telegram", + selectionLabel: "Telegram", + docsPath: "/channels/telegram", + blurb: "Telegram", + }, + capabilities: { chatTypes: ["direct"] }, + config: { + listAccountIds: () => accountIds, + resolveAccount: () => ({}), + }, + }; +} + describe("channels list auth profiles", () => { beforeEach(() => { mocks.readConfigFileSnapshot.mockReset(); @@ -92,21 +110,7 @@ describe("channels list auth profiles", () => { it("includes configured chat channel accounts in JSON output", async () => { const runtime = createTestRuntime(); mocks.listReadOnlyChannelPluginsForConfig.mockReturnValue([ - { - id: "telegram", - meta: { - id: "telegram", - label: "Telegram", - selectionLabel: "Telegram", - docsPath: "/channels/telegram", - blurb: "Telegram", - }, - capabilities: { chatTypes: ["direct"] }, - config: { - listAccountIds: () => ["alerts", "default"], - resolveAccount: () => ({}), - }, - }, + createMockChannelPlugin(["alerts", "default"]), ]); mocks.readConfigFileSnapshot.mockResolvedValue({ ...baseConfigSnapshot, @@ -141,21 +145,7 @@ describe("channels list auth profiles", () => { it("prints configured chat channel accounts before auth providers", async () => { const runtime = createTestRuntime(); mocks.listReadOnlyChannelPluginsForConfig.mockReturnValue([ - { - id: "telegram", - meta: { - id: "telegram", - label: "Telegram", - selectionLabel: "Telegram", - docsPath: "/channels/telegram", - blurb: "Telegram", - }, - capabilities: { chatTypes: ["direct"] }, - config: { - listAccountIds: () => ["default"], - resolveAccount: () => ({}), - }, - }, + createMockChannelPlugin(["default"]), ]); mocks.buildChannelAccountSnapshot.mockResolvedValue({ accountId: "default", diff --git a/ui/src/ui/app-chat.test.ts b/ui/src/ui/app-chat.test.ts index b20c6dcb8fd..abb19e459e3 100644 --- a/ui/src/ui/app-chat.test.ts +++ b/ui/src/ui/app-chat.test.ts @@ -2,6 +2,12 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChatHost } from "./app-chat.ts"; +import { + getChatAttachmentDataUrl, + getChatAttachmentPreviewUrl, + registerChatAttachmentPayload, + resetChatAttachmentPayloadStoreForTest, +} from "./chat/attachment-payload-store.ts"; import type { GatewaySessionRow, SessionsListResult } from "./types.ts"; const { setLastActiveSessionKeyMock } = vi.hoisted(() => ({ @@ -18,6 +24,7 @@ let navigateChatInputHistory: typeof import("./app-chat.ts").navigateChatInputHi let handleAbortChat: typeof import("./app-chat.ts").handleAbortChat; let refreshChatAvatar: typeof import("./app-chat.ts").refreshChatAvatar; let clearPendingQueueItemsForRun: typeof import("./app-chat.ts").clearPendingQueueItemsForRun; +let removeQueuedMessage: typeof import("./app-chat.ts").removeQueuedMessage; async function loadChatHelpers(): Promise { ({ @@ -27,6 +34,7 @@ async function loadChatHelpers(): Promise { handleAbortChat, refreshChatAvatar, clearPendingQueueItemsForRun, + removeQueuedMessage, } = await import("./app-chat.ts")); } @@ -117,6 +125,7 @@ describe("refreshChatAvatar", () => { }); afterEach(() => { + resetChatAttachmentPayloadStoreForTest(); vi.unstubAllGlobals(); }); @@ -729,6 +738,75 @@ describe("handleSendChat", () => { }), ]); }); + + it("drops sent attachment payload bytes while keeping the optimistic preview URL", async () => { + vi.stubGlobal( + "URL", + class extends URL { + static createObjectURL = vi.fn(() => "blob:brief"); + static revokeObjectURL = vi.fn(); + }, + ); + const request = vi.fn(async (method: string) => { + if (method === "chat.send") { + return { status: "started", runId: "run-1" }; + } + throw new Error(`Unexpected request: ${method}`); + }); + const file = new File(["%PDF-1.4\n"], "brief.pdf", { type: "application/pdf" }); + const attachment = registerChatAttachmentPayload({ + attachment: { + id: "att-1", + mimeType: "application/pdf", + fileName: "brief.pdf", + sizeBytes: file.size, + }, + dataUrl: "data:application/pdf;base64,JVBERi0xLjQK", + file, + }); + const host = makeHost({ + client: { request } as unknown as ChatHost["client"], + chatAttachments: [attachment], + chatMessage: "summarize", + }); + + await handleSendChat(host); + + expect(getChatAttachmentDataUrl(attachment)).toBeNull(); + expect(getChatAttachmentPreviewUrl(attachment)).toBe("blob:brief"); + expect(JSON.stringify(host.chatMessages)).not.toContain("JVBERi0xLjQK"); + }); + + it("releases queued attachment payloads when the queued item is removed", async () => { + const revokeObjectURL = vi.fn(); + vi.stubGlobal( + "URL", + class extends URL { + static createObjectURL = vi.fn(() => "blob:queued"); + static revokeObjectURL = revokeObjectURL; + }, + ); + const file = new File(["%PDF-1.4\n"], "brief.pdf", { type: "application/pdf" }); + const attachment = registerChatAttachmentPayload({ + attachment: { + id: "queued-att", + mimeType: "application/pdf", + fileName: "brief.pdf", + sizeBytes: file.size, + }, + dataUrl: "data:application/pdf;base64,JVBERi0xLjQK", + file, + }); + const host = makeHost({ + chatQueue: [{ id: "queued", text: "later", createdAt: 1, attachments: [attachment] }], + }); + + removeQueuedMessage(host, "queued"); + + expect(host.chatQueue).toEqual([]); + expect(getChatAttachmentDataUrl(attachment)).toBeNull(); + expect(revokeObjectURL).toHaveBeenCalledWith("blob:queued"); + }); }); describe("handleAbortChat", () => { diff --git a/ui/src/ui/app-chat.ts b/ui/src/ui/app-chat.ts index 7f56e69e34d..1e8332b4cdc 100644 --- a/ui/src/ui/app-chat.ts +++ b/ui/src/ui/app-chat.ts @@ -1,6 +1,12 @@ import { setLastActiveSessionKey } from "./app-last-active-session.ts"; import { scheduleChatScroll, resetChatScroll } from "./app-scroll.ts"; import { resetToolStream } from "./app-tool-stream.ts"; +import { + cloneChatAttachmentsMetadata, + discardChatAttachmentDataUrls, + getChatAttachmentDataUrl, + releaseChatAttachmentPayloads, +} from "./chat/attachment-payload-store.ts"; import { handleChatDraftChange, handleChatInputHistoryKey, @@ -147,7 +153,7 @@ function enqueueChatMessage( id: generateUUID(), text: trimmed, createdAt: Date.now(), - attachments: hasAttachments ? attachments?.map((att) => ({ ...att })) : undefined, + attachments: hasAttachments ? cloneChatAttachmentsMetadata(attachments ?? []) : undefined, refreshSessions, localCommandArgs: localCommand?.args, localCommandName: localCommand?.name, @@ -173,7 +179,7 @@ function enqueuePendingRunMessage( text: trimmed, createdAt: Date.now(), kind: "steered", - attachments: hasAttachments ? attachments?.map((att) => ({ ...att })) : undefined, + attachments: hasAttachments ? cloneChatAttachmentsMetadata(attachments ?? []) : undefined, pendingRunId, }, ]; @@ -223,16 +229,21 @@ async function sendChatMessageNow( if (ok && opts?.refreshSessions && runId) { host.refreshSessionsAfterChat.add(runId); } + if (ok) { + discardChatAttachmentDataUrls(opts?.attachments); + } return ok; } function attachmentSubmitSignature(attachment: ChatAttachment): string { + const dataUrl = getChatAttachmentDataUrl(attachment); return JSON.stringify([ attachment.id, attachment.mimeType, attachment.fileName ?? "", - attachment.dataUrl.length, - attachment.dataUrl.slice(0, 64), + attachment.sizeBytes ?? 0, + dataUrl?.length ?? 0, + dataUrl?.slice(0, 64) ?? "", ]); } @@ -300,6 +311,7 @@ async function sendDetachedBtwMessage( host as unknown as Parameters[0], host.sessionKey, ); + releaseChatAttachmentPayloads(opts?.attachments); } return ok; } @@ -334,6 +346,7 @@ export async function steerQueuedChatMessage(host: ChatHost, id: string) { host.chatQueue = host.chatQueue.map((entry) => (entry.id === id ? item : entry)); return; } + releaseChatAttachmentPayloads(attachments); setLastActiveSessionKey( host as unknown as Parameters[0], host.sessionKey, @@ -374,14 +387,22 @@ async function flushChatQueue(host: ChatHost) { } export function removeQueuedMessage(host: ChatHost, id: string) { + const removed = host.chatQueue.filter((item) => item.id === id); host.chatQueue = host.chatQueue.filter((item) => item.id !== id); + for (const item of removed) { + releaseChatAttachmentPayloads(item.attachments); + } } export function clearPendingQueueItemsForRun(host: ChatHost, runId: string | undefined) { if (!runId) { return; } + const removed = host.chatQueue.filter((item) => item.pendingRunId === runId); host.chatQueue = host.chatQueue.filter((item) => item.pendingRunId !== runId); + for (const item of removed) { + releaseChatAttachmentPayloads(item.attachments); + } } export async function handleSendChat( diff --git a/ui/src/ui/app-render.helpers.node.test.ts b/ui/src/ui/app-render.helpers.node.test.ts index 94fc11b9087..e9280f3496c 100644 --- a/ui/src/ui/app-render.helpers.node.test.ts +++ b/ui/src/ui/app-render.helpers.node.test.ts @@ -499,7 +499,9 @@ describe("switchChatSession", () => { const state = { sessionKey: "main", chatMessage: "draft", - chatAttachments: [{ mimeType: "image/png", dataUrl: "data:image/png;base64,AAA" }], + chatAttachments: [ + { id: "att-1", mimeType: "image/png", dataUrl: "data:image/png;base64,AAA" }, + ], chatMessages: [{ role: "assistant", content: "old" }], chatToolMessages: [{ id: "tool-1" }], chatStreamSegments: [{ text: "segment", ts: 1 }], diff --git a/ui/src/ui/chat/attachment-payload-store.ts b/ui/src/ui/chat/attachment-payload-store.ts new file mode 100644 index 00000000000..9c6af1809e0 --- /dev/null +++ b/ui/src/ui/chat/attachment-payload-store.ts @@ -0,0 +1,102 @@ +import type { ChatAttachment } from "../ui-types.ts"; + +type AttachmentPayload = { + dataUrl?: string; + previewUrl?: string; +}; + +const payloads = new Map(); + +function createObjectUrl(file: File): string | undefined { + if (typeof URL === "undefined" || typeof URL.createObjectURL !== "function") { + return undefined; + } + return URL.createObjectURL(file); +} + +function revokeObjectUrl(url: string | undefined): void { + if (!url || typeof URL === "undefined" || typeof URL.revokeObjectURL !== "function") { + return; + } + URL.revokeObjectURL(url); +} + +export function registerChatAttachmentPayload(params: { + attachment: ChatAttachment; + dataUrl: string; + file: File; +}): ChatAttachment { + const previous = payloads.get(params.attachment.id); + revokeObjectUrl(previous?.previewUrl); + const objectUrl = createObjectUrl(params.file); + const previewUrl = objectUrl ?? params.attachment.previewUrl; + payloads.set(params.attachment.id, { + dataUrl: params.dataUrl, + ...(previewUrl ? { previewUrl } : {}), + }); + return { + ...params.attachment, + ...(previewUrl ? { previewUrl } : {}), + }; +} + +export function getChatAttachmentDataUrl(attachment: ChatAttachment): string | null { + return attachment.dataUrl ?? payloads.get(attachment.id)?.dataUrl ?? null; +} + +export function getChatAttachmentPreviewUrl(attachment: ChatAttachment): string | null { + return ( + attachment.previewUrl ?? payloads.get(attachment.id)?.previewUrl ?? attachment.dataUrl ?? null + ); +} + +export function cloneChatAttachmentMetadata(attachment: ChatAttachment): ChatAttachment { + const { dataUrl: _dataUrl, ...metadata } = attachment; + return metadata; +} + +export function cloneChatAttachmentsMetadata( + attachments: readonly ChatAttachment[], +): ChatAttachment[] { + return attachments.map(cloneChatAttachmentMetadata); +} + +export function releaseChatAttachmentPayload(id: string): void { + const payload = payloads.get(id); + if (!payload) { + return; + } + revokeObjectUrl(payload.previewUrl); + payloads.delete(id); +} + +export function releaseChatAttachmentPayloads(attachments: readonly ChatAttachment[] = []): void { + for (const attachment of attachments) { + releaseChatAttachmentPayload(attachment.id); + } +} + +export function discardChatAttachmentDataUrl(id: string): void { + const payload = payloads.get(id); + if (!payload) { + return; + } + if (payload.previewUrl) { + payloads.set(id, { previewUrl: payload.previewUrl }); + return; + } + payloads.delete(id); +} + +export function discardChatAttachmentDataUrls(attachments: readonly ChatAttachment[] = []): void { + for (const attachment of attachments) { + discardChatAttachmentDataUrl(attachment.id); + } +} + +export function resetChatAttachmentPayloadStoreForTest(): void { + for (const payload of payloads.values()) { + revokeObjectUrl(payload.previewUrl); + } + payloads.clear(); +} diff --git a/ui/src/ui/controllers/chat.test.ts b/ui/src/ui/controllers/chat.test.ts index e9a7c783fef..edecf302705 100644 --- a/ui/src/ui/controllers/chat.test.ts +++ b/ui/src/ui/controllers/chat.test.ts @@ -1,4 +1,8 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + registerChatAttachmentPayload, + resetChatAttachmentPayloadStoreForTest, +} from "../chat/attachment-payload-store.ts"; import { GatewayRequestError } from "../gateway.ts"; import { abortChatRun, @@ -28,6 +32,10 @@ function createState(overrides: Partial = {}): ChatState { }; } +afterEach(() => { + resetChatAttachmentPayloadStoreForTest(); +}); + function createDeferred() { let resolve!: (value: T) => void; let reject!: (reason?: unknown) => void; @@ -743,6 +751,44 @@ describe("sendChatMessage", () => { }); }); + it("serializes attachments from the side payload store without copying data URLs into chat state", async () => { + const request = vi.fn().mockResolvedValue({ runId: "run-1", status: "started" }); + const state = createState({ + connected: true, + client: { request } as unknown as ChatState["client"], + }); + const pdfBytes = "%PDF-1.4\n"; + const file = new File([pdfBytes], "brief.pdf", { type: "application/pdf" }); + const attachment = registerChatAttachmentPayload({ + attachment: { + id: "att-side-store", + mimeType: "application/pdf", + fileName: "brief.pdf", + sizeBytes: file.size, + }, + dataUrl: `data:application/pdf;base64,${Buffer.from(pdfBytes).toString("base64")}`, + file, + }); + + const result = await sendChatMessage(state, "summarize", [attachment]); + + expect(result).toEqual(expect.any(String)); + expect(request).toHaveBeenCalledWith( + "chat.send", + expect.objectContaining({ + attachments: [ + expect.objectContaining({ + type: "file", + content: Buffer.from(pdfBytes).toString("base64"), + }), + ], + }), + ); + expect(JSON.stringify(state.chatMessages)).not.toContain( + Buffer.from(pdfBytes).toString("base64"), + ); + }); + it("formats structured non-auth connect failures for chat send", async () => { const request = vi.fn().mockRejectedValue( new GatewayRequestError({ diff --git a/ui/src/ui/controllers/chat.ts b/ui/src/ui/controllers/chat.ts index 3e0daf83b3c..3f3f613f4a3 100644 --- a/ui/src/ui/controllers/chat.ts +++ b/ui/src/ui/controllers/chat.ts @@ -1,4 +1,8 @@ import { resetToolStream } from "../app-tool-stream.ts"; +import { + getChatAttachmentDataUrl, + getChatAttachmentPreviewUrl, +} from "../chat/attachment-payload-store.ts"; import { extractText } from "../chat/message-extract.ts"; import { formatConnectError } from "../connect-error.ts"; import { GatewayRequestError, type GatewayBrowserClient } from "../gateway.ts"; @@ -462,7 +466,8 @@ function buildApiAttachments(attachments?: ChatAttachment[]) { return hasAttachments ? attachments .map((att) => { - const parsed = dataUrlToBase64(att.dataUrl); + const dataUrl = getChatAttachmentDataUrl(att); + const parsed = dataUrl ? dataUrlToBase64(dataUrl) : null; if (!parsed) { return null; } @@ -562,6 +567,7 @@ export async function sendChatMessage( const contentBlocks: Array<{ type: string; text?: string; + url?: string; source?: unknown; attachment?: { url: string; @@ -576,17 +582,22 @@ export async function sendChatMessage( // Add image previews to the message for display if (hasAttachments) { for (const att of attachments) { + const previewUrl = getChatAttachmentPreviewUrl(att); + if (!previewUrl) { + continue; + } if (att.mimeType.startsWith("image/")) { contentBlocks.push({ type: "image", - source: { type: "base64", media_type: att.mimeType, data: att.dataUrl }, + url: previewUrl, + source: { type: "url", url: previewUrl }, }); continue; } contentBlocks.push({ type: "attachment", attachment: { - url: att.dataUrl, + url: previewUrl, kind: att.mimeType.startsWith("audio/") ? "audio" : "document", label: att.fileName?.trim() || "Attached file", mimeType: att.mimeType, diff --git a/ui/src/ui/ui-types.ts b/ui/src/ui/ui-types.ts index c13f0123b0e..51206f0f89b 100644 --- a/ui/src/ui/ui-types.ts +++ b/ui/src/ui/ui-types.ts @@ -1,8 +1,10 @@ export type ChatAttachment = { id: string; - dataUrl: string; + dataUrl?: string; + previewUrl?: string; mimeType: string; fileName?: string; + sizeBytes?: number; }; export type ChatQueueItem = { diff --git a/ui/src/ui/views/chat.test.ts b/ui/src/ui/views/chat.test.ts index e9939a4d099..0abf3d54a1a 100644 --- a/ui/src/ui/views/chat.test.ts +++ b/ui/src/ui/views/chat.test.ts @@ -8,6 +8,10 @@ import { createSessionsListResult, DEFAULT_CHAT_MODEL_CATALOG, } from "../chat-model.test-helpers.ts"; +import { + getChatAttachmentDataUrl, + resetChatAttachmentPayloadStoreForTest, +} from "../chat/attachment-payload-store.ts"; import { renderChatQueue } from "../chat/chat-queue.ts"; import { buildRawSidebarContent } from "../chat/chat-sidebar-raw.ts"; import { renderWelcomeState } from "../chat/chat-welcome.ts"; @@ -388,6 +392,7 @@ function renderChatView(overrides: Partial[0]> = { afterEach(() => { loadSessionsMock.mockClear(); refreshVisibleToolsEffectiveForCurrentSessionMock.mockClear(); + resetChatAttachmentPayloadStoreForTest(); vi.unstubAllGlobals(); }); @@ -464,14 +469,15 @@ describe("chat attachment picker", () => { await vi.waitFor(() => { expect(onAttachmentsChange).toHaveBeenCalledWith([ expect.objectContaining({ - dataUrl: expect.stringMatching(/^data:application\/pdf;base64,/), fileName: "brief.pdf", mimeType: "application/pdf", + sizeBytes: file.size, }), ]); }); const nextAttachments = onAttachmentsChange.mock.calls[0]?.[0] ?? []; + expect(getChatAttachmentDataUrl(nextAttachments[0])).toMatch(/^data:application\/pdf;base64,/); const preview = renderChatView({ attachments: nextAttachments }); expect(preview.querySelector(".chat-attachment-thumb--file")).not.toBeNull(); expect(preview.textContent).toContain("brief.pdf"); diff --git a/ui/src/ui/views/chat.ts b/ui/src/ui/views/chat.ts index b98e61e1903..eecc94fc67f 100644 --- a/ui/src/ui/views/chat.ts +++ b/ui/src/ui/views/chat.ts @@ -2,6 +2,11 @@ import { html, nothing, type TemplateResult } from "lit"; import { ref } from "lit/directives/ref.js"; import { repeat } from "lit/directives/repeat.js"; import type { CompactionStatus, FallbackStatus } from "../app-tool-stream.ts"; +import { + getChatAttachmentPreviewUrl, + registerChatAttachmentPayload, + releaseChatAttachmentPayload, +} from "../chat/attachment-payload-store.ts"; import { CHAT_ATTACHMENT_ACCEPT, isSupportedChatAttachmentFile, @@ -205,12 +210,13 @@ function generateAttachmentId(): string { } function chatAttachmentFromFile(file: File, dataUrl: string): ChatAttachment { - return { + const attachment = { id: generateAttachmentId(), - dataUrl, mimeType: file.type || "application/octet-stream", fileName: file.name || undefined, + sizeBytes: file.size, }; + return registerChatAttachmentPayload({ attachment, dataUrl, file }); } function isImageAttachment(att: ChatAttachment): boolean { @@ -318,8 +324,8 @@ function renderAttachmentPreview(props: ChatProps): TemplateResult | typeof noth .filter(Boolean) .join(" ")} > - ${isImageAttachment(att) - ? html`Attachment preview` + ${isImageAttachment(att) && getChatAttachmentPreviewUrl(att) + ? html`Attachment preview` : html`
${icons.paperclip} @@ -334,6 +340,7 @@ function renderAttachmentPreview(props: ChatProps): TemplateResult | typeof noth aria-label="Remove attachment" @click=${() => { const next = (props.attachments ?? []).filter((a) => a.id !== att.id); + releaseChatAttachmentPayload(att.id); props.onAttachmentsChange?.(next); }} >