From a9d77b3eb09d5e59d551921632775b70dec2c0ba Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 4 May 2026 06:48:59 +0100 Subject: [PATCH] fix: scope Control UI assistant media tickets --- CHANGELOG.md | 1 + docs/web/control-ui.md | 10 ++ .../control-ui-assistant-media.e2e.test.ts | 65 ++++++++ src/gateway/control-ui.http.test.ts | 98 +++++++++++- src/gateway/control-ui.ts | 86 ++++++++++- ui/src/ui/chat/grouped-render.test.ts | 142 +++++++++++++++--- ui/src/ui/chat/grouped-render.ts | 124 ++++++++++++--- 7 files changed, 475 insertions(+), 51 deletions(-) create mode 100644 src/gateway/control-ui-assistant-media.e2e.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b5478dd9f0..24882dc0890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ Docs: https://docs.openclaw.ai - Google Meet: log the concrete agent-mode TTS provider, model, voice, output format, and sample rate after speech synthesis, so Meet logs show which voice backend spoke each reply. - Voice Call: mark realtime calls completed when the realtime provider closes normally, so Twilio/OpenAI/Google realtime stop events do not leave active call records behind. Thanks @vincentkoc. - Gateway/update: keep the shutdown close path behind a stable runtime chunk and ship compatibility aliases for recent `server-close-*` hashes, so manual npm package replacement cannot leave an already-running Gateway unable to shut down cleanly. Fixes #77087. Thanks @westlife219. +- Control UI/media: mint short-lived scoped tickets for assistant media fetches and render ticketed URLs instead of exposing long-lived auth tokens in chat image URLs. Fixes #70830 and #77097. Thanks @hclsys. - Exec approvals: treat POSIX `exec` as a command carrier for inline eval, shell-wrapper, and eval/source detection, so approval explanations and command-risk checks do not miss payloads hidden behind `exec`. Thanks @vincentkoc. - Google Meet: log the resolved audio provider model when starting Chrome and paired-node Meet talk-back bridges, so agent-mode joins show the STT model and bidi joins show the realtime voice model. - Diagnostics: handle missing session-tail files in cron recovery context without tripping extension test typecheck. Thanks @vincentkoc. diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index c3d033d6b55..a896f7dcfc8 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -384,6 +384,16 @@ When gateway auth is configured, the Control UI avatar endpoint requires the sam If you disable gateway auth (not recommended on shared hosts), the avatar route also becomes unauthenticated, in line with the rest of the gateway. +## Assistant media route auth + +When gateway auth is configured, assistant local-media previews use a two-step route: + +- `GET /__openclaw__/assistant-media?meta=1&source=` requires the normal Control UI operator auth. The browser sends the gateway token as a bearer header when checking availability. +- Successful metadata responses include a short-lived `mediaTicket` scoped to that exact source path. +- Browser-rendered image, audio, video, and document URLs use `mediaTicket=` instead of the active gateway token or password. The ticket expires quickly and cannot authorize a different source. + +This keeps normal media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs. + ## Building the UI The Gateway serves static files from `dist/control-ui`. Build them with: diff --git a/src/gateway/control-ui-assistant-media.e2e.test.ts b/src/gateway/control-ui-assistant-media.e2e.test.ts new file mode 100644 index 00000000000..dad3c2a84a9 --- /dev/null +++ b/src/gateway/control-ui-assistant-media.e2e.test.ts @@ -0,0 +1,65 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { installGatewayTestHooks, testState, withGatewayServer } from "./test-helpers.js"; + +installGatewayTestHooks({ scope: "suite" }); + +const CONTROL_UI_E2E_TOKEN = "test-gateway-token-1234567890"; + +describe("Control UI assistant media e2e", () => { + test("serves local assistant media through scoped tickets over the gateway HTTP route", async () => { + const stateDir = process.env.OPENCLAW_STATE_DIR; + if (!stateDir) { + throw new Error("OPENCLAW_STATE_DIR is required for gateway e2e media fixtures"); + } + testState.gatewayAuth = { mode: "token", token: CONTROL_UI_E2E_TOKEN }; + + const mediaDir = path.join(stateDir, "media", "control-ui-assistant-media-e2e"); + await fs.mkdir(mediaDir, { recursive: true }); + const filePath = path.join(mediaDir, "ticketed-preview.txt"); + await fs.writeFile(filePath, "ticketed control ui media\n", "utf8"); + + await withGatewayServer( + async ({ port }) => { + const route = `http://127.0.0.1:${port}/__openclaw__/assistant-media`; + const sourceParam = encodeURIComponent(filePath); + + const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, { + headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` }, + }); + expect(metadata.status).toBe(200); + const payload = (await metadata.json()) as { + available?: boolean; + mediaTicket?: string; + mediaTicketExpiresAt?: string; + }; + expect(payload.available).toBe(true); + expect(payload.mediaTicket).toMatch(/^v1\./); + expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN(); + + const withoutTicket = await fetch(`${route}?source=${sourceParam}`); + expect(withoutTicket.status).toBe(401); + + const ticketed = await fetch( + `${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`, + ); + expect(ticketed.status).toBe(200); + expect(await ticketed.text()).toBe("ticketed control ui media\n"); + + const otherFilePath = path.join(mediaDir, "other-preview.txt"); + await fs.writeFile(otherFilePath, "other media\n", "utf8"); + const wrongSource = await fetch( + `${route}?source=${encodeURIComponent(otherFilePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`, + ); + expect(wrongSource.status).toBe(401); + }, + { + serverOptions: { + auth: { mode: "token", token: CONTROL_UI_E2E_TOKEN }, + controlUiEnabled: true, + }, + }, + ); + }); +}); diff --git a/src/gateway/control-ui.http.test.ts b/src/gateway/control-ui.http.test.ts index 26ec371e694..3aa46588aab 100644 --- a/src/gateway/control-ui.http.test.ts +++ b/src/gateway/control-ui.http.test.ts @@ -368,7 +368,14 @@ describe("handleControlUiHttpRequest", () => { }); expect(handled).toBe(true); expect(res.statusCode).toBe(200); - expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true }); + const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as { + available?: boolean; + mediaTicket?: string; + mediaTicketExpiresAt?: string; + }; + expect(payload).toMatchObject({ available: true }); + expect(payload.mediaTicket).toMatch(/^v1\./); + expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN(); } finally { await fs.rm(filePath, { force: true }); } @@ -403,7 +410,94 @@ describe("handleControlUiHttpRequest", () => { }); expect(handled).toBe(true); expect(res.statusCode).toBe(200); - expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true }); + const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as { + available?: boolean; + mediaTicket?: string; + mediaTicketExpiresAt?: string; + }; + expect(payload).toMatchObject({ available: true }); + expect(payload.mediaTicket).toMatch(/^v1\./); + expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN(); + }, + }); + }); + + it("serves assistant local media with a scoped media ticket after metadata auth", async () => { + await withAllowedAssistantMediaRoot({ + prefix: "ui-media-ticket-", + fn: async (tmpRoot) => { + const filePath = path.join(tmpRoot, "photo.png"); + await fs.writeFile(filePath, Buffer.from("not-a-real-png")); + const meta = await runAssistantMediaRequest({ + url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`, + method: "GET", + auth: { mode: "token", token: "test-token", allowTailscale: false }, + headers: { + authorization: "Bearer test-token", + }, + }); + const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as { + mediaTicket?: string; + }; + expect(meta.handled).toBe(true); + expect(meta.res.statusCode).toBe(200); + expect(payload.mediaTicket).toMatch(/^v1\./); + + const media = await runAssistantMediaRequest({ + url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`, + method: "GET", + auth: { mode: "token", token: "test-token", allowTailscale: false }, + }); + expect(media.handled).toBe(true); + expect(media.res.statusCode).toBe(200); + }, + }); + }); + + it("does not refresh assistant media tickets without operator auth", async () => { + await withAllowedAssistantMediaRoot({ + prefix: "ui-media-ticket-refresh-", + fn: async (tmpRoot) => { + const filePath = path.join(tmpRoot, "photo.png"); + await fs.writeFile(filePath, Buffer.from("not-a-real-png")); + const meta = await runAssistantMediaRequest({ + url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`, + method: "GET", + auth: { mode: "token", token: "test-token", allowTailscale: false }, + headers: { + authorization: "Bearer test-token", + }, + }); + const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as { + mediaTicket?: string; + }; + + const refresh = await runAssistantMediaRequest({ + url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`, + method: "GET", + auth: { mode: "token", token: "test-token", allowTailscale: false }, + }); + expect(refresh.handled).toBe(true); + expect(refresh.res.statusCode).toBe(401); + expect(String(refresh.end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized"); + }, + }); + }); + + it("rejects assistant local media with an invalid scoped media ticket", async () => { + await withAllowedAssistantMediaRoot({ + prefix: "ui-media-ticket-invalid-", + fn: async (tmpRoot) => { + const filePath = path.join(tmpRoot, "photo.png"); + await fs.writeFile(filePath, Buffer.from("not-a-real-png")); + const { res, handled, end } = await runAssistantMediaRequest({ + url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=v1.invalid.invalid`, + method: "GET", + auth: { mode: "token", token: "test-token", allowTailscale: false }, + }); + expect(handled).toBe(true); + expect(res.statusCode).toBe(401); + expect(String(end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized"); }, }); }); diff --git a/src/gateway/control-ui.ts b/src/gateway/control-ui.ts index bf7a7e8c334..d74a830cce7 100644 --- a/src/gateway/control-ui.ts +++ b/src/gateway/control-ui.ts @@ -1,3 +1,4 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; import fs from "node:fs"; import type { IncomingMessage, ServerResponse } from "node:http"; import path from "node:path"; @@ -56,10 +57,13 @@ import { resolveRequestClientIp } from "./net.js"; const ROOT_PREFIX = "/"; const CONTROL_UI_ASSISTANT_MEDIA_PREFIX = "/__openclaw__/assistant-media"; +const CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE = "assistant-media"; +const CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS = 5 * 60 * 1000; const CONTROL_UI_ASSETS_MISSING_MESSAGE = "Control UI assets not found. Build them with `pnpm ui:build` (auto-installs UI deps), or run `pnpm ui:dev` during development."; const CONTROL_UI_OPERATOR_READ_SCOPE = "operator.read"; const CONTROL_UI_OPERATOR_ROLE = "operator"; +const controlUiAssistantMediaTicketSecret = randomBytes(32); export type ControlUiRequestOptions = { basePath?: string; @@ -378,6 +382,64 @@ type AssistantMediaAvailability = | { available: true } | { available: false; reason: string; code: string }; +type AssistantMediaTicketPayload = { + scope: typeof CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE; + source: string; + exp: number; +}; + +function signAssistantMediaTicketPayload(encodedPayload: string): string { + return createHmac("sha256", controlUiAssistantMediaTicketSecret) + .update(encodedPayload) + .digest("base64url"); +} + +function createAssistantMediaTicket(source: string, nowMs = Date.now()) { + const exp = nowMs + CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS; + const payload: AssistantMediaTicketPayload = { + scope: CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE, + source, + exp, + }; + const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); + const sig = signAssistantMediaTicketPayload(encodedPayload); + return { + mediaTicket: `v1.${encodedPayload}.${sig}`, + mediaTicketExpiresAt: new Date(exp).toISOString(), + }; +} + +function verifyAssistantMediaTicket(ticket: string | null, source: string, nowMs = Date.now()) { + const parts = ticket?.split("."); + if (!parts || parts.length !== 3 || parts[0] !== "v1") { + return false; + } + const [, encodedPayload, sig] = parts; + if (!encodedPayload || !sig) { + return false; + } + const expectedSig = signAssistantMediaTicketPayload(encodedPayload); + const sigBuffer = Buffer.from(sig, "base64url"); + const expectedBuffer = Buffer.from(expectedSig, "base64url"); + if (sigBuffer.length !== expectedBuffer.length || !timingSafeEqual(sigBuffer, expectedBuffer)) { + return false; + } + try { + const payload = JSON.parse( + Buffer.from(encodedPayload, "base64url").toString("utf8"), + ) as Partial; + return ( + payload.scope === CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE && + payload.source === source && + typeof payload.exp === "number" && + Number.isFinite(payload.exp) && + payload.exp >= nowMs + ); + } catch { + return false; + } +} + function classifyAssistantMediaError(err: unknown): AssistantMediaAvailability { if (err instanceof SafeOpenError) { switch (err.code) { @@ -461,7 +523,16 @@ export async function handleControlUiAssistantMediaRequest( } applyControlUiSecurityHeaders(res); + const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? ""); + if (!source) { + respondControlUiNotFound(res); + return true; + } + const isMetaRequest = url.searchParams.get("meta") === "1"; + const hasValidMediaTicket = + !isMetaRequest && verifyAssistantMediaTicket(url.searchParams.get("mediaTicket"), source); if ( + !hasValidMediaTicket && !(await authorizeControlUiReadRequest(req, res, { auth: opts?.auth, trustedProxies: opts?.trustedProxies, @@ -472,18 +543,19 @@ export async function handleControlUiAssistantMediaRequest( ) { return true; } - const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? ""); - if (!source) { - respondControlUiNotFound(res); - return true; - } const localRoots = opts?.config ? getAgentScopedMediaLocalRoots(opts.config, opts.agentId) : getDefaultLocalRoots(); - if (url.searchParams.get("meta") === "1") { + if (isMetaRequest) { const availability = await resolveAssistantMediaAvailability(source, localRoots); - sendJson(res, 200, availability); + sendJson( + res, + 200, + availability.available + ? { ...availability, ...createAssistantMediaTicket(source) } + : availability, + ); return true; } diff --git a/ui/src/ui/chat/grouped-render.test.ts b/ui/src/ui/chat/grouped-render.test.ts index 4dd75f0060b..0a607c9aabf 100644 --- a/ui/src/ui/chat/grouped-render.test.ts +++ b/ui/src/ui/chat/grouped-render.test.ts @@ -347,6 +347,14 @@ async function flushAssistantAttachmentAvailabilityChecks() { } } +function mediaTicketPayload(mediaTicket: string, ttlMs = 5 * 60 * 1000) { + return { + available: true, + mediaTicket, + mediaTicketExpiresAt: new Date(Date.now() + ttlMs).toISOString(), + }; +} + afterEach(() => { document.querySelectorAll("[data-delete-confirm-fixture]").forEach((element) => { element.remove(); @@ -808,15 +816,30 @@ describe("grouped chat rendering", () => { expect(container.textContent).not.toContain("MEDIA:https://example.com/photo.png"); }); - it("renders allowed transcript and content image variants", () => { + it("renders allowed transcript and content image variants", async () => { + resetAssistantAttachmentAvailabilityCacheForTest(); + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + expect(url).toContain("meta=1"); + const headers = init?.headers as Headers; + expect(headers.get("Authorization")).toBe("Bearer session-token"); + return { + ok: true, + json: async () => mediaTicketPayload("ticket-user"), + }; + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + const renderUserMedia = (message: unknown) => { const container = document.createElement("div"); - renderGroupedMessage(container, message, "user", { - showToolCalls: false, - basePath: "/openclaw", - assistantAttachmentAuthToken: "session-token", - localMediaPreviewRoots: ["/tmp/openclaw"], - }); + const renderMessage = () => + renderGroupedMessage(container, message, "user", { + showToolCalls: false, + basePath: "/openclaw", + assistantAttachmentAuthToken: "session-token", + localMediaPreviewRoots: ["/tmp/openclaw"], + onRequestUpdate: renderMessage, + }); + renderMessage(); return container; }; @@ -827,10 +850,11 @@ describe("grouped chat rendering", () => { MediaPath: "/tmp/openclaw/user-upload.png", timestamp: Date.now(), }); + await flushAssistantAttachmentAvailabilityChecks(); expect( container.querySelector(".chat-message-image")?.getAttribute("src"), ).toBe( - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user", ); container = renderUserMedia({ @@ -841,10 +865,11 @@ describe("grouped chat rendering", () => { MediaType: "application/octet-stream", timestamp: Date.now(), }); + await flushAssistantAttachmentAvailabilityChecks(); expect( container.querySelector(".chat-message-image")?.getAttribute("src"), ).toBe( - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user", ); container = renderUserMedia({ @@ -855,13 +880,14 @@ describe("grouped chat rendering", () => { MediaTypes: ["image/png", "application/octet-stream"], timestamp: Date.now(), }); + await flushAssistantAttachmentAvailabilityChecks(); expect( [...container.querySelectorAll(".chat-message-image")].map((image) => image.getAttribute("src"), ), ).toEqual([ - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&token=session-token", - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&token=session-token", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&mediaTicket=ticket-user", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&mediaTicket=ticket-user", ]); const assistantContainer = document.createElement("div"); @@ -905,6 +931,7 @@ describe("grouped chat rendering", () => { ); expect(documentLink?.textContent).toContain("user-upload.pdf"); expect(documentLink?.getAttribute("href")).toBe("/__openclaw__/media/user-upload.pdf"); + vi.unstubAllGlobals(); }); it("fetches managed chat images with auth and renders blob previews", async () => { @@ -1065,11 +1092,13 @@ describe("grouped chat rendering", () => { it("renders verified local assistant attachments through the Control UI media route", async () => { resetAssistantAttachmentAvailabilityCacheForTest(); - const fetchMock = vi.fn(async (url: string) => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (url.includes("meta=1")) { + const headers = init?.headers as Headers; + expect(headers.get("Authorization")).toBe("Bearer session-token"); return { ok: true, - json: async () => ({ available: true }), + json: async () => mediaTicketPayload("ticket-local"), }; } throw new Error(`Unexpected fetch: ${url}`); @@ -1100,7 +1129,7 @@ describe("grouped chat rendering", () => { await flushAssistantAttachmentAvailabilityChecks(); expect(fetchMock).toHaveBeenCalledWith( - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token&meta=1", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1", expect.objectContaining({ credentials: "same-origin", method: "GET" }), ); @@ -1109,24 +1138,84 @@ describe("grouped chat rendering", () => { ".chat-assistant-attachment-card__link", ); expect(image?.getAttribute("src")).toBe( - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-local", ); expect(docLink?.getAttribute("href")).toBe( - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&token=session-token", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&mediaTicket=ticket-local", ); expect(container.textContent).not.toContain("test image.png"); vi.unstubAllGlobals(); }); + it("refreshes cached local assistant media tickets before they expire without another render", async () => { + resetAssistantAttachmentAvailabilityCacheForTest(); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-04-30T00:00:00Z")); + const fetchMock = vi + .fn< + (url: string, init?: RequestInit) => Promise<{ ok: true; json: () => Promise }> + >() + .mockResolvedValueOnce({ + ok: true, + json: async () => mediaTicketPayload("ticket-old", 31_000), + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => mediaTicketPayload("ticket-new"), + }); + vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); + const container = document.createElement("div"); + const renderMessage = () => + renderAssistantMessage( + container, + { + id: "assistant-local-media-ticket-refresh", + role: "assistant", + content: "Local image\nMEDIA:/tmp/openclaw/test image.png", + timestamp: Date.now(), + }, + { + showToolCalls: false, + basePath: "/openclaw", + assistantAttachmentAuthToken: "session-token", + localMediaPreviewRoots: ["/tmp/openclaw"], + onRequestUpdate: renderMessage, + }, + ); + + renderMessage(); + await flushAssistantAttachmentAvailabilityChecks(); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect( + container.querySelector(".chat-message-image")?.getAttribute("src"), + ).toBe( + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-old", + ); + + vi.advanceTimersByTime(1_001); + await flushAssistantAttachmentAvailabilityChecks(); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect( + container.querySelector(".chat-message-image")?.getAttribute("src"), + ).toBe( + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-new", + ); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + it("rechecks local assistant attachment availability when the auth token changes", async () => { resetAssistantAttachmentAvailabilityCacheForTest(); - const fetchMock = vi.fn(async (url: string) => { + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { if (!url.includes("meta=1")) { throw new Error(`Unexpected fetch: ${url}`); } + const headers = init?.headers as Headers; + const authorized = headers.get("Authorization") === "Bearer fresh-token"; return { ok: true, - json: async () => ({ available: url.includes("token=fresh-token") }), + json: async () => (authorized ? mediaTicketPayload("ticket-fresh") : { available: false }), }; }); vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); @@ -1165,10 +1254,15 @@ describe("grouped chat rendering", () => { ); expect(fetchMock).toHaveBeenNthCalledWith( 2, - "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=fresh-token&meta=1", + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1", expect.objectContaining({ credentials: "same-origin", method: "GET" }), ); expect(container.querySelector(".chat-message-image")).not.toBeNull(); + expect( + container.querySelector(".chat-message-image")?.getAttribute("src"), + ).toBe( + "/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-fresh", + ); expect(container.textContent).not.toContain("Unavailable"); vi.unstubAllGlobals(); }); @@ -1235,7 +1329,7 @@ describe("grouped chat rendering", () => { } return { ok: true, - json: async () => ({ available: true }), + json: async () => mediaTicketPayload("ticket-platform"), }; }); vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); @@ -1321,7 +1415,7 @@ describe("grouped chat rendering", () => { }) .mockResolvedValueOnce({ ok: true, - json: async () => ({ available: true }), + json: async () => mediaTicketPayload("ticket-retry"), }); vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch); const container = document.createElement("div"); @@ -1344,15 +1438,13 @@ describe("grouped chat rendering", () => { ); renderMessage(); - await vi.runAllTimersAsync(); - await Promise.resolve(); + await flushAssistantAttachmentAvailabilityChecks(); expect(fetchMock).toHaveBeenCalledTimes(1); expect(container.textContent).toContain("Unavailable"); vi.advanceTimersByTime(5_001); renderMessage(); - await vi.runAllTimersAsync(); - await Promise.resolve(); + await flushAssistantAttachmentAvailabilityChecks(); expect(fetchMock).toHaveBeenCalledTimes(2); expect(container.querySelector(".chat-message-image")).not.toBeNull(); diff --git a/ui/src/ui/chat/grouped-render.ts b/ui/src/ui/chat/grouped-render.ts index 53fe34d903a..7d565841f6d 100644 --- a/ui/src/ui/chat/grouped-render.ts +++ b/ui/src/ui/chat/grouped-render.ts @@ -32,11 +32,13 @@ import { type AssistantAttachmentAvailability = | { status: "checking" } - | { status: "available" } + | { status: "available"; mediaTicket?: string; mediaTicketExpiresAt?: number } | { status: "unavailable"; reason: string; checkedAt: number }; const assistantAttachmentAvailabilityCache = new Map(); +const assistantAttachmentRefreshTimers = new Map>(); const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000; +const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000; export type ChatTimestampDisplay = { label: string; @@ -87,6 +89,10 @@ function renderChatTimestamp(timestamp: number) { export function resetAssistantAttachmentAvailabilityCacheForTest() { assistantAttachmentAvailabilityCache.clear(); + for (const timer of assistantAttachmentRefreshTimers.values()) { + clearTimeout(timer); + } + assistantAttachmentRefreshTimers.clear(); for (const blobUrl of managedImageBlobUrlResolvedCache.values()) { URL.revokeObjectURL(blobUrl); } @@ -107,6 +113,7 @@ type ImageRenderOptions = { localMediaPreviewRoots?: readonly string[]; basePath?: string; authToken?: string | null; + onRequestUpdate?: () => void; }; type RenderableImageBlock = ImageBlock & { @@ -718,8 +725,20 @@ function resolveRenderableMessageImages( if (isLocalImage && !canProxyLocalImage) { return []; } + const availability = canProxyLocalImage + ? resolveAssistantAttachmentAvailability( + img.url, + opts?.localMediaPreviewRoots ?? [], + opts?.basePath, + opts?.authToken, + opts?.onRequestUpdate, + ) + : { status: "available" as const }; + if (availability.status !== "available") { + return []; + } const displayUrl = canProxyLocalImage - ? buildAssistantAttachmentUrl(img.url, opts?.basePath, opts?.authToken) + ? buildAssistantAttachmentUrl(img.url, opts?.basePath, availability.mediaTicket) : img.url; return [{ ...img, displayUrl }]; }); @@ -871,7 +890,7 @@ function isLocalAttachmentPreviewAllowed( function buildAssistantAttachmentUrl( source: string, basePath?: string, - authToken?: string | null, + mediaTicket?: string | null, ): string { if (!isLocalAssistantAttachmentSource(source)) { return source; @@ -879,9 +898,9 @@ function buildAssistantAttachmentUrl( const normalizedBasePath = basePath && basePath !== "/" ? (basePath.endsWith("/") ? basePath.slice(0, -1) : basePath) : ""; const params = new URLSearchParams({ source }); - const normalizedToken = authToken?.trim(); - if (normalizedToken) { - params.set("token", normalizedToken); + const normalizedMediaTicket = mediaTicket?.trim(); + if (normalizedMediaTicket) { + params.set("mediaTicket", normalizedMediaTicket); } return `${normalizedBasePath}/__openclaw__/assistant-media?${params.toString()}`; } @@ -974,15 +993,51 @@ async function resolveManagedOutgoingImageBlobUrl( return pending; } -function buildAssistantAttachmentMetaUrl( - source: string, - basePath?: string, - authToken?: string | null, -): string { - const attachmentUrl = buildAssistantAttachmentUrl(source, basePath, authToken); +function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string { + const attachmentUrl = buildAssistantAttachmentUrl(source, basePath); return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`; } +function clearAssistantAttachmentRefreshTimer(cacheKey: string) { + const timer = assistantAttachmentRefreshTimers.get(cacheKey); + if (timer) { + clearTimeout(timer); + assistantAttachmentRefreshTimers.delete(cacheKey); + } +} + +function scheduleAssistantAttachmentRefresh( + cacheKey: string, + availability: AssistantAttachmentAvailability, + onRequestUpdate: (() => void) | undefined, +) { + clearAssistantAttachmentRefreshTimer(cacheKey); + if ( + availability.status !== "available" || + !availability.mediaTicket || + !availability.mediaTicketExpiresAt || + !onRequestUpdate + ) { + return; + } + const refreshInMs = Math.max( + 0, + availability.mediaTicketExpiresAt - + Date.now() - + ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS, + ); + const timer = setTimeout(() => { + assistantAttachmentRefreshTimers.delete(cacheKey); + const cached = assistantAttachmentAvailabilityCache.get(cacheKey); + if (cached?.status !== "available" || cached.mediaTicket !== availability.mediaTicket) { + return; + } + assistantAttachmentAvailabilityCache.delete(cacheKey); + onRequestUpdate(); + }, refreshInMs); + assistantAttachmentRefreshTimers.set(cacheKey, timer); +} + function resolveAssistantAttachmentAvailability( source: string, localMediaPreviewRoots: readonly string[], @@ -1000,30 +1055,63 @@ function resolveAssistantAttachmentAvailability( const cacheKey = `${basePath ?? ""}::${normalizedAuthToken}::${source}`; const cached = assistantAttachmentAvailabilityCache.get(cacheKey); if (cached) { + const now = Date.now(); if ( cached.status === "unavailable" && - Date.now() - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS + now - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS + ) { + assistantAttachmentAvailabilityCache.delete(cacheKey); + } else if ( + cached.status === "available" && + cached.mediaTicket && + (!cached.mediaTicketExpiresAt || + cached.mediaTicketExpiresAt - now <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS) ) { assistantAttachmentAvailabilityCache.delete(cacheKey); } else { + scheduleAssistantAttachmentRefresh(cacheKey, cached, onRequestUpdate); return cached; } } + clearAssistantAttachmentRefreshTimer(cacheKey); assistantAttachmentAvailabilityCache.set(cacheKey, { status: "checking" }); if (typeof fetch === "function") { - void fetch(buildAssistantAttachmentMetaUrl(source, basePath, authToken), { + const headers = new Headers({ Accept: "application/json" }); + if (normalizedAuthToken) { + headers.set("Authorization", `Bearer ${normalizedAuthToken}`); + } + void fetch(buildAssistantAttachmentMetaUrl(source, basePath), { method: "GET", - headers: { Accept: "application/json" }, + headers, credentials: "same-origin", }) .then(async (res) => { const payload = (await res.json().catch(() => null)) as { available?: boolean; + mediaTicket?: string; + mediaTicketExpiresAt?: string; reason?: string; } | null; if (payload?.available === true) { - assistantAttachmentAvailabilityCache.set(cacheKey, { status: "available" }); + const mediaTicket = payload.mediaTicket?.trim(); + const mediaTicketExpiresAt = Date.parse(payload.mediaTicketExpiresAt ?? ""); + if (mediaTicket && !Number.isFinite(mediaTicketExpiresAt)) { + clearAssistantAttachmentRefreshTimer(cacheKey); + assistantAttachmentAvailabilityCache.set(cacheKey, { + status: "unavailable", + reason: "Attachment unavailable", + checkedAt: Date.now(), + }); + return; + } + const availability: AssistantAttachmentAvailability = { + status: "available", + ...(mediaTicket ? { mediaTicket, mediaTicketExpiresAt } : {}), + }; + assistantAttachmentAvailabilityCache.set(cacheKey, availability); + scheduleAssistantAttachmentRefresh(cacheKey, availability, onRequestUpdate); } else { + clearAssistantAttachmentRefreshTimer(cacheKey); assistantAttachmentAvailabilityCache.set(cacheKey, { status: "unavailable", reason: payload?.reason?.trim() || "Attachment unavailable", @@ -1032,6 +1120,7 @@ function resolveAssistantAttachmentAvailability( } }) .catch(() => { + clearAssistantAttachmentRefreshTimer(cacheKey); assistantAttachmentAvailabilityCache.set(cacheKey, { status: "unavailable", reason: "Attachment unavailable", @@ -1097,7 +1186,7 @@ function renderAssistantAttachments( ); const attachmentUrl = availability.status === "available" - ? buildAssistantAttachmentUrl(attachment.url, basePath, authToken) + ? buildAssistantAttachmentUrl(attachment.url, basePath, availability.mediaTicket) : null; if (attachment.kind === "image") { if (!attachmentUrl) { @@ -1315,6 +1404,7 @@ function renderGroupedMessage( localMediaPreviewRoots: opts.localMediaPreviewRoots ?? [], basePath: opts.basePath, authToken: opts.assistantAttachmentAuthToken, + onRequestUpdate: opts.onRequestUpdate, }; const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions); const hasImages = images.length > 0;