fix: scope Control UI assistant media tickets

This commit is contained in:
Peter Steinberger
2026-05-04 06:48:59 +01:00
parent bc0b54e844
commit a9d77b3eb0
7 changed files with 475 additions and 51 deletions

View File

@@ -57,6 +57,7 @@ Docs: https://docs.openclaw.ai
- Google Meet: log the concrete agent-mode TTS provider, model, voice, output format, and sample rate after speech synthesis, so Meet logs show which voice backend spoke each reply.
- Voice Call: mark realtime calls completed when the realtime provider closes normally, so Twilio/OpenAI/Google realtime stop events do not leave active call records behind. Thanks @vincentkoc.
- Gateway/update: keep the shutdown close path behind a stable runtime chunk and ship compatibility aliases for recent `server-close-*` hashes, so manual npm package replacement cannot leave an already-running Gateway unable to shut down cleanly. Fixes #77087. Thanks @westlife219.
- Control UI/media: mint short-lived scoped tickets for assistant media fetches and render ticketed URLs instead of exposing long-lived auth tokens in chat image URLs. Fixes #70830 and #77097. Thanks @hclsys.
- Exec approvals: treat POSIX `exec` as a command carrier for inline eval, shell-wrapper, and eval/source detection, so approval explanations and command-risk checks do not miss payloads hidden behind `exec`. Thanks @vincentkoc.
- Google Meet: log the resolved audio provider model when starting Chrome and paired-node Meet talk-back bridges, so agent-mode joins show the STT model and bidi joins show the realtime voice model.
- Diagnostics: handle missing session-tail files in cron recovery context without tripping extension test typecheck. Thanks @vincentkoc.

View File

@@ -384,6 +384,16 @@ When gateway auth is configured, the Control UI avatar endpoint requires the sam
If you disable gateway auth (not recommended on shared hosts), the avatar route also becomes unauthenticated, in line with the rest of the gateway.
## Assistant media route auth
When gateway auth is configured, assistant local-media previews use a two-step route:
- `GET /__openclaw__/assistant-media?meta=1&source=<path>` requires the normal Control UI operator auth. The browser sends the gateway token as a bearer header when checking availability.
- Successful metadata responses include a short-lived `mediaTicket` scoped to that exact source path.
- Browser-rendered image, audio, video, and document URLs use `mediaTicket=<ticket>` instead of the active gateway token or password. The ticket expires quickly and cannot authorize a different source.
This keeps normal media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs.
## Building the UI
The Gateway serves static files from `dist/control-ui`. Build them with:

View File

@@ -0,0 +1,65 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, test } from "vitest";
import { installGatewayTestHooks, testState, withGatewayServer } from "./test-helpers.js";
installGatewayTestHooks({ scope: "suite" });
const CONTROL_UI_E2E_TOKEN = "test-gateway-token-1234567890";
describe("Control UI assistant media e2e", () => {
test("serves local assistant media through scoped tickets over the gateway HTTP route", async () => {
const stateDir = process.env.OPENCLAW_STATE_DIR;
if (!stateDir) {
throw new Error("OPENCLAW_STATE_DIR is required for gateway e2e media fixtures");
}
testState.gatewayAuth = { mode: "token", token: CONTROL_UI_E2E_TOKEN };
const mediaDir = path.join(stateDir, "media", "control-ui-assistant-media-e2e");
await fs.mkdir(mediaDir, { recursive: true });
const filePath = path.join(mediaDir, "ticketed-preview.txt");
await fs.writeFile(filePath, "ticketed control ui media\n", "utf8");
await withGatewayServer(
async ({ port }) => {
const route = `http://127.0.0.1:${port}/__openclaw__/assistant-media`;
const sourceParam = encodeURIComponent(filePath);
const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, {
headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` },
});
expect(metadata.status).toBe(200);
const payload = (await metadata.json()) as {
available?: boolean;
mediaTicket?: string;
mediaTicketExpiresAt?: string;
};
expect(payload.available).toBe(true);
expect(payload.mediaTicket).toMatch(/^v1\./);
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
const withoutTicket = await fetch(`${route}?source=${sourceParam}`);
expect(withoutTicket.status).toBe(401);
const ticketed = await fetch(
`${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
);
expect(ticketed.status).toBe(200);
expect(await ticketed.text()).toBe("ticketed control ui media\n");
const otherFilePath = path.join(mediaDir, "other-preview.txt");
await fs.writeFile(otherFilePath, "other media\n", "utf8");
const wrongSource = await fetch(
`${route}?source=${encodeURIComponent(otherFilePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
);
expect(wrongSource.status).toBe(401);
},
{
serverOptions: {
auth: { mode: "token", token: CONTROL_UI_E2E_TOKEN },
controlUiEnabled: true,
},
},
);
});
});

View File

@@ -368,7 +368,14 @@ describe("handleControlUiHttpRequest", () => {
});
expect(handled).toBe(true);
expect(res.statusCode).toBe(200);
expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true });
const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as {
available?: boolean;
mediaTicket?: string;
mediaTicketExpiresAt?: string;
};
expect(payload).toMatchObject({ available: true });
expect(payload.mediaTicket).toMatch(/^v1\./);
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
} finally {
await fs.rm(filePath, { force: true });
}
@@ -403,7 +410,94 @@ describe("handleControlUiHttpRequest", () => {
});
expect(handled).toBe(true);
expect(res.statusCode).toBe(200);
expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true });
const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as {
available?: boolean;
mediaTicket?: string;
mediaTicketExpiresAt?: string;
};
expect(payload).toMatchObject({ available: true });
expect(payload.mediaTicket).toMatch(/^v1\./);
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
},
});
});
it("serves assistant local media with a scoped media ticket after metadata auth", async () => {
await withAllowedAssistantMediaRoot({
prefix: "ui-media-ticket-",
fn: async (tmpRoot) => {
const filePath = path.join(tmpRoot, "photo.png");
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
const meta = await runAssistantMediaRequest({
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`,
method: "GET",
auth: { mode: "token", token: "test-token", allowTailscale: false },
headers: {
authorization: "Bearer test-token",
},
});
const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as {
mediaTicket?: string;
};
expect(meta.handled).toBe(true);
expect(meta.res.statusCode).toBe(200);
expect(payload.mediaTicket).toMatch(/^v1\./);
const media = await runAssistantMediaRequest({
url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
method: "GET",
auth: { mode: "token", token: "test-token", allowTailscale: false },
});
expect(media.handled).toBe(true);
expect(media.res.statusCode).toBe(200);
},
});
});
it("does not refresh assistant media tickets without operator auth", async () => {
await withAllowedAssistantMediaRoot({
prefix: "ui-media-ticket-refresh-",
fn: async (tmpRoot) => {
const filePath = path.join(tmpRoot, "photo.png");
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
const meta = await runAssistantMediaRequest({
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`,
method: "GET",
auth: { mode: "token", token: "test-token", allowTailscale: false },
headers: {
authorization: "Bearer test-token",
},
});
const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as {
mediaTicket?: string;
};
const refresh = await runAssistantMediaRequest({
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
method: "GET",
auth: { mode: "token", token: "test-token", allowTailscale: false },
});
expect(refresh.handled).toBe(true);
expect(refresh.res.statusCode).toBe(401);
expect(String(refresh.end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized");
},
});
});
it("rejects assistant local media with an invalid scoped media ticket", async () => {
await withAllowedAssistantMediaRoot({
prefix: "ui-media-ticket-invalid-",
fn: async (tmpRoot) => {
const filePath = path.join(tmpRoot, "photo.png");
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
const { res, handled, end } = await runAssistantMediaRequest({
url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=v1.invalid.invalid`,
method: "GET",
auth: { mode: "token", token: "test-token", allowTailscale: false },
});
expect(handled).toBe(true);
expect(res.statusCode).toBe(401);
expect(String(end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized");
},
});
});

View File

@@ -1,3 +1,4 @@
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
import fs from "node:fs";
import type { IncomingMessage, ServerResponse } from "node:http";
import path from "node:path";
@@ -56,10 +57,13 @@ import { resolveRequestClientIp } from "./net.js";
const ROOT_PREFIX = "/";
const CONTROL_UI_ASSISTANT_MEDIA_PREFIX = "/__openclaw__/assistant-media";
const CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE = "assistant-media";
const CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS = 5 * 60 * 1000;
const CONTROL_UI_ASSETS_MISSING_MESSAGE =
"Control UI assets not found. Build them with `pnpm ui:build` (auto-installs UI deps), or run `pnpm ui:dev` during development.";
const CONTROL_UI_OPERATOR_READ_SCOPE = "operator.read";
const CONTROL_UI_OPERATOR_ROLE = "operator";
const controlUiAssistantMediaTicketSecret = randomBytes(32);
export type ControlUiRequestOptions = {
basePath?: string;
@@ -378,6 +382,64 @@ type AssistantMediaAvailability =
| { available: true }
| { available: false; reason: string; code: string };
type AssistantMediaTicketPayload = {
scope: typeof CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE;
source: string;
exp: number;
};
function signAssistantMediaTicketPayload(encodedPayload: string): string {
return createHmac("sha256", controlUiAssistantMediaTicketSecret)
.update(encodedPayload)
.digest("base64url");
}
function createAssistantMediaTicket(source: string, nowMs = Date.now()) {
const exp = nowMs + CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS;
const payload: AssistantMediaTicketPayload = {
scope: CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE,
source,
exp,
};
const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url");
const sig = signAssistantMediaTicketPayload(encodedPayload);
return {
mediaTicket: `v1.${encodedPayload}.${sig}`,
mediaTicketExpiresAt: new Date(exp).toISOString(),
};
}
function verifyAssistantMediaTicket(ticket: string | null, source: string, nowMs = Date.now()) {
const parts = ticket?.split(".");
if (!parts || parts.length !== 3 || parts[0] !== "v1") {
return false;
}
const [, encodedPayload, sig] = parts;
if (!encodedPayload || !sig) {
return false;
}
const expectedSig = signAssistantMediaTicketPayload(encodedPayload);
const sigBuffer = Buffer.from(sig, "base64url");
const expectedBuffer = Buffer.from(expectedSig, "base64url");
if (sigBuffer.length !== expectedBuffer.length || !timingSafeEqual(sigBuffer, expectedBuffer)) {
return false;
}
try {
const payload = JSON.parse(
Buffer.from(encodedPayload, "base64url").toString("utf8"),
) as Partial<AssistantMediaTicketPayload>;
return (
payload.scope === CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE &&
payload.source === source &&
typeof payload.exp === "number" &&
Number.isFinite(payload.exp) &&
payload.exp >= nowMs
);
} catch {
return false;
}
}
function classifyAssistantMediaError(err: unknown): AssistantMediaAvailability {
if (err instanceof SafeOpenError) {
switch (err.code) {
@@ -461,7 +523,16 @@ export async function handleControlUiAssistantMediaRequest(
}
applyControlUiSecurityHeaders(res);
const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? "");
if (!source) {
respondControlUiNotFound(res);
return true;
}
const isMetaRequest = url.searchParams.get("meta") === "1";
const hasValidMediaTicket =
!isMetaRequest && verifyAssistantMediaTicket(url.searchParams.get("mediaTicket"), source);
if (
!hasValidMediaTicket &&
!(await authorizeControlUiReadRequest(req, res, {
auth: opts?.auth,
trustedProxies: opts?.trustedProxies,
@@ -472,18 +543,19 @@ export async function handleControlUiAssistantMediaRequest(
) {
return true;
}
const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? "");
if (!source) {
respondControlUiNotFound(res);
return true;
}
const localRoots = opts?.config
? getAgentScopedMediaLocalRoots(opts.config, opts.agentId)
: getDefaultLocalRoots();
if (url.searchParams.get("meta") === "1") {
if (isMetaRequest) {
const availability = await resolveAssistantMediaAvailability(source, localRoots);
sendJson(res, 200, availability);
sendJson(
res,
200,
availability.available
? { ...availability, ...createAssistantMediaTicket(source) }
: availability,
);
return true;
}

View File

@@ -347,6 +347,14 @@ async function flushAssistantAttachmentAvailabilityChecks() {
}
}
function mediaTicketPayload(mediaTicket: string, ttlMs = 5 * 60 * 1000) {
return {
available: true,
mediaTicket,
mediaTicketExpiresAt: new Date(Date.now() + ttlMs).toISOString(),
};
}
afterEach(() => {
document.querySelectorAll("[data-delete-confirm-fixture]").forEach((element) => {
element.remove();
@@ -808,15 +816,30 @@ describe("grouped chat rendering", () => {
expect(container.textContent).not.toContain("MEDIA:https://example.com/photo.png");
});
it("renders allowed transcript and content image variants", () => {
it("renders allowed transcript and content image variants", async () => {
resetAssistantAttachmentAvailabilityCacheForTest();
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
expect(url).toContain("meta=1");
const headers = init?.headers as Headers;
expect(headers.get("Authorization")).toBe("Bearer session-token");
return {
ok: true,
json: async () => mediaTicketPayload("ticket-user"),
};
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
const renderUserMedia = (message: unknown) => {
const container = document.createElement("div");
renderGroupedMessage(container, message, "user", {
showToolCalls: false,
basePath: "/openclaw",
assistantAttachmentAuthToken: "session-token",
localMediaPreviewRoots: ["/tmp/openclaw"],
});
const renderMessage = () =>
renderGroupedMessage(container, message, "user", {
showToolCalls: false,
basePath: "/openclaw",
assistantAttachmentAuthToken: "session-token",
localMediaPreviewRoots: ["/tmp/openclaw"],
onRequestUpdate: renderMessage,
});
renderMessage();
return container;
};
@@ -827,10 +850,11 @@ describe("grouped chat rendering", () => {
MediaPath: "/tmp/openclaw/user-upload.png",
timestamp: Date.now(),
});
await flushAssistantAttachmentAvailabilityChecks();
expect(
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user",
);
container = renderUserMedia({
@@ -841,10 +865,11 @@ describe("grouped chat rendering", () => {
MediaType: "application/octet-stream",
timestamp: Date.now(),
});
await flushAssistantAttachmentAvailabilityChecks();
expect(
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user",
);
container = renderUserMedia({
@@ -855,13 +880,14 @@ describe("grouped chat rendering", () => {
MediaTypes: ["image/png", "application/octet-stream"],
timestamp: Date.now(),
});
await flushAssistantAttachmentAvailabilityChecks();
expect(
[...container.querySelectorAll<HTMLImageElement>(".chat-message-image")].map((image) =>
image.getAttribute("src"),
),
).toEqual([
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&mediaTicket=ticket-user",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&mediaTicket=ticket-user",
]);
const assistantContainer = document.createElement("div");
@@ -905,6 +931,7 @@ describe("grouped chat rendering", () => {
);
expect(documentLink?.textContent).toContain("user-upload.pdf");
expect(documentLink?.getAttribute("href")).toBe("/__openclaw__/media/user-upload.pdf");
vi.unstubAllGlobals();
});
it("fetches managed chat images with auth and renders blob previews", async () => {
@@ -1065,11 +1092,13 @@ describe("grouped chat rendering", () => {
it("renders verified local assistant attachments through the Control UI media route", async () => {
resetAssistantAttachmentAvailabilityCacheForTest();
const fetchMock = vi.fn(async (url: string) => {
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
if (url.includes("meta=1")) {
const headers = init?.headers as Headers;
expect(headers.get("Authorization")).toBe("Bearer session-token");
return {
ok: true,
json: async () => ({ available: true }),
json: async () => mediaTicketPayload("ticket-local"),
};
}
throw new Error(`Unexpected fetch: ${url}`);
@@ -1100,7 +1129,7 @@ describe("grouped chat rendering", () => {
await flushAssistantAttachmentAvailabilityChecks();
expect(fetchMock).toHaveBeenCalledWith(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token&meta=1",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1",
expect.objectContaining({ credentials: "same-origin", method: "GET" }),
);
@@ -1109,24 +1138,84 @@ describe("grouped chat rendering", () => {
".chat-assistant-attachment-card__link",
);
expect(image?.getAttribute("src")).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-local",
);
expect(docLink?.getAttribute("href")).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&token=session-token",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&mediaTicket=ticket-local",
);
expect(container.textContent).not.toContain("test image.png");
vi.unstubAllGlobals();
});
it("refreshes cached local assistant media tickets before they expire without another render", async () => {
resetAssistantAttachmentAvailabilityCacheForTest();
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-04-30T00:00:00Z"));
const fetchMock = vi
.fn<
(url: string, init?: RequestInit) => Promise<{ ok: true; json: () => Promise<unknown> }>
>()
.mockResolvedValueOnce({
ok: true,
json: async () => mediaTicketPayload("ticket-old", 31_000),
})
.mockResolvedValueOnce({
ok: true,
json: async () => mediaTicketPayload("ticket-new"),
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
const container = document.createElement("div");
const renderMessage = () =>
renderAssistantMessage(
container,
{
id: "assistant-local-media-ticket-refresh",
role: "assistant",
content: "Local image\nMEDIA:/tmp/openclaw/test image.png",
timestamp: Date.now(),
},
{
showToolCalls: false,
basePath: "/openclaw",
assistantAttachmentAuthToken: "session-token",
localMediaPreviewRoots: ["/tmp/openclaw"],
onRequestUpdate: renderMessage,
},
);
renderMessage();
await flushAssistantAttachmentAvailabilityChecks();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-old",
);
vi.advanceTimersByTime(1_001);
await flushAssistantAttachmentAvailabilityChecks();
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-new",
);
vi.useRealTimers();
vi.unstubAllGlobals();
});
it("rechecks local assistant attachment availability when the auth token changes", async () => {
resetAssistantAttachmentAvailabilityCacheForTest();
const fetchMock = vi.fn(async (url: string) => {
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
if (!url.includes("meta=1")) {
throw new Error(`Unexpected fetch: ${url}`);
}
const headers = init?.headers as Headers;
const authorized = headers.get("Authorization") === "Bearer fresh-token";
return {
ok: true,
json: async () => ({ available: url.includes("token=fresh-token") }),
json: async () => (authorized ? mediaTicketPayload("ticket-fresh") : { available: false }),
};
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
@@ -1165,10 +1254,15 @@ describe("grouped chat rendering", () => {
);
expect(fetchMock).toHaveBeenNthCalledWith(
2,
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=fresh-token&meta=1",
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1",
expect.objectContaining({ credentials: "same-origin", method: "GET" }),
);
expect(container.querySelector(".chat-message-image")).not.toBeNull();
expect(
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
).toBe(
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-fresh",
);
expect(container.textContent).not.toContain("Unavailable");
vi.unstubAllGlobals();
});
@@ -1235,7 +1329,7 @@ describe("grouped chat rendering", () => {
}
return {
ok: true,
json: async () => ({ available: true }),
json: async () => mediaTicketPayload("ticket-platform"),
};
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
@@ -1321,7 +1415,7 @@ describe("grouped chat rendering", () => {
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({ available: true }),
json: async () => mediaTicketPayload("ticket-retry"),
});
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
const container = document.createElement("div");
@@ -1344,15 +1438,13 @@ describe("grouped chat rendering", () => {
);
renderMessage();
await vi.runAllTimersAsync();
await Promise.resolve();
await flushAssistantAttachmentAvailabilityChecks();
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(container.textContent).toContain("Unavailable");
vi.advanceTimersByTime(5_001);
renderMessage();
await vi.runAllTimersAsync();
await Promise.resolve();
await flushAssistantAttachmentAvailabilityChecks();
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(container.querySelector(".chat-message-image")).not.toBeNull();

View File

@@ -32,11 +32,13 @@ import {
type AssistantAttachmentAvailability =
| { status: "checking" }
| { status: "available" }
| { status: "available"; mediaTicket?: string; mediaTicketExpiresAt?: number }
| { status: "unavailable"; reason: string; checkedAt: number };
const assistantAttachmentAvailabilityCache = new Map<string, AssistantAttachmentAvailability>();
const assistantAttachmentRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000;
const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000;
export type ChatTimestampDisplay = {
label: string;
@@ -87,6 +89,10 @@ function renderChatTimestamp(timestamp: number) {
export function resetAssistantAttachmentAvailabilityCacheForTest() {
assistantAttachmentAvailabilityCache.clear();
for (const timer of assistantAttachmentRefreshTimers.values()) {
clearTimeout(timer);
}
assistantAttachmentRefreshTimers.clear();
for (const blobUrl of managedImageBlobUrlResolvedCache.values()) {
URL.revokeObjectURL(blobUrl);
}
@@ -107,6 +113,7 @@ type ImageRenderOptions = {
localMediaPreviewRoots?: readonly string[];
basePath?: string;
authToken?: string | null;
onRequestUpdate?: () => void;
};
type RenderableImageBlock = ImageBlock & {
@@ -718,8 +725,20 @@ function resolveRenderableMessageImages(
if (isLocalImage && !canProxyLocalImage) {
return [];
}
const availability = canProxyLocalImage
? resolveAssistantAttachmentAvailability(
img.url,
opts?.localMediaPreviewRoots ?? [],
opts?.basePath,
opts?.authToken,
opts?.onRequestUpdate,
)
: { status: "available" as const };
if (availability.status !== "available") {
return [];
}
const displayUrl = canProxyLocalImage
? buildAssistantAttachmentUrl(img.url, opts?.basePath, opts?.authToken)
? buildAssistantAttachmentUrl(img.url, opts?.basePath, availability.mediaTicket)
: img.url;
return [{ ...img, displayUrl }];
});
@@ -871,7 +890,7 @@ function isLocalAttachmentPreviewAllowed(
function buildAssistantAttachmentUrl(
source: string,
basePath?: string,
authToken?: string | null,
mediaTicket?: string | null,
): string {
if (!isLocalAssistantAttachmentSource(source)) {
return source;
@@ -879,9 +898,9 @@ function buildAssistantAttachmentUrl(
const normalizedBasePath =
basePath && basePath !== "/" ? (basePath.endsWith("/") ? basePath.slice(0, -1) : basePath) : "";
const params = new URLSearchParams({ source });
const normalizedToken = authToken?.trim();
if (normalizedToken) {
params.set("token", normalizedToken);
const normalizedMediaTicket = mediaTicket?.trim();
if (normalizedMediaTicket) {
params.set("mediaTicket", normalizedMediaTicket);
}
return `${normalizedBasePath}/__openclaw__/assistant-media?${params.toString()}`;
}
@@ -974,15 +993,51 @@ async function resolveManagedOutgoingImageBlobUrl(
return pending;
}
function buildAssistantAttachmentMetaUrl(
source: string,
basePath?: string,
authToken?: string | null,
): string {
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath, authToken);
function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string {
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath);
return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`;
}
function clearAssistantAttachmentRefreshTimer(cacheKey: string) {
const timer = assistantAttachmentRefreshTimers.get(cacheKey);
if (timer) {
clearTimeout(timer);
assistantAttachmentRefreshTimers.delete(cacheKey);
}
}
function scheduleAssistantAttachmentRefresh(
cacheKey: string,
availability: AssistantAttachmentAvailability,
onRequestUpdate: (() => void) | undefined,
) {
clearAssistantAttachmentRefreshTimer(cacheKey);
if (
availability.status !== "available" ||
!availability.mediaTicket ||
!availability.mediaTicketExpiresAt ||
!onRequestUpdate
) {
return;
}
const refreshInMs = Math.max(
0,
availability.mediaTicketExpiresAt -
Date.now() -
ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS,
);
const timer = setTimeout(() => {
assistantAttachmentRefreshTimers.delete(cacheKey);
const cached = assistantAttachmentAvailabilityCache.get(cacheKey);
if (cached?.status !== "available" || cached.mediaTicket !== availability.mediaTicket) {
return;
}
assistantAttachmentAvailabilityCache.delete(cacheKey);
onRequestUpdate();
}, refreshInMs);
assistantAttachmentRefreshTimers.set(cacheKey, timer);
}
function resolveAssistantAttachmentAvailability(
source: string,
localMediaPreviewRoots: readonly string[],
@@ -1000,30 +1055,63 @@ function resolveAssistantAttachmentAvailability(
const cacheKey = `${basePath ?? ""}::${normalizedAuthToken}::${source}`;
const cached = assistantAttachmentAvailabilityCache.get(cacheKey);
if (cached) {
const now = Date.now();
if (
cached.status === "unavailable" &&
Date.now() - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
now - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
) {
assistantAttachmentAvailabilityCache.delete(cacheKey);
} else if (
cached.status === "available" &&
cached.mediaTicket &&
(!cached.mediaTicketExpiresAt ||
cached.mediaTicketExpiresAt - now <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS)
) {
assistantAttachmentAvailabilityCache.delete(cacheKey);
} else {
scheduleAssistantAttachmentRefresh(cacheKey, cached, onRequestUpdate);
return cached;
}
}
clearAssistantAttachmentRefreshTimer(cacheKey);
assistantAttachmentAvailabilityCache.set(cacheKey, { status: "checking" });
if (typeof fetch === "function") {
void fetch(buildAssistantAttachmentMetaUrl(source, basePath, authToken), {
const headers = new Headers({ Accept: "application/json" });
if (normalizedAuthToken) {
headers.set("Authorization", `Bearer ${normalizedAuthToken}`);
}
void fetch(buildAssistantAttachmentMetaUrl(source, basePath), {
method: "GET",
headers: { Accept: "application/json" },
headers,
credentials: "same-origin",
})
.then(async (res) => {
const payload = (await res.json().catch(() => null)) as {
available?: boolean;
mediaTicket?: string;
mediaTicketExpiresAt?: string;
reason?: string;
} | null;
if (payload?.available === true) {
assistantAttachmentAvailabilityCache.set(cacheKey, { status: "available" });
const mediaTicket = payload.mediaTicket?.trim();
const mediaTicketExpiresAt = Date.parse(payload.mediaTicketExpiresAt ?? "");
if (mediaTicket && !Number.isFinite(mediaTicketExpiresAt)) {
clearAssistantAttachmentRefreshTimer(cacheKey);
assistantAttachmentAvailabilityCache.set(cacheKey, {
status: "unavailable",
reason: "Attachment unavailable",
checkedAt: Date.now(),
});
return;
}
const availability: AssistantAttachmentAvailability = {
status: "available",
...(mediaTicket ? { mediaTicket, mediaTicketExpiresAt } : {}),
};
assistantAttachmentAvailabilityCache.set(cacheKey, availability);
scheduleAssistantAttachmentRefresh(cacheKey, availability, onRequestUpdate);
} else {
clearAssistantAttachmentRefreshTimer(cacheKey);
assistantAttachmentAvailabilityCache.set(cacheKey, {
status: "unavailable",
reason: payload?.reason?.trim() || "Attachment unavailable",
@@ -1032,6 +1120,7 @@ function resolveAssistantAttachmentAvailability(
}
})
.catch(() => {
clearAssistantAttachmentRefreshTimer(cacheKey);
assistantAttachmentAvailabilityCache.set(cacheKey, {
status: "unavailable",
reason: "Attachment unavailable",
@@ -1097,7 +1186,7 @@ function renderAssistantAttachments(
);
const attachmentUrl =
availability.status === "available"
? buildAssistantAttachmentUrl(attachment.url, basePath, authToken)
? buildAssistantAttachmentUrl(attachment.url, basePath, availability.mediaTicket)
: null;
if (attachment.kind === "image") {
if (!attachmentUrl) {
@@ -1315,6 +1404,7 @@ function renderGroupedMessage(
localMediaPreviewRoots: opts.localMediaPreviewRoots ?? [],
basePath: opts.basePath,
authToken: opts.assistantAttachmentAuthToken,
onRequestUpdate: opts.onRequestUpdate,
};
const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions);
const hasImages = images.length > 0;