mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:30:42 +00:00
fix: scope Control UI assistant media tickets
This commit is contained in:
@@ -57,6 +57,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Google Meet: log the concrete agent-mode TTS provider, model, voice, output format, and sample rate after speech synthesis, so Meet logs show which voice backend spoke each reply.
|
||||
- Voice Call: mark realtime calls completed when the realtime provider closes normally, so Twilio/OpenAI/Google realtime stop events do not leave active call records behind. Thanks @vincentkoc.
|
||||
- Gateway/update: keep the shutdown close path behind a stable runtime chunk and ship compatibility aliases for recent `server-close-*` hashes, so manual npm package replacement cannot leave an already-running Gateway unable to shut down cleanly. Fixes #77087. Thanks @westlife219.
|
||||
- Control UI/media: mint short-lived scoped tickets for assistant media fetches and render ticketed URLs instead of exposing long-lived auth tokens in chat image URLs. Fixes #70830 and #77097. Thanks @hclsys.
|
||||
- Exec approvals: treat POSIX `exec` as a command carrier for inline eval, shell-wrapper, and eval/source detection, so approval explanations and command-risk checks do not miss payloads hidden behind `exec`. Thanks @vincentkoc.
|
||||
- Google Meet: log the resolved audio provider model when starting Chrome and paired-node Meet talk-back bridges, so agent-mode joins show the STT model and bidi joins show the realtime voice model.
|
||||
- Diagnostics: handle missing session-tail files in cron recovery context without tripping extension test typecheck. Thanks @vincentkoc.
|
||||
|
||||
@@ -384,6 +384,16 @@ When gateway auth is configured, the Control UI avatar endpoint requires the sam
|
||||
|
||||
If you disable gateway auth (not recommended on shared hosts), the avatar route also becomes unauthenticated, in line with the rest of the gateway.
|
||||
|
||||
## Assistant media route auth
|
||||
|
||||
When gateway auth is configured, assistant local-media previews use a two-step route:
|
||||
|
||||
- `GET /__openclaw__/assistant-media?meta=1&source=<path>` requires the normal Control UI operator auth. The browser sends the gateway token as a bearer header when checking availability.
|
||||
- Successful metadata responses include a short-lived `mediaTicket` scoped to that exact source path.
|
||||
- Browser-rendered image, audio, video, and document URLs use `mediaTicket=<ticket>` instead of the active gateway token or password. The ticket expires quickly and cannot authorize a different source.
|
||||
|
||||
This keeps normal media rendering compatible with browser-native media elements without putting reusable gateway credentials in visible media URLs.
|
||||
|
||||
## Building the UI
|
||||
|
||||
The Gateway serves static files from `dist/control-ui`. Build them with:
|
||||
|
||||
65
src/gateway/control-ui-assistant-media.e2e.test.ts
Normal file
65
src/gateway/control-ui-assistant-media.e2e.test.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { installGatewayTestHooks, testState, withGatewayServer } from "./test-helpers.js";
|
||||
|
||||
installGatewayTestHooks({ scope: "suite" });
|
||||
|
||||
const CONTROL_UI_E2E_TOKEN = "test-gateway-token-1234567890";
|
||||
|
||||
describe("Control UI assistant media e2e", () => {
|
||||
test("serves local assistant media through scoped tickets over the gateway HTTP route", async () => {
|
||||
const stateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
if (!stateDir) {
|
||||
throw new Error("OPENCLAW_STATE_DIR is required for gateway e2e media fixtures");
|
||||
}
|
||||
testState.gatewayAuth = { mode: "token", token: CONTROL_UI_E2E_TOKEN };
|
||||
|
||||
const mediaDir = path.join(stateDir, "media", "control-ui-assistant-media-e2e");
|
||||
await fs.mkdir(mediaDir, { recursive: true });
|
||||
const filePath = path.join(mediaDir, "ticketed-preview.txt");
|
||||
await fs.writeFile(filePath, "ticketed control ui media\n", "utf8");
|
||||
|
||||
await withGatewayServer(
|
||||
async ({ port }) => {
|
||||
const route = `http://127.0.0.1:${port}/__openclaw__/assistant-media`;
|
||||
const sourceParam = encodeURIComponent(filePath);
|
||||
|
||||
const metadata = await fetch(`${route}?meta=1&source=${sourceParam}`, {
|
||||
headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` },
|
||||
});
|
||||
expect(metadata.status).toBe(200);
|
||||
const payload = (await metadata.json()) as {
|
||||
available?: boolean;
|
||||
mediaTicket?: string;
|
||||
mediaTicketExpiresAt?: string;
|
||||
};
|
||||
expect(payload.available).toBe(true);
|
||||
expect(payload.mediaTicket).toMatch(/^v1\./);
|
||||
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
|
||||
|
||||
const withoutTicket = await fetch(`${route}?source=${sourceParam}`);
|
||||
expect(withoutTicket.status).toBe(401);
|
||||
|
||||
const ticketed = await fetch(
|
||||
`${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
);
|
||||
expect(ticketed.status).toBe(200);
|
||||
expect(await ticketed.text()).toBe("ticketed control ui media\n");
|
||||
|
||||
const otherFilePath = path.join(mediaDir, "other-preview.txt");
|
||||
await fs.writeFile(otherFilePath, "other media\n", "utf8");
|
||||
const wrongSource = await fetch(
|
||||
`${route}?source=${encodeURIComponent(otherFilePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
);
|
||||
expect(wrongSource.status).toBe(401);
|
||||
},
|
||||
{
|
||||
serverOptions: {
|
||||
auth: { mode: "token", token: CONTROL_UI_E2E_TOKEN },
|
||||
controlUiEnabled: true,
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -368,7 +368,14 @@ describe("handleControlUiHttpRequest", () => {
|
||||
});
|
||||
expect(handled).toBe(true);
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true });
|
||||
const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as {
|
||||
available?: boolean;
|
||||
mediaTicket?: string;
|
||||
mediaTicketExpiresAt?: string;
|
||||
};
|
||||
expect(payload).toMatchObject({ available: true });
|
||||
expect(payload.mediaTicket).toMatch(/^v1\./);
|
||||
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
|
||||
} finally {
|
||||
await fs.rm(filePath, { force: true });
|
||||
}
|
||||
@@ -403,7 +410,94 @@ describe("handleControlUiHttpRequest", () => {
|
||||
});
|
||||
expect(handled).toBe(true);
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(JSON.parse(String(end.mock.calls[0]?.[0] ?? ""))).toEqual({ available: true });
|
||||
const payload = JSON.parse(String(end.mock.calls[0]?.[0] ?? "")) as {
|
||||
available?: boolean;
|
||||
mediaTicket?: string;
|
||||
mediaTicketExpiresAt?: string;
|
||||
};
|
||||
expect(payload).toMatchObject({ available: true });
|
||||
expect(payload.mediaTicket).toMatch(/^v1\./);
|
||||
expect(Date.parse(payload.mediaTicketExpiresAt ?? "")).not.toBeNaN();
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("serves assistant local media with a scoped media ticket after metadata auth", async () => {
|
||||
await withAllowedAssistantMediaRoot({
|
||||
prefix: "ui-media-ticket-",
|
||||
fn: async (tmpRoot) => {
|
||||
const filePath = path.join(tmpRoot, "photo.png");
|
||||
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
|
||||
const meta = await runAssistantMediaRequest({
|
||||
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`,
|
||||
method: "GET",
|
||||
auth: { mode: "token", token: "test-token", allowTailscale: false },
|
||||
headers: {
|
||||
authorization: "Bearer test-token",
|
||||
},
|
||||
});
|
||||
const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as {
|
||||
mediaTicket?: string;
|
||||
};
|
||||
expect(meta.handled).toBe(true);
|
||||
expect(meta.res.statusCode).toBe(200);
|
||||
expect(payload.mediaTicket).toMatch(/^v1\./);
|
||||
|
||||
const media = await runAssistantMediaRequest({
|
||||
url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
method: "GET",
|
||||
auth: { mode: "token", token: "test-token", allowTailscale: false },
|
||||
});
|
||||
expect(media.handled).toBe(true);
|
||||
expect(media.res.statusCode).toBe(200);
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not refresh assistant media tickets without operator auth", async () => {
|
||||
await withAllowedAssistantMediaRoot({
|
||||
prefix: "ui-media-ticket-refresh-",
|
||||
fn: async (tmpRoot) => {
|
||||
const filePath = path.join(tmpRoot, "photo.png");
|
||||
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
|
||||
const meta = await runAssistantMediaRequest({
|
||||
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}`,
|
||||
method: "GET",
|
||||
auth: { mode: "token", token: "test-token", allowTailscale: false },
|
||||
headers: {
|
||||
authorization: "Bearer test-token",
|
||||
},
|
||||
});
|
||||
const payload = JSON.parse(String(meta.end.mock.calls[0]?.[0] ?? "")) as {
|
||||
mediaTicket?: string;
|
||||
};
|
||||
|
||||
const refresh = await runAssistantMediaRequest({
|
||||
url: `/__openclaw__/assistant-media?meta=1&source=${encodeURIComponent(filePath)}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
method: "GET",
|
||||
auth: { mode: "token", token: "test-token", allowTailscale: false },
|
||||
});
|
||||
expect(refresh.handled).toBe(true);
|
||||
expect(refresh.res.statusCode).toBe(401);
|
||||
expect(String(refresh.end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized");
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects assistant local media with an invalid scoped media ticket", async () => {
|
||||
await withAllowedAssistantMediaRoot({
|
||||
prefix: "ui-media-ticket-invalid-",
|
||||
fn: async (tmpRoot) => {
|
||||
const filePath = path.join(tmpRoot, "photo.png");
|
||||
await fs.writeFile(filePath, Buffer.from("not-a-real-png"));
|
||||
const { res, handled, end } = await runAssistantMediaRequest({
|
||||
url: `/__openclaw__/assistant-media?source=${encodeURIComponent(filePath)}&mediaTicket=v1.invalid.invalid`,
|
||||
method: "GET",
|
||||
auth: { mode: "token", token: "test-token", allowTailscale: false },
|
||||
});
|
||||
expect(handled).toBe(true);
|
||||
expect(res.statusCode).toBe(401);
|
||||
expect(String(end.mock.calls[0]?.[0] ?? "")).toContain("Unauthorized");
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import path from "node:path";
|
||||
@@ -56,10 +57,13 @@ import { resolveRequestClientIp } from "./net.js";
|
||||
|
||||
const ROOT_PREFIX = "/";
|
||||
const CONTROL_UI_ASSISTANT_MEDIA_PREFIX = "/__openclaw__/assistant-media";
|
||||
const CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE = "assistant-media";
|
||||
const CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS = 5 * 60 * 1000;
|
||||
const CONTROL_UI_ASSETS_MISSING_MESSAGE =
|
||||
"Control UI assets not found. Build them with `pnpm ui:build` (auto-installs UI deps), or run `pnpm ui:dev` during development.";
|
||||
const CONTROL_UI_OPERATOR_READ_SCOPE = "operator.read";
|
||||
const CONTROL_UI_OPERATOR_ROLE = "operator";
|
||||
const controlUiAssistantMediaTicketSecret = randomBytes(32);
|
||||
|
||||
export type ControlUiRequestOptions = {
|
||||
basePath?: string;
|
||||
@@ -378,6 +382,64 @@ type AssistantMediaAvailability =
|
||||
| { available: true }
|
||||
| { available: false; reason: string; code: string };
|
||||
|
||||
type AssistantMediaTicketPayload = {
|
||||
scope: typeof CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE;
|
||||
source: string;
|
||||
exp: number;
|
||||
};
|
||||
|
||||
function signAssistantMediaTicketPayload(encodedPayload: string): string {
|
||||
return createHmac("sha256", controlUiAssistantMediaTicketSecret)
|
||||
.update(encodedPayload)
|
||||
.digest("base64url");
|
||||
}
|
||||
|
||||
function createAssistantMediaTicket(source: string, nowMs = Date.now()) {
|
||||
const exp = nowMs + CONTROL_UI_ASSISTANT_MEDIA_TICKET_TTL_MS;
|
||||
const payload: AssistantMediaTicketPayload = {
|
||||
scope: CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE,
|
||||
source,
|
||||
exp,
|
||||
};
|
||||
const encodedPayload = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url");
|
||||
const sig = signAssistantMediaTicketPayload(encodedPayload);
|
||||
return {
|
||||
mediaTicket: `v1.${encodedPayload}.${sig}`,
|
||||
mediaTicketExpiresAt: new Date(exp).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function verifyAssistantMediaTicket(ticket: string | null, source: string, nowMs = Date.now()) {
|
||||
const parts = ticket?.split(".");
|
||||
if (!parts || parts.length !== 3 || parts[0] !== "v1") {
|
||||
return false;
|
||||
}
|
||||
const [, encodedPayload, sig] = parts;
|
||||
if (!encodedPayload || !sig) {
|
||||
return false;
|
||||
}
|
||||
const expectedSig = signAssistantMediaTicketPayload(encodedPayload);
|
||||
const sigBuffer = Buffer.from(sig, "base64url");
|
||||
const expectedBuffer = Buffer.from(expectedSig, "base64url");
|
||||
if (sigBuffer.length !== expectedBuffer.length || !timingSafeEqual(sigBuffer, expectedBuffer)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const payload = JSON.parse(
|
||||
Buffer.from(encodedPayload, "base64url").toString("utf8"),
|
||||
) as Partial<AssistantMediaTicketPayload>;
|
||||
return (
|
||||
payload.scope === CONTROL_UI_ASSISTANT_MEDIA_TICKET_SCOPE &&
|
||||
payload.source === source &&
|
||||
typeof payload.exp === "number" &&
|
||||
Number.isFinite(payload.exp) &&
|
||||
payload.exp >= nowMs
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function classifyAssistantMediaError(err: unknown): AssistantMediaAvailability {
|
||||
if (err instanceof SafeOpenError) {
|
||||
switch (err.code) {
|
||||
@@ -461,7 +523,16 @@ export async function handleControlUiAssistantMediaRequest(
|
||||
}
|
||||
|
||||
applyControlUiSecurityHeaders(res);
|
||||
const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? "");
|
||||
if (!source) {
|
||||
respondControlUiNotFound(res);
|
||||
return true;
|
||||
}
|
||||
const isMetaRequest = url.searchParams.get("meta") === "1";
|
||||
const hasValidMediaTicket =
|
||||
!isMetaRequest && verifyAssistantMediaTicket(url.searchParams.get("mediaTicket"), source);
|
||||
if (
|
||||
!hasValidMediaTicket &&
|
||||
!(await authorizeControlUiReadRequest(req, res, {
|
||||
auth: opts?.auth,
|
||||
trustedProxies: opts?.trustedProxies,
|
||||
@@ -472,18 +543,19 @@ export async function handleControlUiAssistantMediaRequest(
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const source = normalizeAssistantMediaSource(url.searchParams.get("source") ?? "");
|
||||
if (!source) {
|
||||
respondControlUiNotFound(res);
|
||||
return true;
|
||||
}
|
||||
const localRoots = opts?.config
|
||||
? getAgentScopedMediaLocalRoots(opts.config, opts.agentId)
|
||||
: getDefaultLocalRoots();
|
||||
|
||||
if (url.searchParams.get("meta") === "1") {
|
||||
if (isMetaRequest) {
|
||||
const availability = await resolveAssistantMediaAvailability(source, localRoots);
|
||||
sendJson(res, 200, availability);
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
availability.available
|
||||
? { ...availability, ...createAssistantMediaTicket(source) }
|
||||
: availability,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -347,6 +347,14 @@ async function flushAssistantAttachmentAvailabilityChecks() {
|
||||
}
|
||||
}
|
||||
|
||||
function mediaTicketPayload(mediaTicket: string, ttlMs = 5 * 60 * 1000) {
|
||||
return {
|
||||
available: true,
|
||||
mediaTicket,
|
||||
mediaTicketExpiresAt: new Date(Date.now() + ttlMs).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
document.querySelectorAll("[data-delete-confirm-fixture]").forEach((element) => {
|
||||
element.remove();
|
||||
@@ -808,15 +816,30 @@ describe("grouped chat rendering", () => {
|
||||
expect(container.textContent).not.toContain("MEDIA:https://example.com/photo.png");
|
||||
});
|
||||
|
||||
it("renders allowed transcript and content image variants", () => {
|
||||
it("renders allowed transcript and content image variants", async () => {
|
||||
resetAssistantAttachmentAvailabilityCacheForTest();
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
expect(url).toContain("meta=1");
|
||||
const headers = init?.headers as Headers;
|
||||
expect(headers.get("Authorization")).toBe("Bearer session-token");
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => mediaTicketPayload("ticket-user"),
|
||||
};
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
|
||||
const renderUserMedia = (message: unknown) => {
|
||||
const container = document.createElement("div");
|
||||
renderGroupedMessage(container, message, "user", {
|
||||
showToolCalls: false,
|
||||
basePath: "/openclaw",
|
||||
assistantAttachmentAuthToken: "session-token",
|
||||
localMediaPreviewRoots: ["/tmp/openclaw"],
|
||||
});
|
||||
const renderMessage = () =>
|
||||
renderGroupedMessage(container, message, "user", {
|
||||
showToolCalls: false,
|
||||
basePath: "/openclaw",
|
||||
assistantAttachmentAuthToken: "session-token",
|
||||
localMediaPreviewRoots: ["/tmp/openclaw"],
|
||||
onRequestUpdate: renderMessage,
|
||||
});
|
||||
renderMessage();
|
||||
return container;
|
||||
};
|
||||
|
||||
@@ -827,10 +850,11 @@ describe("grouped chat rendering", () => {
|
||||
MediaPath: "/tmp/openclaw/user-upload.png",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
expect(
|
||||
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
|
||||
).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user",
|
||||
);
|
||||
|
||||
container = renderUserMedia({
|
||||
@@ -841,10 +865,11 @@ describe("grouped chat rendering", () => {
|
||||
MediaType: "application/octet-stream",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
expect(
|
||||
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
|
||||
).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fuser-upload.png&mediaTicket=ticket-user",
|
||||
);
|
||||
|
||||
container = renderUserMedia({
|
||||
@@ -855,13 +880,14 @@ describe("grouped chat rendering", () => {
|
||||
MediaTypes: ["image/png", "application/octet-stream"],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
expect(
|
||||
[...container.querySelectorAll<HTMLImageElement>(".chat-message-image")].map((image) =>
|
||||
image.getAttribute("src"),
|
||||
),
|
||||
).toEqual([
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ffirst.png&mediaTicket=ticket-user",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Fsecond.jpg&mediaTicket=ticket-user",
|
||||
]);
|
||||
|
||||
const assistantContainer = document.createElement("div");
|
||||
@@ -905,6 +931,7 @@ describe("grouped chat rendering", () => {
|
||||
);
|
||||
expect(documentLink?.textContent).toContain("user-upload.pdf");
|
||||
expect(documentLink?.getAttribute("href")).toBe("/__openclaw__/media/user-upload.pdf");
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("fetches managed chat images with auth and renders blob previews", async () => {
|
||||
@@ -1065,11 +1092,13 @@ describe("grouped chat rendering", () => {
|
||||
|
||||
it("renders verified local assistant attachments through the Control UI media route", async () => {
|
||||
resetAssistantAttachmentAvailabilityCacheForTest();
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (url.includes("meta=1")) {
|
||||
const headers = init?.headers as Headers;
|
||||
expect(headers.get("Authorization")).toBe("Bearer session-token");
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ available: true }),
|
||||
json: async () => mediaTicketPayload("ticket-local"),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
@@ -1100,7 +1129,7 @@ describe("grouped chat rendering", () => {
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token&meta=1",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1",
|
||||
expect.objectContaining({ credentials: "same-origin", method: "GET" }),
|
||||
);
|
||||
|
||||
@@ -1109,24 +1138,84 @@ describe("grouped chat rendering", () => {
|
||||
".chat-assistant-attachment-card__link",
|
||||
);
|
||||
expect(image?.getAttribute("src")).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-local",
|
||||
);
|
||||
expect(docLink?.getAttribute("href")).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&token=session-token",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest-doc.pdf&mediaTicket=ticket-local",
|
||||
);
|
||||
expect(container.textContent).not.toContain("test image.png");
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("refreshes cached local assistant media tickets before they expire without another render", async () => {
|
||||
resetAssistantAttachmentAvailabilityCacheForTest();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-04-30T00:00:00Z"));
|
||||
const fetchMock = vi
|
||||
.fn<
|
||||
(url: string, init?: RequestInit) => Promise<{ ok: true; json: () => Promise<unknown> }>
|
||||
>()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mediaTicketPayload("ticket-old", 31_000),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => mediaTicketPayload("ticket-new"),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
const container = document.createElement("div");
|
||||
const renderMessage = () =>
|
||||
renderAssistantMessage(
|
||||
container,
|
||||
{
|
||||
id: "assistant-local-media-ticket-refresh",
|
||||
role: "assistant",
|
||||
content: "Local image\nMEDIA:/tmp/openclaw/test image.png",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{
|
||||
showToolCalls: false,
|
||||
basePath: "/openclaw",
|
||||
assistantAttachmentAuthToken: "session-token",
|
||||
localMediaPreviewRoots: ["/tmp/openclaw"],
|
||||
onRequestUpdate: renderMessage,
|
||||
},
|
||||
);
|
||||
|
||||
renderMessage();
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
|
||||
).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-old",
|
||||
);
|
||||
|
||||
vi.advanceTimersByTime(1_001);
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
|
||||
).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-new",
|
||||
);
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("rechecks local assistant attachment availability when the auth token changes", async () => {
|
||||
resetAssistantAttachmentAvailabilityCacheForTest();
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
const fetchMock = vi.fn(async (url: string, init?: RequestInit) => {
|
||||
if (!url.includes("meta=1")) {
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
}
|
||||
const headers = init?.headers as Headers;
|
||||
const authorized = headers.get("Authorization") === "Bearer fresh-token";
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ available: url.includes("token=fresh-token") }),
|
||||
json: async () => (authorized ? mediaTicketPayload("ticket-fresh") : { available: false }),
|
||||
};
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
@@ -1165,10 +1254,15 @@ describe("grouped chat rendering", () => {
|
||||
);
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&token=fresh-token&meta=1",
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&meta=1",
|
||||
expect.objectContaining({ credentials: "same-origin", method: "GET" }),
|
||||
);
|
||||
expect(container.querySelector(".chat-message-image")).not.toBeNull();
|
||||
expect(
|
||||
container.querySelector<HTMLImageElement>(".chat-message-image")?.getAttribute("src"),
|
||||
).toBe(
|
||||
"/openclaw/__openclaw__/assistant-media?source=%2Ftmp%2Fopenclaw%2Ftest+image.png&mediaTicket=ticket-fresh",
|
||||
);
|
||||
expect(container.textContent).not.toContain("Unavailable");
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
@@ -1235,7 +1329,7 @@ describe("grouped chat rendering", () => {
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({ available: true }),
|
||||
json: async () => mediaTicketPayload("ticket-platform"),
|
||||
};
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
@@ -1321,7 +1415,7 @@ describe("grouped chat rendering", () => {
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ available: true }),
|
||||
json: async () => mediaTicketPayload("ticket-retry"),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock as unknown as typeof fetch);
|
||||
const container = document.createElement("div");
|
||||
@@ -1344,15 +1438,13 @@ describe("grouped chat rendering", () => {
|
||||
);
|
||||
|
||||
renderMessage();
|
||||
await vi.runAllTimersAsync();
|
||||
await Promise.resolve();
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(container.textContent).toContain("Unavailable");
|
||||
|
||||
vi.advanceTimersByTime(5_001);
|
||||
renderMessage();
|
||||
await vi.runAllTimersAsync();
|
||||
await Promise.resolve();
|
||||
await flushAssistantAttachmentAvailabilityChecks();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
expect(container.querySelector(".chat-message-image")).not.toBeNull();
|
||||
|
||||
@@ -32,11 +32,13 @@ import {
|
||||
|
||||
type AssistantAttachmentAvailability =
|
||||
| { status: "checking" }
|
||||
| { status: "available" }
|
||||
| { status: "available"; mediaTicket?: string; mediaTicketExpiresAt?: number }
|
||||
| { status: "unavailable"; reason: string; checkedAt: number };
|
||||
|
||||
const assistantAttachmentAvailabilityCache = new Map<string, AssistantAttachmentAvailability>();
|
||||
const assistantAttachmentRefreshTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
||||
const ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS = 5_000;
|
||||
const ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS = 30_000;
|
||||
|
||||
export type ChatTimestampDisplay = {
|
||||
label: string;
|
||||
@@ -87,6 +89,10 @@ function renderChatTimestamp(timestamp: number) {
|
||||
|
||||
export function resetAssistantAttachmentAvailabilityCacheForTest() {
|
||||
assistantAttachmentAvailabilityCache.clear();
|
||||
for (const timer of assistantAttachmentRefreshTimers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
assistantAttachmentRefreshTimers.clear();
|
||||
for (const blobUrl of managedImageBlobUrlResolvedCache.values()) {
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
}
|
||||
@@ -107,6 +113,7 @@ type ImageRenderOptions = {
|
||||
localMediaPreviewRoots?: readonly string[];
|
||||
basePath?: string;
|
||||
authToken?: string | null;
|
||||
onRequestUpdate?: () => void;
|
||||
};
|
||||
|
||||
type RenderableImageBlock = ImageBlock & {
|
||||
@@ -718,8 +725,20 @@ function resolveRenderableMessageImages(
|
||||
if (isLocalImage && !canProxyLocalImage) {
|
||||
return [];
|
||||
}
|
||||
const availability = canProxyLocalImage
|
||||
? resolveAssistantAttachmentAvailability(
|
||||
img.url,
|
||||
opts?.localMediaPreviewRoots ?? [],
|
||||
opts?.basePath,
|
||||
opts?.authToken,
|
||||
opts?.onRequestUpdate,
|
||||
)
|
||||
: { status: "available" as const };
|
||||
if (availability.status !== "available") {
|
||||
return [];
|
||||
}
|
||||
const displayUrl = canProxyLocalImage
|
||||
? buildAssistantAttachmentUrl(img.url, opts?.basePath, opts?.authToken)
|
||||
? buildAssistantAttachmentUrl(img.url, opts?.basePath, availability.mediaTicket)
|
||||
: img.url;
|
||||
return [{ ...img, displayUrl }];
|
||||
});
|
||||
@@ -871,7 +890,7 @@ function isLocalAttachmentPreviewAllowed(
|
||||
function buildAssistantAttachmentUrl(
|
||||
source: string,
|
||||
basePath?: string,
|
||||
authToken?: string | null,
|
||||
mediaTicket?: string | null,
|
||||
): string {
|
||||
if (!isLocalAssistantAttachmentSource(source)) {
|
||||
return source;
|
||||
@@ -879,9 +898,9 @@ function buildAssistantAttachmentUrl(
|
||||
const normalizedBasePath =
|
||||
basePath && basePath !== "/" ? (basePath.endsWith("/") ? basePath.slice(0, -1) : basePath) : "";
|
||||
const params = new URLSearchParams({ source });
|
||||
const normalizedToken = authToken?.trim();
|
||||
if (normalizedToken) {
|
||||
params.set("token", normalizedToken);
|
||||
const normalizedMediaTicket = mediaTicket?.trim();
|
||||
if (normalizedMediaTicket) {
|
||||
params.set("mediaTicket", normalizedMediaTicket);
|
||||
}
|
||||
return `${normalizedBasePath}/__openclaw__/assistant-media?${params.toString()}`;
|
||||
}
|
||||
@@ -974,15 +993,51 @@ async function resolveManagedOutgoingImageBlobUrl(
|
||||
return pending;
|
||||
}
|
||||
|
||||
function buildAssistantAttachmentMetaUrl(
|
||||
source: string,
|
||||
basePath?: string,
|
||||
authToken?: string | null,
|
||||
): string {
|
||||
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath, authToken);
|
||||
function buildAssistantAttachmentMetaUrl(source: string, basePath?: string): string {
|
||||
const attachmentUrl = buildAssistantAttachmentUrl(source, basePath);
|
||||
return `${attachmentUrl}${attachmentUrl.includes("?") ? "&" : "?"}meta=1`;
|
||||
}
|
||||
|
||||
function clearAssistantAttachmentRefreshTimer(cacheKey: string) {
|
||||
const timer = assistantAttachmentRefreshTimers.get(cacheKey);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
assistantAttachmentRefreshTimers.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleAssistantAttachmentRefresh(
|
||||
cacheKey: string,
|
||||
availability: AssistantAttachmentAvailability,
|
||||
onRequestUpdate: (() => void) | undefined,
|
||||
) {
|
||||
clearAssistantAttachmentRefreshTimer(cacheKey);
|
||||
if (
|
||||
availability.status !== "available" ||
|
||||
!availability.mediaTicket ||
|
||||
!availability.mediaTicketExpiresAt ||
|
||||
!onRequestUpdate
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const refreshInMs = Math.max(
|
||||
0,
|
||||
availability.mediaTicketExpiresAt -
|
||||
Date.now() -
|
||||
ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS,
|
||||
);
|
||||
const timer = setTimeout(() => {
|
||||
assistantAttachmentRefreshTimers.delete(cacheKey);
|
||||
const cached = assistantAttachmentAvailabilityCache.get(cacheKey);
|
||||
if (cached?.status !== "available" || cached.mediaTicket !== availability.mediaTicket) {
|
||||
return;
|
||||
}
|
||||
assistantAttachmentAvailabilityCache.delete(cacheKey);
|
||||
onRequestUpdate();
|
||||
}, refreshInMs);
|
||||
assistantAttachmentRefreshTimers.set(cacheKey, timer);
|
||||
}
|
||||
|
||||
function resolveAssistantAttachmentAvailability(
|
||||
source: string,
|
||||
localMediaPreviewRoots: readonly string[],
|
||||
@@ -1000,30 +1055,63 @@ function resolveAssistantAttachmentAvailability(
|
||||
const cacheKey = `${basePath ?? ""}::${normalizedAuthToken}::${source}`;
|
||||
const cached = assistantAttachmentAvailabilityCache.get(cacheKey);
|
||||
if (cached) {
|
||||
const now = Date.now();
|
||||
if (
|
||||
cached.status === "unavailable" &&
|
||||
Date.now() - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
|
||||
now - cached.checkedAt >= ASSISTANT_ATTACHMENT_UNAVAILABLE_RETRY_MS
|
||||
) {
|
||||
assistantAttachmentAvailabilityCache.delete(cacheKey);
|
||||
} else if (
|
||||
cached.status === "available" &&
|
||||
cached.mediaTicket &&
|
||||
(!cached.mediaTicketExpiresAt ||
|
||||
cached.mediaTicketExpiresAt - now <= ASSISTANT_ATTACHMENT_MEDIA_TICKET_REFRESH_SKEW_MS)
|
||||
) {
|
||||
assistantAttachmentAvailabilityCache.delete(cacheKey);
|
||||
} else {
|
||||
scheduleAssistantAttachmentRefresh(cacheKey, cached, onRequestUpdate);
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
clearAssistantAttachmentRefreshTimer(cacheKey);
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, { status: "checking" });
|
||||
if (typeof fetch === "function") {
|
||||
void fetch(buildAssistantAttachmentMetaUrl(source, basePath, authToken), {
|
||||
const headers = new Headers({ Accept: "application/json" });
|
||||
if (normalizedAuthToken) {
|
||||
headers.set("Authorization", `Bearer ${normalizedAuthToken}`);
|
||||
}
|
||||
void fetch(buildAssistantAttachmentMetaUrl(source, basePath), {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
headers,
|
||||
credentials: "same-origin",
|
||||
})
|
||||
.then(async (res) => {
|
||||
const payload = (await res.json().catch(() => null)) as {
|
||||
available?: boolean;
|
||||
mediaTicket?: string;
|
||||
mediaTicketExpiresAt?: string;
|
||||
reason?: string;
|
||||
} | null;
|
||||
if (payload?.available === true) {
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, { status: "available" });
|
||||
const mediaTicket = payload.mediaTicket?.trim();
|
||||
const mediaTicketExpiresAt = Date.parse(payload.mediaTicketExpiresAt ?? "");
|
||||
if (mediaTicket && !Number.isFinite(mediaTicketExpiresAt)) {
|
||||
clearAssistantAttachmentRefreshTimer(cacheKey);
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, {
|
||||
status: "unavailable",
|
||||
reason: "Attachment unavailable",
|
||||
checkedAt: Date.now(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const availability: AssistantAttachmentAvailability = {
|
||||
status: "available",
|
||||
...(mediaTicket ? { mediaTicket, mediaTicketExpiresAt } : {}),
|
||||
};
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, availability);
|
||||
scheduleAssistantAttachmentRefresh(cacheKey, availability, onRequestUpdate);
|
||||
} else {
|
||||
clearAssistantAttachmentRefreshTimer(cacheKey);
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, {
|
||||
status: "unavailable",
|
||||
reason: payload?.reason?.trim() || "Attachment unavailable",
|
||||
@@ -1032,6 +1120,7 @@ function resolveAssistantAttachmentAvailability(
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
clearAssistantAttachmentRefreshTimer(cacheKey);
|
||||
assistantAttachmentAvailabilityCache.set(cacheKey, {
|
||||
status: "unavailable",
|
||||
reason: "Attachment unavailable",
|
||||
@@ -1097,7 +1186,7 @@ function renderAssistantAttachments(
|
||||
);
|
||||
const attachmentUrl =
|
||||
availability.status === "available"
|
||||
? buildAssistantAttachmentUrl(attachment.url, basePath, authToken)
|
||||
? buildAssistantAttachmentUrl(attachment.url, basePath, availability.mediaTicket)
|
||||
: null;
|
||||
if (attachment.kind === "image") {
|
||||
if (!attachmentUrl) {
|
||||
@@ -1315,6 +1404,7 @@ function renderGroupedMessage(
|
||||
localMediaPreviewRoots: opts.localMediaPreviewRoots ?? [],
|
||||
basePath: opts.basePath,
|
||||
authToken: opts.assistantAttachmentAuthToken,
|
||||
onRequestUpdate: opts.onRequestUpdate,
|
||||
};
|
||||
const images = resolveRenderableMessageImages(extractImages(message), imageRenderOptions);
|
||||
const hasImages = images.length > 0;
|
||||
|
||||
Reference in New Issue
Block a user