From ee316dbc4bac668f88e6f54879b08b3dc926b27e Mon Sep 17 00:00:00 2001 From: Devin Robison Date: Tue, 21 Apr 2026 15:57:49 -0600 Subject: [PATCH] fix(tlon): guard memex upload target (#69794) * fix(tlon): guard memex upload target * fix(tlon): harden guarded memex upload * fix(tlon): validate hosted memex upload targets * fix(tlon): tighten hosted domain matching * fix(tlon): reject non-standard memex upload ports * fix(tlon): disable memex upload redirects * test(tlon): drop redundant mock resets in memex upload test * chore(lint): update tlon raw-fetch allowlist for guarded memex upload * fix(tlon): reject unparseable ship URLs in hosted-ship classifier * fix(lint): point tlon raw-fetch allowlist at fetch callee lines * fix(tlon): guard custom-S3 upload through fetchWithSsrFGuard * fix(tlon): preserve scheme-less hosted ship routing and allow explicit :443 * docs(changelog): note tlon upload guard * fix(tlon): guard memex lookup and private s3 opt-in * fix(tlon): validate upload result URLs --- CHANGELOG.md | 1 + extensions/tlon/src/tlon-api.test.ts | 587 +++++++++++++++++++++++++ extensions/tlon/src/tlon-api.ts | 188 ++++++-- scripts/check-no-raw-channel-fetch.mjs | 3 - 4 files changed, 731 insertions(+), 48 deletions(-) create mode 100644 extensions/tlon/src/tlon-api.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 04fe9b8a413..b366602b819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Docs: https://docs.openclaw.ai - Ollama/media understanding: register Ollama as an image-capable media-understanding provider so `agents.defaults.imageModel.primary` values like `ollama/qwen2.5vl:7b` route through the Ollama plugin instead of failing as unknown models. (#69816) Thanks @soloclz. - CLI/media understanding: make `openclaw infer image describe --model ` execute the explicit image model instead of skipping description when that model supports native vision. - Usage/providers: keep plugin-owned usage auth enabled when manifest-declared provider auth env vars such as `MINIMAX_CODE_PLAN_KEY` are present, so `/usage` can resolve MiniMax billing credentials through the provider plugin. +- Tlon/uploads: route both hosted Memex upload targets and custom-S3 presigned upload URLs through the shared SSRF guard so blocked private or loopback destinations fail before upload, while public upload URLs continue through the existing hosted upload flow. (#69794) Thanks @drobison00. ## 2026.4.20 diff --git a/extensions/tlon/src/tlon-api.test.ts b/extensions/tlon/src/tlon-api.test.ts new file mode 100644 index 00000000000..be352d132e1 --- /dev/null +++ b/extensions/tlon/src/tlon-api.test.ts @@ -0,0 +1,587 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { authenticate } from "./urbit/auth.js"; +import { scryUrbitPath } from "./urbit/channel-ops.js"; + +const { mockFetchGuard, mockRelease, mockGetSignedUrl } = vi.hoisted(() => ({ + mockFetchGuard: vi.fn(), + mockRelease: vi.fn(async () => {}), + mockGetSignedUrl: vi.fn(), +})); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => { + const original = (await vi.importActual("openclaw/plugin-sdk/ssrf-runtime")) as Record< + string, + unknown + >; + return { + ...original, + fetchWithSsrFGuard: mockFetchGuard, + }; +}); + +vi.mock("@aws-sdk/s3-request-presigner", () => ({ + getSignedUrl: mockGetSignedUrl, +})); + +vi.mock("./urbit/auth.js", () => ({ + authenticate: vi.fn(), +})); + +vi.mock("./urbit/channel-ops.js", () => ({ + scryUrbitPath: vi.fn(), +})); + +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; +import { configureClient, uploadFile } from "./tlon-api.js"; + +const mockAuthenticate = vi.mocked(authenticate); +const mockScryUrbitPath = vi.mocked(scryUrbitPath); +const mockGuardedFetch = vi.mocked(fetchWithSsrFGuard); + +function createMemexResponse( + uploadUrl: string, + filePath = "https://memex.tlon.network/files/uploaded.png", +): Response { + return new Response( + JSON.stringify({ + url: uploadUrl, + filePath, + }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ); +} + +function createGuardedResult(response: Response, finalUrl: string) { + return { + response, + finalUrl, + release: mockRelease, + }; +} + +describe("uploadFile memex upload hardening", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", vi.fn()); + mockAuthenticate.mockResolvedValue("urbauth-~zod=fake-cookie"); + configureClient({ + shipUrl: "https://groups.tlon.network", + shipName: "~zod", + verbose: false, + getCode: async () => "123456", + }); + mockScryUrbitPath.mockImplementation(async (_deps, params) => { + if (params.path === "/storage/configuration.json") { + return { + currentBucket: "uploads", + buckets: ["uploads"], + publicUrlBase: "https://files.tlon.network/", + presignedUrl: "https://files.tlon.network/presigned", + region: "us-east-1", + service: "presigned-url", + }; + } + if (params.path === "/storage/credentials.json") { + return { "storage-update": {} }; + } + if (params.path === "/genuine/secret.json") { + return { secret: "genuine-secret" }; + } + throw new Error(`Unexpected scry path: ${params.path}`); + }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("routes the memex upload URL through the SSRF guard", async () => { + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://uploads.tlon.network/put", + ), + ); + + const result = await uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }); + + expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" }); + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledTimes(2); + expect(mockGuardedFetch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + url: "https://memex.tlon.network/v1/zod/upload", + init: expect.objectContaining({ + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: expect.any(String), + }), + auditContext: "tlon-memex-upload-url", + capture: false, + maxRedirects: 0, + }), + ); + expect(mockGuardedFetch).toHaveBeenNthCalledWith(2, { + url: "https://uploads.tlon.network/put", + init: expect.objectContaining({ + method: "PUT", + body: expect.any(Blob), + headers: expect.objectContaining({ + "Cache-Control": "public, max-age=3600", + "Content-Type": "image/png", + }), + }), + auditContext: "tlon-memex-upload", + capture: false, + maxRedirects: 0, + }); + const firstCall = mockGuardedFetch.mock.calls[0]?.[0]; + const firstBody = JSON.parse(String(firstCall?.init?.body)) as Record; + expect(firstBody).toMatchObject({ + token: "genuine-secret", + contentLength: 11, + contentType: "image/png", + }); + expect(typeof firstBody.fileName).toBe("string"); + expect(mockRelease).toHaveBeenCalledTimes(2); + }); + + it("surfaces guarded upload failures for hosted Memex targets", async () => { + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockRejectedValueOnce(new Error("Blocked upload target")); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Blocked upload target"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://uploads.tlon.network/put", + auditContext: "tlon-memex-upload", + capture: false, + maxRedirects: 0, + }), + ); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); + + it("rejects Memex upload targets outside the hosted Tlon domain allowlist", async () => { + mockGuardedFetch.mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://eviltlon.network/upload"), + "https://memex.tlon.network/v1/zod/upload", + ), + ); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Memex upload URL must target a trusted hosted Tlon domain"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); + + it("rejects Memex hosted result URLs outside the hosted Tlon domain allowlist", async () => { + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse( + "https://uploads.tlon.network/put", + "https://evil.example/files/uploaded.png", + ), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://uploads.tlon.network/put", + ), + ); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Memex hosted URL must target a trusted hosted Tlon domain"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledTimes(2); + expect(mockRelease).toHaveBeenCalledTimes(2); + }); + + it("rejects Memex upload targets with a non-standard port", async () => { + mockGuardedFetch.mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network:8443/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Memex upload URL must not specify a non-standard port"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); + + it("disables redirects for Memex upload targets", async () => { + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockRejectedValueOnce(new Error("Too many redirects (limit: 0)")); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Too many redirects (limit: 0)"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://uploads.tlon.network/put", + auditContext: "tlon-memex-upload", + capture: false, + maxRedirects: 0, + }), + ); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); + + it("routes scheme-less hosted ship URLs through the Memex upload path", async () => { + configureClient({ + shipUrl: "foo.tlon.network", + shipName: "~zod", + verbose: false, + getCode: async () => "123456", + }); + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://uploads.tlon.network/put", + ), + ); + + const result = await uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }); + + expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" }); + expect(mockGuardedFetch).toHaveBeenCalledTimes(2); + expect(mockRelease).toHaveBeenCalledTimes(2); + }); + + it("rejects truly unparseable ship URLs as not hosted", async () => { + configureClient({ + shipUrl: " ", + shipName: "~zod", + verbose: false, + getCode: async () => "123456", + }); + mockScryUrbitPath.mockImplementation(async (_deps, params) => { + if (params.path === "/storage/configuration.json") { + return { + currentBucket: "uploads", + buckets: ["uploads"], + publicUrlBase: "https://files.tlon.network/", + presignedUrl: "https://files.tlon.network/presigned", + region: "us-east-1", + service: "presigned-url", + }; + } + if (params.path === "/storage/credentials.json") { + return { "storage-update": {} }; + } + throw new Error(`Unexpected scry path: ${params.path}`); + }); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("No storage credentials configured"); + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).not.toHaveBeenCalled(); + expect(mockRelease).not.toHaveBeenCalled(); + }); + + it("accepts hosted Memex upload URLs with an explicit :443 port", async () => { + mockGuardedFetch + .mockResolvedValueOnce( + createGuardedResult( + createMemexResponse("https://uploads.tlon.network:443/put"), + "https://memex.tlon.network/v1/zod/upload", + ), + ) + .mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://uploads.tlon.network:443/put", + ), + ); + + const result = await uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }); + + expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" }); + expect(mockGuardedFetch).toHaveBeenCalledTimes(2); + expect(mockRelease).toHaveBeenCalledTimes(2); + }); + + it("disables redirects for the Memex upload URL lookup", async () => { + mockGuardedFetch.mockRejectedValueOnce(new Error("Too many redirects (limit: 0)")); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Too many redirects (limit: 0)"); + + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockGuardedFetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://memex.tlon.network/v1/zod/upload", + auditContext: "tlon-memex-upload-url", + capture: false, + maxRedirects: 0, + }), + ); + expect(mockRelease).not.toHaveBeenCalled(); + }); +}); + +describe("uploadFile custom S3 upload hardening", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", vi.fn()); + mockAuthenticate.mockResolvedValue("urbauth-~zod=fake-cookie"); + configureClient({ + shipUrl: "https://ship.example.com", + shipName: "~zod", + verbose: false, + getCode: async () => "123456", + }); + mockScryUrbitPath.mockImplementation(async (_deps, params) => { + if (params.path === "/storage/configuration.json") { + return { + currentBucket: "uploads", + buckets: ["uploads"], + publicUrlBase: "https://files.example.com/", + presignedUrl: "", + region: "us-east-1", + service: "custom", + }; + } + if (params.path === "/storage/credentials.json") { + return { + "storage-update": { + credentials: { + endpoint: "https://s3.example.com", + accessKeyId: "AKIAFAKE", + secretAccessKey: "fake-secret", + }, + }, + }; + } + throw new Error(`Unexpected scry path: ${params.path}`); + }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("routes the custom S3 signed URL through the SSRF guard", async () => { + mockGetSignedUrl.mockResolvedValueOnce("https://s3.example.com/uploads/file?sig=abc"); + mockGuardedFetch.mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://s3.example.com/uploads/file?sig=abc", + ), + ); + + const result = await uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }); + + expect(result.url.startsWith("https://files.example.com/")).toBe(true); + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockGuardedFetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://s3.example.com/uploads/file?sig=abc", + auditContext: "tlon-custom-s3-upload", + capture: false, + maxRedirects: 0, + }), + ); + expect(mockRelease).toHaveBeenCalledTimes(1); + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + }); + + it("surfaces guarded upload failures for custom S3 targets without calling release", async () => { + mockGetSignedUrl.mockResolvedValueOnce("https://169.254.169.254/uploads/file?sig=abc"); + mockGuardedFetch.mockRejectedValueOnce(new Error("Blocked private network target")); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Blocked private network target"); + + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockRelease).not.toHaveBeenCalled(); + expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled(); + }); + + it("passes the private-network opt-in to guarded custom S3 uploads", async () => { + configureClient({ + shipUrl: "https://ship.example.com", + shipName: "~zod", + verbose: false, + getCode: async () => "123456", + dangerouslyAllowPrivateNetwork: true, + }); + mockGetSignedUrl.mockResolvedValueOnce("https://10.0.0.15/uploads/file?sig=abc"); + mockGuardedFetch.mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://10.0.0.15/uploads/file?sig=abc", + ), + ); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).resolves.toEqual({ + url: expect.stringMatching(/^https:\/\/files\.example\.com\//), + }); + + expect(mockGuardedFetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://10.0.0.15/uploads/file?sig=abc", + auditContext: "tlon-custom-s3-upload", + capture: false, + maxRedirects: 0, + policy: { allowPrivateNetwork: true }, + }), + ); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); + + it("rejects custom S3 result URLs that are not http(s)", async () => { + mockScryUrbitPath.mockImplementation(async (_deps, params) => { + if (params.path === "/storage/configuration.json") { + return { + currentBucket: "uploads", + buckets: ["uploads"], + publicUrlBase: "ftp://files.example.com/", + presignedUrl: "", + region: "us-east-1", + service: "custom", + }; + } + if (params.path === "/storage/credentials.json") { + return { + "storage-update": { + credentials: { + endpoint: "https://s3.example.com", + accessKeyId: "AKIAFAKE", + secretAccessKey: "fake-secret", + }, + }, + }; + } + throw new Error(`Unexpected scry path: ${params.path}`); + }); + mockGetSignedUrl.mockResolvedValueOnce("https://s3.example.com/uploads/file?sig=abc"); + mockGuardedFetch.mockResolvedValueOnce( + createGuardedResult( + new Response(null, { status: 200 }), + "https://s3.example.com/uploads/file?sig=abc", + ), + ); + + await expect( + uploadFile({ + blob: new Blob(["image-bytes"], { type: "image/png" }), + fileName: "avatar.png", + contentType: "image/png", + }), + ).rejects.toThrow("Upload result URL must use http or https"); + + expect(mockGuardedFetch).toHaveBeenCalledTimes(1); + expect(mockRelease).toHaveBeenCalledTimes(1); + }); +}); diff --git a/extensions/tlon/src/tlon-api.ts b/extensions/tlon/src/tlon-api.ts index 01ba8718f4d..2e85bb2efe1 100644 --- a/extensions/tlon/src/tlon-api.ts +++ b/extensions/tlon/src/tlon-api.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { authenticate } from "./urbit/auth.js"; import { scryUrbitPath } from "./urbit/channel-ops.js"; @@ -94,13 +95,71 @@ function isStorageCredentials(value: unknown): value is StorageCredentials { ); } +function hostnameMatchesDomainBoundary(hostname: string, domain: string): boolean { + return hostname === domain || hostname.endsWith(`.${domain}`); +} + function isHostedShipUrl(shipUrl: string): boolean { - try { - const { hostname } = new URL(shipUrl); - return hostname.endsWith("tlon.network") || hostname.endsWith(".test.tlon.systems"); - } catch { - return shipUrl.endsWith("tlon.network") || shipUrl.endsWith(".test.tlon.systems"); + const hostname = extractShipHostname(shipUrl); + return hostname !== null && isHostedTlonHostname(hostname); +} + +function extractShipHostname(shipUrl: string): string | null { + const trimmed = shipUrl.trim(); + if (!trimmed) { + return null; } + const normalized = /^[a-zA-Z][\w+.-]*:\/\//.test(trimmed) ? trimmed : `https://${trimmed}`; + try { + return new URL(normalized).hostname; + } catch { + return null; + } +} + +function isHostedTlonHostname(hostname: string): boolean { + return ( + hostnameMatchesDomainBoundary(hostname, "tlon.network") || + hostnameMatchesDomainBoundary(hostname, "test.tlon.systems") + ); +} + +function assertTrustedMemexUploadUrl(rawUrl: string, label: string): string { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new Error(`${label} must be a valid https URL`); + } + + if (parsed.protocol !== "https:") { + throw new Error(`${label} must use https`); + } + + if (!isHostedTlonHostname(parsed.hostname)) { + throw new Error(`${label} must target a trusted hosted Tlon domain`); + } + + if (parsed.port && parsed.port !== "443") { + throw new Error(`${label} must not specify a non-standard port`); + } + + return parsed.toString(); +} + +function assertSafeUploadResultUrl(rawUrl: string, label: string): string { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch { + throw new Error(`${label} must be a valid http(s) URL`); + } + + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new Error(`${label} must use http or https`); + } + + return parsed.toString(); } function prefixEndpoint(endpoint: string): string { @@ -182,32 +241,46 @@ async function getMemexUploadUrl(params: { } const endpoint = `${MEMEX_BASE_URL}/v1/${params.config.shipName}/upload`; - const response = await fetch(endpoint, { - method: "PUT", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - token: resolvedToken, - contentLength: params.contentLength, - contentType: params.contentType, - fileName: params.fileName, - }), - }); + let release: (() => Promise) | undefined; + try { + const guarded = await fetchWithSsrFGuard({ + url: endpoint, + init: { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + token: resolvedToken, + contentLength: params.contentLength, + contentType: params.contentType, + fileName: params.fileName, + }), + }, + auditContext: "tlon-memex-upload-url", + capture: false, + maxRedirects: 0, + }); + release = guarded.release; + if (!guarded.response.ok) { + throw new Error(`Memex upload request failed: ${guarded.response.status}`); + } - if (!response.ok) { - throw new Error(`Memex upload request failed: ${response.status}`); + const data = (await guarded.response.json()) as { url?: string; filePath?: string } | null; + if (!data?.url || !data.filePath) { + throw new Error("Invalid response from Memex"); + } + + return { hostedUrl: data.filePath, uploadUrl: data.url }; + } finally { + await release?.(); } - - const data = (await response.json()) as { url?: string; filePath?: string } | null; - if (!data?.url || !data.filePath) { - throw new Error("Invalid response from Memex"); - } - - return { hostedUrl: data.filePath, uploadUrl: data.url }; } export async function uploadFile(params: UploadFileParams): Promise { const config = requireClientConfig(); const cookie = await getAuthCookie(config); + const privateNetworkPolicy = ssrfPolicyFromDangerouslyAllowPrivateNetwork( + config.dangerouslyAllowPrivateNetwork, + ); const [storageConfig, credentials] = await Promise.all([ getStorageConfiguration(config, cookie), @@ -231,21 +304,34 @@ export async function uploadFile(params: UploadFileParams): Promise Promise) | undefined; + try { + const guarded = await fetchWithSsrFGuard({ + url: trustedUploadUrl, + init: { + method: "PUT", + body: params.blob, + headers: { + "Cache-Control": "public, max-age=3600", + "Content-Type": contentType, + }, + }, + auditContext: "tlon-memex-upload", + capture: false, + maxRedirects: 0, + }); + release = guarded.release; + assertTrustedMemexUploadUrl(guarded.finalUrl, "Memex final upload URL"); + if (!guarded.response.ok) { + throw new Error(`Upload failed: ${guarded.response.status}`); + } + } finally { + await release?.(); } - return { url: hostedUrl }; + return { url: assertTrustedMemexUploadUrl(hostedUrl, "Memex hosted URL") }; } if (!hasCustomS3Creds(credentials)) { @@ -286,19 +372,31 @@ export async function uploadFile(params: UploadFileParams): Promise Promise) | undefined; + try { + const guarded = await fetchWithSsrFGuard({ + url: signedUrl, + init: { + method: "PUT", + body: params.blob, + headers: signedUrl.includes("digitaloceanspaces.com") ? headers : undefined, + }, + auditContext: "tlon-custom-s3-upload", + capture: false, + maxRedirects: 0, + policy: privateNetworkPolicy, + }); + release = guarded.release; + if (!guarded.response.ok) { + throw new Error(`Upload failed: ${guarded.response.status}`); + } + } finally { + await release?.(); } const publicUrl = storageConfig.publicUrlBase ? new URL(fileKey, storageConfig.publicUrlBase).toString() : signedUrl.split("?")[0]; - return { url: publicUrl }; + return { url: assertSafeUploadResultUrl(publicUrl, "Upload result URL") }; } diff --git a/scripts/check-no-raw-channel-fetch.mjs b/scripts/check-no-raw-channel-fetch.mjs index 5c9b8dcec8b..bd2d9907721 100644 --- a/scripts/check-no-raw-channel-fetch.mjs +++ b/scripts/check-no-raw-channel-fetch.mjs @@ -62,9 +62,6 @@ const allowedRawFetchCallsites = new Set([ bundledPluginCallsite("slack", "src/monitor/media.ts", 99), bundledPluginCallsite("slack", "src/monitor/media.ts", 118), bundledPluginCallsite("slack", "src/monitor/media.ts", 123), - bundledPluginCallsite("tlon", "src/tlon-api.ts", 185), - bundledPluginCallsite("tlon", "src/tlon-api.ts", 235), - bundledPluginCallsite("tlon", "src/tlon-api.ts", 289), bundledPluginCallsite("venice", "models.ts", 552), bundledPluginCallsite("vercel-ai-gateway", "models.ts", 181), bundledPluginCallsite("voice-call", "src/providers/twilio/api.ts", 23),