mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:30:42 +00:00
fix(tlon): guard memex upload target (#69794)
* fix(tlon): guard memex upload target * fix(tlon): harden guarded memex upload * fix(tlon): validate hosted memex upload targets * fix(tlon): tighten hosted domain matching * fix(tlon): reject non-standard memex upload ports * fix(tlon): disable memex upload redirects * test(tlon): drop redundant mock resets in memex upload test * chore(lint): update tlon raw-fetch allowlist for guarded memex upload * fix(tlon): reject unparseable ship URLs in hosted-ship classifier * fix(lint): point tlon raw-fetch allowlist at fetch callee lines * fix(tlon): guard custom-S3 upload through fetchWithSsrFGuard * fix(tlon): preserve scheme-less hosted ship routing and allow explicit :443 * docs(changelog): note tlon upload guard * fix(tlon): guard memex lookup and private s3 opt-in * fix(tlon): validate upload result URLs
This commit is contained in:
@@ -37,6 +37,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Ollama/media understanding: register Ollama as an image-capable media-understanding provider so `agents.defaults.imageModel.primary` values like `ollama/qwen2.5vl:7b` route through the Ollama plugin instead of failing as unknown models. (#69816) Thanks @soloclz.
|
||||
- CLI/media understanding: make `openclaw infer image describe --model <provider/model>` execute the explicit image model instead of skipping description when that model supports native vision.
|
||||
- Usage/providers: keep plugin-owned usage auth enabled when manifest-declared provider auth env vars such as `MINIMAX_CODE_PLAN_KEY` are present, so `/usage` can resolve MiniMax billing credentials through the provider plugin.
|
||||
- Tlon/uploads: route both hosted Memex upload targets and custom-S3 presigned upload URLs through the shared SSRF guard so blocked private or loopback destinations fail before upload, while public upload URLs continue through the existing hosted upload flow. (#69794) Thanks @drobison00.
|
||||
|
||||
## 2026.4.20
|
||||
|
||||
|
||||
587
extensions/tlon/src/tlon-api.test.ts
Normal file
587
extensions/tlon/src/tlon-api.test.ts
Normal file
@@ -0,0 +1,587 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { authenticate } from "./urbit/auth.js";
|
||||
import { scryUrbitPath } from "./urbit/channel-ops.js";
|
||||
|
||||
const { mockFetchGuard, mockRelease, mockGetSignedUrl } = vi.hoisted(() => ({
|
||||
mockFetchGuard: vi.fn(),
|
||||
mockRelease: vi.fn(async () => {}),
|
||||
mockGetSignedUrl: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => {
|
||||
const original = (await vi.importActual("openclaw/plugin-sdk/ssrf-runtime")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
return {
|
||||
...original,
|
||||
fetchWithSsrFGuard: mockFetchGuard,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@aws-sdk/s3-request-presigner", () => ({
|
||||
getSignedUrl: mockGetSignedUrl,
|
||||
}));
|
||||
|
||||
vi.mock("./urbit/auth.js", () => ({
|
||||
authenticate: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./urbit/channel-ops.js", () => ({
|
||||
scryUrbitPath: vi.fn(),
|
||||
}));
|
||||
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { configureClient, uploadFile } from "./tlon-api.js";
|
||||
|
||||
const mockAuthenticate = vi.mocked(authenticate);
|
||||
const mockScryUrbitPath = vi.mocked(scryUrbitPath);
|
||||
const mockGuardedFetch = vi.mocked(fetchWithSsrFGuard);
|
||||
|
||||
function createMemexResponse(
|
||||
uploadUrl: string,
|
||||
filePath = "https://memex.tlon.network/files/uploaded.png",
|
||||
): Response {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
url: uploadUrl,
|
||||
filePath,
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function createGuardedResult(response: Response, finalUrl: string) {
|
||||
return {
|
||||
response,
|
||||
finalUrl,
|
||||
release: mockRelease,
|
||||
};
|
||||
}
|
||||
|
||||
describe("uploadFile memex upload hardening", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", vi.fn());
|
||||
mockAuthenticate.mockResolvedValue("urbauth-~zod=fake-cookie");
|
||||
configureClient({
|
||||
shipUrl: "https://groups.tlon.network",
|
||||
shipName: "~zod",
|
||||
verbose: false,
|
||||
getCode: async () => "123456",
|
||||
});
|
||||
mockScryUrbitPath.mockImplementation(async (_deps, params) => {
|
||||
if (params.path === "/storage/configuration.json") {
|
||||
return {
|
||||
currentBucket: "uploads",
|
||||
buckets: ["uploads"],
|
||||
publicUrlBase: "https://files.tlon.network/",
|
||||
presignedUrl: "https://files.tlon.network/presigned",
|
||||
region: "us-east-1",
|
||||
service: "presigned-url",
|
||||
};
|
||||
}
|
||||
if (params.path === "/storage/credentials.json") {
|
||||
return { "storage-update": {} };
|
||||
}
|
||||
if (params.path === "/genuine/secret.json") {
|
||||
return { secret: "genuine-secret" };
|
||||
}
|
||||
throw new Error(`Unexpected scry path: ${params.path}`);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("routes the memex upload URL through the SSRF guard", async () => {
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://uploads.tlon.network/put",
|
||||
),
|
||||
);
|
||||
|
||||
const result = await uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" });
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(2);
|
||||
expect(mockGuardedFetch).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://memex.tlon.network/v1/zod/upload",
|
||||
init: expect.objectContaining({
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: expect.any(String),
|
||||
}),
|
||||
auditContext: "tlon-memex-upload-url",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
}),
|
||||
);
|
||||
expect(mockGuardedFetch).toHaveBeenNthCalledWith(2, {
|
||||
url: "https://uploads.tlon.network/put",
|
||||
init: expect.objectContaining({
|
||||
method: "PUT",
|
||||
body: expect.any(Blob),
|
||||
headers: expect.objectContaining({
|
||||
"Cache-Control": "public, max-age=3600",
|
||||
"Content-Type": "image/png",
|
||||
}),
|
||||
}),
|
||||
auditContext: "tlon-memex-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
});
|
||||
const firstCall = mockGuardedFetch.mock.calls[0]?.[0];
|
||||
const firstBody = JSON.parse(String(firstCall?.init?.body)) as Record<string, unknown>;
|
||||
expect(firstBody).toMatchObject({
|
||||
token: "genuine-secret",
|
||||
contentLength: 11,
|
||||
contentType: "image/png",
|
||||
});
|
||||
expect(typeof firstBody.fileName).toBe("string");
|
||||
expect(mockRelease).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("surfaces guarded upload failures for hosted Memex targets", async () => {
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockRejectedValueOnce(new Error("Blocked upload target"));
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Blocked upload target");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://uploads.tlon.network/put",
|
||||
auditContext: "tlon-memex-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
}),
|
||||
);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects Memex upload targets outside the hosted Tlon domain allowlist", async () => {
|
||||
mockGuardedFetch.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://eviltlon.network/upload"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Memex upload URL must target a trusted hosted Tlon domain");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects Memex hosted result URLs outside the hosted Tlon domain allowlist", async () => {
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse(
|
||||
"https://uploads.tlon.network/put",
|
||||
"https://evil.example/files/uploaded.png",
|
||||
),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://uploads.tlon.network/put",
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Memex hosted URL must target a trusted hosted Tlon domain");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(2);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rejects Memex upload targets with a non-standard port", async () => {
|
||||
mockGuardedFetch.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network:8443/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Memex upload URL must not specify a non-standard port");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("disables redirects for Memex upload targets", async () => {
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockRejectedValueOnce(new Error("Too many redirects (limit: 0)"));
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Too many redirects (limit: 0)");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://uploads.tlon.network/put",
|
||||
auditContext: "tlon-memex-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
}),
|
||||
);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("routes scheme-less hosted ship URLs through the Memex upload path", async () => {
|
||||
configureClient({
|
||||
shipUrl: "foo.tlon.network",
|
||||
shipName: "~zod",
|
||||
verbose: false,
|
||||
getCode: async () => "123456",
|
||||
});
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://uploads.tlon.network/put",
|
||||
),
|
||||
);
|
||||
|
||||
const result = await uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" });
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(2);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rejects truly unparseable ship URLs as not hosted", async () => {
|
||||
configureClient({
|
||||
shipUrl: " ",
|
||||
shipName: "~zod",
|
||||
verbose: false,
|
||||
getCode: async () => "123456",
|
||||
});
|
||||
mockScryUrbitPath.mockImplementation(async (_deps, params) => {
|
||||
if (params.path === "/storage/configuration.json") {
|
||||
return {
|
||||
currentBucket: "uploads",
|
||||
buckets: ["uploads"],
|
||||
publicUrlBase: "https://files.tlon.network/",
|
||||
presignedUrl: "https://files.tlon.network/presigned",
|
||||
region: "us-east-1",
|
||||
service: "presigned-url",
|
||||
};
|
||||
}
|
||||
if (params.path === "/storage/credentials.json") {
|
||||
return { "storage-update": {} };
|
||||
}
|
||||
throw new Error(`Unexpected scry path: ${params.path}`);
|
||||
});
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("No storage credentials configured");
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).not.toHaveBeenCalled();
|
||||
expect(mockRelease).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts hosted Memex upload URLs with an explicit :443 port", async () => {
|
||||
mockGuardedFetch
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
createMemexResponse("https://uploads.tlon.network:443/put"),
|
||||
"https://memex.tlon.network/v1/zod/upload",
|
||||
),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://uploads.tlon.network:443/put",
|
||||
),
|
||||
);
|
||||
|
||||
const result = await uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ url: "https://memex.tlon.network/files/uploaded.png" });
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(2);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("disables redirects for the Memex upload URL lookup", async () => {
|
||||
mockGuardedFetch.mockRejectedValueOnce(new Error("Too many redirects (limit: 0)"));
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Too many redirects (limit: 0)");
|
||||
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockGuardedFetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://memex.tlon.network/v1/zod/upload",
|
||||
auditContext: "tlon-memex-upload-url",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
}),
|
||||
);
|
||||
expect(mockRelease).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("uploadFile custom S3 upload hardening", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.stubGlobal("fetch", vi.fn());
|
||||
mockAuthenticate.mockResolvedValue("urbauth-~zod=fake-cookie");
|
||||
configureClient({
|
||||
shipUrl: "https://ship.example.com",
|
||||
shipName: "~zod",
|
||||
verbose: false,
|
||||
getCode: async () => "123456",
|
||||
});
|
||||
mockScryUrbitPath.mockImplementation(async (_deps, params) => {
|
||||
if (params.path === "/storage/configuration.json") {
|
||||
return {
|
||||
currentBucket: "uploads",
|
||||
buckets: ["uploads"],
|
||||
publicUrlBase: "https://files.example.com/",
|
||||
presignedUrl: "",
|
||||
region: "us-east-1",
|
||||
service: "custom",
|
||||
};
|
||||
}
|
||||
if (params.path === "/storage/credentials.json") {
|
||||
return {
|
||||
"storage-update": {
|
||||
credentials: {
|
||||
endpoint: "https://s3.example.com",
|
||||
accessKeyId: "AKIAFAKE",
|
||||
secretAccessKey: "fake-secret",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected scry path: ${params.path}`);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("routes the custom S3 signed URL through the SSRF guard", async () => {
|
||||
mockGetSignedUrl.mockResolvedValueOnce("https://s3.example.com/uploads/file?sig=abc");
|
||||
mockGuardedFetch.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://s3.example.com/uploads/file?sig=abc",
|
||||
),
|
||||
);
|
||||
|
||||
const result = await uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
});
|
||||
|
||||
expect(result.url.startsWith("https://files.example.com/")).toBe(true);
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockGuardedFetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://s3.example.com/uploads/file?sig=abc",
|
||||
auditContext: "tlon-custom-s3-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
}),
|
||||
);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("surfaces guarded upload failures for custom S3 targets without calling release", async () => {
|
||||
mockGetSignedUrl.mockResolvedValueOnce("https://169.254.169.254/uploads/file?sig=abc");
|
||||
mockGuardedFetch.mockRejectedValueOnce(new Error("Blocked private network target"));
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Blocked private network target");
|
||||
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockRelease).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(globalThis.fetch)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes the private-network opt-in to guarded custom S3 uploads", async () => {
|
||||
configureClient({
|
||||
shipUrl: "https://ship.example.com",
|
||||
shipName: "~zod",
|
||||
verbose: false,
|
||||
getCode: async () => "123456",
|
||||
dangerouslyAllowPrivateNetwork: true,
|
||||
});
|
||||
mockGetSignedUrl.mockResolvedValueOnce("https://10.0.0.15/uploads/file?sig=abc");
|
||||
mockGuardedFetch.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://10.0.0.15/uploads/file?sig=abc",
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
url: expect.stringMatching(/^https:\/\/files\.example\.com\//),
|
||||
});
|
||||
|
||||
expect(mockGuardedFetch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://10.0.0.15/uploads/file?sig=abc",
|
||||
auditContext: "tlon-custom-s3-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
policy: { allowPrivateNetwork: true },
|
||||
}),
|
||||
);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects custom S3 result URLs that are not http(s)", async () => {
|
||||
mockScryUrbitPath.mockImplementation(async (_deps, params) => {
|
||||
if (params.path === "/storage/configuration.json") {
|
||||
return {
|
||||
currentBucket: "uploads",
|
||||
buckets: ["uploads"],
|
||||
publicUrlBase: "ftp://files.example.com/",
|
||||
presignedUrl: "",
|
||||
region: "us-east-1",
|
||||
service: "custom",
|
||||
};
|
||||
}
|
||||
if (params.path === "/storage/credentials.json") {
|
||||
return {
|
||||
"storage-update": {
|
||||
credentials: {
|
||||
endpoint: "https://s3.example.com",
|
||||
accessKeyId: "AKIAFAKE",
|
||||
secretAccessKey: "fake-secret",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected scry path: ${params.path}`);
|
||||
});
|
||||
mockGetSignedUrl.mockResolvedValueOnce("https://s3.example.com/uploads/file?sig=abc");
|
||||
mockGuardedFetch.mockResolvedValueOnce(
|
||||
createGuardedResult(
|
||||
new Response(null, { status: 200 }),
|
||||
"https://s3.example.com/uploads/file?sig=abc",
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
uploadFile({
|
||||
blob: new Blob(["image-bytes"], { type: "image/png" }),
|
||||
fileName: "avatar.png",
|
||||
contentType: "image/png",
|
||||
}),
|
||||
).rejects.toThrow("Upload result URL must use http or https");
|
||||
|
||||
expect(mockGuardedFetch).toHaveBeenCalledTimes(1);
|
||||
expect(mockRelease).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import crypto from "node:crypto";
|
||||
import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { authenticate } from "./urbit/auth.js";
|
||||
import { scryUrbitPath } from "./urbit/channel-ops.js";
|
||||
@@ -94,13 +95,71 @@ function isStorageCredentials(value: unknown): value is StorageCredentials {
|
||||
);
|
||||
}
|
||||
|
||||
function hostnameMatchesDomainBoundary(hostname: string, domain: string): boolean {
|
||||
return hostname === domain || hostname.endsWith(`.${domain}`);
|
||||
}
|
||||
|
||||
function isHostedShipUrl(shipUrl: string): boolean {
|
||||
try {
|
||||
const { hostname } = new URL(shipUrl);
|
||||
return hostname.endsWith("tlon.network") || hostname.endsWith(".test.tlon.systems");
|
||||
} catch {
|
||||
return shipUrl.endsWith("tlon.network") || shipUrl.endsWith(".test.tlon.systems");
|
||||
const hostname = extractShipHostname(shipUrl);
|
||||
return hostname !== null && isHostedTlonHostname(hostname);
|
||||
}
|
||||
|
||||
function extractShipHostname(shipUrl: string): string | null {
|
||||
const trimmed = shipUrl.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
const normalized = /^[a-zA-Z][\w+.-]*:\/\//.test(trimmed) ? trimmed : `https://${trimmed}`;
|
||||
try {
|
||||
return new URL(normalized).hostname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isHostedTlonHostname(hostname: string): boolean {
|
||||
return (
|
||||
hostnameMatchesDomainBoundary(hostname, "tlon.network") ||
|
||||
hostnameMatchesDomainBoundary(hostname, "test.tlon.systems")
|
||||
);
|
||||
}
|
||||
|
||||
function assertTrustedMemexUploadUrl(rawUrl: string, label: string): string {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new Error(`${label} must be a valid https URL`);
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "https:") {
|
||||
throw new Error(`${label} must use https`);
|
||||
}
|
||||
|
||||
if (!isHostedTlonHostname(parsed.hostname)) {
|
||||
throw new Error(`${label} must target a trusted hosted Tlon domain`);
|
||||
}
|
||||
|
||||
if (parsed.port && parsed.port !== "443") {
|
||||
throw new Error(`${label} must not specify a non-standard port`);
|
||||
}
|
||||
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
function assertSafeUploadResultUrl(rawUrl: string, label: string): string {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new Error(`${label} must be a valid http(s) URL`);
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error(`${label} must use http or https`);
|
||||
}
|
||||
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
function prefixEndpoint(endpoint: string): string {
|
||||
@@ -182,32 +241,46 @@ async function getMemexUploadUrl(params: {
|
||||
}
|
||||
|
||||
const endpoint = `${MEMEX_BASE_URL}/v1/${params.config.shipName}/upload`;
|
||||
const response = await fetch(endpoint, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
token: resolvedToken,
|
||||
contentLength: params.contentLength,
|
||||
contentType: params.contentType,
|
||||
fileName: params.fileName,
|
||||
}),
|
||||
});
|
||||
let release: (() => Promise<void>) | undefined;
|
||||
try {
|
||||
const guarded = await fetchWithSsrFGuard({
|
||||
url: endpoint,
|
||||
init: {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
token: resolvedToken,
|
||||
contentLength: params.contentLength,
|
||||
contentType: params.contentType,
|
||||
fileName: params.fileName,
|
||||
}),
|
||||
},
|
||||
auditContext: "tlon-memex-upload-url",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
});
|
||||
release = guarded.release;
|
||||
if (!guarded.response.ok) {
|
||||
throw new Error(`Memex upload request failed: ${guarded.response.status}`);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Memex upload request failed: ${response.status}`);
|
||||
const data = (await guarded.response.json()) as { url?: string; filePath?: string } | null;
|
||||
if (!data?.url || !data.filePath) {
|
||||
throw new Error("Invalid response from Memex");
|
||||
}
|
||||
|
||||
return { hostedUrl: data.filePath, uploadUrl: data.url };
|
||||
} finally {
|
||||
await release?.();
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { url?: string; filePath?: string } | null;
|
||||
if (!data?.url || !data.filePath) {
|
||||
throw new Error("Invalid response from Memex");
|
||||
}
|
||||
|
||||
return { hostedUrl: data.filePath, uploadUrl: data.url };
|
||||
}
|
||||
|
||||
export async function uploadFile(params: UploadFileParams): Promise<UploadResult> {
|
||||
const config = requireClientConfig();
|
||||
const cookie = await getAuthCookie(config);
|
||||
const privateNetworkPolicy = ssrfPolicyFromDangerouslyAllowPrivateNetwork(
|
||||
config.dangerouslyAllowPrivateNetwork,
|
||||
);
|
||||
|
||||
const [storageConfig, credentials] = await Promise.all([
|
||||
getStorageConfiguration(config, cookie),
|
||||
@@ -231,21 +304,34 @@ export async function uploadFile(params: UploadFileParams): Promise<UploadResult
|
||||
contentType,
|
||||
fileName: fileKey,
|
||||
});
|
||||
const trustedUploadUrl = assertTrustedMemexUploadUrl(uploadUrl, "Memex upload URL");
|
||||
|
||||
const response = await fetch(uploadUrl, {
|
||||
method: "PUT",
|
||||
body: params.blob,
|
||||
headers: {
|
||||
"Cache-Control": "public, max-age=3600",
|
||||
"Content-Type": contentType,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Upload failed: ${response.status}`);
|
||||
let release: (() => Promise<void>) | undefined;
|
||||
try {
|
||||
const guarded = await fetchWithSsrFGuard({
|
||||
url: trustedUploadUrl,
|
||||
init: {
|
||||
method: "PUT",
|
||||
body: params.blob,
|
||||
headers: {
|
||||
"Cache-Control": "public, max-age=3600",
|
||||
"Content-Type": contentType,
|
||||
},
|
||||
},
|
||||
auditContext: "tlon-memex-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
});
|
||||
release = guarded.release;
|
||||
assertTrustedMemexUploadUrl(guarded.finalUrl, "Memex final upload URL");
|
||||
if (!guarded.response.ok) {
|
||||
throw new Error(`Upload failed: ${guarded.response.status}`);
|
||||
}
|
||||
} finally {
|
||||
await release?.();
|
||||
}
|
||||
|
||||
return { url: hostedUrl };
|
||||
return { url: assertTrustedMemexUploadUrl(hostedUrl, "Memex hosted URL") };
|
||||
}
|
||||
|
||||
if (!hasCustomS3Creds(credentials)) {
|
||||
@@ -286,19 +372,31 @@ export async function uploadFile(params: UploadFileParams): Promise<UploadResult
|
||||
signableHeaders: new Set(Object.keys(headers)),
|
||||
});
|
||||
|
||||
const response = await fetch(signedUrl, {
|
||||
method: "PUT",
|
||||
body: params.blob,
|
||||
headers: signedUrl.includes("digitaloceanspaces.com") ? headers : undefined,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Upload failed: ${response.status}`);
|
||||
let release: (() => Promise<void>) | undefined;
|
||||
try {
|
||||
const guarded = await fetchWithSsrFGuard({
|
||||
url: signedUrl,
|
||||
init: {
|
||||
method: "PUT",
|
||||
body: params.blob,
|
||||
headers: signedUrl.includes("digitaloceanspaces.com") ? headers : undefined,
|
||||
},
|
||||
auditContext: "tlon-custom-s3-upload",
|
||||
capture: false,
|
||||
maxRedirects: 0,
|
||||
policy: privateNetworkPolicy,
|
||||
});
|
||||
release = guarded.release;
|
||||
if (!guarded.response.ok) {
|
||||
throw new Error(`Upload failed: ${guarded.response.status}`);
|
||||
}
|
||||
} finally {
|
||||
await release?.();
|
||||
}
|
||||
|
||||
const publicUrl = storageConfig.publicUrlBase
|
||||
? new URL(fileKey, storageConfig.publicUrlBase).toString()
|
||||
: signedUrl.split("?")[0];
|
||||
|
||||
return { url: publicUrl };
|
||||
return { url: assertSafeUploadResultUrl(publicUrl, "Upload result URL") };
|
||||
}
|
||||
|
||||
@@ -62,9 +62,6 @@ const allowedRawFetchCallsites = new Set([
|
||||
bundledPluginCallsite("slack", "src/monitor/media.ts", 99),
|
||||
bundledPluginCallsite("slack", "src/monitor/media.ts", 118),
|
||||
bundledPluginCallsite("slack", "src/monitor/media.ts", 123),
|
||||
bundledPluginCallsite("tlon", "src/tlon-api.ts", 185),
|
||||
bundledPluginCallsite("tlon", "src/tlon-api.ts", 235),
|
||||
bundledPluginCallsite("tlon", "src/tlon-api.ts", 289),
|
||||
bundledPluginCallsite("venice", "models.ts", 552),
|
||||
bundledPluginCallsite("vercel-ai-gateway", "models.ts", 181),
|
||||
bundledPluginCallsite("voice-call", "src/providers/twilio/api.ts", 23),
|
||||
|
||||
Reference in New Issue
Block a user