mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-12 07:20:45 +00:00
refactor: extract archive staging helpers
This commit is contained in:
Peter Steinberger
218
src/infra/archive-staging.ts
Normal file
218
src/infra/archive-staging.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { copyFileWithinRoot } from "./fs-safe.js";
|
||||
import { isNotFoundPathError, isPathInside } from "./path-guards.js";
|
||||
|
||||
const ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK = "archive entry traverses symlink in destination";
|
||||
|
||||
export type ArchiveSecurityErrorCode =
|
||||
| "destination-not-directory"
|
||||
| "destination-symlink"
|
||||
| "destination-symlink-traversal";
|
||||
|
||||
export class ArchiveSecurityError extends Error {
|
||||
code: ArchiveSecurityErrorCode;
|
||||
|
||||
constructor(code: ArchiveSecurityErrorCode, message: string, options?: ErrorOptions) {
|
||||
super(message, options);
|
||||
this.code = code;
|
||||
this.name = "ArchiveSecurityError";
|
||||
}
|
||||
}
|
||||
|
||||
function symlinkTraversalError(originalPath: string): ArchiveSecurityError {
|
||||
return new ArchiveSecurityError(
|
||||
"destination-symlink-traversal",
|
||||
`${ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK}: ${originalPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function prepareArchiveDestinationDir(destDir: string): Promise<string> {
|
||||
const stat = await fs.lstat(destDir);
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new ArchiveSecurityError("destination-symlink", "archive destination is a symlink");
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
throw new ArchiveSecurityError(
|
||||
"destination-not-directory",
|
||||
"archive destination is not a directory",
|
||||
);
|
||||
}
|
||||
return await fs.realpath(destDir);
|
||||
}
|
||||
|
||||
async function assertNoSymlinkTraversal(params: {
|
||||
rootDir: string;
|
||||
relPath: string;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
const parts = params.relPath.split(/[\\/]+/).filter(Boolean);
|
||||
let current = path.resolve(params.rootDir);
|
||||
for (const part of parts) {
|
||||
current = path.join(current, part);
|
||||
let stat: Awaited<ReturnType<typeof fs.lstat>>;
|
||||
try {
|
||||
stat = await fs.lstat(current);
|
||||
} catch (err) {
|
||||
if (isNotFoundPathError(err)) {
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw symlinkTraversalError(params.originalPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function assertResolvedInsideDestination(params: {
|
||||
destinationRealDir: string;
|
||||
targetPath: string;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = await fs.realpath(params.targetPath);
|
||||
} catch (err) {
|
||||
if (isNotFoundPathError(err)) {
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!isPathInside(params.destinationRealDir, resolved)) {
|
||||
throw symlinkTraversalError(params.originalPath);
|
||||
}
|
||||
}
|
||||
|
||||
export async function prepareArchiveOutputPath(params: {
|
||||
destinationDir: string;
|
||||
destinationRealDir: string;
|
||||
relPath: string;
|
||||
outPath: string;
|
||||
originalPath: string;
|
||||
isDirectory: boolean;
|
||||
}): Promise<void> {
|
||||
await assertNoSymlinkTraversal({
|
||||
rootDir: params.destinationDir,
|
||||
relPath: params.relPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
|
||||
if (params.isDirectory) {
|
||||
await fs.mkdir(params.outPath, { recursive: true });
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: params.outPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const parentDir = path.dirname(params.outPath);
|
||||
await fs.mkdir(parentDir, { recursive: true });
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: parentDir,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
}
|
||||
|
||||
async function applyStagedEntryMode(params: {
|
||||
destinationRealDir: string;
|
||||
relPath: string;
|
||||
mode: number;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
const destinationPath = path.join(params.destinationRealDir, params.relPath);
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: destinationPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
if (params.mode !== 0) {
|
||||
await fs.chmod(destinationPath, params.mode).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function withStagedArchiveDestination<T>(params: {
|
||||
destinationRealDir: string;
|
||||
run: (stagingDir: string) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const stagingDir = await fs.mkdtemp(path.join(params.destinationRealDir, ".openclaw-archive-"));
|
||||
try {
|
||||
return await params.run(stagingDir);
|
||||
} finally {
|
||||
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function mergeExtractedTreeIntoDestination(params: {
|
||||
sourceDir: string;
|
||||
destinationDir: string;
|
||||
destinationRealDir: string;
|
||||
}): Promise<void> {
|
||||
const walk = async (currentSourceDir: string): Promise<void> => {
|
||||
const entries = await fs.readdir(currentSourceDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const sourcePath = path.join(currentSourceDir, entry.name);
|
||||
const relPath = path.relative(params.sourceDir, sourcePath);
|
||||
const originalPath = relPath.split(path.sep).join("/");
|
||||
const destinationPath = path.join(params.destinationDir, relPath);
|
||||
const sourceStat = await fs.lstat(sourcePath);
|
||||
|
||||
if (sourceStat.isSymbolicLink()) {
|
||||
throw symlinkTraversalError(originalPath);
|
||||
}
|
||||
|
||||
if (sourceStat.isDirectory()) {
|
||||
await prepareArchiveOutputPath({
|
||||
destinationDir: params.destinationDir,
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
outPath: destinationPath,
|
||||
originalPath,
|
||||
isDirectory: true,
|
||||
});
|
||||
await walk(sourcePath);
|
||||
await applyStagedEntryMode({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
mode: sourceStat.mode & 0o777,
|
||||
originalPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!sourceStat.isFile()) {
|
||||
throw new Error(`archive staging contains unsupported entry: ${originalPath}`);
|
||||
}
|
||||
|
||||
await prepareArchiveOutputPath({
|
||||
destinationDir: params.destinationDir,
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
outPath: destinationPath,
|
||||
originalPath,
|
||||
isDirectory: false,
|
||||
});
|
||||
await copyFileWithinRoot({
|
||||
sourcePath,
|
||||
rootDir: params.destinationRealDir,
|
||||
relativePath: relPath,
|
||||
mkdir: true,
|
||||
});
|
||||
await applyStagedEntryMode({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
mode: sourceStat.mode & 0o777,
|
||||
originalPath,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
await walk(params.sourceDir);
|
||||
}
|
||||
|
||||
export function createArchiveSymlinkTraversalError(originalPath: string): ArchiveSecurityError {
|
||||
return symlinkTraversalError(originalPath);
|
||||
}
|
||||
@@ -13,14 +13,16 @@ import {
|
||||
stripArchivePath,
|
||||
validateArchiveEntryPath,
|
||||
} from "./archive-path.js";
|
||||
import { sameFileIdentity } from "./file-identity.js";
|
||||
import {
|
||||
copyFileWithinRoot,
|
||||
openFileWithinRoot,
|
||||
openWritableFileWithinRoot,
|
||||
SafeOpenError,
|
||||
} from "./fs-safe.js";
|
||||
import { isNotFoundPathError, isPathInside } from "./path-guards.js";
|
||||
createArchiveSymlinkTraversalError,
|
||||
mergeExtractedTreeIntoDestination,
|
||||
prepareArchiveDestinationDir,
|
||||
prepareArchiveOutputPath,
|
||||
withStagedArchiveDestination,
|
||||
} from "./archive-staging.js";
|
||||
import { sameFileIdentity } from "./file-identity.js";
|
||||
import { openFileWithinRoot, openWritableFileWithinRoot, SafeOpenError } from "./fs-safe.js";
|
||||
import { isNotFoundPathError } from "./path-guards.js";
|
||||
|
||||
export type ArchiveKind = "tar" | "zip";
|
||||
|
||||
@@ -42,20 +44,13 @@ export type ArchiveExtractLimits = {
|
||||
maxEntryBytes?: number;
|
||||
};
|
||||
|
||||
export type ArchiveSecurityErrorCode =
|
||||
| "destination-not-directory"
|
||||
| "destination-symlink"
|
||||
| "destination-symlink-traversal";
|
||||
|
||||
export class ArchiveSecurityError extends Error {
|
||||
code: ArchiveSecurityErrorCode;
|
||||
|
||||
constructor(code: ArchiveSecurityErrorCode, message: string, options?: ErrorOptions) {
|
||||
super(message, options);
|
||||
this.code = code;
|
||||
this.name = "ArchiveSecurityError";
|
||||
}
|
||||
}
|
||||
export { ArchiveSecurityError, type ArchiveSecurityErrorCode } from "./archive-staging.js";
|
||||
export {
|
||||
mergeExtractedTreeIntoDestination,
|
||||
prepareArchiveDestinationDir,
|
||||
prepareArchiveOutputPath,
|
||||
withStagedArchiveDestination,
|
||||
} from "./archive-staging.js";
|
||||
|
||||
/** @internal */
|
||||
export const DEFAULT_MAX_ARCHIVE_BYTES_ZIP = 256 * 1024 * 1024;
|
||||
@@ -71,7 +66,6 @@ const ERROR_ARCHIVE_ENTRY_COUNT_EXCEEDS_LIMIT = "archive entry count exceeds lim
|
||||
const ERROR_ARCHIVE_ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT =
|
||||
"archive entry extracted size exceeds limit";
|
||||
const ERROR_ARCHIVE_EXTRACTED_SIZE_EXCEEDS_LIMIT = "archive extracted size exceeds limit";
|
||||
const ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK = "archive entry traverses symlink in destination";
|
||||
const SUPPORTS_NOFOLLOW = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants;
|
||||
const OPEN_WRITE_CREATE_FLAGS =
|
||||
fsConstants.O_WRONLY |
|
||||
@@ -222,197 +216,8 @@ function createExtractBudgetTransform(params: {
|
||||
});
|
||||
}
|
||||
|
||||
function symlinkTraversalError(originalPath: string): ArchiveSecurityError {
|
||||
return new ArchiveSecurityError(
|
||||
"destination-symlink-traversal",
|
||||
`${ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK}: ${originalPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function prepareArchiveDestinationDir(destDir: string): Promise<string> {
|
||||
const stat = await fs.lstat(destDir);
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new ArchiveSecurityError("destination-symlink", "archive destination is a symlink");
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
throw new ArchiveSecurityError(
|
||||
"destination-not-directory",
|
||||
"archive destination is not a directory",
|
||||
);
|
||||
}
|
||||
return await fs.realpath(destDir);
|
||||
}
|
||||
|
||||
async function assertNoSymlinkTraversal(params: {
|
||||
rootDir: string;
|
||||
relPath: string;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
const parts = params.relPath.split(/[\\/]+/).filter(Boolean);
|
||||
let current = path.resolve(params.rootDir);
|
||||
for (const part of parts) {
|
||||
current = path.join(current, part);
|
||||
let stat: Awaited<ReturnType<typeof fs.lstat>>;
|
||||
try {
|
||||
stat = await fs.lstat(current);
|
||||
} catch (err) {
|
||||
if (isNotFoundPathError(err)) {
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw symlinkTraversalError(params.originalPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function assertResolvedInsideDestination(params: {
|
||||
destinationRealDir: string;
|
||||
targetPath: string;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = await fs.realpath(params.targetPath);
|
||||
} catch (err) {
|
||||
if (isNotFoundPathError(err)) {
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!isPathInside(params.destinationRealDir, resolved)) {
|
||||
throw symlinkTraversalError(params.originalPath);
|
||||
}
|
||||
}
|
||||
|
||||
async function prepareArchiveOutputPath(params: {
|
||||
destinationDir: string;
|
||||
destinationRealDir: string;
|
||||
relPath: string;
|
||||
outPath: string;
|
||||
originalPath: string;
|
||||
isDirectory: boolean;
|
||||
}): Promise<void> {
|
||||
await assertNoSymlinkTraversal({
|
||||
rootDir: params.destinationDir,
|
||||
relPath: params.relPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
|
||||
if (params.isDirectory) {
|
||||
await fs.mkdir(params.outPath, { recursive: true });
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: params.outPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const parentDir = path.dirname(params.outPath);
|
||||
await fs.mkdir(parentDir, { recursive: true });
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: parentDir,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
}
|
||||
|
||||
async function applyStagedEntryMode(params: {
|
||||
destinationRealDir: string;
|
||||
relPath: string;
|
||||
mode: number;
|
||||
originalPath: string;
|
||||
}): Promise<void> {
|
||||
const destinationPath = path.join(params.destinationRealDir, params.relPath);
|
||||
await assertResolvedInsideDestination({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
targetPath: destinationPath,
|
||||
originalPath: params.originalPath,
|
||||
});
|
||||
if (params.mode !== 0) {
|
||||
await fs.chmod(destinationPath, params.mode).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function withStagedArchiveDestination<T>(params: {
|
||||
destinationRealDir: string;
|
||||
run: (stagingDir: string) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const stagingDir = await fs.mkdtemp(path.join(params.destinationRealDir, ".openclaw-archive-"));
|
||||
try {
|
||||
return await params.run(stagingDir);
|
||||
} finally {
|
||||
await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function mergeExtractedTreeIntoDestination(params: {
|
||||
sourceDir: string;
|
||||
destinationDir: string;
|
||||
destinationRealDir: string;
|
||||
}): Promise<void> {
|
||||
const walk = async (currentSourceDir: string): Promise<void> => {
|
||||
const entries = await fs.readdir(currentSourceDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const sourcePath = path.join(currentSourceDir, entry.name);
|
||||
const relPath = path.relative(params.sourceDir, sourcePath);
|
||||
const originalPath = relPath.split(path.sep).join("/");
|
||||
const destinationPath = path.join(params.destinationDir, relPath);
|
||||
const sourceStat = await fs.lstat(sourcePath);
|
||||
|
||||
if (sourceStat.isSymbolicLink()) {
|
||||
throw symlinkTraversalError(originalPath);
|
||||
}
|
||||
|
||||
if (sourceStat.isDirectory()) {
|
||||
await prepareArchiveOutputPath({
|
||||
destinationDir: params.destinationDir,
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
outPath: destinationPath,
|
||||
originalPath,
|
||||
isDirectory: true,
|
||||
});
|
||||
await walk(sourcePath);
|
||||
await applyStagedEntryMode({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
mode: sourceStat.mode & 0o777,
|
||||
originalPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!sourceStat.isFile()) {
|
||||
throw new Error(`archive staging contains unsupported entry: ${originalPath}`);
|
||||
}
|
||||
|
||||
await prepareArchiveOutputPath({
|
||||
destinationDir: params.destinationDir,
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
outPath: destinationPath,
|
||||
originalPath,
|
||||
isDirectory: false,
|
||||
});
|
||||
await copyFileWithinRoot({
|
||||
sourcePath,
|
||||
rootDir: params.destinationRealDir,
|
||||
relativePath: relPath,
|
||||
mkdir: true,
|
||||
});
|
||||
await applyStagedEntryMode({
|
||||
destinationRealDir: params.destinationRealDir,
|
||||
relPath,
|
||||
mode: sourceStat.mode & 0o777,
|
||||
originalPath,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
await walk(params.sourceDir);
|
||||
function symlinkTraversalError(originalPath: string) {
|
||||
return createArchiveSymlinkTraversalError(originalPath);
|
||||
}
|
||||
|
||||
type OpenZipOutputFileResult = {
|
||||
|
||||
Reference in New Issue
Block a user