mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-16 19:20:45 +00:00
[codex] Extract filesystem safety primitives (#77918)
* refactor: extract filesystem safety primitives * refactor: use fs-safe for file access helpers * refactor: reuse fs-safe for media reads * refactor: use fs-safe for image reads * refactor: reuse fs-safe in qqbot media opener * refactor: reuse fs-safe for local media checks * refactor: consume cleaner fs-safe api * refactor: align fs-safe json option names * fix: preserve fs-safe migration contracts * refactor: use fs-safe primitive subpaths * refactor: use grouped fs-safe subpaths * refactor: align fs-safe api usage * refactor: adapt private state store api * chore: refresh proof gate * refactor: follow fs-safe json api split * refactor: follow reduced fs-safe surface * build: default fs-safe python helper off * fix: preserve fs-safe plugin sdk aliases * refactor: consolidate fs-safe usage * refactor: unify fs-safe store usage * refactor: trim fs-safe temp workspace usage * refactor: hide low-level fs-safe primitives * build: use published fs-safe package * fix: preserve outbound recovery durability after rebase * chore: refresh pr checks
This commit is contained in:
committed by
GitHub
parent
61481eb34f
commit
538605ff44
@@ -1,9 +1,10 @@
|
||||
import { appendFile, mkdir } from "node:fs/promises";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { readAcpSessionEntry } from "../acp/runtime/session-meta.js";
|
||||
import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js";
|
||||
import { onAgentEvent } from "../infra/agent-events.js";
|
||||
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
|
||||
import { appendRegularFile } from "../infra/regular-file.js";
|
||||
import { enqueueSystemEvent } from "../infra/system-events.js";
|
||||
import { scopedHeartbeatWakeOptions } from "../routing/session-key.js";
|
||||
import { normalizeAssistantPhase } from "../shared/chat-message-content.js";
|
||||
@@ -130,10 +131,7 @@ export function startAcpSpawnParentStreamRelay(params: {
|
||||
});
|
||||
logDirReady = true;
|
||||
}
|
||||
await appendFile(logPath, chunk, {
|
||||
encoding: "utf-8",
|
||||
mode: 0o600,
|
||||
});
|
||||
await appendRegularFile({ filePath: logPath, content: chunk });
|
||||
})
|
||||
.catch(() => {
|
||||
// Best-effort diagnostics; never break relay flow.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isPathInside } from "../infra/path-guards.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
import { lowercasePreservingWhitespace } from "../shared/string-coerce.js";
|
||||
import { listAgentEntries, resolveAgentWorkspaceDir } from "./agent-scope.js";
|
||||
@@ -19,17 +20,11 @@ function normalizeWorkspacePathForComparison(input: string): string {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function isPathWithinRoot(candidatePath: string, rootPath: string): boolean {
|
||||
const relative = path.relative(rootPath, candidatePath);
|
||||
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
||||
}
|
||||
|
||||
function workspacePathsOverlap(left: string, right: string): boolean {
|
||||
const normalizedLeft = normalizeWorkspacePathForComparison(left);
|
||||
const normalizedRight = normalizeWorkspacePathForComparison(right);
|
||||
return (
|
||||
isPathWithinRoot(normalizedLeft, normalizedRight) ||
|
||||
isPathWithinRoot(normalizedRight, normalizedLeft)
|
||||
isPathInside(normalizedRight, normalizedLeft) || isPathInside(normalizedLeft, normalizedRight)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js";
|
||||
import type { AgentModelConfig } from "../config/types.agents-shared.js";
|
||||
import type { AgentConfig } from "../config/types.agents.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { isPathInside } from "../infra/path-guards.js";
|
||||
import {
|
||||
normalizeAgentId,
|
||||
parseAgentSessionKey,
|
||||
@@ -239,11 +240,6 @@ function normalizePathForComparison(input: string): string {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function isPathWithinRoot(candidatePath: string, rootPath: string): boolean {
|
||||
const relative = path.relative(rootPath, candidatePath);
|
||||
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
|
||||
}
|
||||
|
||||
export function resolveAgentIdsByWorkspacePath(
|
||||
cfg: OpenClawConfig,
|
||||
workspacePath: string,
|
||||
@@ -255,7 +251,7 @@ export function resolveAgentIdsByWorkspacePath(
|
||||
for (let index = 0; index < ids.length; index += 1) {
|
||||
const id = ids[index];
|
||||
const workspaceDir = normalizePathForComparison(resolveAgentWorkspaceDir(cfg, id));
|
||||
if (!isPathWithinRoot(normalizedWorkspacePath, workspaceDir)) {
|
||||
if (!isPathInside(workspaceDir, normalizedWorkspacePath)) {
|
||||
continue;
|
||||
}
|
||||
matches.push({ id, workspaceDir, order: index });
|
||||
|
||||
@@ -9,111 +9,6 @@ import {
|
||||
import { applyPatch } from "./apply-patch.js";
|
||||
import type { SandboxFsBridge } from "./sandbox/fs-bridge.js";
|
||||
|
||||
const pinnedPathHelper = vi.hoisted(() => {
|
||||
const fs = require("node:fs/promises") as typeof import("node:fs/promises");
|
||||
const path = require("node:path") as typeof import("node:path");
|
||||
const { pipeline } = require("node:stream/promises") as typeof import("node:stream/promises");
|
||||
|
||||
async function resolvePinnedParent(params: {
|
||||
rootPath: string;
|
||||
relativeParentPath?: string;
|
||||
mkdir?: boolean;
|
||||
}): Promise<string> {
|
||||
let current = params.rootPath;
|
||||
for (const segment of (params.relativeParentPath ?? "").split("/").filter(Boolean)) {
|
||||
const next = path.join(current, segment);
|
||||
try {
|
||||
const stat = await fs.lstat(next);
|
||||
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
||||
throw new Error("symbolic link or non-directory path segment");
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !params.mkdir) {
|
||||
throw error;
|
||||
}
|
||||
await fs.mkdir(next);
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
return {
|
||||
runPinnedPathHelper: vi.fn(
|
||||
async (params: {
|
||||
operation: "mkdirp" | "remove";
|
||||
rootPath: string;
|
||||
relativePath: string;
|
||||
}) => {
|
||||
const segments = params.relativePath.split("/").filter(Boolean);
|
||||
const targetPath = path.join(params.rootPath, ...segments);
|
||||
if (params.operation === "mkdirp") {
|
||||
await resolvePinnedParent({
|
||||
rootPath: params.rootPath,
|
||||
relativeParentPath: params.relativePath,
|
||||
mkdir: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await resolvePinnedParent({
|
||||
rootPath: params.rootPath,
|
||||
relativeParentPath: segments.slice(0, -1).join("/"),
|
||||
mkdir: false,
|
||||
});
|
||||
const stat = await fs.lstat(targetPath);
|
||||
if (stat.isDirectory() && !stat.isSymbolicLink()) {
|
||||
await fs.rmdir(targetPath);
|
||||
return;
|
||||
}
|
||||
await fs.unlink(targetPath);
|
||||
},
|
||||
),
|
||||
runPinnedWriteHelper: vi.fn(
|
||||
async (params: {
|
||||
rootPath: string;
|
||||
relativeParentPath: string;
|
||||
basename: string;
|
||||
mkdir: boolean;
|
||||
mode: number;
|
||||
input:
|
||||
| { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding }
|
||||
| { kind: "stream"; stream: NodeJS.ReadableStream };
|
||||
}) => {
|
||||
const parentPath = await resolvePinnedParent({
|
||||
rootPath: params.rootPath,
|
||||
relativeParentPath: params.relativeParentPath,
|
||||
mkdir: params.mkdir,
|
||||
});
|
||||
const targetPath = path.join(parentPath, params.basename);
|
||||
if (params.input.kind === "buffer") {
|
||||
await fs.writeFile(targetPath, params.input.data, {
|
||||
encoding: params.input.encoding,
|
||||
mode: params.mode,
|
||||
});
|
||||
} else {
|
||||
const handle = await fs.open(targetPath, "w", params.mode);
|
||||
try {
|
||||
await pipeline(params.input.stream, handle.createWriteStream());
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
const stat = await fs.stat(targetPath);
|
||||
return { dev: stat.dev, ino: stat.ino };
|
||||
},
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../infra/fs-pinned-path-helper.js", () => ({
|
||||
isPinnedPathHelperSpawnError: () => false,
|
||||
runPinnedPathHelper: pinnedPathHelper.runPinnedPathHelper,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/fs-pinned-write-helper.js", () => ({
|
||||
runPinnedWriteHelper: pinnedPathHelper.runPinnedWriteHelper,
|
||||
}));
|
||||
|
||||
async function withTempDir<T>(fn: (dir: string) => Promise<T>) {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-patch-"));
|
||||
try {
|
||||
|
||||
@@ -3,12 +3,8 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { AgentTool } from "@mariozechner/pi-agent-core";
|
||||
import { Type } from "typebox";
|
||||
import { openBoundaryFile, type BoundaryFileOpenResult } from "../infra/boundary-file-read.js";
|
||||
import {
|
||||
mkdirPathWithinRoot,
|
||||
removePathWithinRoot,
|
||||
writeFileWithinRoot,
|
||||
} from "../infra/fs-safe.js";
|
||||
import { openRootFile, type RootFileOpenResult } from "../infra/boundary-file-read.js";
|
||||
import { root as fsRoot } from "../infra/fs-safe.js";
|
||||
import { PATH_ALIAS_POLICIES, type PathAliasPolicy } from "../infra/path-alias-guards.js";
|
||||
import { applyUpdateHunk } from "./apply-patch-update.js";
|
||||
import { toRelativeSandboxPath, resolvePathFromInput } from "./path-policy.js";
|
||||
@@ -244,12 +240,13 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps {
|
||||
};
|
||||
}
|
||||
const workspaceOnly = options.workspaceOnly !== false;
|
||||
const rootPromise = workspaceOnly ? fsRoot(options.cwd) : undefined;
|
||||
return {
|
||||
readFile: async (filePath) => {
|
||||
if (!workspaceOnly) {
|
||||
return await fs.readFile(filePath, "utf8");
|
||||
}
|
||||
const opened = await openBoundaryFile({
|
||||
const opened = await openRootFile({
|
||||
absolutePath: filePath,
|
||||
rootPath: options.cwd,
|
||||
boundaryLabel: "workspace root",
|
||||
@@ -267,12 +264,7 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps {
|
||||
return;
|
||||
}
|
||||
const relative = toRelativeSandboxPath(options.cwd, filePath);
|
||||
await writeFileWithinRoot({
|
||||
rootDir: options.cwd,
|
||||
relativePath: relative,
|
||||
data: content,
|
||||
encoding: "utf8",
|
||||
});
|
||||
await (await rootPromise)?.write(relative, content, { encoding: "utf8" });
|
||||
},
|
||||
remove: async (filePath) => {
|
||||
if (!workspaceOnly) {
|
||||
@@ -280,10 +272,7 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps {
|
||||
return;
|
||||
}
|
||||
const relative = toRelativeSandboxPath(options.cwd, filePath);
|
||||
await removePathWithinRoot({
|
||||
rootDir: options.cwd,
|
||||
relativePath: relative,
|
||||
});
|
||||
await (await rootPromise)?.remove(relative);
|
||||
},
|
||||
mkdirp: async (dir) => {
|
||||
if (!workspaceOnly) {
|
||||
@@ -291,11 +280,15 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps {
|
||||
return;
|
||||
}
|
||||
const relative = toRelativeSandboxPath(options.cwd, dir, { allowRoot: true });
|
||||
await mkdirPathWithinRoot({
|
||||
rootDir: options.cwd,
|
||||
relativePath: relative,
|
||||
allowRoot: true,
|
||||
});
|
||||
const root = await rootPromise;
|
||||
if (!root) {
|
||||
return;
|
||||
}
|
||||
if (relative === "" || relative === ".") {
|
||||
await root.ensureRoot();
|
||||
return;
|
||||
}
|
||||
await root.mkdir(relative);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -352,9 +345,9 @@ async function resolvePatchPath(
|
||||
}
|
||||
|
||||
function assertBoundaryRead(
|
||||
opened: BoundaryFileOpenResult,
|
||||
opened: RootFileOpenResult,
|
||||
targetPath: string,
|
||||
): asserts opened is Extract<BoundaryFileOpenResult, { ok: true }> {
|
||||
): asserts opened is Extract<RootFileOpenResult, { ok: true }> {
|
||||
if (opened.ok) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { constants as fsConstants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { __setFsSafeTestHooksForTest } from "../infra/fs-safe.js";
|
||||
import { __setFsSafeTestHooksForTest } from "../infra/fs-safe-test-hooks.js";
|
||||
import { withTempDir } from "../test-utils/temp-dir.js";
|
||||
import { __testing, createExecTool } from "./bash-tools.exec.js";
|
||||
|
||||
|
||||
@@ -139,9 +139,9 @@ async function loadFsSafeModule(): Promise<FsSafeModule> {
|
||||
|
||||
function shouldSkipScriptPreflightPathError(
|
||||
error: unknown,
|
||||
SafeOpenError: FsSafeModule["SafeOpenError"],
|
||||
FsSafeError: FsSafeModule["FsSafeError"],
|
||||
): boolean {
|
||||
if (error instanceof SafeOpenError) {
|
||||
if (error instanceof FsSafeError) {
|
||||
return true;
|
||||
}
|
||||
const errorCode = getNodeErrorCode(error);
|
||||
@@ -155,8 +155,8 @@ function resolvePreflightRelativePath(params: { rootDir: string; absPath: string
|
||||
if (/^\.\.(?:[\\/]|$)/u.test(relative) || path.isAbsolute(relative)) {
|
||||
return null;
|
||||
}
|
||||
// Preserve literal "~" path segments under the workdir. `readFileWithinRoot`
|
||||
// expands home prefixes for relative paths, so normalize `~/...` to `./~/...`.
|
||||
// Preserve literal "~" path segments under the workdir. Root reads
|
||||
// expand home prefixes for relative paths, so normalize `~/...` to `./~/...`.
|
||||
return /^~(?:$|[\\/])/u.test(relative) ? `.${path.sep}${relative}` : relative;
|
||||
}
|
||||
|
||||
@@ -973,7 +973,8 @@ async function validateScriptFileForShellBleed(params: {
|
||||
return;
|
||||
}
|
||||
|
||||
const { SafeOpenError, readFileWithinRoot } = await loadFsSafeModule();
|
||||
const { FsSafeError, root: fsRoot } = await loadFsSafeModule();
|
||||
const workspaceRoot = await fsRoot(params.workdir);
|
||||
for (const relOrAbsPath of target.relOrAbsPaths) {
|
||||
const absPath = path.isAbsolute(relOrAbsPath)
|
||||
? path.resolve(relOrAbsPath)
|
||||
@@ -992,16 +993,14 @@ async function validateScriptFileForShellBleed(params: {
|
||||
// Use non-blocking open to avoid stalls if a path is swapped to a FIFO.
|
||||
let content: string;
|
||||
try {
|
||||
const safeRead = await readFileWithinRoot({
|
||||
rootDir: params.workdir,
|
||||
relativePath,
|
||||
const safeRead = await workspaceRoot.read(relativePath, {
|
||||
nonBlockingRead: true,
|
||||
allowSymlinkTargetWithinRoot: true,
|
||||
symlinks: "follow-within-root",
|
||||
maxBytes: 512 * 1024,
|
||||
});
|
||||
content = safeRead.buffer.toString("utf-8");
|
||||
} catch (error) {
|
||||
if (shouldSkipScriptPreflightPathError(error, SafeOpenError)) {
|
||||
if (shouldSkipScriptPreflightPathError(error, FsSafeError)) {
|
||||
// Preflight validation is best-effort: skip path/read failures and
|
||||
// continue to execute the command normally.
|
||||
continue;
|
||||
|
||||
@@ -10,6 +10,8 @@ import type { SourceReplyDeliveryMode } from "../../auto-reply/get-reply-options
|
||||
import type { ThinkLevel } from "../../auto-reply/thinking.js";
|
||||
import type { CliBackendConfig } from "../../config/types.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { privateFileStore } from "../../infra/private-file-store.js";
|
||||
import { tempWorkspace } from "../../infra/private-temp-workspace.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js";
|
||||
import { MAX_IMAGE_BYTES } from "../../media/constants.js";
|
||||
import { extensionForMime } from "../../media/mime.js";
|
||||
@@ -283,14 +285,14 @@ export async function writeCliImages(params: {
|
||||
workspaceDir: params.workspaceDir,
|
||||
});
|
||||
await fs.mkdir(imageRoot, { recursive: true, mode: 0o700 });
|
||||
const store = privateFileStore(imageRoot);
|
||||
const paths: string[] = [];
|
||||
for (let i = 0; i < params.images.length; i += 1) {
|
||||
const image = params.images[i];
|
||||
const fileName = path.basename(resolveCliImagePath(image));
|
||||
const filePath = path.join(imageRoot, fileName);
|
||||
const buffer = Buffer.from(image.data, "base64");
|
||||
await fs.writeFile(filePath, buffer, { mode: 0o600 });
|
||||
paths.push(filePath);
|
||||
await store.writeText(fileName, buffer);
|
||||
paths.push(store.path(fileName));
|
||||
}
|
||||
// Keep content-addressed image paths stable across Claude CLI runs so prompt
|
||||
// text and argv don't churn on every turn with fresh temp-dir suffixes.
|
||||
@@ -308,19 +310,17 @@ export async function writeCliSystemPromptFile(params: {
|
||||
) {
|
||||
return { cleanup: async () => {} };
|
||||
}
|
||||
const tempDir = await fs.mkdtemp(
|
||||
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-system-prompt-"),
|
||||
);
|
||||
const filePath = path.join(tempDir, "system-prompt.md");
|
||||
await fs.writeFile(filePath, stripSystemPromptCacheBoundary(params.systemPrompt), {
|
||||
encoding: "utf-8",
|
||||
mode: 0o600,
|
||||
const workspace = await tempWorkspace({
|
||||
rootDir: resolvePreferredOpenClawTmpDir(),
|
||||
prefix: "openclaw-cli-system-prompt-",
|
||||
});
|
||||
const filePath = await workspace.write(
|
||||
"system-prompt.md",
|
||||
stripSystemPromptCacheBoundary(params.systemPrompt),
|
||||
);
|
||||
return {
|
||||
filePath,
|
||||
cleanup: async () => {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
},
|
||||
cleanup: async () => await workspace.cleanup(),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
resolveSessionFilePathOptions,
|
||||
} from "../../config/sessions/paths.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { isPathInside } from "../../infra/path-guards.js";
|
||||
import { resolveSessionAgentIds } from "../agent-scope.js";
|
||||
import {
|
||||
limitAgentHookHistoryMessages,
|
||||
@@ -108,11 +109,6 @@ async function safeRealpath(filePath: string): Promise<string | undefined> {
|
||||
}
|
||||
}
|
||||
|
||||
function isPathWithinBase(basePath: string, targetPath: string): boolean {
|
||||
const relative = path.relative(basePath, targetPath);
|
||||
return Boolean(relative) && !relative.startsWith("..") && !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function resolveSafeCliSessionFile(params: {
|
||||
sessionId: string;
|
||||
sessionFile: string;
|
||||
@@ -155,7 +151,11 @@ async function loadCliSessionEntries(params: {
|
||||
}
|
||||
const realSessionsDir = (await safeRealpath(sessionsDir)) ?? path.resolve(sessionsDir);
|
||||
const realSessionFile = await safeRealpath(sessionFile);
|
||||
if (!realSessionFile || !isPathWithinBase(realSessionsDir, realSessionFile)) {
|
||||
if (
|
||||
!realSessionFile ||
|
||||
realSessionFile === realSessionsDir ||
|
||||
!isPathInside(realSessionsDir, realSessionFile)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const stat = await fsp.stat(realSessionFile);
|
||||
|
||||
@@ -1,14 +1,5 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from "node:fs";
|
||||
import {
|
||||
createServer,
|
||||
request as httpRequest,
|
||||
@@ -20,6 +11,7 @@ import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { resolveOpenClawPackageRootSync } from "../../infra/openclaw-root.js";
|
||||
import { privateFileStoreSync } from "../../infra/private-file-store.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { PluginApprovalResolutions } from "../../plugins/types.js";
|
||||
import { runBeforeToolCallHook } from "../pi-tools.before-tool-call.js";
|
||||
@@ -823,18 +815,10 @@ function writeNativeHookRelayBridgeRecord(
|
||||
registryPath: string,
|
||||
record: NativeHookRelayBridgeRecord,
|
||||
): void {
|
||||
const tempPath = path.join(
|
||||
path.dirname(registryPath),
|
||||
`.${path.basename(registryPath)}.${process.pid}.${randomUUID()}.tmp`,
|
||||
privateFileStoreSync(path.dirname(registryPath)).writeText(
|
||||
path.basename(registryPath),
|
||||
`${JSON.stringify(record)}\n`,
|
||||
);
|
||||
try {
|
||||
writeFileSync(tempPath, `${JSON.stringify(record)}\n`, { mode: 0o600, flag: "wx" });
|
||||
renameSync(tempPath, registryPath);
|
||||
chmodSync(registryPath, 0o600);
|
||||
} catch (error) {
|
||||
rmSync(tempPath, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function nativeHookRelayBridgeRegistryPath(relayId: string): string {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type OpenClawConfig,
|
||||
} from "../config/config.js";
|
||||
import { createConfigRuntimeEnv } from "../config/env-vars.js";
|
||||
import { privateFileStore } from "../infra/private-file-store.js";
|
||||
import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
|
||||
import { resolveInstalledManifestRegistryIndexFingerprint } from "../plugins/manifest-registry-installed.js";
|
||||
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
|
||||
@@ -85,7 +86,15 @@ async function readExistingModelsFile(pathname: string): Promise<{
|
||||
parsed: unknown;
|
||||
}> {
|
||||
try {
|
||||
const raw = await fs.readFile(pathname, "utf8");
|
||||
const raw = await privateFileStore(path.dirname(pathname)).readTextIfExists(
|
||||
path.basename(pathname),
|
||||
);
|
||||
if (raw === null) {
|
||||
return {
|
||||
raw: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
raw,
|
||||
parsed: JSON.parse(raw) as unknown,
|
||||
@@ -108,9 +117,7 @@ export async function writeModelsFileAtomicForModelsJson(
|
||||
targetPath: string,
|
||||
contents: string,
|
||||
): Promise<void> {
|
||||
const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tempPath, contents, { mode: 0o600 });
|
||||
await fs.rename(tempPath, targetPath);
|
||||
await privateFileStore(path.dirname(targetPath)).writeText(path.basename(targetPath), contents);
|
||||
}
|
||||
|
||||
function resolveModelsConfigInput(config?: OpenClawConfig): {
|
||||
|
||||
@@ -12,6 +12,10 @@ import {
|
||||
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
|
||||
|
||||
const planOpenClawModelsJsonMock = vi.fn();
|
||||
const writePrivateStoreTextWriteMock = vi.fn();
|
||||
let actualPrivateFileStore:
|
||||
| typeof import("../infra/private-file-store.js").privateFileStore
|
||||
| undefined;
|
||||
|
||||
installModelsConfigTestHooks();
|
||||
|
||||
@@ -66,6 +70,27 @@ beforeAll(async () => {
|
||||
vi.doMock("./models-config.plan.js", () => ({
|
||||
planOpenClawModelsJson: (...args: unknown[]) => planOpenClawModelsJsonMock(...args),
|
||||
}));
|
||||
vi.doMock("../infra/private-file-store.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../infra/private-file-store.js")>(
|
||||
"../infra/private-file-store.js",
|
||||
);
|
||||
actualPrivateFileStore = actual.privateFileStore;
|
||||
return {
|
||||
...actual,
|
||||
privateFileStore: (rootDir: string) => {
|
||||
const store = actual.privateFileStore(rootDir);
|
||||
return {
|
||||
...store,
|
||||
writeText: (relativePath: string, content: string | Uint8Array) =>
|
||||
writePrivateStoreTextWriteMock({
|
||||
rootDir,
|
||||
filePath: path.join(rootDir, relativePath),
|
||||
content,
|
||||
}),
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
({ ensureOpenClawModelsJson } = await import("./models-config.js"));
|
||||
({ clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot } =
|
||||
await import("../plugins/current-plugin-metadata-snapshot.js"));
|
||||
@@ -73,6 +98,19 @@ beforeAll(async () => {
|
||||
|
||||
beforeEach(() => {
|
||||
clearCurrentPluginMetadataSnapshot();
|
||||
writePrivateStoreTextWriteMock
|
||||
.mockReset()
|
||||
.mockImplementation(
|
||||
async (params: { filePath: string; rootDir: string; content: string | Uint8Array }) => {
|
||||
if (!actualPrivateFileStore) {
|
||||
throw new Error("private file store mock not initialized");
|
||||
}
|
||||
return await actualPrivateFileStore(params.rootDir).writeText(
|
||||
path.basename(params.filePath),
|
||||
params.content,
|
||||
);
|
||||
},
|
||||
);
|
||||
planOpenClawModelsJsonMock
|
||||
.mockReset()
|
||||
.mockImplementation(async (params: { cfg?: typeof CUSTOM_PROXY_MODELS_CONFIG }) => ({
|
||||
@@ -207,42 +245,35 @@ describe("models-config write serialization", () => {
|
||||
firstModel.name = "Proxy A";
|
||||
secondModel.name = "Proxy B with longer name";
|
||||
|
||||
const originalWriteFile = fs.writeFile.bind(fs);
|
||||
let inFlightWrites = 0;
|
||||
let maxInFlightWrites = 0;
|
||||
const writeSpy = vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => {
|
||||
const targetArg = args[0];
|
||||
const targetPath =
|
||||
typeof targetArg === "string"
|
||||
? targetArg
|
||||
: targetArg instanceof URL
|
||||
? targetArg.pathname
|
||||
: undefined;
|
||||
const isModelsTempWrite =
|
||||
typeof targetPath === "string" &&
|
||||
path.basename(targetPath).startsWith("models.json.") &&
|
||||
targetPath.endsWith(".tmp");
|
||||
if (isModelsTempWrite) {
|
||||
inFlightWrites += 1;
|
||||
if (inFlightWrites > maxInFlightWrites) {
|
||||
maxInFlightWrites = inFlightWrites;
|
||||
writePrivateStoreTextWriteMock.mockImplementation(
|
||||
async (params: { filePath: string; rootDir: string; content: string | Uint8Array }) => {
|
||||
const isModelsWrite = path.basename(params.filePath) === "models.json";
|
||||
if (isModelsWrite) {
|
||||
inFlightWrites += 1;
|
||||
if (inFlightWrites > maxInFlightWrites) {
|
||||
maxInFlightWrites = inFlightWrites;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
try {
|
||||
return await originalWriteFile(...args);
|
||||
} finally {
|
||||
if (isModelsTempWrite) {
|
||||
inFlightWrites -= 1;
|
||||
try {
|
||||
if (!actualPrivateFileStore) {
|
||||
throw new Error("private file store mock not initialized");
|
||||
}
|
||||
return await actualPrivateFileStore(params.rootDir).writeText(
|
||||
path.basename(params.filePath),
|
||||
params.content,
|
||||
);
|
||||
} finally {
|
||||
if (isModelsWrite) {
|
||||
inFlightWrites -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
await Promise.all([ensureOpenClawModelsJson(first), ensureOpenClawModelsJson(second)]);
|
||||
} finally {
|
||||
writeSpy.mockRestore();
|
||||
}
|
||||
await Promise.all([ensureOpenClawModelsJson(first), ensureOpenClawModelsJson(second)]);
|
||||
|
||||
expect(maxInFlightWrites).toBe(1);
|
||||
const parsed = await readGeneratedModelsJson<{
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { z } from "zod";
|
||||
import { safeParseJsonWithSchema, safeParseWithSchema } from "../utils/zod-parse.js";
|
||||
import { privateFileStore } from "../infra/private-file-store.js";
|
||||
import { safeParseWithSchema } from "../utils/zod-parse.js";
|
||||
import { ensureAuthProfileStore } from "./auth-profiles/store.js";
|
||||
import {
|
||||
piCredentialsEqual,
|
||||
@@ -26,10 +26,12 @@ const PiCredentialSchema: z.ZodType<PiCredential> = z.discriminatedUnion("type",
|
||||
|
||||
const AuthJsonShapeSchema = z.record(z.string(), z.unknown());
|
||||
|
||||
async function readAuthJson(filePath: string): Promise<AuthJsonShape> {
|
||||
async function readAuthJson(rootDir: string, filePath: string): Promise<AuthJsonShape> {
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, "utf8");
|
||||
return safeParseJsonWithSchema(AuthJsonShapeSchema, raw) ?? {};
|
||||
const parsed = await privateFileStore(rootDir).readJsonIfExists(
|
||||
path.relative(rootDir, filePath),
|
||||
);
|
||||
return safeParseWithSchema(AuthJsonShapeSchema, parsed) ?? {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
@@ -58,7 +60,7 @@ export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promis
|
||||
return { wrote: false, authPath };
|
||||
}
|
||||
|
||||
const existing = await readAuthJson(authPath);
|
||||
const existing = await readAuthJson(agentDir, authPath);
|
||||
let changed = false;
|
||||
|
||||
for (const [provider, cred] of Object.entries(providerCredentials)) {
|
||||
@@ -73,8 +75,9 @@ export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promis
|
||||
return { wrote: false, authPath };
|
||||
}
|
||||
|
||||
await fs.mkdir(agentDir, { recursive: true, mode: 0o700 });
|
||||
await fs.writeFile(authPath, `${JSON.stringify(existing, null, 2)}\n`, { mode: 0o600 });
|
||||
await privateFileStore(agentDir).writeJson(path.basename(authPath), existing, {
|
||||
trailingNewline: true,
|
||||
});
|
||||
|
||||
return { wrote: true, authPath };
|
||||
}
|
||||
|
||||
@@ -18,11 +18,12 @@
|
||||
* capabilities instead of the text-only fallback.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { resolveStateDir } from "../../config/paths.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { resolveProxyFetchFromEnv } from "../../infra/net/proxy-fetch.js";
|
||||
import { privateFileStoreSync } from "../../infra/private-file-store.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
|
||||
const log = createSubsystemLogger("openrouter-model-capabilities");
|
||||
@@ -89,14 +90,11 @@ function resolveDiskCachePath(): string {
|
||||
|
||||
function writeDiskCache(map: Map<string, OpenRouterModelCapabilities>): void {
|
||||
try {
|
||||
const cacheDir = resolveDiskCacheDir();
|
||||
if (!existsSync(cacheDir)) {
|
||||
mkdirSync(cacheDir, { recursive: true });
|
||||
}
|
||||
const cachePath = resolveDiskCachePath();
|
||||
const payload: DiskCachePayload = {
|
||||
models: Object.fromEntries(map),
|
||||
};
|
||||
writeFileSync(resolveDiskCachePath(), JSON.stringify(payload), "utf-8");
|
||||
privateFileStoreSync(dirname(cachePath)).writeJson(basename(cachePath), payload);
|
||||
} catch (err: unknown) {
|
||||
const message = formatErrorMessage(err);
|
||||
log.debug(`Failed to write OpenRouter disk cache: ${message}`);
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type SessionEntry,
|
||||
type SessionHeader,
|
||||
} from "@mariozechner/pi-coding-agent";
|
||||
import { appendRegularFile } from "../../infra/fs-safe.js";
|
||||
import { privateFileStore } from "../../infra/private-file-store.js";
|
||||
|
||||
type BranchSummaryEntry = Extract<SessionEntry, { type: "branch_summary" }>;
|
||||
type CompactionEntry = Extract<SessionEntry, { type: "compaction" }>;
|
||||
@@ -293,20 +295,10 @@ export async function writeTranscriptFileAtomic(
|
||||
filePath: string,
|
||||
entries: Array<SessionHeader | SessionEntry>,
|
||||
): Promise<void> {
|
||||
const dir = path.dirname(filePath);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const tmpFile = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`);
|
||||
try {
|
||||
await fs.writeFile(tmpFile, serializeTranscriptFileEntries(entries), {
|
||||
encoding: "utf-8",
|
||||
mode: 0o600,
|
||||
flag: "wx",
|
||||
});
|
||||
await fs.rename(tmpFile, filePath);
|
||||
} catch (err) {
|
||||
await fs.unlink(tmpFile).catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
await privateFileStore(path.dirname(filePath)).writeText(
|
||||
path.basename(filePath),
|
||||
serializeTranscriptFileEntries(entries),
|
||||
);
|
||||
}
|
||||
|
||||
export async function persistTranscriptStateMutation(params: {
|
||||
@@ -324,9 +316,9 @@ export async function persistTranscriptStateMutation(params: {
|
||||
]);
|
||||
return;
|
||||
}
|
||||
await fs.appendFile(
|
||||
params.sessionFile,
|
||||
params.appendedEntries.map((entry) => JSON.stringify(entry)).join("\n") + "\n",
|
||||
"utf-8",
|
||||
);
|
||||
await appendRegularFile({
|
||||
filePath: params.sessionFile,
|
||||
content: `${params.appendedEntries.map((entry) => JSON.stringify(entry)).join("\n")}\n`,
|
||||
rejectSymlinkParents: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { isTruthyEnvValue } from "../infra/env.js";
|
||||
import { appendRegularFile } from "../infra/fs-safe.js";
|
||||
|
||||
let rawStreamReady = false;
|
||||
|
||||
@@ -30,7 +31,11 @@ export function appendRawStream(payload: Record<string, unknown>) {
|
||||
}
|
||||
}
|
||||
try {
|
||||
void fs.promises.appendFile(rawStreamPath, `${JSON.stringify(payload)}\n`);
|
||||
void appendRegularFile({
|
||||
filePath: rawStreamPath,
|
||||
content: `${JSON.stringify(payload)}\n`,
|
||||
rejectSymlinkParents: true,
|
||||
});
|
||||
} catch {
|
||||
// ignore raw stream write failures
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import type { AgentMessage } from "@mariozechner/pi-agent-core";
|
||||
import type { ExtensionAPI, ExtensionContext, FileOperations } from "@mariozechner/pi-coding-agent";
|
||||
import { extractSections } from "../../auto-reply/reply/post-compaction-context.js";
|
||||
import { openBoundaryFile } from "../../infra/boundary-file-read.js";
|
||||
import { openRootFile } from "../../infra/boundary-file-read.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { isAbortError } from "../../infra/unhandled-rejections.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
@@ -735,7 +735,7 @@ async function readWorkspaceContextForSummary(): Promise<string> {
|
||||
const agentsPath = path.join(workspaceDir, "AGENTS.md");
|
||||
|
||||
try {
|
||||
const opened = await openBoundaryFile({
|
||||
const opened = await openRootFile({
|
||||
absolutePath: agentsPath,
|
||||
rootPath: workspaceDir,
|
||||
boundaryLabel: "workspace root",
|
||||
|
||||
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import type { SettingsManager } from "@mariozechner/pi-coding-agent";
|
||||
import { applyMergePatch } from "../config/merge-patch.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { openBoundaryFileSync } from "../infra/boundary-file-read.js";
|
||||
import { openRootFileSync } from "../infra/boundary-file-read.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import type { BundleMcpServerConfig } from "../plugins/bundle-mcp.js";
|
||||
import {
|
||||
@@ -43,7 +43,7 @@ function loadBundleSettingsFile(params: {
|
||||
relativePath: string;
|
||||
}): PiSettingsSnapshot | null {
|
||||
const absolutePath = path.join(params.rootDir, params.relativePath);
|
||||
const opened = openBoundaryFileSync({
|
||||
const opened = openRootFileSync({
|
||||
absolutePath,
|
||||
rootPath: params.rootDir,
|
||||
boundaryLabel: "plugin root",
|
||||
|
||||
@@ -6,7 +6,7 @@ import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
|
||||
vi.mock("../infra/boundary-file-read.js", async () => {
|
||||
const fs = await import("node:fs");
|
||||
return {
|
||||
openBoundaryFileSync: ({ absolutePath }: { absolutePath: string }) => ({
|
||||
openRootFileSync: ({ absolutePath }: { absolutePath: string }) => ({
|
||||
ok: true,
|
||||
fd: fs.openSync(absolutePath, "r"),
|
||||
}),
|
||||
|
||||
@@ -4,13 +4,7 @@ import { URL } from "node:url";
|
||||
import type { AgentToolResult } from "@mariozechner/pi-agent-core";
|
||||
import { createEditTool, createReadTool, createWriteTool } from "@mariozechner/pi-coding-agent";
|
||||
import { isWindowsDrivePath } from "../infra/archive-path.js";
|
||||
import {
|
||||
appendFileWithinRoot,
|
||||
SafeOpenError,
|
||||
openFileWithinRoot,
|
||||
readFileWithinRoot,
|
||||
writeFileWithinRoot,
|
||||
} from "../infra/fs-safe.js";
|
||||
import { root as fsRoot, FsSafeError } from "../infra/fs-safe.js";
|
||||
import { expandHomePrefix, resolveOsHomeDir } from "../infra/home-dir.js";
|
||||
import { hasEncodedFileUrlSeparator, trySafeFileURLToPath } from "../infra/local-file-access.js";
|
||||
import { detectMime } from "../media/mime.js";
|
||||
@@ -491,10 +485,8 @@ async function appendMemoryFlushContent(params: {
|
||||
signal?: AbortSignal;
|
||||
}) {
|
||||
if (!params.sandbox) {
|
||||
await appendFileWithinRoot({
|
||||
rootDir: params.root,
|
||||
relativePath: params.relativePath,
|
||||
data: params.content,
|
||||
const root = await fsRoot(params.root);
|
||||
await root.append(params.relativePath, params.content, {
|
||||
mkdir: true,
|
||||
prependNewlineIfNeeded: true,
|
||||
});
|
||||
@@ -769,6 +761,7 @@ function createHostWriteOperations(root: string, options?: { workspaceOnly?: boo
|
||||
}
|
||||
|
||||
// When workspaceOnly is true, enforce workspace boundary
|
||||
const rootPromise = fsRoot(root);
|
||||
return {
|
||||
mkdir: async (dir: string) => {
|
||||
const relative = toRelativeWorkspacePath(root, dir, { allowRoot: true });
|
||||
@@ -778,12 +771,7 @@ function createHostWriteOperations(root: string, options?: { workspaceOnly?: boo
|
||||
},
|
||||
writeFile: async (absolutePath: string, content: string) => {
|
||||
const relative = toRelativeWorkspacePath(root, absolutePath);
|
||||
await writeFileWithinRoot({
|
||||
rootDir: root,
|
||||
relativePath: relative,
|
||||
data: content,
|
||||
mkdir: true,
|
||||
});
|
||||
await (await rootPromise).write(relative, content, { mkdir: true });
|
||||
},
|
||||
} as const;
|
||||
}
|
||||
@@ -807,23 +795,16 @@ function createHostEditOperations(root: string, options?: { workspaceOnly?: bool
|
||||
}
|
||||
|
||||
// When workspaceOnly is true, enforce workspace boundary
|
||||
const rootPromise = fsRoot(root);
|
||||
return {
|
||||
readFile: async (absolutePath: string) => {
|
||||
const relative = toRelativeWorkspacePath(root, absolutePath);
|
||||
const safeRead = await readFileWithinRoot({
|
||||
rootDir: root,
|
||||
relativePath: relative,
|
||||
});
|
||||
const safeRead = await (await rootPromise).read(relative);
|
||||
return safeRead.buffer;
|
||||
},
|
||||
writeFile: async (absolutePath: string, content: string) => {
|
||||
const relative = toRelativeWorkspacePath(root, absolutePath);
|
||||
await writeFileWithinRoot({
|
||||
rootDir: root,
|
||||
relativePath: relative,
|
||||
data: content,
|
||||
mkdir: true,
|
||||
});
|
||||
await (await rootPromise).write(relative, content, { mkdir: true });
|
||||
},
|
||||
access: async (absolutePath: string) => {
|
||||
let relative: string;
|
||||
@@ -838,16 +819,13 @@ function createHostEditOperations(root: string, options?: { workspaceOnly?: bool
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const opened = await openFileWithinRoot({
|
||||
rootDir: root,
|
||||
relativePath: relative,
|
||||
});
|
||||
const opened = await (await rootPromise).open(relative);
|
||||
await opened.handle.close().catch(() => {});
|
||||
} catch (error) {
|
||||
if (error instanceof SafeOpenError && error.code === "not-found") {
|
||||
if (error instanceof FsSafeError && error.code === "not-found") {
|
||||
throw createFsAccessError("ENOENT", absolutePath);
|
||||
}
|
||||
if (error instanceof SafeOpenError && error.code === "outside-workspace") {
|
||||
if (error instanceof FsSafeError && error.code === "outside-workspace") {
|
||||
// Don't throw here – see the comment above about the upstream
|
||||
// library swallowing access errors as "File not found".
|
||||
return;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import nodeFs from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { appendRegularFile, resolveRegularFileAppendFlags } from "../infra/fs-safe.js";
|
||||
|
||||
export type QueuedFileWriteResult = "queued" | "dropped";
|
||||
|
||||
@@ -16,103 +16,19 @@ type QueuedFileWriterOptions = {
|
||||
yieldBeforeWrite?: boolean;
|
||||
};
|
||||
|
||||
type QueuedFileAppendFlagConstants = Pick<
|
||||
typeof nodeFs.constants,
|
||||
"O_APPEND" | "O_CREAT" | "O_WRONLY"
|
||||
> &
|
||||
Partial<Pick<typeof nodeFs.constants, "O_NOFOLLOW">>;
|
||||
|
||||
export function resolveQueuedFileAppendFlags(
|
||||
constants: QueuedFileAppendFlagConstants = nodeFs.constants,
|
||||
): number {
|
||||
const noFollow = constants.O_NOFOLLOW;
|
||||
return (
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
constants.O_WRONLY |
|
||||
(typeof noFollow === "number" ? noFollow : 0)
|
||||
);
|
||||
}
|
||||
|
||||
async function assertNoSymlinkParents(filePath: string): Promise<void> {
|
||||
const resolvedDir = path.resolve(path.dirname(filePath));
|
||||
const parsed = path.parse(resolvedDir);
|
||||
const relativeParts = path.relative(parsed.root, resolvedDir).split(path.sep).filter(Boolean);
|
||||
let current = parsed.root;
|
||||
for (const part of relativeParts) {
|
||||
current = path.join(current, part);
|
||||
const stat = await fs.lstat(current);
|
||||
if (stat.isSymbolicLink()) {
|
||||
if (path.dirname(current) === parsed.root) {
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Refusing to write queued log under symlinked directory: ${current}`);
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error(`Refusing to write queued log under non-directory: ${current}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function verifyStableOpenedFile(params: {
|
||||
preOpenStat?: nodeFs.Stats;
|
||||
postOpenStat: nodeFs.Stats;
|
||||
filePath: string;
|
||||
}): void {
|
||||
if (!params.postOpenStat.isFile()) {
|
||||
throw new Error(`Refusing to write queued log to non-file: ${params.filePath}`);
|
||||
}
|
||||
if (params.postOpenStat.nlink > 1) {
|
||||
throw new Error(`Refusing to write queued log to hardlinked file: ${params.filePath}`);
|
||||
}
|
||||
const pre = params.preOpenStat;
|
||||
if (pre && (pre.dev !== params.postOpenStat.dev || pre.ino !== params.postOpenStat.ino)) {
|
||||
throw new Error(`Refusing to write queued log after file changed: ${params.filePath}`);
|
||||
}
|
||||
}
|
||||
export const resolveQueuedFileAppendFlags = resolveRegularFileAppendFlags;
|
||||
|
||||
async function safeAppendFile(
|
||||
filePath: string,
|
||||
line: string,
|
||||
options: QueuedFileWriterOptions,
|
||||
): Promise<void> {
|
||||
await assertNoSymlinkParents(filePath);
|
||||
|
||||
let preOpenStat: nodeFs.Stats | undefined;
|
||||
try {
|
||||
const stat = await fs.lstat(filePath);
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new Error(`Refusing to write queued log through symlink: ${filePath}`);
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
throw new Error(`Refusing to write queued log to non-file: ${filePath}`);
|
||||
}
|
||||
preOpenStat = stat;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const lineBytes = Buffer.byteLength(line, "utf8");
|
||||
if (
|
||||
options.maxFileBytes !== undefined &&
|
||||
(preOpenStat?.size ?? 0) + lineBytes > options.maxFileBytes
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const handle = await fs.open(filePath, resolveQueuedFileAppendFlags(), 0o600);
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
verifyStableOpenedFile({ preOpenStat, postOpenStat: stat, filePath });
|
||||
if (options.maxFileBytes !== undefined && stat.size + lineBytes > options.maxFileBytes) {
|
||||
return;
|
||||
}
|
||||
await handle.chmod(0o600);
|
||||
await handle.appendFile(line, "utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
await appendRegularFile({
|
||||
filePath,
|
||||
content: line,
|
||||
maxFileBytes: options.maxFileBytes,
|
||||
rejectSymlinkParents: true,
|
||||
});
|
||||
}
|
||||
|
||||
function waitForImmediate(): Promise<void> {
|
||||
|
||||
@@ -108,10 +108,15 @@ function isManagedMediaPathUnderRoot(candidate: string): boolean {
|
||||
return false;
|
||||
}
|
||||
const mediaRoot = path.join(resolveConfigDir(), "media");
|
||||
const relative = path.relative(path.resolve(mediaRoot), path.resolve(expanded));
|
||||
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) {
|
||||
const resolvedMediaRoot = path.resolve(mediaRoot);
|
||||
const resolvedExpanded = path.resolve(expanded);
|
||||
if (
|
||||
resolvedExpanded === resolvedMediaRoot ||
|
||||
!isPathInside(resolvedMediaRoot, resolvedExpanded)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const relative = path.relative(resolvedMediaRoot, resolvedExpanded);
|
||||
const firstSegment = relative.split(path.sep)[0] ?? "";
|
||||
return MANAGED_MEDIA_SUBDIRS.has(firstSegment) || firstSegment.startsWith("tool-");
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
export { openBoundaryFile, type BoundaryFileOpenResult } from "../../infra/boundary-file-read.js";
|
||||
export { openRootFile, type RootFileOpenResult } from "../../infra/boundary-file-read.js";
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { PathAliasPolicy } from "../../infra/path-alias-guards.js";
|
||||
import type { SafeOpenSyncAllowedType } from "../../infra/safe-open-sync.js";
|
||||
import { openBoundaryFile, type BoundaryFileOpenResult } from "./fs-bridge-path-safety.runtime.js";
|
||||
import { openRootFile, type RootFileOpenResult } from "./fs-bridge-path-safety.runtime.js";
|
||||
import type { SandboxResolvedFsPath, SandboxFsMount } from "./fs-paths.js";
|
||||
import { isPathInsideContainerRoot, normalizeContainerPath } from "./path-utils.js";
|
||||
|
||||
type BoundaryAllowedType = "file" | "directory";
|
||||
|
||||
export type PathSafetyOptions = {
|
||||
action: string;
|
||||
aliasPolicy?: PathAliasPolicy;
|
||||
requireWritable?: boolean;
|
||||
allowedType?: SafeOpenSyncAllowedType;
|
||||
allowedType?: BoundaryAllowedType;
|
||||
};
|
||||
|
||||
export type PathSafetyCheck = {
|
||||
@@ -69,7 +70,7 @@ export class SandboxFsPathGuard {
|
||||
|
||||
async openReadableFile(
|
||||
target: SandboxResolvedFsPath,
|
||||
): Promise<BoundaryFileOpenResult & { ok: true }> {
|
||||
): Promise<RootFileOpenResult & { ok: true }> {
|
||||
const opened = await this.openBoundaryWithinRequiredMount(target, "read files");
|
||||
if (!opened.ok) {
|
||||
throw opened.error instanceof Error
|
||||
@@ -110,7 +111,7 @@ export class SandboxFsPathGuard {
|
||||
private async assertGuardedPathSafety(
|
||||
target: SandboxResolvedFsPath,
|
||||
options: PathSafetyOptions,
|
||||
guarded: BoundaryFileOpenResult,
|
||||
guarded: RootFileOpenResult,
|
||||
) {
|
||||
if (!guarded.ok) {
|
||||
if (guarded.reason !== "path") {
|
||||
@@ -145,11 +146,11 @@ export class SandboxFsPathGuard {
|
||||
action: string,
|
||||
options?: {
|
||||
aliasPolicy?: PathAliasPolicy;
|
||||
allowedType?: SafeOpenSyncAllowedType;
|
||||
allowedType?: BoundaryAllowedType;
|
||||
},
|
||||
): Promise<BoundaryFileOpenResult> {
|
||||
): Promise<RootFileOpenResult> {
|
||||
const lexicalMount = this.resolveRequiredMount(target.containerPath, action);
|
||||
const guarded = await openBoundaryFile({
|
||||
const guarded = await openRootFile({
|
||||
absolutePath: target.hostPath,
|
||||
rootPath: lexicalMount.hostRoot,
|
||||
boundaryLabel: "sandbox mount root",
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
getScriptsFromCalls,
|
||||
installFsBridgeTestHarness,
|
||||
mockedExecDockerRaw,
|
||||
mockedOpenBoundaryFile,
|
||||
mockedOpenRootFile,
|
||||
withTempDir,
|
||||
} from "./fs-bridge.test-helpers.js";
|
||||
|
||||
@@ -159,7 +159,7 @@ describe("sandbox fs bridge shell compatibility", () => {
|
||||
});
|
||||
|
||||
it("re-validates target before the pinned write helper runs", async () => {
|
||||
mockedOpenBoundaryFile
|
||||
mockedOpenRootFile
|
||||
.mockImplementationOnce(async () => ({ ok: false, reason: "path" }))
|
||||
.mockImplementationOnce(async () => ({
|
||||
ok: false,
|
||||
|
||||
@@ -4,21 +4,21 @@ import path from "node:path";
|
||||
import { beforeEach, expect, vi, type Mock } from "vitest";
|
||||
|
||||
type ExecDockerRawFn = typeof import("./docker.js").execDockerRaw;
|
||||
type OpenBoundaryFileFn = typeof import("./fs-bridge-path-safety.runtime.js").openBoundaryFile;
|
||||
type OpenRootFileFn = typeof import("./fs-bridge-path-safety.runtime.js").openRootFile;
|
||||
type ExecDockerArgs = Parameters<ExecDockerRawFn>[0];
|
||||
type ExecDockerRawMock = Mock<ExecDockerRawFn>;
|
||||
type OpenBoundaryFileMock = Mock<OpenBoundaryFileFn>;
|
||||
type OpenRootFileMock = Mock<OpenRootFileFn>;
|
||||
type FsBridgeHoisted = {
|
||||
execDockerRaw: ExecDockerRawMock;
|
||||
openBoundaryFile: OpenBoundaryFileMock;
|
||||
openRootFile: OpenRootFileMock;
|
||||
};
|
||||
|
||||
let actualOpenBoundaryFile: OpenBoundaryFileFn | undefined;
|
||||
let actualOpenRootFile: OpenRootFileFn | undefined;
|
||||
|
||||
const hoisted = vi.hoisted(
|
||||
(): FsBridgeHoisted => ({
|
||||
execDockerRaw: vi.fn(),
|
||||
openBoundaryFile: vi.fn(),
|
||||
openRootFile: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -31,11 +31,10 @@ vi.mock("./fs-bridge-path-safety.runtime.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("./fs-bridge-path-safety.runtime.js")>(
|
||||
"./fs-bridge-path-safety.runtime.js",
|
||||
);
|
||||
actualOpenBoundaryFile = actual.openBoundaryFile;
|
||||
actualOpenRootFile = actual.openRootFile;
|
||||
return {
|
||||
...actual,
|
||||
openBoundaryFile: (params: Parameters<OpenBoundaryFileFn>[0]) =>
|
||||
hoisted.openBoundaryFile(params),
|
||||
openRootFile: (params: Parameters<OpenRootFileFn>[0]) => hoisted.openRootFile(params),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -54,11 +53,10 @@ async function loadFreshFsBridgeModuleForTest() {
|
||||
const actual = await vi.importActual<typeof import("./fs-bridge-path-safety.runtime.js")>(
|
||||
"./fs-bridge-path-safety.runtime.js",
|
||||
);
|
||||
actualOpenBoundaryFile = actual.openBoundaryFile;
|
||||
actualOpenRootFile = actual.openRootFile;
|
||||
return {
|
||||
...actual,
|
||||
openBoundaryFile: (params: Parameters<OpenBoundaryFileFn>[0]) =>
|
||||
hoisted.openBoundaryFile(params),
|
||||
openRootFile: (params: Parameters<OpenRootFileFn>[0]) => hoisted.openRootFile(params),
|
||||
};
|
||||
});
|
||||
({ createSandboxFsBridge: createSandboxFsBridgeImpl } = await import("./fs-bridge.js"));
|
||||
@@ -74,7 +72,7 @@ export function createSandboxFsBridge(
|
||||
}
|
||||
|
||||
export const mockedExecDockerRaw: ExecDockerRawMock = hoisted.execDockerRaw;
|
||||
export const mockedOpenBoundaryFile: OpenBoundaryFileMock = hoisted.openBoundaryFile;
|
||||
export const mockedOpenRootFile: OpenRootFileMock = hoisted.openRootFile;
|
||||
const DOCKER_SCRIPT_INDEX = 5;
|
||||
const DOCKER_FIRST_SCRIPT_ARG_INDEX = 7;
|
||||
|
||||
@@ -206,7 +204,7 @@ export async function expectMkdirpAllowsExistingDirectory(params?: {
|
||||
await fs.mkdir(nestedDir, { recursive: true });
|
||||
|
||||
if (params?.forceBoundaryIoFallback) {
|
||||
mockedOpenBoundaryFile.mockImplementationOnce(async () => ({
|
||||
mockedOpenRootFile.mockImplementationOnce(async () => ({
|
||||
ok: false,
|
||||
reason: "io",
|
||||
error: Object.assign(new Error("EISDIR"), { code: "EISDIR" }),
|
||||
@@ -239,9 +237,9 @@ export function installFsBridgeTestHarness() {
|
||||
beforeEach(async () => {
|
||||
await loadFreshFsBridgeModuleForTest();
|
||||
mockedExecDockerRaw.mockClear();
|
||||
mockedOpenBoundaryFile.mockClear();
|
||||
if (actualOpenBoundaryFile) {
|
||||
mockedOpenBoundaryFile.mockImplementation(actualOpenBoundaryFile);
|
||||
mockedOpenRootFile.mockClear();
|
||||
if (actualOpenRootFile) {
|
||||
mockedOpenRootFile.mockImplementation(actualOpenRootFile);
|
||||
}
|
||||
installDockerReadMock();
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import path from "node:path";
|
||||
import { isPathInside } from "../../infra/path-guards.js";
|
||||
import { normalizeOptionalLowercaseString } from "../../shared/string-coerce.js";
|
||||
import { resolveSandboxInputPath, resolveSandboxPath } from "../sandbox-paths.js";
|
||||
import type { SandboxFsBridgeContext } from "./backend-handle.types.js";
|
||||
@@ -228,11 +229,7 @@ function isPathInsideHost(root: string, target: string): boolean {
|
||||
path.dirname(resolvedTarget),
|
||||
);
|
||||
const canonicalTarget = path.resolve(canonicalTargetParent, path.basename(resolvedTarget));
|
||||
const rel = path.relative(canonicalRoot, canonicalTarget);
|
||||
if (!rel) {
|
||||
return true;
|
||||
}
|
||||
return !(rel.startsWith("..") || path.isAbsolute(rel));
|
||||
return isPathInside(canonicalRoot, canonicalTarget);
|
||||
}
|
||||
|
||||
function toHostSegments(relativePosix: string): string[] {
|
||||
|
||||
@@ -46,10 +46,10 @@ vi.mock("../../infra/json-files.js", async () => {
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
writeJsonAtomic: async (
|
||||
writeJson: async (
|
||||
filePath: string,
|
||||
value: unknown,
|
||||
options?: Parameters<typeof actual.writeJsonAtomic>[2],
|
||||
options?: Parameters<typeof actual.writeJson>[2],
|
||||
) => {
|
||||
const payload = JSON.stringify(value);
|
||||
const gate = writeGateState.active;
|
||||
@@ -64,7 +64,7 @@ vi.mock("../../infra/json-files.js", async () => {
|
||||
}
|
||||
await gate.waitForRelease;
|
||||
}
|
||||
await actual.writeJsonAtomic(filePath, value, options);
|
||||
await actual.writeJson(filePath, value, options);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { z } from "zod";
|
||||
import { writeJsonAtomic } from "../../infra/json-files.js";
|
||||
import { writeJson } from "../../infra/json-files.js";
|
||||
import { safeParseJsonWithSchema } from "../../utils/zod-parse.js";
|
||||
import { acquireSessionWriteLock } from "../session-write-lock.js";
|
||||
import {
|
||||
@@ -171,7 +171,7 @@ async function readShardedEntry<T extends RegistryEntry>(
|
||||
|
||||
async function writeShardedEntry(dir: string, entry: RegistryEntryPayload): Promise<void> {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await writeJsonAtomic(shardedEntryFilePath(dir, entry.containerName), entry, {
|
||||
await writeJson(shardedEntryFilePath(dir, entry.containerName), entry, {
|
||||
trailingNewline: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { resolveBoundaryPath } from "../../infra/boundary-path.js";
|
||||
import { resolveRootPath } from "../../infra/boundary-path.js";
|
||||
import { parseSshTarget } from "../../infra/ssh-tunnel.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js";
|
||||
import { resolveUserPath } from "../../utils.js";
|
||||
@@ -349,7 +349,7 @@ async function assertSafeUploadSymlinks(localDir: string): Promise<void> {
|
||||
const entryPath = path.join(currentDir, entry.name);
|
||||
if (entry.isSymbolicLink()) {
|
||||
try {
|
||||
await resolveBoundaryPath({
|
||||
await resolveRootPath({
|
||||
absolutePath: entryPath,
|
||||
rootPath: rootDir,
|
||||
boundaryLabel: "SSH sandbox upload tree",
|
||||
|
||||
@@ -2,7 +2,7 @@ import syncFs from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OptionalBootstrapFileName } from "../../config/types.agent-defaults.js";
|
||||
import { openBoundaryFile } from "../../infra/boundary-file-read.js";
|
||||
import { openRootFile } from "../../infra/boundary-file-read.js";
|
||||
import { resolveUserPath } from "../../utils.js";
|
||||
import {
|
||||
DEFAULT_AGENTS_FILENAME,
|
||||
@@ -40,7 +40,7 @@ export async function ensureSandboxWorkspace(
|
||||
await fs.access(dest);
|
||||
} catch {
|
||||
try {
|
||||
const opened = await openBoundaryFile({
|
||||
const opened = await openRootFile({
|
||||
absolutePath: src,
|
||||
rootPath: seed,
|
||||
boundaryLabel: "sandbox seed workspace",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { replaceFileAtomic } from "../infra/replace-file.js";
|
||||
import { STREAM_ERROR_FALLBACK_TEXT } from "./stream-message-shared.js";
|
||||
|
||||
/** Placeholder for blank user messages — preserves the user turn so strict
|
||||
@@ -278,28 +279,19 @@ export async function repairSessionFileIfNeeded(params: {
|
||||
|
||||
const cleaned = `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`;
|
||||
const backupPath = `${sessionFile}.bak-${process.pid}-${Date.now()}`;
|
||||
const tmpPath = `${sessionFile}.repair-${process.pid}-${Date.now()}.tmp`;
|
||||
try {
|
||||
const stat = await fs.stat(sessionFile).catch(() => null);
|
||||
await fs.writeFile(backupPath, content, "utf-8");
|
||||
if (stat) {
|
||||
await fs.chmod(backupPath, stat.mode);
|
||||
}
|
||||
await fs.writeFile(tmpPath, cleaned, "utf-8");
|
||||
if (stat) {
|
||||
await fs.chmod(tmpPath, stat.mode);
|
||||
}
|
||||
await fs.rename(tmpPath, sessionFile);
|
||||
await replaceFileAtomic({
|
||||
filePath: sessionFile,
|
||||
content: cleaned,
|
||||
preserveExistingMode: true,
|
||||
tempPrefix: `${path.basename(sessionFile)}.repair`,
|
||||
});
|
||||
} catch (err) {
|
||||
try {
|
||||
await fs.unlink(tmpPath);
|
||||
} catch (cleanupErr) {
|
||||
params.warn?.(
|
||||
`session file repair cleanup failed: ${cleanupErr instanceof Error ? cleanupErr.message : "unknown error"} (${path.basename(
|
||||
tmpPath,
|
||||
)})`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
repaired: false,
|
||||
droppedLines,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import fsSync from "node:fs";
|
||||
import "../infra/fs-safe-defaults.js";
|
||||
import type fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { createFileLockManager } from "../infra/file-lock-manager.js";
|
||||
import { getProcessStartTime, isPidAlive } from "../shared/pid-alive.js";
|
||||
import { resolveProcessScopedMap } from "../shared/process-scoped-map.js";
|
||||
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
|
||||
|
||||
type LockFilePayload = {
|
||||
@@ -16,19 +17,6 @@ function isValidLockNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
||||
}
|
||||
|
||||
type HeldLock = {
|
||||
count: number;
|
||||
handle: fs.FileHandle;
|
||||
lockPath: string;
|
||||
acquiredAt: number;
|
||||
maxHoldMs: number;
|
||||
releasePromise?: Promise<void>;
|
||||
};
|
||||
|
||||
type SyncClosableFileHandle = fs.FileHandle & {
|
||||
[key: symbol]: unknown;
|
||||
};
|
||||
|
||||
export type SessionLockInspection = {
|
||||
lockPath: string;
|
||||
pid: number | null;
|
||||
@@ -43,7 +31,6 @@ export type SessionLockInspection = {
|
||||
const CLEANUP_SIGNALS = ["SIGINT", "SIGTERM", "SIGQUIT", "SIGABRT"] as const;
|
||||
type CleanupSignal = (typeof CLEANUP_SIGNALS)[number];
|
||||
const CLEANUP_STATE_KEY = Symbol.for("openclaw.sessionWriteLockCleanupState");
|
||||
const HELD_LOCKS_KEY = Symbol.for("openclaw.sessionWriteLockHeldLocks");
|
||||
const WATCHDOG_STATE_KEY = Symbol.for("openclaw.sessionWriteLockWatchdogState");
|
||||
|
||||
const DEFAULT_STALE_MS = 30 * 60 * 1000;
|
||||
@@ -73,7 +60,7 @@ type LockInspectionDetails = Pick<
|
||||
"pid" | "pidAlive" | "createdAt" | "ageMs" | "stale" | "staleReasons"
|
||||
>;
|
||||
|
||||
const HELD_LOCKS = resolveProcessScopedMap<HeldLock>(HELD_LOCKS_KEY);
|
||||
const SESSION_LOCKS = createFileLockManager("openclaw.session-write-lock");
|
||||
|
||||
export type SessionWriteLockAcquireTimeoutConfig = {
|
||||
session?: {
|
||||
@@ -151,105 +138,30 @@ export function resolveSessionLockMaxHoldFromTimeout(params: {
|
||||
return Math.min(MAX_LOCK_HOLD_MS, Math.max(minMs, timeoutMs + graceMs));
|
||||
}
|
||||
|
||||
async function releaseHeldLock(
|
||||
normalizedSessionFile: string,
|
||||
held: HeldLock,
|
||||
opts: { force?: boolean } = {},
|
||||
): Promise<boolean> {
|
||||
const current = HELD_LOCKS.get(normalizedSessionFile);
|
||||
if (current !== held) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (opts.force) {
|
||||
held.count = 0;
|
||||
} else {
|
||||
held.count -= 1;
|
||||
if (held.count > 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (held.releasePromise) {
|
||||
await held.releasePromise.catch(() => undefined);
|
||||
return true;
|
||||
}
|
||||
|
||||
HELD_LOCKS.delete(normalizedSessionFile);
|
||||
held.releasePromise = (async () => {
|
||||
try {
|
||||
await held.handle.close();
|
||||
} catch {
|
||||
// Ignore errors during cleanup - best effort.
|
||||
}
|
||||
try {
|
||||
await fs.rm(held.lockPath, { force: true });
|
||||
} catch {
|
||||
// Ignore errors during cleanup - best effort.
|
||||
}
|
||||
})();
|
||||
|
||||
try {
|
||||
await held.releasePromise;
|
||||
return true;
|
||||
} finally {
|
||||
held.releasePromise = undefined;
|
||||
if (HELD_LOCKS.size === 0) {
|
||||
stopWatchdogTimer();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronously release all held locks.
|
||||
* Used during process exit when async operations aren't reliable.
|
||||
*/
|
||||
function releaseAllLocksSync(): void {
|
||||
for (const [sessionFile, held] of HELD_LOCKS) {
|
||||
closeFileHandleSyncBestEffort(held.handle);
|
||||
try {
|
||||
fsSync.rmSync(held.lockPath, { force: true });
|
||||
} catch {
|
||||
// Ignore errors during cleanup - best effort
|
||||
}
|
||||
HELD_LOCKS.delete(sessionFile);
|
||||
}
|
||||
if (HELD_LOCKS.size === 0) {
|
||||
stopWatchdogTimer();
|
||||
}
|
||||
}
|
||||
|
||||
function closeFileHandleSyncBestEffort(handle: fs.FileHandle): void {
|
||||
const syncCloseSymbol = Object.getOwnPropertySymbols(Object.getPrototypeOf(handle)).find(
|
||||
(symbol) => symbol.description === "kCloseSync",
|
||||
);
|
||||
if (syncCloseSymbol) {
|
||||
const closeSync = (handle as SyncClosableFileHandle)[syncCloseSymbol];
|
||||
if (typeof closeSync === "function") {
|
||||
try {
|
||||
closeSync.call(handle);
|
||||
return;
|
||||
} catch {
|
||||
// Fall back to async close below.
|
||||
}
|
||||
}
|
||||
}
|
||||
void handle.close().catch(() => undefined);
|
||||
SESSION_LOCKS.reset();
|
||||
stopWatchdogTimer();
|
||||
}
|
||||
|
||||
async function runLockWatchdogCheck(nowMs = Date.now()): Promise<number> {
|
||||
let released = 0;
|
||||
for (const [sessionFile, held] of HELD_LOCKS.entries()) {
|
||||
for (const held of SESSION_LOCKS.heldEntries()) {
|
||||
const maxHoldMs =
|
||||
typeof held.metadata.maxHoldMs === "number" ? held.metadata.maxHoldMs : DEFAULT_MAX_HOLD_MS;
|
||||
const heldForMs = nowMs - held.acquiredAt;
|
||||
if (heldForMs <= held.maxHoldMs) {
|
||||
if (heldForMs <= maxHoldMs) {
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stderr.write(
|
||||
`[session-write-lock] releasing lock held for ${heldForMs}ms (max=${held.maxHoldMs}ms): ${held.lockPath}\n`,
|
||||
`[session-write-lock] releasing lock held for ${heldForMs}ms (max=${maxHoldMs}ms): ${held.lockPath}\n`,
|
||||
);
|
||||
|
||||
const didRelease = await releaseHeldLock(sessionFile, held, { force: true });
|
||||
const didRelease = await held.forceRelease();
|
||||
if (didRelease) {
|
||||
released += 1;
|
||||
}
|
||||
@@ -458,14 +370,14 @@ async function shouldReclaimContendedLockFile(
|
||||
|
||||
function shouldTreatAsOrphanSelfLock(params: {
|
||||
payload: LockFilePayload | null;
|
||||
normalizedSessionFile: string;
|
||||
heldByThisProcess: boolean;
|
||||
reclaimLockWithoutStarttime: boolean;
|
||||
}): boolean {
|
||||
const pid = isValidLockNumber(params.payload?.pid) ? params.payload.pid : null;
|
||||
if (pid !== process.pid) {
|
||||
return false;
|
||||
}
|
||||
if (HELD_LOCKS.has(params.normalizedSessionFile)) {
|
||||
if (params.heldByThisProcess) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -484,14 +396,14 @@ function inspectLockPayloadForSession(params: {
|
||||
payload: LockFilePayload | null;
|
||||
staleMs: number;
|
||||
nowMs: number;
|
||||
normalizedSessionFile: string;
|
||||
heldByThisProcess: boolean;
|
||||
reclaimLockWithoutStarttime: boolean;
|
||||
}): LockInspectionDetails {
|
||||
const inspected = inspectLockPayload(params.payload, params.staleMs, params.nowMs);
|
||||
if (
|
||||
!shouldTreatAsOrphanSelfLock({
|
||||
payload: params.payload,
|
||||
normalizedSessionFile: params.normalizedSessionFile,
|
||||
heldByThisProcess: params.heldByThisProcess,
|
||||
reclaimLockWithoutStarttime: params.reclaimLockWithoutStarttime,
|
||||
})
|
||||
) {
|
||||
@@ -541,13 +453,11 @@ export async function cleanStaleLockFiles(params: {
|
||||
for (const entry of lockEntries) {
|
||||
const lockPath = path.join(sessionsDir, entry.name);
|
||||
const payload = await readLockPayload(lockPath);
|
||||
const sessionFile = lockPath.slice(0, -".lock".length);
|
||||
const normalizedSessionFile = await resolveNormalizedSessionFile(sessionFile);
|
||||
const inspected = inspectLockPayloadForSession({
|
||||
payload,
|
||||
staleMs,
|
||||
nowMs,
|
||||
normalizedSessionFile,
|
||||
heldByThisProcess: false,
|
||||
reclaimLockWithoutStarttime: false,
|
||||
});
|
||||
const lockInfo: SessionLockInspection = {
|
||||
@@ -589,97 +499,46 @@ export async function acquireSessionWriteLock(params: {
|
||||
const maxHoldMs = resolvePositiveMs(params.maxHoldMs, DEFAULT_MAX_HOLD_MS);
|
||||
const sessionFile = path.resolve(params.sessionFile);
|
||||
const sessionDir = path.dirname(sessionFile);
|
||||
await fs.mkdir(sessionDir, { recursive: true });
|
||||
const normalizedSessionFile = await resolveNormalizedSessionFile(sessionFile);
|
||||
const lockPath = `${normalizedSessionFile}.lock`;
|
||||
|
||||
const held = HELD_LOCKS.get(normalizedSessionFile);
|
||||
if (allowReentrant && held) {
|
||||
held.count += 1;
|
||||
return {
|
||||
release: async () => {
|
||||
await releaseHeldLock(normalizedSessionFile, held);
|
||||
await fs.mkdir(sessionDir, { recursive: true });
|
||||
try {
|
||||
const lock = await SESSION_LOCKS.acquire(sessionFile, {
|
||||
staleMs,
|
||||
timeoutMs,
|
||||
retry: { minTimeout: 50, maxTimeout: 1000, factor: 1 },
|
||||
allowReentrant,
|
||||
metadata: { maxHoldMs },
|
||||
payload: () => {
|
||||
const createdAt = new Date().toISOString();
|
||||
const starttime = getProcessStartTime(process.pid);
|
||||
const lockPayload: LockFilePayload = { pid: process.pid, createdAt };
|
||||
if (starttime !== null) {
|
||||
lockPayload.starttime = starttime;
|
||||
}
|
||||
return lockPayload as Record<string, unknown>;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
let attempt = 0;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
attempt += 1;
|
||||
let handle: fs.FileHandle | null = null;
|
||||
try {
|
||||
handle = await fs.open(lockPath, "wx");
|
||||
const createdHeld: HeldLock = {
|
||||
count: 1,
|
||||
handle,
|
||||
lockPath,
|
||||
acquiredAt: Date.now(),
|
||||
maxHoldMs,
|
||||
};
|
||||
HELD_LOCKS.set(normalizedSessionFile, createdHeld);
|
||||
const createdAt = new Date().toISOString();
|
||||
const starttime = getProcessStartTime(process.pid);
|
||||
const lockPayload: LockFilePayload = { pid: process.pid, createdAt };
|
||||
if (starttime !== null) {
|
||||
lockPayload.starttime = starttime;
|
||||
}
|
||||
await handle.writeFile(JSON.stringify(lockPayload, null, 2), "utf8");
|
||||
return {
|
||||
release: async () => {
|
||||
await releaseHeldLock(normalizedSessionFile, createdHeld);
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
if (handle) {
|
||||
const currentHeld = HELD_LOCKS.get(normalizedSessionFile);
|
||||
if (currentHeld?.handle === handle) {
|
||||
HELD_LOCKS.delete(normalizedSessionFile);
|
||||
if (HELD_LOCKS.size === 0) {
|
||||
stopWatchdogTimer();
|
||||
}
|
||||
}
|
||||
try {
|
||||
await handle.close();
|
||||
} catch {
|
||||
// Ignore cleanup errors on failed lock initialization.
|
||||
}
|
||||
try {
|
||||
await fs.rm(lockPath, { force: true });
|
||||
} catch {
|
||||
// Ignore cleanup errors on failed lock initialization.
|
||||
}
|
||||
}
|
||||
const code = (err as { code?: unknown }).code;
|
||||
if (code !== "EEXIST") {
|
||||
throw err;
|
||||
}
|
||||
const payload = await readLockPayload(lockPath);
|
||||
const nowMs = Date.now();
|
||||
const inspected = inspectLockPayloadForSession({
|
||||
payload,
|
||||
staleMs,
|
||||
nowMs,
|
||||
normalizedSessionFile,
|
||||
reclaimLockWithoutStarttime: true,
|
||||
});
|
||||
if (await shouldReclaimContendedLockFile(lockPath, inspected, staleMs, nowMs)) {
|
||||
await fs.rm(lockPath, { force: true });
|
||||
continue;
|
||||
}
|
||||
|
||||
const remainingMs = timeoutMs - (Date.now() - startedAt);
|
||||
if (remainingMs <= 0) {
|
||||
break;
|
||||
}
|
||||
const delay = Math.min(1000, 50 * attempt, remainingMs);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
shouldReclaim: async ({ payload, nowMs, heldByThisProcess }) => {
|
||||
const inspected = inspectLockPayloadForSession({
|
||||
payload: payload as LockFilePayload | null,
|
||||
staleMs,
|
||||
nowMs,
|
||||
heldByThisProcess,
|
||||
reclaimLockWithoutStarttime: true,
|
||||
});
|
||||
return await shouldReclaimContendedLockFile(lockPath, inspected, staleMs, nowMs);
|
||||
},
|
||||
});
|
||||
return { release: lock.release };
|
||||
} catch (err) {
|
||||
if ((err as { code?: unknown }).code !== "file_lock_timeout") {
|
||||
throw err;
|
||||
}
|
||||
const timeoutLockPath = (err as { lockPath?: string }).lockPath ?? lockPath;
|
||||
const payload = await readLockPayload(timeoutLockPath);
|
||||
const owner = typeof payload?.pid === "number" ? `pid=${payload.pid}` : "unknown";
|
||||
throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath: timeoutLockPath });
|
||||
}
|
||||
|
||||
const payload = await readLockPayload(lockPath);
|
||||
const owner = typeof payload?.pid === "number" ? `pid=${payload.pid}` : "unknown";
|
||||
throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath });
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
@@ -690,9 +549,7 @@ export const __testing = {
|
||||
};
|
||||
|
||||
export async function drainSessionWriteLockStateForTest(): Promise<void> {
|
||||
for (const [sessionFile, held] of Array.from(HELD_LOCKS.entries())) {
|
||||
await releaseHeldLock(sessionFile, held, { force: true }).catch(() => undefined);
|
||||
}
|
||||
await SESSION_LOCKS.drain();
|
||||
stopWatchdogTimer();
|
||||
unregisterCleanupHandlers();
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ const searchClawHubSkillsMock = vi.fn();
|
||||
const archiveCleanupMock = vi.fn();
|
||||
const withExtractedArchiveRootMock = vi.fn();
|
||||
const installPackageDirMock = vi.fn();
|
||||
const fileExistsMock = vi.fn();
|
||||
const pathExistsMock = vi.fn();
|
||||
|
||||
vi.mock("../infra/clawhub.js", () => ({
|
||||
fetchClawHubSkillDetail: fetchClawHubSkillDetailMock,
|
||||
@@ -29,8 +29,8 @@ vi.mock("../infra/install-package-dir.js", () => ({
|
||||
installPackageDir: installPackageDirMock,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/archive.js", () => ({
|
||||
fileExists: fileExistsMock,
|
||||
vi.mock("../infra/fs-safe.js", () => ({
|
||||
pathExists: pathExistsMock,
|
||||
}));
|
||||
|
||||
const { installSkillFromClawHub, searchSkillsFromClawHub, updateSkillsFromClawHub } =
|
||||
@@ -46,10 +46,10 @@ describe("skills-clawhub", () => {
|
||||
archiveCleanupMock.mockReset();
|
||||
withExtractedArchiveRootMock.mockReset();
|
||||
installPackageDirMock.mockReset();
|
||||
fileExistsMock.mockReset();
|
||||
pathExistsMock.mockReset();
|
||||
|
||||
resolveClawHubBaseUrlMock.mockReturnValue("https://clawhub.ai");
|
||||
fileExistsMock.mockImplementation(async (input: string) => input.endsWith("SKILL.md"));
|
||||
pathExistsMock.mockImplementation(async (input: string) => input.endsWith("SKILL.md"));
|
||||
fetchClawHubSkillDetailMock.mockResolvedValue({
|
||||
skill: {
|
||||
slug: "agentreceipt",
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileExists } from "../infra/archive.js";
|
||||
import {
|
||||
downloadClawHubSkillArchive,
|
||||
fetchClawHubSkillDetail,
|
||||
@@ -10,6 +9,7 @@ import {
|
||||
type ClawHubSkillSearchResult,
|
||||
} from "../infra/clawhub.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { pathExists } from "../infra/fs-safe.js";
|
||||
import { withExtractedArchiveRoot } from "../infra/install-flow.js";
|
||||
import { installPackageDir } from "../infra/install-package-dir.js";
|
||||
import { resolveSafeInstallDir } from "../infra/install-safe-path.js";
|
||||
@@ -133,7 +133,7 @@ function resolveSkillInstallDir(workspaceDir: string, slug: string): string {
|
||||
|
||||
async function ensureSkillRoot(rootDir: string): Promise<void> {
|
||||
for (const candidate of ["SKILL.md", "skill.md", "skills.md", "SKILL.MD"]) {
|
||||
if (await fileExists(path.join(rootDir, candidate))) {
|
||||
if (await pathExists(path.join(rootDir, candidate))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -274,7 +274,7 @@ async function performClawHubSkillInstall(
|
||||
baseUrl: params.baseUrl,
|
||||
});
|
||||
const targetDir = resolveSkillInstallDir(params.workspaceDir, params.slug);
|
||||
if (!params.force && (await fileExists(targetDir))) {
|
||||
if (!params.force && (await pathExists(targetDir))) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Skill already exists at ${targetDir}. Re-run with force/update.`,
|
||||
|
||||
@@ -6,7 +6,7 @@ import { pipeline } from "node:stream/promises";
|
||||
import type { ReadableStream as NodeReadableStream } from "node:stream/web";
|
||||
import { isWindowsDrivePath } from "../infra/archive-path.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { writeFileFromPathWithinRoot } from "../infra/fs-safe.js";
|
||||
import { root as fsRoot } from "../infra/fs-safe.js";
|
||||
import { assertCanonicalPathWithinBase } from "../infra/install-safe-path.js";
|
||||
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
|
||||
import { isWithinDir } from "../infra/path-safety.js";
|
||||
@@ -29,21 +29,21 @@ function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream {
|
||||
}
|
||||
|
||||
function resolveDownloadTargetDir(entry: SkillEntry, spec: SkillInstallSpec): string {
|
||||
const safeRoot = resolveSkillToolsRootDir(entry);
|
||||
const root = resolveSkillToolsRootDir(entry);
|
||||
const raw = spec.targetDir?.trim();
|
||||
if (!raw) {
|
||||
return safeRoot;
|
||||
return root;
|
||||
}
|
||||
|
||||
// Treat non-absolute paths as relative to the per-skill tools root.
|
||||
const resolved =
|
||||
raw.startsWith("~") || path.isAbsolute(raw) || isWindowsDrivePath(raw)
|
||||
? resolveUserPath(raw)
|
||||
: path.resolve(safeRoot, raw);
|
||||
: path.resolve(root, raw);
|
||||
|
||||
if (!isWithinDir(safeRoot, resolved)) {
|
||||
if (!isWithinDir(root, resolved)) {
|
||||
throw new Error(
|
||||
`Refusing to install outside the skill tools directory. targetDir="${raw}" resolves to "${resolved}". Allowed root: "${safeRoot}".`,
|
||||
`Refusing to install outside the skill tools directory. targetDir="${raw}" resolves to "${resolved}". Allowed root: "${root}".`,
|
||||
);
|
||||
}
|
||||
return resolved;
|
||||
@@ -99,11 +99,8 @@ async function downloadFile(params: {
|
||||
? body
|
||||
: Readable.fromWeb(body as NodeReadableStream);
|
||||
await pipeline(readable, file);
|
||||
await writeFileFromPathWithinRoot({
|
||||
rootDir: params.rootDir,
|
||||
relativePath: params.relativePath,
|
||||
sourcePath: tempPath,
|
||||
});
|
||||
const root = await fsRoot(params.rootDir);
|
||||
await root.copyIn(params.relativePath, tempPath);
|
||||
const stat = await fs.promises.stat(destPath);
|
||||
return { bytes: stat.size };
|
||||
} finally {
|
||||
@@ -118,7 +115,7 @@ export async function installDownloadSpec(params: {
|
||||
timeoutMs: number;
|
||||
}): Promise<SkillInstallResult> {
|
||||
const { entry, spec, timeoutMs } = params;
|
||||
const safeRoot = resolveSkillToolsRootDir(entry);
|
||||
const root = resolveSkillToolsRootDir(entry);
|
||||
const url = spec.url?.trim();
|
||||
if (!url) {
|
||||
return {
|
||||
@@ -141,33 +138,33 @@ export async function installDownloadSpec(params: {
|
||||
filename = "download";
|
||||
}
|
||||
|
||||
let canonicalSafeRoot = "";
|
||||
let canonicalRoot = "";
|
||||
let targetDir = "";
|
||||
try {
|
||||
await ensureDir(safeRoot);
|
||||
await ensureDir(root);
|
||||
await assertCanonicalPathWithinBase({
|
||||
baseDir: safeRoot,
|
||||
candidatePath: safeRoot,
|
||||
baseDir: root,
|
||||
candidatePath: root,
|
||||
boundaryLabel: "skill tools directory",
|
||||
});
|
||||
canonicalSafeRoot = await fs.promises.realpath(safeRoot);
|
||||
canonicalRoot = await fs.promises.realpath(root);
|
||||
|
||||
const requestedTargetDir = resolveDownloadTargetDir(entry, spec);
|
||||
await ensureDir(requestedTargetDir);
|
||||
await assertCanonicalPathWithinBase({
|
||||
baseDir: safeRoot,
|
||||
baseDir: root,
|
||||
candidatePath: requestedTargetDir,
|
||||
boundaryLabel: "skill tools directory",
|
||||
});
|
||||
const targetRelativePath = path.relative(safeRoot, requestedTargetDir);
|
||||
targetDir = path.join(canonicalSafeRoot, targetRelativePath);
|
||||
const targetRelativePath = path.relative(root, requestedTargetDir);
|
||||
targetDir = path.join(canonicalRoot, targetRelativePath);
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
return { ok: false, message, stdout: "", stderr: message, code: null };
|
||||
}
|
||||
|
||||
const archivePath = path.join(targetDir, filename);
|
||||
const archiveRelativePath = path.relative(canonicalSafeRoot, archivePath);
|
||||
const archiveRelativePath = path.relative(canonicalRoot, archivePath);
|
||||
if (
|
||||
!archiveRelativePath ||
|
||||
archiveRelativePath === ".." ||
|
||||
@@ -186,7 +183,7 @@ export async function installDownloadSpec(params: {
|
||||
try {
|
||||
const result = await downloadFile({
|
||||
url,
|
||||
rootDir: canonicalSafeRoot,
|
||||
rootDir: canonicalRoot,
|
||||
relativePath: archiveRelativePath,
|
||||
timeoutMs,
|
||||
});
|
||||
@@ -220,7 +217,7 @@ export async function installDownloadSpec(params: {
|
||||
|
||||
try {
|
||||
await assertCanonicalPathWithinBase({
|
||||
baseDir: canonicalSafeRoot,
|
||||
baseDir: canonicalRoot,
|
||||
candidatePath: targetDir,
|
||||
boundaryLabel: "skill tools directory",
|
||||
});
|
||||
|
||||
@@ -22,60 +22,6 @@ vi.mock("../infra/net/fetch-guard.js", () => ({
|
||||
fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args),
|
||||
}));
|
||||
|
||||
// Download tests cover installer path handling; fs-safe has dedicated pinned-helper coverage.
|
||||
vi.mock("../infra/fs-pinned-write-helper.js", async () => {
|
||||
const fs = await import("node:fs/promises");
|
||||
const path = await import("node:path");
|
||||
const { pipeline } = await import("node:stream/promises");
|
||||
|
||||
type PinnedWriteParams = {
|
||||
rootPath: string;
|
||||
relativeParentPath: string;
|
||||
basename: string;
|
||||
mkdir: boolean;
|
||||
mode: number;
|
||||
input:
|
||||
| { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding }
|
||||
| { kind: "stream"; stream: NodeJS.ReadableStream };
|
||||
};
|
||||
|
||||
async function resolveParentPath(params: PinnedWriteParams): Promise<string> {
|
||||
const parentPath = params.relativeParentPath
|
||||
? path.join(params.rootPath, ...params.relativeParentPath.split("/"))
|
||||
: params.rootPath;
|
||||
if (params.mkdir) {
|
||||
await fs.mkdir(parentPath, { recursive: true });
|
||||
}
|
||||
return parentPath;
|
||||
}
|
||||
|
||||
async function writePinnedTarget(params: PinnedWriteParams, targetPath: string) {
|
||||
if (params.input.kind === "buffer") {
|
||||
await fs.writeFile(targetPath, params.input.data, {
|
||||
encoding: params.input.encoding,
|
||||
mode: params.mode,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const handle = await fs.open(targetPath, "w", params.mode);
|
||||
try {
|
||||
await pipeline(params.input.stream, handle.createWriteStream());
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
runPinnedWriteHelper: async (params: PinnedWriteParams) => {
|
||||
const parentPath = await resolveParentPath(params);
|
||||
const targetPath = path.join(parentPath, params.basename);
|
||||
await writePinnedTarget(params, targetPath);
|
||||
const stat = await fs.stat(targetPath);
|
||||
return { dev: stat.dev, ino: stat.ino };
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./skills.js", () => ({
|
||||
hasBinary: (bin: string) => hasBinaryMock(bin),
|
||||
}));
|
||||
@@ -262,7 +208,7 @@ describe("installDownloadSpec extraction safety", () => {
|
||||
"fails closed when the lexical tools root is rebound before the final copy",
|
||||
async () => {
|
||||
const entry = buildEntry("base-rebind");
|
||||
const safeRoot = resolveSkillToolsRootDir(entry);
|
||||
const safeToolsRoot = resolveSkillToolsRootDir(entry);
|
||||
const outsideRoot = path.join(workspaceDir, "outside-root");
|
||||
await fs.mkdir(outsideRoot, { recursive: true });
|
||||
|
||||
@@ -274,9 +220,9 @@ describe("installDownloadSpec extraction safety", () => {
|
||||
body: Readable.from(
|
||||
(async function* () {
|
||||
yield Buffer.from("payload");
|
||||
const reboundRoot = `${safeRoot}-rebound`;
|
||||
await fs.rename(safeRoot, reboundRoot);
|
||||
await fs.symlink(outsideRoot, safeRoot);
|
||||
const reboundRoot = `${safeToolsRoot}-rebound`;
|
||||
await fs.rename(safeToolsRoot, reboundRoot);
|
||||
await fs.symlink(outsideRoot, safeToolsRoot);
|
||||
})(),
|
||||
),
|
||||
},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { openVerifiedFileSync } from "../../infra/safe-open-sync.js";
|
||||
import { openRootFileSync } from "../../infra/boundary-file-read.js";
|
||||
import { parseFrontmatter, resolveSkillInvocationPolicy } from "./frontmatter.js";
|
||||
import { createSyntheticSourceInfo, type Skill } from "./skill-contract.js";
|
||||
import type { ParsedSkillFrontmatter } from "./types.js";
|
||||
@@ -10,31 +10,22 @@ type LoadedLocalSkill = {
|
||||
frontmatter: ParsedSkillFrontmatter;
|
||||
};
|
||||
|
||||
function isPathWithinRoot(rootRealPath: string, candidatePath: string): boolean {
|
||||
const relative = path.relative(rootRealPath, candidatePath);
|
||||
return (
|
||||
relative === "" ||
|
||||
(!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative))
|
||||
);
|
||||
}
|
||||
|
||||
function readSkillFileSync(params: {
|
||||
rootRealPath: string;
|
||||
filePath: string;
|
||||
maxBytes?: number;
|
||||
}): string | null {
|
||||
const opened = openVerifiedFileSync({
|
||||
filePath: params.filePath,
|
||||
rejectPathSymlink: true,
|
||||
const opened = openRootFileSync({
|
||||
absolutePath: params.filePath,
|
||||
rootPath: params.rootRealPath,
|
||||
rootRealPath: params.rootRealPath,
|
||||
boundaryLabel: "skill root",
|
||||
maxBytes: params.maxBytes,
|
||||
});
|
||||
if (!opened.ok) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
if (!isPathWithinRoot(params.rootRealPath, opened.path)) {
|
||||
return null;
|
||||
}
|
||||
return fs.readFileSync(opened.fd, "utf8");
|
||||
} finally {
|
||||
fs.closeSync(opened.fd);
|
||||
|
||||
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { isAcpRuntimeSpawnAvailable } from "../../acp/runtime/availability.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { walkDirectorySync } from "../../infra/fs-safe.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import {
|
||||
normalizePluginsConfigWithResolver,
|
||||
@@ -130,15 +131,13 @@ function collectSkillTargets(dir: string, targets: Map<string, string>): void {
|
||||
return;
|
||||
}
|
||||
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const entries = walkDirectorySync(dir, {
|
||||
maxDepth: 1,
|
||||
symlinks: "skip",
|
||||
include: (entry) => entry.kind === "directory",
|
||||
}).entries;
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const childPath = path.join(dir, entry.name);
|
||||
const childPath = entry.path;
|
||||
if (!hasPublishableSkillFile({ skillDir: childPath, rootDir: dir })) continue;
|
||||
const basename = entry.name;
|
||||
const existing = targets.get(basename);
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import fs, { type Dirent } from "node:fs";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { walkDirectorySync } from "../../infra/fs-safe.js";
|
||||
import { resolveOsHomeDir } from "../../infra/home-dir.js";
|
||||
import { isPathInside } from "../../infra/path-guards.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
@@ -181,44 +182,21 @@ function listChildDirectories(
|
||||
opts?.maxRawEntriesToScan === undefined
|
||||
? resolveRawEntryScanLimit(opts?.maxCandidateDirs)
|
||||
: Math.max(0, opts.maxRawEntriesToScan);
|
||||
try {
|
||||
const dirs: string[] = [];
|
||||
let scannedEntryCount = 0;
|
||||
let truncated = false;
|
||||
const handle = fs.opendirSync(dir);
|
||||
try {
|
||||
let entry: Dirent | null;
|
||||
while ((entry = handle.readSync()) !== null) {
|
||||
if (scannedEntryCount >= maxRawEntriesToScan) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
scannedEntryCount += 1;
|
||||
|
||||
if (entry.name.startsWith(".")) continue;
|
||||
if (entry.name === "node_modules") continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
dirs.push(entry.name);
|
||||
continue;
|
||||
}
|
||||
if (entry.isSymbolicLink()) {
|
||||
try {
|
||||
if (fs.statSync(fullPath).isDirectory()) {
|
||||
dirs.push(entry.name);
|
||||
}
|
||||
} catch {
|
||||
// ignore broken symlinks
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
handle.closeSync();
|
||||
}
|
||||
return { dirs, scannedEntryCount, truncated };
|
||||
} catch {
|
||||
const scan = walkDirectorySync(dir, {
|
||||
maxDepth: 1,
|
||||
maxEntries: maxRawEntriesToScan,
|
||||
symlinks: "follow",
|
||||
include: (entry) =>
|
||||
entry.kind === "directory" && !entry.name.startsWith(".") && entry.name !== "node_modules",
|
||||
});
|
||||
if (scan.scannedEntryCount === 0 && scan.entries.length === 0) {
|
||||
return { dirs: [], scannedEntryCount: 0, truncated: false };
|
||||
}
|
||||
return {
|
||||
dirs: scan.entries.map((entry) => entry.name),
|
||||
scannedEntryCount: scan.scannedEntryCount,
|
||||
truncated: scan.truncated,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRawEntryScanLimit(maxCandidateDirs: number | undefined): number {
|
||||
|
||||
@@ -2,6 +2,7 @@ import crypto from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { privateFileStore } from "../infra/private-file-store.js";
|
||||
import { normalizeOptionalString } from "../shared/string-coerce.js";
|
||||
import { resolveAgentWorkspaceDir } from "./agent-scope.js";
|
||||
|
||||
@@ -131,6 +132,7 @@ export async function materializeSubagentAttachments(params: {
|
||||
|
||||
try {
|
||||
await fs.mkdir(absDir, { recursive: true, mode: 0o700 });
|
||||
const store = privateFileStore(absDir);
|
||||
|
||||
const seen = new Set<string>();
|
||||
const files: SubagentAttachmentReceiptFile[] = [];
|
||||
@@ -192,14 +194,11 @@ export async function materializeSubagentAttachments(params: {
|
||||
}
|
||||
|
||||
const sha256 = crypto.createHash("sha256").update(buf).digest("hex");
|
||||
const outPath = path.join(absDir, name);
|
||||
writeJobs.push({ outPath, buf });
|
||||
writeJobs.push({ outPath: name, buf });
|
||||
files.push({ name, bytes, sha256 });
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
writeJobs.map(({ outPath, buf }) => fs.writeFile(outPath, buf, { mode: 0o600, flag: "wx" })),
|
||||
);
|
||||
await Promise.all(writeJobs.map(({ outPath, buf }) => store.writeText(outPath, buf)));
|
||||
|
||||
const manifest = {
|
||||
relDir,
|
||||
@@ -207,14 +206,7 @@ export async function materializeSubagentAttachments(params: {
|
||||
totalBytes,
|
||||
files,
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(absDir, ".manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
{
|
||||
mode: 0o600,
|
||||
flag: "wx",
|
||||
},
|
||||
);
|
||||
await store.writeJson(".manifest.json", manifest, { trailingNewline: true });
|
||||
|
||||
return {
|
||||
status: "ok",
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { Type } from "typebox";
|
||||
import { writeBase64ToFile } from "../../cli/nodes-camera.js";
|
||||
import { canvasSnapshotTempPath, parseCanvasSnapshotPayload } from "../../cli/nodes-canvas.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../../globals.js";
|
||||
import { isInboundPathAllowed } from "../../media/inbound-path-policy.js";
|
||||
import { readLocalFileFromRoots } from "../../infra/fs-safe.js";
|
||||
import { getDefaultMediaLocalRoots } from "../../media/local-roots.js";
|
||||
import { imageMimeFromFormat } from "../../media/mime.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js";
|
||||
@@ -33,22 +31,19 @@ async function readJsonlFromPath(jsonlPath: string): Promise<string> {
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
const resolved = path.resolve(trimmed);
|
||||
const roots = getDefaultMediaLocalRoots();
|
||||
if (!isInboundPathAllowed({ filePath: resolved, roots })) {
|
||||
const result = await readLocalFileFromRoots({
|
||||
filePath: trimmed,
|
||||
roots,
|
||||
label: "canvas jsonlPath",
|
||||
});
|
||||
if (!result) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${resolved}`);
|
||||
logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${trimmed}`);
|
||||
}
|
||||
throw new Error("jsonlPath outside allowed roots");
|
||||
}
|
||||
const canonical = await fs.realpath(resolved).catch(() => resolved);
|
||||
if (!isInboundPathAllowed({ filePath: canonical, roots })) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${canonical}`);
|
||||
}
|
||||
throw new Error("jsonlPath outside allowed roots");
|
||||
}
|
||||
return await fs.readFile(canonical, "utf8");
|
||||
return result.buffer.toString("utf8");
|
||||
}
|
||||
|
||||
// Flattened schema: runtime validates per-action requirements.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import fs from "node:fs/promises";
|
||||
import type {
|
||||
AgentTool,
|
||||
AgentToolResult,
|
||||
AgentToolUpdateCallback,
|
||||
} from "@mariozechner/pi-agent-core";
|
||||
import type { TSchema } from "typebox";
|
||||
import { readLocalFileSafely } from "../../infra/fs-safe.js";
|
||||
import { detectMime } from "../../media/mime.js";
|
||||
import { readSnakeCaseParamRaw } from "../../param-key.js";
|
||||
import type { ImageSanitizationLimits } from "../image-sanitization.js";
|
||||
@@ -345,7 +345,7 @@ export async function imageResultFromFile(params: {
|
||||
details?: Record<string, unknown>;
|
||||
imageSanitization?: ImageSanitizationLimits;
|
||||
}): Promise<AgentToolResult<unknown>> {
|
||||
const buf = await fs.readFile(params.path);
|
||||
const buf = (await readLocalFileSafely({ filePath: params.path })).buffer;
|
||||
const mimeType = (await detectMime({ buffer: buf.slice(0, 256) })) ?? "image/png";
|
||||
return await imageResult({
|
||||
label: params.label,
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import syncFs from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { openBoundaryFile } from "../infra/boundary-file-read.js";
|
||||
import { openRootFile } from "../infra/boundary-file-read.js";
|
||||
import { pathExists } from "../infra/fs-safe.js";
|
||||
import { replaceFileAtomic } from "../infra/replace-file.js";
|
||||
import {
|
||||
CANONICAL_ROOT_MEMORY_FILENAME,
|
||||
exactWorkspaceEntryExists,
|
||||
@@ -55,7 +57,7 @@ async function readWorkspaceFileWithGuards(params: {
|
||||
filePath: string;
|
||||
workspaceDir: string;
|
||||
}): Promise<WorkspaceGuardedReadResult> {
|
||||
const opened = await openBoundaryFile({
|
||||
const opened = await openRootFile({
|
||||
absolutePath: params.filePath,
|
||||
rootPath: params.workspaceDir,
|
||||
boundaryLabel: "workspace root",
|
||||
@@ -197,15 +199,6 @@ async function writeFileIfMissing(filePath: string, content: string): Promise<bo
|
||||
}
|
||||
}
|
||||
|
||||
async function fileExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function fileContentDiffersFromTemplate(
|
||||
filePath: string,
|
||||
template: string,
|
||||
@@ -274,7 +267,7 @@ async function reconcileWorkspaceBootstrapCompletionState(params: {
|
||||
state: WorkspaceSetupState;
|
||||
bootstrapExists?: boolean;
|
||||
}): Promise<WorkspaceBootstrapCompletionReconcileResult> {
|
||||
const bootstrapExists = params.bootstrapExists ?? (await fileExists(params.bootstrapPath));
|
||||
const bootstrapExists = params.bootstrapExists ?? (await pathExists(params.bootstrapPath));
|
||||
if (
|
||||
typeof params.state.setupCompletedAt === "string" &&
|
||||
params.state.setupCompletedAt.trim().length > 0
|
||||
@@ -384,7 +377,7 @@ export async function resolveWorkspaceBootstrapStatus(
|
||||
return "complete";
|
||||
}
|
||||
const bootstrapPath = path.join(resolvedDir, DEFAULT_BOOTSTRAP_FILENAME);
|
||||
const bootstrapExists = await fileExists(bootstrapPath);
|
||||
const bootstrapExists = await pathExists(bootstrapPath);
|
||||
if (!bootstrapExists) {
|
||||
return "complete";
|
||||
}
|
||||
@@ -416,16 +409,11 @@ async function writeWorkspaceSetupState(
|
||||
statePath: string,
|
||||
state: WorkspaceSetupState,
|
||||
): Promise<void> {
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
const payload = `${JSON.stringify(state, null, 2)}\n`;
|
||||
const tmpPath = `${statePath}.tmp-${process.pid}-${Date.now().toString(36)}`;
|
||||
try {
|
||||
await fs.writeFile(tmpPath, payload, { encoding: "utf-8" });
|
||||
await fs.rename(tmpPath, statePath);
|
||||
} catch (err) {
|
||||
await fs.unlink(tmpPath).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
await replaceFileAtomic({
|
||||
filePath: statePath,
|
||||
content: `${JSON.stringify(state, null, 2)}\n`,
|
||||
tempPrefix: ".workspace-state",
|
||||
});
|
||||
}
|
||||
|
||||
async function hasGitRepo(dir: string): Promise<boolean> {
|
||||
@@ -561,7 +549,7 @@ export async function ensureAgentWorkspace(params?: {
|
||||
};
|
||||
const nowIso = () => new Date().toISOString();
|
||||
|
||||
let bootstrapExists = await fileExists(bootstrapPath);
|
||||
let bootstrapExists = await pathExists(bootstrapPath);
|
||||
if (!state.bootstrapSeededAt && bootstrapExists) {
|
||||
markState({ bootstrapSeededAt: nowIso() });
|
||||
}
|
||||
@@ -596,7 +584,7 @@ export async function ensureAgentWorkspace(params?: {
|
||||
const bootstrapTemplate = await loadTemplate(DEFAULT_BOOTSTRAP_FILENAME);
|
||||
const wroteBootstrap = await writeFileIfMissing(bootstrapPath, bootstrapTemplate);
|
||||
if (!wroteBootstrap) {
|
||||
bootstrapExists = await fileExists(bootstrapPath);
|
||||
bootstrapExists = await pathExists(bootstrapPath);
|
||||
} else {
|
||||
bootstrapExists = true;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user