From 538605ff44d23e4e57ecfd72a8f0b20159ffebbf Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 02:15:17 +0100 Subject: [PATCH] [codex] Extract filesystem safety primitives (#77918) * refactor: extract filesystem safety primitives * refactor: use fs-safe for file access helpers * refactor: reuse fs-safe for media reads * refactor: use fs-safe for image reads * refactor: reuse fs-safe in qqbot media opener * refactor: reuse fs-safe for local media checks * refactor: consume cleaner fs-safe api * refactor: align fs-safe json option names * fix: preserve fs-safe migration contracts * refactor: use fs-safe primitive subpaths * refactor: use grouped fs-safe subpaths * refactor: align fs-safe api usage * refactor: adapt private state store api * chore: refresh proof gate * refactor: follow fs-safe json api split * refactor: follow reduced fs-safe surface * build: default fs-safe python helper off * fix: preserve fs-safe plugin sdk aliases * refactor: consolidate fs-safe usage * refactor: unify fs-safe store usage * refactor: trim fs-safe temp workspace usage * refactor: hide low-level fs-safe primitives * build: use published fs-safe package * fix: preserve outbound recovery durability after rebase * chore: refresh pr checks --- CHANGELOG.md | 2 + .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/.i18n/glossary.zh-CN.json | 8 + docs/docs.json | 1 + docs/gateway/security/index.md | 6 + .../security/secure-file-operations.md | 76 ++ docs/plugins/sdk-migration.md | 2 +- docs/plugins/sdk-subpaths.md | 4 +- docs/refactor/fs-cleanup.md | 448 +++++++ extensions/active-memory/index.test.ts | 13 +- extensions/active-memory/index.ts | 30 +- extensions/bluebubbles/src/attachments.ts | 5 +- extensions/bluebubbles/src/media-send.ts | 130 +- .../browser/src/browser/output-atomic.ts | 50 +- extensions/browser/src/browser/paths.ts | 249 +--- .../src/browser/routes/agent.act.hooks.ts | 10 +- .../src/browser/routes/output-paths.ts | 12 +- .../browser/src/browser/routes/path-output.ts | 1 + .../browser/src/browser/safe-filename.ts | 28 +- extensions/browser/src/infra/fs-safe.ts | 6 +- .../browser/src/sdk-security-runtime.ts | 13 +- .../codex/src/app-server/run-attempt.ts | 15 +- extensions/codex/src/app-server/timeout.ts | 19 +- extensions/codex/src/app-server/trajectory.ts | 89 +- extensions/codex/src/migration/helpers.ts | 8 +- extensions/device-pair/notify.ts | 8 +- extensions/diffs/src/store.ts | 32 +- .../discord/src/internal/command-deploy.ts | 36 +- extensions/discord/src/send.voice.ts | 22 +- extensions/discord/src/voice/audio.ts | 14 +- extensions/feishu/src/media.test.ts | 3 +- extensions/feishu/src/media.ts | 91 +- extensions/feishu/src/outbound.ts | 12 +- .../file-transfer/src/node-host/dir-fetch.ts | 61 +- .../file-transfer/src/node-host/dir-list.ts | 86 +- .../file-transfer/src/node-host/file-fetch.ts | 175 +-- .../file-transfer/src/node-host/file-write.ts | 187 ++- extensions/file-transfer/src/shared/audit.ts | 7 +- .../src/tools/file-write-tool.ts | 16 +- .../google/video-generation-provider.ts | 36 +- extensions/irc/src/client.ts | 19 +- extensions/llm-task/api.ts | 2 +- extensions/llm-task/src/llm-task-tool.ts | 124 +- extensions/llm-task/src/runtime-api.ts | 2 +- .../memory-core/src/dreaming-narrative.ts | 43 +- extensions/memory-core/src/dreaming-phases.ts | 37 +- .../memory-core/src/memory/qmd-manager.ts | 41 +- .../memory-core/src/public-artifacts.ts | 10 +- .../memory-core/src/short-term-promotion.ts | 40 +- extensions/memory-wiki/src/apply.ts | 21 +- extensions/memory-wiki/src/compile.ts | 35 +- extensions/memory-wiki/src/ingest.ts | 8 +- extensions/memory-wiki/src/log.ts | 7 +- .../memory-wiki/src/source-page-shared.ts | 163 +-- extensions/memory-wiki/src/status.ts | 10 +- extensions/memory-wiki/src/vault.ts | 44 +- extensions/microsoft/speech-provider.ts | 14 +- extensions/migrate-claude/helpers.ts | 14 +- extensions/migrate-hermes/helpers.ts | 14 +- .../msteams/src/feedback-reflection-store.ts | 9 +- extensions/msteams/src/monitor-handler.ts | 8 +- extensions/msteams/src/store-fs.ts | 6 +- extensions/msteams/src/token.ts | 9 +- extensions/nostr/src/nostr-state-store.ts | 43 +- extensions/openshell/src/backend.ts | 105 +- extensions/openshell/src/fs-bridge.ts | 240 +--- extensions/openshell/src/mirror.ts | 22 +- .../openshell/src/openshell-core.test.ts | 191 +-- extensions/phone-control/index.ts | 8 +- extensions/qa-lab/src/cli-paths.ts | 74 +- .../mantis/desktop-browser-smoke.runtime.ts | 10 +- .../src/mantis/slack-desktop-smoke.runtime.ts | 11 +- .../qa-lab/src/mantis/visual-task.runtime.ts | 10 +- extensions/qa-lab/src/multipass.runtime.ts | 5 +- extensions/qa-lab/src/temp-dir.test-helper.ts | 21 +- .../contract/scenario-runtime-config.ts | 13 +- .../src/engine/api/media-chunked.test.ts | 48 + .../qqbot/src/engine/api/media-chunked.ts | 293 ++-- .../src/engine/config/credential-backup.ts | 20 +- .../src/engine/messaging/media-source.ts | 67 +- .../engine/messaging/outbound-media-send.ts | 84 +- .../qqbot/src/engine/messaging/sender.ts | 60 +- extensions/qqbot/src/engine/ref/store.ts | 11 +- .../qqbot/src/engine/session/known-users.ts | 17 +- .../qqbot/src/engine/session/session-store.ts | 13 +- extensions/qqbot/src/engine/utils/audio.ts | 13 +- .../qqbot/src/engine/utils/file-utils.test.ts | 19 +- .../qqbot/src/engine/utils/file-utils.ts | 27 +- extensions/skill-workshop/src/skills.ts | 46 +- extensions/skill-workshop/src/store.ts | 70 +- extensions/speech-core/src/audio-transcode.ts | 24 +- extensions/speech-core/src/tts.ts | 40 +- .../bot/delivery.resolve-media-retry.test.ts | 27 +- .../src/bot/delivery.resolve-media.ts | 7 +- extensions/telegram/src/sent-message-cache.ts | 11 +- extensions/telegram/src/state-migrations.ts | 4 +- extensions/telegram/src/topic-name-cache.ts | 11 +- extensions/tts-local-cli/speech-provider.ts | 30 +- extensions/voice-call/src/manager/store.ts | 27 +- .../voice-call/src/realtime-fast-context.ts | 18 +- extensions/whatsapp/src/auth-store.ts | 20 +- extensions/whatsapp/src/creds-persistence.ts | 51 +- .../whatsapp/src/outbound-media-contract.ts | 70 +- extensions/whatsapp/src/session.test.ts | 92 +- extensions/whatsapp/src/state-migrations.ts | 3 +- extensions/zalo/src/outbound-media.ts | 26 +- extensions/zalouser/src/zalo-js.ts | 87 +- package.json | 1 + .../memory-host-sdk/src/engine-foundation.ts | 3 +- .../src/host/backend-config.ts | 4 +- packages/memory-host-sdk/src/host/fs-utils.ts | 28 +- packages/memory-host-sdk/src/host/internal.ts | 103 +- .../src/host/openclaw-runtime-io.ts | 2 +- .../src/host/openclaw-runtime.ts | 2 +- .../memory-host-sdk/src/host/read-file.ts | 10 +- .../memory-host-sdk/src/host/session-files.ts | 9 +- pnpm-lock.yaml | 12 + pnpm-workspace.yaml | 1 + scripts/openclaw-npm-release-check.ts | 10 + src/acp/approval-classifier.ts | 5 +- src/agents/acp-spawn-parent-stream.ts | 8 +- src/agents/agent-delete-safety.ts | 9 +- src/agents/agent-scope.ts | 8 +- src/agents/apply-patch.test.ts | 105 -- src/agents/apply-patch.ts | 41 +- .../bash-tools.exec.script-preflight.test.ts | 2 +- src/agents/bash-tools.exec.ts | 19 +- src/agents/cli-runner/helpers.ts | 26 +- src/agents/cli-runner/session-history.ts | 12 +- src/agents/harness/native-hook-relay.ts | 26 +- src/agents/models-config.ts | 15 +- .../models-config.write-serialization.test.ts | 93 +- src/agents/pi-auth-json.ts | 19 +- .../openrouter-model-capabilities.ts | 12 +- .../transcript-file-state.ts | 30 +- .../pi-embedded-subscribe.raw-stream.ts | 7 +- src/agents/pi-hooks/compaction-safeguard.ts | 4 +- src/agents/pi-project-settings-snapshot.ts | 4 +- src/agents/pi-project-settings.bundle.test.ts | 2 +- src/agents/pi-tools.read.ts | 44 +- src/agents/queued-file-writer.ts | 100 +- src/agents/sandbox-paths.ts | 9 +- .../sandbox/fs-bridge-path-safety.runtime.ts | 2 +- src/agents/sandbox/fs-bridge-path-safety.ts | 17 +- src/agents/sandbox/fs-bridge.shell.test.ts | 4 +- src/agents/sandbox/fs-bridge.test-helpers.ts | 30 +- src/agents/sandbox/fs-paths.ts | 7 +- src/agents/sandbox/registry.test.ts | 6 +- src/agents/sandbox/registry.ts | 4 +- src/agents/sandbox/ssh.ts | 4 +- src/agents/sandbox/workspace.ts | 4 +- src/agents/session-file-repair.ts | 22 +- src/agents/session-write-lock.ts | 249 +--- src/agents/skills-clawhub.test.ts | 10 +- src/agents/skills-clawhub.ts | 6 +- src/agents/skills-install-download.ts | 43 +- src/agents/skills-install.download.test.ts | 62 +- src/agents/skills/local-loader.ts | 21 +- src/agents/skills/plugin-skills.ts | 15 +- src/agents/skills/workspace.ts | 52 +- src/agents/subagent-attachments.ts | 18 +- src/agents/tools/canvas-tool.ts | 23 +- src/agents/tools/common.ts | 4 +- src/agents/workspace.ts | 38 +- ...bound-media-into-sandbox-workspace.test.ts | 30 +- .../reply/commands-export-session.ts | 13 +- .../reply/commands-export-trajectory.test.ts | 27 + .../reply/commands-export-trajectory.ts | 13 +- .../reply/post-compaction-context.ts | 4 +- src/auto-reply/reply/session-fork.runtime.ts | 14 +- src/auto-reply/reply/stage-sandbox-media.ts | 10 +- src/auto-reply/reply/startup-context.ts | 4 +- src/canvas-host/file-resolver.test.ts | 49 + src/canvas-host/file-resolver.ts | 24 +- .../plugins/bundled.shape-guard.test.ts | 2 +- src/channels/plugins/module-loader.ts | 4 +- src/cli/update-cli/update-command.ts | 10 +- src/commands/agents.commands.add.ts | 14 +- src/commands/cleanup-utils.ts | 4 +- src/commands/doctor-state-migrations.test.ts | 6 +- src/commands/export-trajectory.ts | 13 +- src/commands/status.agent-local.ts | 13 +- src/commitments/store.ts | 20 +- src/config/doc-baseline.ts | 10 +- src/config/includes.ts | 6 +- src/config/io.ts | 96 +- src/config/mutate.ts | 40 +- src/config/validation.ts | 5 +- src/crestodian/audit.ts | 8 +- src/cron/run-log.ts | 54 +- src/cron/store.ts | 43 +- src/daemon/service-layout.ts | 10 +- src/gateway/canvas-documents.ts | 61 +- src/gateway/control-ui.ts | 45 +- src/gateway/managed-image-attachments.ts | 9 +- .../server-methods/agents-mutate.test.ts | 394 +++--- src/gateway/server-methods/agents.ts | 72 +- .../server-methods/chat-webchat-media.test.ts | 26 +- .../server-methods/chat-webchat-media.ts | 81 +- src/gateway/server.auth.control-ui.suite.ts | 16 +- src/gateway/session-utils.ts | 4 +- src/hooks/bundled/command-logger/handler.ts | 7 +- src/hooks/bundled/session-memory/handler.ts | 10 +- src/hooks/install.runtime.ts | 8 +- src/hooks/loader.ts | 6 +- src/hooks/workspace.ts | 20 +- src/infra/archive-helpers.test.ts | 13 +- src/infra/archive-path.ts | 71 +- src/infra/archive-staging.ts | 228 +--- src/infra/archive.ts | 908 +------------ src/infra/boundary-file-read.test.ts | 246 +--- src/infra/boundary-file-read.ts | 236 +--- src/infra/boundary-path.test.ts | 22 +- src/infra/boundary-path.ts | 870 +----------- src/infra/device-auth-store.ts | 11 +- src/infra/device-bootstrap.ts | 11 +- src/infra/device-identity.ts | 24 +- src/infra/device-pairing.ts | 16 +- src/infra/diagnostics-timeline.ts | 5 +- src/infra/exec-approvals.ts | 51 +- src/infra/file-identity.test.ts | 2 +- src/infra/file-identity.ts | 25 - src/infra/file-lock-manager.ts | 7 + src/infra/file-store.ts | 8 + src/infra/fs-pinned-path-helper.ts | 168 --- src/infra/fs-pinned-write-helper.test.ts | 86 -- src/infra/fs-pinned-write-helper.ts | 262 ---- src/infra/fs-safe-advanced.ts | 11 + src/infra/fs-safe-defaults.test.ts | 44 + src/infra/fs-safe-defaults.ts | 8 + src/infra/fs-safe-import-boundary.test.ts | 51 + src/infra/fs-safe-test-hooks.ts | 2 + src/infra/fs-safe.test.ts | 265 +--- src/infra/fs-safe.ts | 1183 +---------------- src/infra/hardlink-guards.test.ts | 2 +- src/infra/hardlink-guards.ts | 38 - src/infra/home-dir.ts | 76 +- src/infra/install-flow.ts | 5 +- src/infra/install-package-dir.ts | 4 +- src/infra/install-safe-path.ts | 121 +- src/infra/install-source-utils.ts | 15 +- src/infra/install-target.test.ts | 12 +- src/infra/install-target.ts | 4 +- src/infra/json-file.ts | 147 +- src/infra/json-files.test.ts | 13 +- src/infra/json-files.ts | 179 +-- src/infra/local-file-access.ts | 84 +- src/infra/node-pairing.ts | 12 +- src/infra/outbound/delivery-queue-recovery.ts | 10 +- src/infra/outbound/delivery-queue-storage.ts | 9 +- .../delivery-queue.reconnect-drain.test.ts | 17 + .../outbound/delivery-queue.recovery.test.ts | 59 +- src/infra/outbound/message-action-params.ts | 11 +- src/infra/package-update-steps.ts | 10 +- src/infra/package-update-utils.ts | 4 +- src/infra/pairing-files.ts | 7 +- src/infra/path-alias-guards.ts | 40 +- src/infra/path-guards.ts | 80 +- src/infra/path-safety.ts | 28 +- src/infra/permissions.ts | 21 + src/infra/private-file-store.ts | 19 + src/infra/private-temp-workspace.ts | 10 + src/infra/push-apns.ts | 8 +- src/infra/push-web.ts | 10 +- src/infra/regular-file.ts | 12 + src/infra/replace-file.ts | 14 + src/infra/restart-sentinel.ts | 4 +- src/infra/restart.ts | 27 +- src/infra/root-paths.ts | 10 + src/infra/safe-open-sync.test.ts | 182 --- src/infra/safe-open-sync.ts | 101 -- src/infra/secret-file.test.ts | 82 +- src/infra/secret-file.ts | 265 +--- src/infra/session-cost-usage.ts | 10 +- src/infra/session-delivery-queue-storage.ts | 9 +- src/infra/sibling-temp-file.ts | 6 + src/infra/temp-download.ts | 83 +- src/infra/tls/gateway.ts | 16 +- src/infra/tmp-openclaw-dir.ts | 217 +-- src/infra/update-startup.ts | 4 +- src/infra/voicewake-routing.ts | 6 +- src/infra/voicewake.ts | 6 +- src/logging/diagnostic-stability-bundle.ts | 11 +- src/logging/diagnostic-support-bundle.ts | 4 +- src/logging/logger.ts | 3 +- src/media-understanding/attachments.cache.ts | 110 +- src/media-understanding/fs.ts | 12 +- src/media-understanding/runtime.ts | 4 +- src/media/audio-transcode.test.ts | 9 +- src/media/audio-transcode.ts | 73 +- src/media/file-context.ts | 3 +- src/media/image-ops.ts | 47 +- src/media/local-media-access.ts | 5 +- src/media/qr-image.ts | 20 +- src/media/store.outside-workspace.test.ts | 4 +- src/media/store.runtime.ts | 13 +- src/media/store.test.ts | 41 + src/media/store.ts | 283 ++-- src/media/web-media.ts | 4 +- src/memory-host-sdk/events.ts | 7 +- src/node-host/config.ts | 4 +- src/node-host/invoke-system-run-plan.ts | 2 +- src/node-host/invoke-system-run.test.ts | 8 +- src/plugin-sdk/browser-trash.ts | 150 +-- src/plugin-sdk/channel-entry-contract.test.ts | 2 +- src/plugin-sdk/channel-entry-contract.ts | 8 +- src/plugin-sdk/facade-loader.ts | 6 +- src/plugin-sdk/file-access-runtime.ts | 7 +- src/plugin-sdk/file-lock.ts | 226 +--- src/plugin-sdk/fs-safe-compat.test.ts | 49 + src/plugin-sdk/json-store.ts | 33 +- src/plugin-sdk/media-store.ts | 2 +- .../memory-core-host-engine-foundation.ts | 2 +- src/plugin-sdk/migration-runtime.ts | 8 +- src/plugin-sdk/sandbox.ts | 9 + src/plugin-sdk/security-runtime.ts | 95 +- src/plugin-sdk/temp-path.test.ts | 10 +- src/plugin-sdk/temp-path.ts | 9 + src/plugins/bundle-commands.ts | 4 +- src/plugins/bundle-config-shared.ts | 10 +- src/plugins/bundle-lsp.ts | 4 +- src/plugins/bundle-manifest.ts | 6 +- src/plugins/bundle-mcp.ts | 4 +- src/plugins/bundled-capability-runtime.ts | 4 +- src/plugins/bundled-dir.ts | 4 +- src/plugins/clawhub.ts | 2 +- src/plugins/conversation-binding.ts | 4 +- src/plugins/discovery.ts | 4 +- src/plugins/git-install.ts | 35 +- src/plugins/install.runtime.ts | 11 +- src/plugins/install.ts | 7 +- .../installed-plugin-index-record-reader.ts | 6 +- src/plugins/installed-plugin-index-store.ts | 10 +- src/plugins/loader.ts | 8 +- src/plugins/manifest.ts | 6 +- src/plugins/marketplace.ts | 10 +- src/plugins/package-entry-resolution.ts | 20 +- src/plugins/path-safety.ts | 49 +- src/plugins/public-surface-loader.ts | 6 +- src/plugins/source-display.ts | 13 +- src/secrets/channel-contract-api.ts | 4 +- src/secrets/resolve.test.ts | 18 +- src/secrets/resolve.ts | 38 +- src/secrets/shared.ts | 23 +- src/security/audit-fs.ts | 214 +-- src/security/scan-paths.ts | 37 +- src/security/windows-acl.ts | 427 +----- src/shared/avatar-policy.ts | 7 +- src/trajectory/cleanup.ts | 4 +- src/trajectory/command-export.ts | 20 +- src/trajectory/paths.ts | 4 +- src/tui/tui-last-session.ts | 13 +- src/utils.ts | 19 +- src/utils/with-timeout.ts | 15 +- test/openclaw-npm-release-check.test.ts | 15 + tsdown.config.ts | 5 + 356 files changed, 4918 insertions(+), 11913 deletions(-) create mode 100644 docs/gateway/security/secure-file-operations.md create mode 100644 docs/refactor/fs-cleanup.md create mode 100644 src/canvas-host/file-resolver.test.ts delete mode 100644 src/infra/file-identity.ts create mode 100644 src/infra/file-lock-manager.ts create mode 100644 src/infra/file-store.ts delete mode 100644 src/infra/fs-pinned-path-helper.ts delete mode 100644 src/infra/fs-pinned-write-helper.test.ts delete mode 100644 src/infra/fs-pinned-write-helper.ts create mode 100644 src/infra/fs-safe-advanced.ts create mode 100644 src/infra/fs-safe-defaults.test.ts create mode 100644 src/infra/fs-safe-defaults.ts create mode 100644 src/infra/fs-safe-import-boundary.test.ts create mode 100644 src/infra/fs-safe-test-hooks.ts delete mode 100644 src/infra/hardlink-guards.ts create mode 100644 src/infra/permissions.ts create mode 100644 src/infra/private-file-store.ts create mode 100644 src/infra/private-temp-workspace.ts create mode 100644 src/infra/regular-file.ts create mode 100644 src/infra/replace-file.ts create mode 100644 src/infra/root-paths.ts delete mode 100644 src/infra/safe-open-sync.test.ts delete mode 100644 src/infra/safe-open-sync.ts create mode 100644 src/infra/sibling-temp-file.ts create mode 100644 src/plugin-sdk/fs-safe-compat.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eb38be1a79..aed86c081a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ Docs: https://docs.openclaw.ai - Gateway/performance: defer non-readiness sidecars until after the ready signal, avoid hot-path channel plugin barrel imports, and fast-path trusted bundled plugin metadata during Gateway startup. - Gateway/performance: avoid importing `jiti` on native-loadable plugin startup paths, so compiled bundled plugin surfaces do not pay source-transform loader cost unless fallback loading is actually needed. - Plugins/loader: preserve real compiled plugin module evaluation errors on the native fast path instead of treating every thrown `.js` module as a source-transform fallback miss. Thanks @vincentkoc. +- Plugin SDK/fs-safe: expose reusable atomic replacement, sibling-temp writes, and cross-device move fallback helpers through `plugin-sdk/security-runtime`, and move OpenClaw's duplicated safe filesystem write paths onto the shared `@openclaw/fs-safe` package. +- Plugin SDK/fs-safe: rename the public temp workspace helpers to `tempWorkspace`, `withTempWorkspace`, `tempWorkspaceSync`, and `withTempWorkspaceSync`, matching the cleaner `@openclaw/fs-safe` API before the package is published. - Providers/OpenRouter: add opt-in response caching params that send OpenRouter's `X-OpenRouter-Cache`, `X-OpenRouter-Cache-TTL`, and cache-clear headers only on verified OpenRouter routes. Thanks @vincentkoc. - Providers/OpenRouter: expand app-attribution categories so OpenClaw advertises coding, programming, writing, chat, and personal-agent usage on verified OpenRouter routes. Thanks @vincentkoc. - Agents/performance: pass the resolved workspace through BTW, compaction, embedded-run model generation, and PDF model setup so explicit agent-dir model refreshes can reuse the current workspace-scoped plugin metadata snapshot instead of falling back to cold plugin metadata scans. (#77519, #77532) diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 32846382977..c356254ceed 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -fe061b6f35adb2b152d8f48244a94d4934b335143cc5f5aebb8cc96e5ba8b287 plugin-sdk-api-baseline.json -495248d5981456192aaf7da2ed23d5951eaa6d9e59d70c716ab91c3da3620e73 plugin-sdk-api-baseline.jsonl +1a06492fe05d1c9dc3194677f52d57ec90468b93023b70d0852ef01d87c7eae3 plugin-sdk-api-baseline.json +c950a1923c0dc7d31120a3010e24217bcf22fd9cacbe102d3ae19b0120c0f648 plugin-sdk-api-baseline.jsonl diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index bdd856eab0b..4090e11e413 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -59,6 +59,10 @@ "source": "Gateway RPC reference", "target": "Gateway RPC 参考" }, + { + "source": "Secure file operations", + "target": "安全文件操作" + }, { "source": "Sessions", "target": "会话" @@ -758,5 +762,9 @@ { "source": "/cli/config", "target": "/cli/config" + }, + { + "source": "fs-safe Cleanup Plan", + "target": "fs-safe Cleanup Plan" } ] diff --git a/docs/docs.json b/docs/docs.json index e6c2c3743d1..6dd32dfe62a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1501,6 +1501,7 @@ "group": "Security and sandboxing", "pages": [ "gateway/security/index", + "gateway/security/secure-file-operations", "gateway/security/audit-checks", "gateway/operator-scopes", "gateway/sandboxing", diff --git a/docs/gateway/security/index.md b/docs/gateway/security/index.md index e52c4534fa9..fbff59e604f 100644 --- a/docs/gateway/security/index.md +++ b/docs/gateway/security/index.md @@ -65,6 +65,12 @@ OpenClaw assumes the host and config boundary are trusted: - Session identifiers (`sessionKey`, session IDs, labels) are routing selectors, not authorization tokens. - If several people can message one tool-enabled agent, each of them can steer that same permission set. Per-user session/memory isolation helps privacy, but does not convert a shared agent into per-user host authorization. +### Secure file operations + +OpenClaw uses `@openclaw/fs-safe` for root-bounded file access, atomic writes, archive extraction, temp workspaces, and secret-file helpers. OpenClaw defaults fs-safe's optional POSIX Python helper to **off**; set `OPENCLAW_FS_SAFE_PYTHON_MODE=auto` or `require` only when you want the extra fd-relative mutation hardening and can support a Python runtime. + +Details: [Secure file operations](/gateway/security/secure-file-operations). + ### Shared Slack workspace: real risk If "everyone in Slack can message the bot," the core risk is delegated tool authority: diff --git a/docs/gateway/security/secure-file-operations.md b/docs/gateway/security/secure-file-operations.md new file mode 100644 index 00000000000..a846c65bfec --- /dev/null +++ b/docs/gateway/security/secure-file-operations.md @@ -0,0 +1,76 @@ +--- +summary: "How OpenClaw handles local file access safely, and why the optional fs-safe Python helper is off by default" +read_when: + - Changing file access, archive extraction, workspace storage, or plugin filesystem helpers +title: "Secure file operations" +--- + +OpenClaw uses [`@openclaw/fs-safe`](https://github.com/openclaw/fs-safe) for security-sensitive local file operations: root-bounded reads/writes, atomic replacement, archive extraction, temp workspaces, JSON state, and secret-file handling. + +The goal is a consistent **library guardrail** for trusted OpenClaw code that receives untrusted path names. It is not a sandbox. Host filesystem permissions, OS users, containers, and the agent/tool policy still define the real blast radius. + +## Default: no Python helper + +OpenClaw defaults the fs-safe POSIX Python helper to **off**. + +Why: + +- the gateway should not spawn a persistent Python sidecar unless an operator opted into it; +- many installs do not need the extra parent-directory mutation hardening; +- disabling Python keeps package/runtime behavior more predictable across desktop, Docker, CI, and bundled app environments. + +OpenClaw only changes the default. If you explicitly set a mode, fs-safe honors it: + +```bash +# Default OpenClaw behavior: Node-only fs-safe fallbacks. +OPENCLAW_FS_SAFE_PYTHON_MODE=off + +# Opt into the helper when available, falling back if unavailable. +OPENCLAW_FS_SAFE_PYTHON_MODE=auto + +# Fail closed if the helper cannot start. +OPENCLAW_FS_SAFE_PYTHON_MODE=require + +# Optional explicit interpreter. +OPENCLAW_FS_SAFE_PYTHON=/usr/bin/python3 +``` + +The generic fs-safe names also work: `FS_SAFE_PYTHON_MODE` and `FS_SAFE_PYTHON`. + +## What stays protected without Python + +With the helper off, OpenClaw still uses fs-safe's Node paths for: + +- rejecting relative-path escapes such as `..`, absolute paths, and path separators where only names are allowed; +- resolving operations through a trusted root handle instead of ad-hoc `path.resolve(...).startsWith(...)` checks; +- refusing symlink and hardlink patterns on APIs that require that policy; +- opening files with identity checks where the API returns or consumes file contents; +- atomic sibling-temp writes for state/config files; +- byte limits for reads and archive extraction; +- private modes for secrets and state files where the API requires them. + +These protections cover the normal OpenClaw threat model: trusted gateway code handling untrusted model/plugin/channel path input inside a single trusted operator boundary. + +## What Python adds + +On POSIX, fs-safe's optional helper keeps one persistent Python process and uses fd-relative filesystem operations for parent-directory mutations such as rename, remove, mkdir, stat/list, and some write paths. + +That narrows same-UID race windows where another process can swap a parent directory between validation and mutation. It is defense in depth for hosts where untrusted local processes can modify the same directories OpenClaw is operating in. + +If your deployment has that risk and Python is guaranteed to exist, use: + +```bash +OPENCLAW_FS_SAFE_PYTHON_MODE=require +``` + +Use `require` rather than `auto` when the helper is part of your security posture; `auto` intentionally falls back to Node-only behavior if the helper is unavailable. + +## Plugin and core guidance + +- Plugin-facing file access should go through `openclaw/plugin-sdk/*` helpers, not raw `fs`, when a path comes from a message, model output, config, or plugin input. +- Core code should use the local fs-safe wrappers under `src/infra/*` so OpenClaw's process policy is applied consistently. +- Archive extraction should use the fs-safe archive helpers with explicit size, entry-count, link, and destination limits. +- Secrets should use OpenClaw secret helpers or fs-safe secret/private-state helpers; do not hand-roll mode checks around `fs.writeFile`. +- If you need hostile local-user isolation, do not rely on fs-safe alone. Run separate gateways under separate OS users/hosts or use sandboxing. + +Related: [Security](/gateway/security), [Sandboxing](/gateway/sandboxing), [Exec approvals](/tools/exec-approvals), [Secrets](/gateway/secrets). diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 9ce2c8d3f72..0db0a8de0bc 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -425,7 +425,7 @@ releases. | `plugin-sdk/approval-native-runtime` | Approval target helpers | Native approval target/account binding helpers | | `plugin-sdk/approval-reply-runtime` | Approval reply helpers | Exec/plugin approval reply payload helpers | | `plugin-sdk/channel-runtime-context` | Channel runtime-context helpers | Generic channel runtime-context register/get/watch helpers | - | `plugin-sdk/security-runtime` | Security helpers | Shared trust, DM gating, external-content, and secret-collection helpers | + | `plugin-sdk/security-runtime` | Security helpers | Shared trust, DM gating, root-bounded file/path helpers, external-content, and secret-collection helpers | | `plugin-sdk/ssrf-policy` | SSRF policy helpers | Host allowlist and private-network policy helpers | | `plugin-sdk/ssrf-runtime` | SSRF runtime helpers | Pinned-dispatcher, guarded fetch, SSRF policy helpers | | `plugin-sdk/system-event-runtime` | System event helpers | `enqueueSystemEvent`, `peekSystemEventEntries` | diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index abb522b3e9f..016bf556ab4 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -161,7 +161,7 @@ For the plugin authoring guide, see [Plugin SDK overview](/plugins/sdk-overview) | `plugin-sdk/allow-from` | `formatAllowFromLowercase` | | `plugin-sdk/channel-secret-runtime` | Narrow secret-contract collection helpers for channel/plugin secret surfaces | | `plugin-sdk/secret-ref-runtime` | Narrow `coerceSecretRef` and SecretRef typing helpers for secret-contract/config parsing | - | `plugin-sdk/security-runtime` | Shared trust, DM gating, external-content, sensitive text redaction, constant-time secret comparison, and secret-collection helpers | + | `plugin-sdk/security-runtime` | Shared trust, DM gating, root-bounded file/path helpers including create-only writes, sync/async atomic file replacement, sibling temp writes, cross-device move fallback, private file-store helpers, symlink-parent guards, external-content, sensitive text redaction, constant-time secret comparison, and secret-collection helpers | | `plugin-sdk/ssrf-policy` | Host allowlist and private-network SSRF policy helpers | | `plugin-sdk/ssrf-dispatcher` | Narrow pinned-dispatcher helpers without the broad infra runtime surface | | `plugin-sdk/ssrf-runtime` | Pinned-dispatcher, SSRF-guarded fetch, SSRF error, and SSRF policy helpers | @@ -210,7 +210,7 @@ For the plugin authoring guide, see [Plugin SDK overview](/plugins/sdk-overview) | `plugin-sdk/param-readers` | Common tool/CLI param readers | | `plugin-sdk/tool-payload` | Extract normalized payloads from tool result objects | | `plugin-sdk/tool-send` | Extract canonical send target fields from tool args | - | `plugin-sdk/temp-path` | Shared temp-download path helpers | + | `plugin-sdk/temp-path` | Shared temp-download path helpers and private secure temp workspaces | | `plugin-sdk/logging-core` | Subsystem logger and redaction helpers | | `plugin-sdk/markdown-table-runtime` | Markdown table mode and conversion helpers | | `plugin-sdk/model-session-runtime` | Model/session override helpers such as `applyModelOverrideToSessionEntry` and `resolveAgentMaxConcurrent` | diff --git a/docs/refactor/fs-cleanup.md b/docs/refactor/fs-cleanup.md new file mode 100644 index 00000000000..0e5481c6bdc --- /dev/null +++ b/docs/refactor/fs-cleanup.md @@ -0,0 +1,448 @@ +--- +title: "fs-safe Cleanup Plan" +summary: "Plan for consolidating OpenClaw filesystem helpers around @openclaw/fs-safe" +read_when: + - You are refactoring OpenClaw filesystem helpers + - You are changing @openclaw/fs-safe imports, wrappers, or plugin SDK file APIs + - You are deciding whether a local file helper belongs in OpenClaw or fs-safe +--- + +## Status + +Implemented on `codex/extract-fs-safe-primitives`. Keep this file as the +cleanup checklist for follow-up reviews and future fs-safe surface changes. + +## Goal + +Make OpenClaw's filesystem access boring and predictable: + +- Core code uses one small set of OpenClaw wrappers that apply OpenClaw policy. +- Plugin SDK compatibility aliases stay deliberate and documented. +- fs-safe keeps a small public story centered on `root()`, with lower-level + primitives behind explicit subpaths. +- Duplicate JSON, temp, private-store, and path helper names disappear from + OpenClaw internals. +- Security-sensitive behavior keeps regression tests before names move. + +## Non-goals + +- Do not remove public plugin SDK exports in this cleanup. Keep deprecated + aliases until a versioned SDK migration removes them. +- Do not make fs-safe a sandbox. It remains a library guardrail for local file + access, not OS isolation. +- Do not convert all absolute-path reads to root-bounded reads. Some OpenClaw + paths are trusted absolute paths and should stay explicit. +- Do not chase cosmetic import churn without reducing helper count or clarifying + trust boundaries. + +## fs-safe Package Pin + +`@openclaw/fs-safe` is published on npm and consumed through a semver range. +Fresh checkouts and CI runners should install the package from the public +registry, not from a local `link:../fs-safe` checkout or a GitHub tarball. + +Current range: + +- `^0.1.0` + +The published package ships built `dist` files, so OpenClaw should not list it +in `pnpm.onlyBuiltDependencies`. + +## Current Shape + +fs-safe's main entry is intentionally narrow: + +- `root` +- `FsSafeError` +- `categorizeFsSafeError` +- root option/result types +- Python helper configuration + +The wider surface lives behind subpaths: + +- `/json` +- `/store` +- `/temp` +- `/atomic` +- `/root` +- `/advanced` +- `/archive` +- `/walk` + +OpenClaw now keeps fs-safe behind a small wrapper boundary: + +- local `src/infra/*` wrappers for core policy defaults +- public plugin SDK aliases, including older names from before fs-safe +- package-local utility exports where importing `src/infra` would cross a + package boundary + +An import-boundary test rejects new direct fs-safe imports outside those +allowed areas. + +## Usage Map + +### Root-bounded access + +Representative use: + +- `src/gateway/server-methods/agents.ts` +- `src/agents/pi-tools.read.ts` +- `src/agents/apply-patch.ts` +- `src/plugins/install.ts` +- `src/auto-reply/reply/stage-sandbox-media.ts` +- `src/gateway/canvas-documents.ts` + +Keep this family. `root()` is the fs-safe product surface OpenClaw should push +callers toward. + +### JSON helpers + +OpenClaw still uses many names for the same operations: + +- `readJsonFile` +- `readJsonFileStrict` +- `readDurableJsonFile` +- `writeJsonAtomic` +- `loadJsonFile` +- `saveJsonFile` +- `readJsonFileWithFallback` +- `writeJsonFileAtomically` + +fs-safe's canonical names are clearer: + +- `tryReadJson` +- `readJson` +- `readJsonIfExists` +- `writeJson` +- `readJsonSync` +- `tryReadJsonSync` +- `writeJsonSync` + +This was the highest-value cleanup because it removed naming drift without +changing semantics. Compatibility aliases stay in `src/infra/json-files.ts` and +plugin SDK barrels. + +### Private state and stores + +Representative use: + +- `src/commitments/store.ts` +- `src/agents/models-config.ts` +- `src/agents/pi-auth-json.ts` +- `src/cron/run-log.ts` +- `src/secrets/shared.ts` +- `src/infra/device-auth-store.ts` +- `src/infra/device-identity.ts` + +Current overlap: + +- `fileStore` +- `fileStore({ private: true })` +- plugin SDK private-state aliases + +The concepts are now one family. fs-safe exposes private mode through +`fileStore({ private: true })`; OpenClaw internals and bundled plugins use +store-shaped wrappers instead of standalone private JSON/text helpers. + +### Temp workspaces + +Representative use: + +- `src/media/qr-image.ts` +- `extensions/discord/src/send.voice.ts` +- `extensions/discord/src/voice/audio.ts` +- `extensions/qa-lab/src/temp-dir.test-helper.ts` + +`tempWorkspace` is the stable useful primitive. One-shot temp targets and +sibling-temp helpers are lower-level implementation tools. + +### Atomic writes + +Representative use: + +- config and session stores +- cron stores +- plugin install paths +- extension state files + +Keep atomic replacement as a public fs-safe subpath. OpenClaw should use the +same canonical JSON/text helpers where possible instead of hand-picking lower +level atomic calls for ordinary JSON state. + +### Regular, secure, and root file reads + +These are not true duplicates: + +- `root()` protects root-relative untrusted paths. +- regular-file helpers read trusted absolute paths with regular-file checks. +- secure-file helpers add ownership and mode checks for secret references. + +Keep them separate. Document the trust boundary instead of hiding it behind one +generic "read file" helper. + +### Archive helpers + +Representative use: + +- plugin install +- skill install +- marketplace and ClawHub archive flows + +Keep as a separate fs-safe subpath. Do not leak archive entry plumbing into +OpenClaw core call sites unless the caller is actually validating archive +metadata. + +## Target Design + +### OpenClaw imports + +Core OpenClaw code should use local policy wrappers: + +- `src/infra/fs-safe.ts` for common root/error helpers +- `src/infra/json-files.ts` for the temporary JSON compatibility layer +- `src/infra/private-file-store.ts` until private stores are unified +- `src/infra/replace-file.ts` for low-level atomic replacement +- `src/infra/boundary-file-read.ts` for loader/package boundary reads +- `src/infra/archive.ts` for archive extraction policy +- `src/infra/file-lock-manager.ts` for the rare core service that needs + manager-style lock lifecycle/diagnostics + +New direct imports from `@openclaw/fs-safe/*` should be reserved for: + +- package-level utilities outside core that cannot import `src/infra` +- compatibility shims +- code that intentionally consumes a narrow fs-safe subpath, such as + `openclaw/plugin-sdk/file-lock` using `@openclaw/fs-safe/file-lock` + +### Plugin SDK exports + +Plugin SDK exports are contractual. Keep aliases even when OpenClaw internals +move to canonical names. + +Mark older names as deprecated in types/docs when the replacement is stable: + +- `readJsonFileWithFallback` -> `readJsonIfExists` or a store method +- `writeJsonFileAtomically` -> `writeJson` +- `loadJsonFile` -> `tryReadJson` +- `saveJsonFile` -> `writeJson` +- `readFileWithinRoot` -> `root(...).read*` +- `writeFileWithinRoot` -> `root(...).write` + +### fs-safe stores + +Move toward one store family: + +```ts +const store = fileStore({ + rootDir, + private: true, + mode: 0o600, + dirMode: 0o700, +}); +``` + +or a thin alias: + +```ts +const store = stateStore({ rootDir, private: true }); +``` + +The store family should cover: + +- `read` +- `readText` +- `readJson` +- `readTextIfExists` +- `readJsonIfExists` +- `write` +- `writeJson` +- `remove` +- `exists` +- `open` +- `copyIn` +- `writeStream` +- `pruneExpired` + +This cleanup added that store shape in fs-safe, removed the unshipped +`privateStateStore` surface, and moved OpenClaw internals and bundled plugins +onto explicit store reads/writes. + +### Temp + +Keep stable public temp surface small: + +```ts +await using workspace = await tempWorkspace({ prefix: "openclaw-" }); +const target = workspace.path("payload.bin"); +``` + +Move one-shot temp target helpers and sibling-temp helpers to advanced/internal +unless a concrete OpenClaw caller needs the public contract. + +## Refactor Phases + +### Phase 1: Inventory and Guards + +- Add a small import-boundary test that lists allowed direct + `@openclaw/fs-safe/*` imports in OpenClaw core. +- Add regression tests for the JSON symlink behavior kept by + `src/infra/json-file.ts`. +- Add regression tests for public plugin SDK aliases that must keep resolving. +- Add a doc note to the plugin SDK runtime docs once aliases are marked + deprecated. + +Exit criteria: + +- The current compatibility surface is executable-tested. +- New direct fs-safe imports are visible in review. + +### Phase 2: JSON Name Cleanup + +- Convert OpenClaw internal callers from old JSON names to canonical fs-safe + names where the semantics are identical. +- Keep plugin SDK aliases unchanged. +- Collapse `src/infra/json-file.ts` and `src/infra/json-files.ts` into one + compatibility module if that reduces indirection without losing symlink + semantics. +- Keep `saveJsonFile` symlink-target behavior until every caller/test is + intentionally migrated. + +Exit criteria: + +- Core internal code no longer imports `readJsonFileStrict`, + `readDurableJsonFile`, or `writeJsonAtomic` unless it is a compatibility shim. +- Plugin SDK aliases still pass import/type tests. + +### Phase 3: Store Unification + +- Add the unified private mode to fs-safe's store API. +- Remove the unshipped `privateStateStore` surface instead of keeping a second + store family. +- Migrate OpenClaw private-state internals to the unified store shape in small + groups: + - auth/profile state + - device identity and device auth + - cron/run logs + - commitments + - extension state +- Regenerate the plugin SDK API baseline for the intentional pre-release + private-helper removal. + +Exit criteria: + +- OpenClaw internals and bundled plugins do not call standalone private + JSON/text helpers. +- `fileStore({ private: true })` is the only private multi-file store API. + +### Phase 4: Temp Simplification + +- Replace OpenClaw one-shot temp target call sites with `tempWorkspace`. +- Keep `resolvePreferredOpenClawTmpDir` as OpenClaw policy. +- Move one-shot temp and sibling-temp helpers out of the curated OpenClaw + wrapper surface. + +Exit criteria: + +- OpenClaw uses `tempWorkspace` for temporary file lifetimes unless a low-level + atomic helper owns the temp path. + +### Phase 5: Shim Reduction + +- Group one-line fs-safe shims into a smaller number of named OpenClaw policy + modules. +- Delete shims that are no longer imported. +- Keep shims that preserve public SDK names or OpenClaw-specific defaults. + +Candidate stable shims: + +- `src/infra/fs-safe.ts` +- `src/infra/json-files.ts` +- `src/infra/private-file-store.ts` +- `src/infra/replace-file.ts` +- `src/infra/boundary-file-read.ts` +- `src/infra/archive.ts` + +Candidate advanced-only grouping: + +- path guards +- symlink parent guards +- hardlink guards +- move-path helpers +- file identity helpers +- sibling temp helpers + +Exit criteria: + +- The local wrapper list has policy meaning, not one file per fs-safe module. + +### Phase 6: fs-safe Public Surface Finalization + +- Keep `@openclaw/fs-safe` main entry curated. +- Keep `root()` as the primary README/API story. +- Keep `openPinnedFileSync` internal. Use `readSecureFile`, `root().open`, or + `openRootFile*` wrappers instead of exposing the fd-level pinned primitive. +- Keep `createSidecarLockManager` internal. Public callers should use + `acquireFileLock` / `withFileLock`; `createFileLockManager` is subpath-only + for long-lived services that need held-lock inspection or drain/reset. +- Move rare root escape hatches such as `openWritable` to advanced only if API + checks show no supported caller needs the main root interface. +- Keep `regular-file`, `secure-file`, archive, and root helpers separate + because their trust models differ. +- Remove or mark unstable any standalone helper that is fully covered by root or + store methods. + +Exit criteria: + +- fs-safe has a stable pre-1.0 public surface. +- OpenClaw imports only stable fs-safe APIs outside compatibility shims. + +## Verification + +Use targeted proof per phase: + +- JSON cleanup: + - JSON symlink tests + - plugin SDK JSON-store import tests + - representative extension tests that use JSON store aliases +- Store unification: + - private mode tests in fs-safe + - auth profile persistence tests + - device identity tests + - cron/run-log tests +- Temp cleanup: + - media temp tests + - Discord voice temp tests + - QA-lab temp helper tests +- Shim reduction: + - plugin SDK API generation/check + - import-boundary tests + - `pnpm build` + +Before merging a broad cleanup batch, run the changed gate and build: + +```sh +pnpm check:changed +pnpm build +``` + +Implementation proof from this cleanup: + +- `pnpm test src/infra/fs-safe-import-boundary.test.ts src/plugin-sdk/temp-path.test.ts src/agents/models-config.write-serialization.test.ts src/infra/json-file.test.ts src/infra/json-files.test.ts` +- `pnpm test src/infra/fs-safe-import-boundary.test.ts src/infra/device-auth-store.test.ts src/infra/device-identity.test.ts src/infra/exec-approvals.test.ts src/agents/models-config.write-serialization.test.ts src/agents/pi-embedded-runner/openrouter-model-capabilities.test.ts src/agents/harness/native-hook-relay.test.ts` +- `pnpm test src/infra/fs-safe-import-boundary.test.ts src/infra/hardlink-guards.test.ts src/infra/file-identity.test.ts src/plugin-sdk/fs-safe-compat.test.ts src/plugin-sdk/temp-path.test.ts` +- `pnpm plugin-sdk:api:check` +- `pnpm build` +- Blacksmith Testbox `pnpm install --frozen-lockfile --config.minimum-release-age=0 && pnpm check:changed` +- In `../fs-safe`: `pnpm docs:site && pnpm build && pnpm test test/api-coverage.test.ts test/new-primitives.test.ts` + +## Review Checklist + +- Does this change reduce a public name, local wrapper, or duplicated semantic + family? +- Is the old name public plugin SDK surface? If yes, keep a deprecated alias. +- Does the replacement preserve symlink, hardlink, mode, and missing-file + behavior? +- Is the caller using an untrusted relative path, trusted absolute path, secret + path, archive entry, or temp lifetime? Pick the helper that says that out + loud. +- Are docs and plugin SDK API snapshots updated when exported names change? diff --git a/extensions/active-memory/index.test.ts b/extensions/active-memory/index.test.ts index 55b94a678f9..550c789b320 100644 --- a/extensions/active-memory/index.test.ts +++ b/extensions/active-memory/index.test.ts @@ -3369,10 +3369,8 @@ describe("active-memory plugin", () => { }); it("keeps subagent transcripts off disk by default by using a temp session file", async () => { - const mkdtempSpy = vi - .spyOn(fs, "mkdtemp") - .mockResolvedValue("/tmp/openclaw-active-memory-temp"); - const rmSpy = vi.spyOn(fs, "rm").mockResolvedValue(undefined); + const mkdtempSpy = vi.spyOn(fs, "mkdtemp"); + const rmSpy = vi.spyOn(fs, "rm"); await hooks.before_prompt_build( { prompt: "what wings should i order? temp transcript path", messages: [] }, @@ -3385,10 +3383,9 @@ describe("active-memory plugin", () => { ); expect(mkdtempSpy).toHaveBeenCalled(); - expect(runEmbeddedPiAgent.mock.calls.at(-1)?.[0]?.sessionFile).toBe( - "/tmp/openclaw-active-memory-temp/session.jsonl", - ); - expect(rmSpy).toHaveBeenCalledWith("/tmp/openclaw-active-memory-temp", { + const sessionFile = runEmbeddedPiAgent.mock.calls.at(-1)?.[0]?.sessionFile; + expect(sessionFile).toMatch(/openclaw-active-memory-.*\/session\.jsonl$/); + expect(rmSpy).toHaveBeenCalledWith(path.dirname(sessionFile), { recursive: true, force: true, }); diff --git a/extensions/active-memory/index.ts b/extensions/active-memory/index.ts index 7e7c9bcd047..53292a29114 100644 --- a/extensions/active-memory/index.ts +++ b/extensions/active-memory/index.ts @@ -18,11 +18,12 @@ import { } from "openclaw/plugin-sdk/plugin-config-runtime"; import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { parseAgentSessionKey, parseThreadSessionSuffix } from "openclaw/plugin-sdk/routing"; +import { isPathInside, replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { resolveSessionStoreEntry, updateSessionStore, } from "openclaw/plugin-sdk/session-store-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; const DEFAULT_TIMEOUT_MS = 15_000; const DEFAULT_AGENT_ID = "main"; @@ -422,7 +423,7 @@ function resolveSafeTranscriptDir(baseSessionsDir: string, transcriptDir: string } const resolvedBase = path.resolve(baseSessionsDir); const candidate = path.resolve(resolvedBase, normalized); - if (candidate !== resolvedBase && !candidate.startsWith(resolvedBase + path.sep)) { + if (!isPathInside(resolvedBase, candidate)) { return path.resolve(resolvedBase, DEFAULT_TRANSCRIPT_DIR); } return candidate; @@ -664,14 +665,11 @@ async function readToggleStore(statePath: string): Promise { - await fs.mkdir(path.dirname(statePath), { recursive: true }); - const tempPath = `${statePath}.${process.pid}.${Date.now()}.${crypto.randomUUID()}.tmp`; - try { - await fs.writeFile(tempPath, `${JSON.stringify(store, null, 2)}\n`, "utf8"); - await fs.rename(tempPath, statePath); - } finally { - await fs.rm(tempPath, { force: true }).catch(() => undefined); - } + await replaceFileAtomic({ + filePath: statePath, + content: `${JSON.stringify(store, null, 2)}\n`, + tempPrefix: ".active-memory", + }); } async function isSessionActiveMemoryDisabled(params: { @@ -2378,9 +2376,13 @@ async function runRecallSubagent(params: { const subagentSessionKey = parentSessionKey ? `${parentSessionKey}:${subagentSuffix}` : `agent:${params.agentId}:${subagentSuffix}`; - const tempDir = params.config.persistTranscripts + const transientWorkspace = params.config.persistTranscripts ? undefined - : await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), "openclaw-active-memory-")); + : await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "openclaw-active-memory-", + }); + const tempDir = transientWorkspace?.dir; const persistedDir = params.config.persistTranscripts ? resolveSafeTranscriptDir( resolvePersistentTranscriptBaseDir(params.api, params.agentId), @@ -2479,9 +2481,7 @@ async function runRecallSubagent(params: { } throw error; } finally { - if (tempDir) { - await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {}); - } + await transientWorkspace?.cleanup(); } } diff --git a/extensions/bluebubbles/src/attachments.ts b/extensions/bluebubbles/src/attachments.ts index 88f5fd3fb89..f8678a0bb86 100644 --- a/extensions/bluebubbles/src/attachments.ts +++ b/extensions/bluebubbles/src/attachments.ts @@ -1,5 +1,6 @@ import crypto from "node:crypto"; import path from "node:path"; +import { sanitizeUntrustedFileName } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, @@ -35,9 +36,7 @@ const AUDIO_MIME_MP3 = new Set(["audio/mpeg", "audio/mp3"]); const AUDIO_MIME_CAF = new Set(["audio/x-caf", "audio/caf"]); function sanitizeFilename(input: string | undefined, fallback: string): string { - const trimmed = input?.trim() ?? ""; - const base = trimmed ? path.basename(trimmed) : ""; - const name = base || fallback; + const name = sanitizeUntrustedFileName(input ?? "", fallback); // Strip characters that could enable multipart header injection (CWE-93) return name.replace(/[\r\n"\\]/g, "_"); } diff --git a/extensions/bluebubbles/src/media-send.ts b/extensions/bluebubbles/src/media-send.ts index e9dac6f04ce..d1b04c6b1c0 100644 --- a/extensions/bluebubbles/src/media-send.ts +++ b/extensions/bluebubbles/src/media-send.ts @@ -1,13 +1,8 @@ -import { constants as fsConstants } from "node:fs"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; import { basenameFromMediaSource, - safeFileURLToPath, + readLocalFileFromRoots, } from "openclaw/plugin-sdk/file-access-runtime"; import { resolveChannelMediaMaxBytes } from "openclaw/plugin-sdk/media-runtime"; -import { lowercasePreservingWhitespace } from "openclaw/plugin-sdk/text-runtime"; import { resolveBlueBubblesAccount } from "./accounts.js"; import { sendBlueBubblesAttachment } from "./attachments.js"; import { resolveBlueBubblesMessageId } from "./monitor-reply-cache.js"; @@ -31,61 +26,6 @@ function assertMediaWithinLimit(sizeBytes: number, maxBytes?: number): void { throw new Error(`Media exceeds ${maxLabel}MB limit (got ${sizeLabel}MB)`); } -function resolveLocalMediaPath(source: string): string { - if (!source.startsWith("file://")) { - return source; - } - try { - return safeFileURLToPath(source); - } catch { - throw new Error(`Invalid file:// URL: ${source}`); - } -} - -function expandHomePath(input: string): string { - if (input === "~") { - return os.homedir(); - } - if (input.startsWith("~/") || input.startsWith(`~${path.sep}`)) { - return path.join(os.homedir(), input.slice(2)); - } - return input; -} - -function resolveConfiguredPath(input: string): string { - const trimmed = input.trim(); - if (!trimmed) { - throw new Error("Empty mediaLocalRoots entry is not allowed"); - } - if (trimmed.startsWith("file://")) { - try { - return safeFileURLToPath(trimmed); - } catch { - throw new Error(`Invalid file:// URL in mediaLocalRoots: ${input}`); - } - } - const resolved = expandHomePath(trimmed); - if (!path.isAbsolute(resolved)) { - throw new Error(`mediaLocalRoots entries must be absolute paths: ${input}`); - } - return resolved; -} - -function isPathInsideRoot(candidate: string, root: string): boolean { - const normalizedCandidate = path.normalize(candidate); - const normalizedRoot = path.normalize(root); - const rootWithSep = normalizedRoot.endsWith(path.sep) - ? normalizedRoot - : normalizedRoot + path.sep; - if (process.platform === "win32") { - const candidateLower = lowercasePreservingWhitespace(normalizedCandidate); - const rootLower = lowercasePreservingWhitespace(normalizedRoot); - const rootWithSepLower = lowercasePreservingWhitespace(rootWithSep); - return candidateLower === rootLower || candidateLower.startsWith(rootWithSepLower); - } - return normalizedCandidate === normalizedRoot || normalizedCandidate.startsWith(rootWithSep); -} - function resolveMediaLocalRoots(params: { cfg: OpenClawConfig; accountId?: string }): string[] { const account = resolveBlueBubblesAccount({ cfg: params.cfg, @@ -111,60 +51,17 @@ async function assertLocalMediaPathAllowed(params: { ); } - const resolvedLocalPath = path.resolve(params.localPath); - const supportsNoFollow = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants; - const openFlags = fsConstants.O_RDONLY | (supportsNoFollow ? fsConstants.O_NOFOLLOW : 0); - - for (const rootEntry of params.localRoots) { - const resolvedRootInput = resolveConfiguredPath(rootEntry); - const relativeToRoot = path.relative(resolvedRootInput, resolvedLocalPath); - if ( - relativeToRoot.startsWith("..") || - path.isAbsolute(relativeToRoot) || - relativeToRoot === "" - ) { - continue; - } - - let rootReal: string; - try { - rootReal = await fs.realpath(resolvedRootInput); - } catch { - rootReal = path.resolve(resolvedRootInput); - } - const candidatePath = path.resolve(rootReal, relativeToRoot); - - if (!isPathInsideRoot(candidatePath, rootReal)) { - continue; - } - - let handle: Awaited> | null = null; - try { - handle = await fs.open(candidatePath, openFlags); - const realPath = await fs.realpath(candidatePath); - if (!isPathInsideRoot(realPath, rootReal)) { - continue; - } - - const stat = await handle.stat(); - if (!stat.isFile()) { - continue; - } - const realStat = await fs.stat(realPath); - if (stat.ino !== realStat.ino || stat.dev !== realStat.dev) { - continue; - } - - const data = await handle.readFile(); - return { data, realPath, sizeBytes: stat.size }; - } catch { - // Try next configured root. - continue; - } finally { - if (handle) { - await handle.close().catch(() => {}); - } - } + const localFile = await readLocalFileFromRoots({ + filePath: params.localPath, + roots: params.localRoots, + label: "mediaLocalRoots", + }); + if (localFile) { + return { + data: localFile.buffer, + realPath: localFile.realPath, + sizeBytes: localFile.stat.size, + }; } throw new Error( @@ -244,9 +141,8 @@ export async function sendBlueBubblesMedia(params: { resolvedContentType = resolvedContentType ?? fetched.contentType ?? undefined; resolvedFilename = resolvedFilename ?? fetched.fileName; } else { - const localPath = expandHomePath(resolveLocalMediaPath(source)); const localFile = await assertLocalMediaPathAllowed({ - localPath, + localPath: source, localRoots: mediaLocalRoots, accountId, }); diff --git a/extensions/browser/src/browser/output-atomic.ts b/extensions/browser/src/browser/output-atomic.ts index 541ad0901b6..e92c5a6abfd 100644 --- a/extensions/browser/src/browser/output-atomic.ts +++ b/extensions/browser/src/browser/output-atomic.ts @@ -1,51 +1,13 @@ -import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { writeFileFromPathWithinRoot } from "../infra/fs-safe.js"; -import { sanitizeUntrustedFileName } from "./safe-filename.js"; - -function buildSiblingTempPath(targetPath: string): string { - const id = crypto.randomUUID(); - const safeTail = sanitizeUntrustedFileName(path.basename(targetPath), "output.bin"); - return path.join(path.dirname(targetPath), `.openclaw-output-${id}-${safeTail}.part`); -} +import { writeViaSiblingTempPath as writeViaSiblingTempPathBase } from "../sdk-security-runtime.js"; export async function writeViaSiblingTempPath(params: { rootDir: string; targetPath: string; writeTemp: (tempPath: string) => Promise; }): Promise { - const rootDir = await fs - .realpath(path.resolve(params.rootDir)) - .catch(() => path.resolve(params.rootDir)); - const requestedTargetPath = path.resolve(params.targetPath); - const targetPath = await fs - .realpath(path.dirname(requestedTargetPath)) - .then((realDir) => path.join(realDir, path.basename(requestedTargetPath))) - .catch(() => requestedTargetPath); - const relativeTargetPath = path.relative(rootDir, targetPath); - if ( - !relativeTargetPath || - relativeTargetPath === ".." || - relativeTargetPath.startsWith(`..${path.sep}`) || - path.isAbsolute(relativeTargetPath) - ) { - throw new Error("Target path is outside the allowed root"); - } - const tempPath = buildSiblingTempPath(targetPath); - let renameSucceeded = false; - try { - await params.writeTemp(tempPath); - await writeFileFromPathWithinRoot({ - rootDir, - relativePath: relativeTargetPath, - sourcePath: tempPath, - mkdir: false, - }); - renameSucceeded = true; - } finally { - if (!renameSucceeded) { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } - } + await writeViaSiblingTempPathBase({ + ...params, + fallbackFileName: "output.bin", + tempPrefix: ".openclaw-output-", + }); } diff --git a/extensions/browser/src/browser/paths.ts b/extensions/browser/src/browser/paths.ts index d32fb5b7858..fa5f5543f7c 100644 --- a/extensions/browser/src/browser/paths.ts +++ b/extensions/browser/src/browser/paths.ts @@ -1,8 +1,13 @@ -import fs from "node:fs/promises"; import path from "node:path"; -import { SafeOpenError, openFileWithinRoot } from "../infra/fs-safe.js"; -import { isNotFoundPathError, isPathInside } from "../infra/path-guards.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; +export { + resolveExistingPathsWithinRoot, + pathScope, + resolvePathsWithinRoot, + resolvePathWithinRoot, + resolveStrictExistingPathsWithinRoot, + resolveWritablePathWithinRoot, +} from "../sdk-security-runtime.js"; const DEFAULT_FALLBACK_BROWSER_TMP_DIR = "/tmp/openclaw"; @@ -28,241 +33,3 @@ const DEFAULT_BROWSER_TMP_DIR = canUseNodeFs() export const DEFAULT_TRACE_DIR = DEFAULT_BROWSER_TMP_DIR; export const DEFAULT_DOWNLOAD_DIR = path.join(DEFAULT_BROWSER_TMP_DIR, "downloads"); export const DEFAULT_UPLOAD_DIR = path.join(DEFAULT_BROWSER_TMP_DIR, "uploads"); - -type InvalidPathResult = { ok: false; error: string }; -type ResolvePathsWithinRootParams = { - rootDir: string; - requestedPaths: string[]; - scopeLabel: string; -}; -type ResolvePathsWithinRootResult = { ok: true; paths: string[] } | InvalidPathResult; - -function invalidPath(scopeLabel: string): InvalidPathResult { - return { - ok: false, - error: `Invalid path: must stay within ${scopeLabel}`, - }; -} - -async function resolveRealPathIfExists(targetPath: string): Promise { - try { - return await fs.realpath(targetPath); - } catch { - return undefined; - } -} - -async function resolveTrustedRootRealPath(rootDir: string): Promise { - try { - const rootLstat = await fs.lstat(rootDir); - if (!rootLstat.isDirectory() || rootLstat.isSymbolicLink()) { - return undefined; - } - return await fs.realpath(rootDir); - } catch { - return undefined; - } -} - -async function validateCanonicalPathWithinRoot(params: { - rootRealPath: string; - candidatePath: string; - expect: "directory" | "file"; -}): Promise<"ok" | "not-found" | "invalid"> { - try { - const candidateLstat = await fs.lstat(params.candidatePath); - if (candidateLstat.isSymbolicLink()) { - return "invalid"; - } - if (params.expect === "directory" && !candidateLstat.isDirectory()) { - return "invalid"; - } - if (params.expect === "file" && !candidateLstat.isFile()) { - return "invalid"; - } - if (params.expect === "file" && candidateLstat.nlink > 1) { - return "invalid"; - } - const candidateRealPath = await fs.realpath(params.candidatePath); - return isPathInside(params.rootRealPath, candidateRealPath) ? "ok" : "invalid"; - } catch (err) { - return isNotFoundPathError(err) ? "not-found" : "invalid"; - } -} - -export function resolvePathWithinRoot(params: { - rootDir: string; - requestedPath: string; - scopeLabel: string; - defaultFileName?: string; -}): { ok: true; path: string } | { ok: false; error: string } { - const root = path.resolve(params.rootDir); - const raw = params.requestedPath.trim(); - if (!raw) { - if (!params.defaultFileName) { - return { ok: false, error: "path is required" }; - } - return { ok: true, path: path.join(root, params.defaultFileName) }; - } - const resolved = path.resolve(root, raw); - const rel = path.relative(root, resolved); - if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) { - return { ok: false, error: `Invalid path: must stay within ${params.scopeLabel}` }; - } - return { ok: true, path: resolved }; -} - -export async function resolveWritablePathWithinRoot(params: { - rootDir: string; - requestedPath: string; - scopeLabel: string; - defaultFileName?: string; -}): Promise<{ ok: true; path: string } | { ok: false; error: string }> { - const lexical = resolvePathWithinRoot(params); - if (!lexical.ok) { - return lexical; - } - - const rootDir = path.resolve(params.rootDir); - const rootRealPath = await resolveTrustedRootRealPath(rootDir); - if (!rootRealPath) { - return invalidPath(params.scopeLabel); - } - - const requestedPath = lexical.path; - const parentDir = path.dirname(requestedPath); - const parentStatus = await validateCanonicalPathWithinRoot({ - rootRealPath, - candidatePath: parentDir, - expect: "directory", - }); - if (parentStatus !== "ok") { - return invalidPath(params.scopeLabel); - } - - const targetStatus = await validateCanonicalPathWithinRoot({ - rootRealPath, - candidatePath: requestedPath, - expect: "file", - }); - if (targetStatus === "invalid") { - return invalidPath(params.scopeLabel); - } - - return lexical; -} - -export function resolvePathsWithinRoot( - params: ResolvePathsWithinRootParams, -): ResolvePathsWithinRootResult { - const resolvedPaths: string[] = []; - for (const raw of params.requestedPaths) { - const pathResult = resolvePathWithinRoot({ - rootDir: params.rootDir, - requestedPath: raw, - scopeLabel: params.scopeLabel, - }); - if (!pathResult.ok) { - return { ok: false, error: pathResult.error }; - } - resolvedPaths.push(pathResult.path); - } - return { ok: true, paths: resolvedPaths }; -} - -export async function resolveExistingPathsWithinRoot( - params: ResolvePathsWithinRootParams, -): Promise { - return await resolveCheckedPathsWithinRoot(params, true); -} - -export async function resolveStrictExistingPathsWithinRoot( - params: ResolvePathsWithinRootParams, -): Promise { - return await resolveCheckedPathsWithinRoot(params, false); -} - -async function resolveCheckedPathsWithinRoot( - params: ResolvePathsWithinRootParams, - allowMissingFallback: boolean, -): Promise { - const rootDir = path.resolve(params.rootDir); - // Keep historical behavior for missing roots and rely on openFileWithinRoot for final checks. - const rootRealPath = await resolveRealPathIfExists(rootDir); - - const isInRoot = (relativePath: string) => - Boolean(relativePath) && !relativePath.startsWith("..") && !path.isAbsolute(relativePath); - - const resolveExistingRelativePath = async ( - requestedPath: string, - ): Promise< - { ok: true; relativePath: string; fallbackPath: string } | { ok: false; error: string } - > => { - const raw = requestedPath.trim(); - const lexicalPathResult = resolvePathWithinRoot({ - rootDir, - requestedPath, - scopeLabel: params.scopeLabel, - }); - if (lexicalPathResult.ok) { - return { - ok: true, - relativePath: path.relative(rootDir, lexicalPathResult.path), - fallbackPath: lexicalPathResult.path, - }; - } - if (!rootRealPath || !raw || !path.isAbsolute(raw)) { - return lexicalPathResult; - } - try { - const resolvedExistingPath = await fs.realpath(raw); - const relativePath = path.relative(rootRealPath, resolvedExistingPath); - if (!isInRoot(relativePath)) { - return lexicalPathResult; - } - return { - ok: true, - relativePath, - fallbackPath: resolvedExistingPath, - }; - } catch { - return lexicalPathResult; - } - }; - - const resolvedPaths: string[] = []; - for (const raw of params.requestedPaths) { - const pathResult = await resolveExistingRelativePath(raw); - if (!pathResult.ok) { - return { ok: false, error: pathResult.error }; - } - - let opened: Awaited> | undefined; - try { - opened = await openFileWithinRoot({ - rootDir, - relativePath: pathResult.relativePath, - }); - resolvedPaths.push(opened.realPath); - } catch (err) { - if (allowMissingFallback && err instanceof SafeOpenError && err.code === "not-found") { - // Preserve historical behavior for paths that do not exist yet. - resolvedPaths.push(pathResult.fallbackPath); - continue; - } - if (err instanceof SafeOpenError && err.code === "outside-workspace") { - return { - ok: false, - error: `File is outside ${params.scopeLabel}`, - }; - } - return { - ok: false, - error: `Invalid path: must stay within ${params.scopeLabel} and be a regular non-symlink file`, - }; - } finally { - await opened?.handle.close().catch(() => {}); - } - } - return { ok: true, paths: resolvedPaths }; -} diff --git a/extensions/browser/src/browser/routes/agent.act.hooks.ts b/extensions/browser/src/browser/routes/agent.act.hooks.ts index 48b03ad93ad..791d3070971 100644 --- a/extensions/browser/src/browser/routes/agent.act.hooks.ts +++ b/extensions/browser/src/browser/routes/agent.act.hooks.ts @@ -8,7 +8,7 @@ import { withRouteTabContext, } from "./agent.shared.js"; import { EXISTING_SESSION_LIMITS } from "./existing-session-limits.js"; -import { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./path-output.js"; +import { DEFAULT_UPLOAD_DIR, pathScope } from "./path-output.js"; import type { BrowserRouteRegistrar } from "./types.js"; import { asyncBrowserRoute, @@ -43,11 +43,9 @@ export function registerBrowserAgentActHookRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - const uploadPathsResult = await resolveExistingPathsWithinRoot({ - rootDir: DEFAULT_UPLOAD_DIR, - requestedPaths: paths, - scopeLabel: `uploads directory (${DEFAULT_UPLOAD_DIR})`, - }); + const uploadPathsResult = await pathScope(DEFAULT_UPLOAD_DIR, { + label: `uploads directory (${DEFAULT_UPLOAD_DIR})`, + }).existing(paths); if (!uploadPathsResult.ok) { res.status(400).json({ error: uploadPathsResult.error }); return; diff --git a/extensions/browser/src/browser/routes/output-paths.ts b/extensions/browser/src/browser/routes/output-paths.ts index 4a11d3dc816..505c4ec6322 100644 --- a/extensions/browser/src/browser/routes/output-paths.ts +++ b/extensions/browser/src/browser/routes/output-paths.ts @@ -1,5 +1,5 @@ import fs from "node:fs/promises"; -import { resolveWritablePathWithinRoot } from "./path-output.js"; +import { pathScope } from "./path-output.js"; import type { BrowserResponse } from "./types.js"; export async function ensureOutputRootDir(rootDir: string): Promise { @@ -17,12 +17,10 @@ export async function resolveWritableOutputPathOrRespond(params: { if (params.ensureRootDir) { await ensureOutputRootDir(params.rootDir); } - const pathResult = await resolveWritablePathWithinRoot({ - rootDir: params.rootDir, - requestedPath: params.requestedPath, - scopeLabel: params.scopeLabel, - defaultFileName: params.defaultFileName, - }); + const pathResult = await pathScope(params.rootDir, { label: params.scopeLabel }).writable( + params.requestedPath, + { defaultName: params.defaultFileName }, + ); if (!pathResult.ok) { params.res.status(400).json({ error: pathResult.error }); return null; diff --git a/extensions/browser/src/browser/routes/path-output.ts b/extensions/browser/src/browser/routes/path-output.ts index 5f997316c61..cf77288d05b 100644 --- a/extensions/browser/src/browser/routes/path-output.ts +++ b/extensions/browser/src/browser/routes/path-output.ts @@ -2,6 +2,7 @@ export { DEFAULT_DOWNLOAD_DIR, DEFAULT_TRACE_DIR, DEFAULT_UPLOAD_DIR, + pathScope, resolveExistingPathsWithinRoot, resolveWritablePathWithinRoot, } from "../paths.js"; diff --git a/extensions/browser/src/browser/safe-filename.ts b/extensions/browser/src/browser/safe-filename.ts index f8ea9b3cbc4..c0010111a5c 100644 --- a/extensions/browser/src/browser/safe-filename.ts +++ b/extensions/browser/src/browser/safe-filename.ts @@ -1,27 +1 @@ -import path from "node:path"; -import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; - -export function sanitizeUntrustedFileName(fileName: string, fallbackName: string): string { - const trimmed = normalizeOptionalString(fileName) ?? ""; - if (!trimmed) { - return fallbackName; - } - let base = path.posix.basename(trimmed); - base = path.win32.basename(base); - let cleaned = ""; - for (let i = 0; i < base.length; i++) { - const code = base.charCodeAt(i); - if (code < 0x20 || code === 0x7f) { - continue; - } - cleaned += base[i]; - } - base = cleaned.trim(); - if (!base || base === "." || base === "..") { - return fallbackName; - } - if (base.length > 200) { - base = base.slice(0, 200); - } - return base; -} +export { sanitizeUntrustedFileName } from "../sdk-security-runtime.js"; diff --git a/extensions/browser/src/infra/fs-safe.ts b/extensions/browser/src/infra/fs-safe.ts index 3e9b84b51f1..9b90290a2a1 100644 --- a/extensions/browser/src/infra/fs-safe.ts +++ b/extensions/browser/src/infra/fs-safe.ts @@ -1,5 +1 @@ -export { - SafeOpenError, - openFileWithinRoot, - writeFileFromPathWithinRoot, -} from "../sdk-security-runtime.js"; +export { root, FsSafeError } from "../sdk-security-runtime.js"; diff --git a/extensions/browser/src/sdk-security-runtime.ts b/extensions/browser/src/sdk-security-runtime.ts index 37edfa4fe18..e39265146a4 100644 --- a/extensions/browser/src/sdk-security-runtime.ts +++ b/extensions/browser/src/sdk-security-runtime.ts @@ -9,13 +9,20 @@ export { isPrivateNetworkAllowedByPolicy, matchesHostnameAllowlist, normalizeHostname, - openFileWithinRoot, + pathScope, redactSensitiveText, + resolveExistingPathsWithinRoot, resolvePinnedHostnameWithPolicy, + resolvePathsWithinRoot, + resolvePathWithinRoot, + root, safeEqualSecret, - SafeOpenError, + sanitizeUntrustedFileName, + resolveStrictExistingPathsWithinRoot, + resolveWritablePathWithinRoot, + FsSafeError, SsrFBlockedError, + writeViaSiblingTempPath, wrapExternalContent, - writeFileFromPathWithinRoot, } from "openclaw/plugin-sdk/security-runtime"; export type { LookupFn, SsrFPolicy } from "openclaw/plugin-sdk/security-runtime"; diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 3147e54a48a..8556a823f9a 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -40,6 +40,7 @@ import { } from "openclaw/plugin-sdk/agent-harness-runtime"; import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime"; import { emitTrustedDiagnosticEvent } from "openclaw/plugin-sdk/diagnostic-runtime"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { handleCodexAppServerApprovalRequest } from "./approval-bridge.js"; import { refreshCodexAppServerAuthTokens, @@ -444,7 +445,7 @@ export async function runCodexAppServerAttempt( runId: params.runId, }, }); - const hadSessionFile = await fileExists(params.sessionFile); + const hadSessionFile = await pathExists(params.sessionFile); let historyMessages = (await readMirroredSessionHistoryMessages(params.sessionFile)) ?? []; const hookContext = { runId: params.runId, @@ -1927,18 +1928,6 @@ async function mirrorTranscriptBestEffort(params: { } } -async function fileExists(filePath: string): Promise { - try { - await fs.stat(filePath); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return false; - } - throw error; - } -} - function isNonEmptyString(value: unknown): value is string { return typeof value === "string" && value.length > 0; } diff --git a/extensions/codex/src/app-server/timeout.ts b/extensions/codex/src/app-server/timeout.ts index f87c4695a68..b1d1d1b5c86 100644 --- a/extensions/codex/src/app-server/timeout.ts +++ b/extensions/codex/src/app-server/timeout.ts @@ -1,22 +1,9 @@ +import { withTimeout as withSharedTimeout } from "openclaw/plugin-sdk/security-runtime"; + export async function withTimeout( promise: Promise, timeoutMs: number, timeoutMessage: string, ): Promise { - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { - return await promise; - } - let timeout: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error(timeoutMessage)), Math.max(1, timeoutMs)); - }), - ]); - } finally { - if (timeout) { - clearTimeout(timeout); - } - } + return await withSharedTimeout(promise, timeoutMs, { message: timeoutMessage }); } diff --git a/extensions/codex/src/app-server/trajectory.ts b/extensions/codex/src/app-server/trajectory.ts index a924fdd6432..0e2ce06f1e7 100644 --- a/extensions/codex/src/app-server/trajectory.ts +++ b/extensions/codex/src/app-server/trajectory.ts @@ -6,6 +6,10 @@ import type { EmbeddedRunAttemptResult, } from "openclaw/plugin-sdk/agent-harness"; import { resolveUserPath } from "openclaw/plugin-sdk/agent-harness"; +import { + appendRegularFile, + resolveRegularFileAppendFlags, +} from "openclaw/plugin-sdk/security-runtime"; type CodexTrajectoryRecorder = { filePath: string; @@ -39,13 +43,7 @@ type CodexTrajectoryOpenFlagConstants = Pick< export function resolveCodexTrajectoryAppendFlags( constants: CodexTrajectoryOpenFlagConstants = nodeFs.constants, ): number { - const noFollow = constants.O_NOFOLLOW; - return ( - constants.O_CREAT | - constants.O_APPEND | - constants.O_WRONLY | - (typeof noFollow === "number" ? noFollow : 0) - ); + return resolveRegularFileAppendFlags(constants); } export function resolveCodexTrajectoryPointerFlags( @@ -60,78 +58,13 @@ export function resolveCodexTrajectoryPointerFlags( ); } -async function assertNoSymlinkParents(filePath: string): Promise { - const resolvedDir = path.resolve(path.dirname(filePath)); - const parsed = path.parse(resolvedDir); - const relativeParts = path.relative(parsed.root, resolvedDir).split(path.sep).filter(Boolean); - let current = parsed.root; - for (const part of relativeParts) { - current = path.join(current, part); - const stat = await fs.lstat(current); - if (stat.isSymbolicLink()) { - if (path.dirname(current) === parsed.root) { - continue; - } - throw new Error(`Refusing to write trajectory under symlinked directory: ${current}`); - } - if (!stat.isDirectory()) { - throw new Error(`Refusing to write trajectory under non-directory: ${current}`); - } - } -} - -function verifyStableOpenedTrajectoryFile(params: { - preOpenStat?: nodeFs.Stats; - postOpenStat: nodeFs.Stats; - filePath: string; -}): void { - if (!params.postOpenStat.isFile()) { - throw new Error(`Refusing to write trajectory to non-file: ${params.filePath}`); - } - if (params.postOpenStat.nlink > 1) { - throw new Error(`Refusing to write trajectory to hardlinked file: ${params.filePath}`); - } - const pre = params.preOpenStat; - if (pre && (pre.dev !== params.postOpenStat.dev || pre.ino !== params.postOpenStat.ino)) { - throw new Error(`Refusing to write trajectory after file changed: ${params.filePath}`); - } -} - async function safeAppendTrajectoryFile(filePath: string, line: string): Promise { - await assertNoSymlinkParents(filePath); - - let preOpenStat: nodeFs.Stats | undefined; - try { - const stat = await fs.lstat(filePath); - if (stat.isSymbolicLink()) { - throw new Error(`Refusing to write trajectory through symlink: ${filePath}`); - } - if (!stat.isFile()) { - throw new Error(`Refusing to write trajectory to non-file: ${filePath}`); - } - preOpenStat = stat; - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - } - const lineBytes = Buffer.byteLength(line, "utf8"); - if ((preOpenStat?.size ?? 0) + lineBytes > TRAJECTORY_RUNTIME_FILE_MAX_BYTES) { - return; - } - - const handle = await fs.open(filePath, resolveCodexTrajectoryAppendFlags(), 0o600); - try { - const stat = await handle.stat(); - verifyStableOpenedTrajectoryFile({ preOpenStat, postOpenStat: stat, filePath }); - if (stat.size + lineBytes > TRAJECTORY_RUNTIME_FILE_MAX_BYTES) { - return; - } - await handle.chmod(0o600); - await handle.appendFile(line, "utf8"); - } finally { - await handle.close(); - } + await appendRegularFile({ + filePath, + content: line, + maxFileBytes: TRAJECTORY_RUNTIME_FILE_MAX_BYTES, + rejectSymlinkParents: true, + }); } function boundedTrajectoryLine(event: Record): string | undefined { diff --git a/extensions/codex/src/migration/helpers.ts b/extensions/codex/src/migration/helpers.ts index 3929565856e..0a91b2bca5f 100644 --- a/extensions/codex/src/migration/helpers.ts +++ b/extensions/codex/src/migration/helpers.ts @@ -1,14 +1,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; export async function exists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } + return await pathExists(filePath); } export async function isDirectory(filePath: string | undefined): Promise { diff --git a/extensions/device-pair/notify.ts b/extensions/device-pair/notify.ts index b341bad4076..70dbd26d5c0 100644 --- a/extensions/device-pair/notify.ts +++ b/extensions/device-pair/notify.ts @@ -4,6 +4,7 @@ import type { OpenClawPluginService } from "openclaw/plugin-sdk/core"; import { listDevicePairing } from "openclaw/plugin-sdk/device-bootstrap"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; const NOTIFY_STATE_FILE = "device-pair-notify.json"; @@ -145,9 +146,12 @@ async function readNotifyState(filePath: string): Promise { } async function writeNotifyState(filePath: string, state: NotifyStateFile): Promise { - await fs.mkdir(path.dirname(filePath), { recursive: true }); const content = JSON.stringify(state, null, 2); - await fs.writeFile(filePath, `${content}\n`, "utf8"); + await replaceFileAtomic({ + filePath, + content: `${content}\n`, + tempPrefix: ".device-pair-notify", + }); } function notifySubscriberKey(subscriber: { diff --git a/extensions/diffs/src/store.ts b/extensions/diffs/src/store.ts index bab0b13abb3..ac38e2d42d8 100644 --- a/extensions/diffs/src/store.ts +++ b/extensions/diffs/src/store.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { PluginLogger } from "../api.js"; import type { DiffArtifactContext, DiffArtifactMeta, DiffOutputFormat } from "./types.js"; @@ -36,6 +37,7 @@ type StandaloneFileMeta = { }; type ArtifactMetaFileName = "meta.json" | "file-meta.json"; +type ArtifactRoot = Awaited>; export class DiffArtifactStore { private readonly rootDir: string; @@ -76,8 +78,9 @@ export class DiffArtifactStore { ...(params.context ? { context: params.context } : {}), }; - await fs.mkdir(artifactDir, { recursive: true }); - await fs.writeFile(htmlPath, params.html, "utf8"); + const root = await this.artifactRoot(); + await root.mkdir(id); + await root.write(path.posix.join(id, "viewer.html"), params.html); await this.writeMeta(meta); this.scheduleCleanup(); return meta; @@ -104,7 +107,7 @@ export class DiffArtifactStore { throw new Error(`Diff artifact not found: ${id}`); } const htmlPath = this.normalizeStoredPath(meta.htmlPath, "htmlPath"); - return await fs.readFile(htmlPath, "utf8"); + return await (await this.artifactRoot()).readText(this.relativeStoredPath(htmlPath)); } async updateFilePath(id: string, filePath: string): Promise { @@ -151,7 +154,7 @@ export class DiffArtifactStore { ...(params.context ? { context: params.context } : {}), }; - await fs.mkdir(artifactDir, { recursive: true }); + await (await this.artifactRoot()).mkdir(id); await this.writeStandaloneMeta(meta); this.scheduleCleanup(); return { @@ -212,6 +215,11 @@ export class DiffArtifactStore { await fs.mkdir(this.rootDir, { recursive: true }); } + private async artifactRoot(): Promise { + await this.ensureRoot(); + return await fsRoot(this.rootDir); + } + private maybeCleanupExpired(): void { const now = Date.now(); if (this.cleanupInFlight || now < this.nextCleanupAt) { @@ -283,16 +291,12 @@ export class DiffArtifactStore { } } - private metaFilePath(id: string, fileName: ArtifactMetaFileName): string { - return path.join(this.artifactDir(id), fileName); - } - private async writeJsonMeta( id: string, fileName: ArtifactMetaFileName, data: unknown, ): Promise { - await fs.writeFile(this.metaFilePath(id, fileName), JSON.stringify(data, null, 2), "utf8"); + await (await this.artifactRoot()).writeJson(path.posix.join(id, fileName), data, { space: 2 }); } private async readJsonMeta( @@ -301,7 +305,7 @@ export class DiffArtifactStore { context: string, ): Promise { try { - const raw = await fs.readFile(this.metaFilePath(id, fileName), "utf8"); + const raw = await (await this.artifactRoot()).readText(path.posix.join(id, fileName)); return JSON.parse(raw) as unknown; } catch (error) { if (isFileNotFound(error)) { @@ -330,6 +334,11 @@ export class DiffArtifactStore { return candidate; } + private relativeStoredPath(storedPath: string): string { + const relativePath = path.relative(this.rootDir, this.normalizeStoredPath(storedPath, "path")); + return relativePath.split(path.sep).join(path.posix.sep); + } + private assertWithinRoot(candidate: string, label = "path"): void { const relative = path.relative(this.rootDir, candidate); if ( @@ -362,7 +371,8 @@ function isExpired(meta: { expiresAt: string }): boolean { } function isFileNotFound(error: unknown): boolean { - return error instanceof Error && "code" in error && error.code === "ENOENT"; + const code = error instanceof Error && "code" in error ? error.code : undefined; + return code === "ENOENT" || code === "not-found"; } function normalizeArtifactContext(value: unknown): DiffArtifactContext | undefined { diff --git a/extensions/discord/src/internal/command-deploy.ts b/extensions/discord/src/internal/command-deploy.ts index 1965d3caa1e..39499fc5977 100644 --- a/extensions/discord/src/internal/command-deploy.ts +++ b/extensions/discord/src/internal/command-deploy.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; import path from "node:path"; import { ApplicationCommandType, type APIApplicationCommand } from "discord-api-types/v10"; +import { privateFileStore } from "openclaw/plugin-sdk/security-runtime"; import { createApplicationCommand, deleteApplicationCommand, @@ -147,9 +147,10 @@ export class DiscordCommandDeployer { return; } try { - const raw = await fs.readFile(storePath, "utf8"); - const parsed = JSON.parse(raw) as { hashes?: unknown }; - if (!parsed.hashes || typeof parsed.hashes !== "object") { + const parsed = await privateFileStore(path.dirname(storePath)).readJsonIfExists<{ + hashes?: unknown; + }>(path.basename(storePath)); + if (!parsed?.hashes || typeof parsed.hashes !== "object") { return; } for (const [key, value] of Object.entries(parsed.hashes)) { @@ -168,24 +169,17 @@ export class DiscordCommandDeployer { return; } try { - await fs.mkdir(path.dirname(storePath), { recursive: true }); - const tmpPath = `${storePath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile( - tmpPath, - `${JSON.stringify( - { - version: 1, - updatedAt: new Date().toISOString(), - hashes: Object.fromEntries( - [...this.hashes.entries()].toSorted(([left], [right]) => left.localeCompare(right)), - ), - }, - null, - 2, - )}\n`, - "utf8", + await privateFileStore(path.dirname(storePath)).writeJson( + path.basename(storePath), + { + version: 1, + updatedAt: new Date().toISOString(), + hashes: Object.fromEntries( + [...this.hashes.entries()].toSorted(([left], [right]) => left.localeCompare(right)), + ), + }, + { trailingNewline: true }, ); - await fs.rename(tmpPath, storePath); } catch { // The cache is only an optimization to avoid redundant Discord writes. } diff --git a/extensions/discord/src/send.voice.ts b/extensions/discord/src/send.voice.ts index 34f63373e16..b18c3e3fc59 100644 --- a/extensions/discord/src/send.voice.ts +++ b/extensions/discord/src/send.voice.ts @@ -1,4 +1,3 @@ -import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime"; @@ -10,7 +9,7 @@ import { } from "openclaw/plugin-sdk/media-runtime"; import { requireRuntimeConfig } from "openclaw/plugin-sdk/plugin-config-runtime"; import type { RetryConfig } from "openclaw/plugin-sdk/retry-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { loadWebMediaRaw } from "openclaw/plugin-sdk/web-media"; import { resolveDiscordAccount } from "./accounts.js"; import type { RequestClient } from "./internal/discord.js"; @@ -46,17 +45,21 @@ function toDiscordSendResult( }); } -async function materializeVoiceMessageInput(mediaUrl: string): Promise<{ filePath: string }> { +async function materializeVoiceMessageInput( + mediaUrl: string, +): Promise<{ filePath: string; cleanup: () => Promise }> { // Security: reuse the standard media loader so we apply SSRF guards + allowed-local-root checks. // Then write to a private temp file so ffmpeg/ffprobe never sees the original URL/path string. const media = await loadWebMediaRaw(mediaUrl, maxBytesForKind("audio")); const extFromName = media.fileName ? path.extname(media.fileName) : ""; const extFromMime = media.contentType ? extensionForMime(media.contentType) : ""; const ext = extFromName || extFromMime || ".bin"; - const tempDir = resolvePreferredOpenClawTmpDir(); - const filePath = path.join(tempDir, `voice-src-${crypto.randomUUID()}${ext}`); - await fs.writeFile(filePath, media.buffer, { mode: 0o600 }); - return { filePath }; + const workspace = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "voice-src-", + }); + const filePath = await workspace.write(`input${ext}`, media.buffer); + return { filePath, cleanup: async () => await workspace.cleanup() }; } /** @@ -74,7 +77,8 @@ export async function sendVoiceMessageDiscord( audioPath: string, opts: VoiceMessageOpts, ): Promise { - const { filePath: localInputPath } = await materializeVoiceMessageInput(audioPath); + const { filePath: localInputPath, cleanup: cleanupLocalInput } = + await materializeVoiceMessageInput(audioPath); let oggPath: string | null = null; let oggCleanup = false; let token: string | undefined; @@ -131,6 +135,6 @@ export async function sendVoiceMessageDiscord( throw err; } finally { await unlinkIfExists(oggCleanup ? oggPath : null); - await unlinkIfExists(localInputPath); + await cleanupLocalInput(); } } diff --git a/extensions/discord/src/voice/audio.ts b/extensions/discord/src/voice/audio.ts index 625d77cb062..fe305059295 100644 --- a/extensions/discord/src/voice/audio.ts +++ b/extensions/discord/src/voice/audio.ts @@ -1,11 +1,9 @@ -import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import { createRequire } from "node:module"; -import path from "node:path"; import type { Readable } from "node:stream"; import { logVerbose, shouldLogVerbose } from "openclaw/plugin-sdk/runtime-env"; import { formatErrorMessage } from "openclaw/plugin-sdk/ssrf-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; const require = createRequire(import.meta.url); @@ -153,11 +151,13 @@ function estimateDurationSeconds(pcm: Buffer): number { export async function writeVoiceWavFile( pcm: Buffer, ): Promise<{ path: string; durationSeconds: number }> { - const tempDir = await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), "discord-voice-")); - const filePath = path.join(tempDir, `segment-${randomUUID()}.wav`); + const workspace = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "discord-voice-", + }); const wav = buildWavBuffer(pcm); - await fs.writeFile(filePath, wav); - scheduleTempCleanup(tempDir); + const filePath = await workspace.write("segment.wav", wav); + scheduleTempCleanup(workspace.dir); return { path: filePath, durationSeconds: estimateDurationSeconds(pcm) }; } diff --git a/extensions/feishu/src/media.test.ts b/extensions/feishu/src/media.test.ts index 5a3b3bd9994..192b9a3def7 100644 --- a/extensions/feishu/src/media.test.ts +++ b/extensions/feishu/src/media.test.ts @@ -1,3 +1,4 @@ +import { realpathSync } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; @@ -61,7 +62,7 @@ function expectPathIsolatedToTmpRoot(pathValue: string, key: string): void { expect(pathValue).not.toContain(key); expect(pathValue).not.toContain(".."); - const tmpRoot = path.resolve(resolvePreferredOpenClawTmpDir()); + const tmpRoot = realpathSync(resolvePreferredOpenClawTmpDir()); const resolved = path.resolve(pathValue); const rel = path.relative(tmpRoot, resolved); expect(rel === ".." || rel.startsWith(`..${path.sep}`)).toBe(false); diff --git a/extensions/feishu/src/media.ts b/extensions/feishu/src/media.ts index c6d5895636f..21e9ffd3f01 100644 --- a/extensions/feishu/src/media.ts +++ b/extensions/feishu/src/media.ts @@ -5,8 +5,10 @@ import type * as Lark from "@larksuiteoapi/node-sdk"; import type { MessageReceipt } from "openclaw/plugin-sdk/channel-message"; import { mediaKindFromMime } from "openclaw/plugin-sdk/media-mime"; import { MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, runFfmpeg } from "openclaw/plugin-sdk/media-runtime"; +import { readRegularFile } from "openclaw/plugin-sdk/security-runtime"; import { resolvePreferredOpenClawTmpDir, + withTempWorkspace, withTempDownloadPath, } from "openclaw/plugin-sdk/temp-path"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; @@ -421,10 +423,11 @@ export async function uploadImageFeishu(params: { const { cfg, image, imageType = "message", accountId } = params; const { client } = createConfiguredFeishuMediaClient({ cfg, accountId }); - // SDK accepts Buffer directly or fs.ReadStream for file paths - // Using Readable.from(buffer) causes issues with form-data library + // SDK accepts Buffer directly. Keep string path support on this helper, but + // verify the path as a regular local file before uploading it. // See: https://github.com/larksuite/node-sdk/issues/121 - const imageData = typeof image === "string" ? fs.createReadStream(image) : image; + const imageData = + typeof image === "string" ? (await readRegularFile({ filePath: image })).buffer : image; const response = await requestFeishuApi( () => @@ -475,10 +478,11 @@ export async function uploadFileFeishu(params: { const { cfg, file, fileName, fileType, duration, accountId } = params; const { client } = createConfiguredFeishuMediaClient({ cfg, accountId }); - // SDK accepts Buffer directly or fs.ReadStream for file paths - // Using Readable.from(buffer) causes issues with form-data library + // SDK accepts Buffer directly. Keep string path support on this helper, but + // verify the path as a regular local file before uploading it. // See: https://github.com/larksuite/node-sdk/issues/121 - const fileData = typeof file === "string" ? fs.createReadStream(file) : file; + const fileData = + typeof file === "string" ? (await readRegularFile({ filePath: file })).buffer : file; const safeFileName = sanitizeFileNameForUpload(fileName); @@ -747,45 +751,42 @@ async function transcodeToFeishuVoiceOpus(params: { fileName: string; contentType?: string; }): Promise<{ buffer: Buffer; fileName: string; contentType: string }> { - const tempRoot = resolvePreferredOpenClawTmpDir(); - await fs.promises.mkdir(tempRoot, { recursive: true, mode: 0o700 }); - const tempDir = await fs.promises.mkdtemp(path.join(tempRoot, "feishu-voice-")); - try { - const ext = normalizeLowercaseStringOrEmpty(path.extname(params.fileName)); - const inputExt = ext && ext.length <= 12 ? ext : ".audio"; - const inputPath = path.join(tempDir, `input${inputExt}`); - const outputPath = path.join(tempDir, FEISHU_VOICE_FILE_NAME); - await fs.promises.writeFile(inputPath, params.buffer, { mode: 0o600 }); - await runFfmpeg([ - "-hide_banner", - "-loglevel", - "error", - "-y", - "-i", - inputPath, - "-vn", - "-sn", - "-dn", - "-t", - String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), - "-ar", - String(FEISHU_VOICE_SAMPLE_RATE_HZ), - "-ac", - "1", - "-c:a", - "libopus", - "-b:a", - FEISHU_VOICE_BITRATE, - outputPath, - ]); - return { - buffer: await fs.promises.readFile(outputPath), - fileName: FEISHU_VOICE_FILE_NAME, - contentType: "audio/ogg", - }; - } finally { - await fs.promises.rm(tempDir, { recursive: true, force: true }); - } + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "feishu-voice-" }, + async (workspace) => { + const ext = normalizeLowercaseStringOrEmpty(path.extname(params.fileName)); + const inputExt = ext && ext.length <= 12 ? ext : ".audio"; + const inputPath = await workspace.write(`input${inputExt}`, params.buffer); + const outputPath = workspace.path(FEISHU_VOICE_FILE_NAME); + await runFfmpeg([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + inputPath, + "-vn", + "-sn", + "-dn", + "-t", + String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), + "-ar", + String(FEISHU_VOICE_SAMPLE_RATE_HZ), + "-ac", + "1", + "-c:a", + "libopus", + "-b:a", + FEISHU_VOICE_BITRATE, + outputPath, + ]); + return { + buffer: await workspace.read(FEISHU_VOICE_FILE_NAME), + fileName: FEISHU_VOICE_FILE_NAME, + contentType: "audio/ogg", + }; + }, + ); } async function prepareFeishuVoiceMedia(params: { diff --git a/extensions/feishu/src/outbound.ts b/extensions/feishu/src/outbound.ts index 3624a49ff1e..6753380f576 100644 --- a/extensions/feishu/src/outbound.ts +++ b/extensions/feishu/src/outbound.ts @@ -1,4 +1,3 @@ -import fs from "node:fs"; import path from "node:path"; import { attachChannelToResult, @@ -18,6 +17,7 @@ import { sendPayloadMediaSequenceAndFinalize, sendTextMediaPayload, } from "openclaw/plugin-sdk/reply-payload"; +import { statRegularFileSync } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { resolveFeishuAccount } from "./accounts.js"; import { createFeishuCardInteractionEnvelope } from "./card-interaction.js"; @@ -66,18 +66,12 @@ function normalizePossibleLocalImagePath(text: string | undefined): string | nul if (!path.isAbsolute(raw)) { return null; } - if (!fs.existsSync(raw)) { - return null; - } - - // Fix race condition: wrap statSync in try-catch to handle file deletion - // between existsSync and statSync try { - if (!fs.statSync(raw).isFile()) { + const stat = statRegularFileSync(raw); + if (stat.missing) { return null; } } catch { - // File may have been deleted or became inaccessible between checks return null; } diff --git a/extensions/file-transfer/src/node-host/dir-fetch.ts b/extensions/file-transfer/src/node-host/dir-fetch.ts index ab13461c6fb..a1bc7d3614f 100644 --- a/extensions/file-transfer/src/node-host/dir-fetch.ts +++ b/extensions/file-transfer/src/node-host/dir-fetch.ts @@ -2,6 +2,11 @@ import { spawn } from "node:child_process"; import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { + FsSafeError, + resolveAbsolutePathForRead, + root as fsRoot, +} from "openclaw/plugin-sdk/security-runtime"; const DIR_FETCH_HARD_MAX_BYTES = 16 * 1024 * 1024; const DIR_FETCH_DEFAULT_MAX_BYTES = 8 * 1024 * 1024; @@ -50,6 +55,17 @@ function clampMaxBytes(input: unknown): number { } function classifyFsError(err: unknown): DirFetchErrCode { + if (err instanceof FsSafeError) { + if (err.code === "not-found") { + return "NOT_FOUND"; + } + if (err.code === "symlink") { + return "SYMLINK_REDIRECT"; + } + if (err.code === "invalid-path") { + return "INVALID_PATH"; + } + } const code = (err as { code?: string } | null)?.code; if (code === "ENOENT") { return "NOT_FOUND"; @@ -145,18 +161,18 @@ async function listTarEntries(tarBuffer: Buffer): Promise { async function listTreeEntries(root: string, maxEntries: number): Promise { const results: string[] = []; - async function visit(dir: string): Promise { - const entries = await fs.readdir(dir, { withFileTypes: true }); + const rootHandle = await fsRoot(root); + async function visit(relativeDir: string): Promise { + const entries = await rootHandle.list(relativeDir, { withFileTypes: true }); entries.sort((left, right) => left.name.localeCompare(right.name)); for (const entry of entries) { - const abs = path.join(dir, entry.name); - const rel = path.relative(root, abs).replace(/\\/gu, "/"); + const rel = path.posix.join(relativeDir === "." ? "" : relativeDir, entry.name); results.push(rel); if (results.length > maxEntries) { return false; } - if (entry.isDirectory()) { - const ok = await visit(abs); + if (entry.isDirectory) { + const ok = await visit(rel); if (!ok) { return false; } @@ -164,7 +180,7 @@ async function listTreeEntries(root: string, maxEntries: number): Promise { @@ -186,22 +202,31 @@ export async function handleDirFetch(params: DirFetchParams): Promise.followSymlinks=true to allow, or update allowReadPaths to the canonical path)`, - canonicalPath: canonical, + message: + code === "NOT_FOUND" + ? "directory not found" + : code === "SYMLINK_REDIRECT" + ? "path traverses a symlink; refusing because followSymlinks=false (set plugins.entries.file-transfer.config.nodes..followSymlinks=true to allow, or update allowReadPaths to the canonical path)" + : `realpath failed: ${String(err)}`, + ...(canonicalPath ? { canonicalPath } : {}), }; } diff --git a/extensions/file-transfer/src/node-host/dir-list.ts b/extensions/file-transfer/src/node-host/dir-list.ts index f6bc587f87d..a0a44afa5c7 100644 --- a/extensions/file-transfer/src/node-host/dir-list.ts +++ b/extensions/file-transfer/src/node-host/dir-list.ts @@ -1,5 +1,10 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { + FsSafeError, + resolveAbsolutePathForRead, + root, +} from "openclaw/plugin-sdk/security-runtime"; import { mimeFromExtension } from "../shared/mime.js"; export const DIR_LIST_DEFAULT_MAX_ENTRIES = 200; @@ -54,6 +59,17 @@ function clampMaxEntries(input: unknown): number { } function classifyFsError(err: unknown): DirListErrCode { + if (err instanceof FsSafeError) { + if (err.code === "not-found") { + return "NOT_FOUND"; + } + if (err.code === "symlink") { + return "SYMLINK_REDIRECT"; + } + if (err.code === "invalid-path") { + return "INVALID_PATH"; + } + } const code = (err as { code?: string } | null)?.code; if (code === "ENOENT") { return "NOT_FOUND"; @@ -86,22 +102,31 @@ export async function handleDirList(params: DirListParams): Promise.followSymlinks=true to allow, or update allowReadPaths to the canonical path)`, - canonicalPath: canonical, + message: + code === "NOT_FOUND" + ? "path not found" + : code === "SYMLINK_REDIRECT" + ? "path traverses a symlink; refusing because followSymlinks=false (set plugins.entries.file-transfer.config.nodes..followSymlinks=true to allow, or update allowReadPaths to the canonical path)" + : `realpath failed: ${String(err)}`, + ...(canonicalPath ? { canonicalPath } : {}), }; } @@ -122,50 +147,39 @@ export async function handleDirList(params: DirListParams): Promise a.localeCompare(b)); + listedEntries.sort((a, b) => a.name.localeCompare(b.name)); - const total = names.length; - const page = names.slice(offset, offset + maxEntries); + const total = listedEntries.length; + const page = listedEntries.slice(offset, offset + maxEntries); const truncated = offset + maxEntries < total; const nextPageToken = truncated ? String(offset + maxEntries) : undefined; const entries: DirListEntry[] = []; - for (const name of page) { - const entryPath = path.join(canonical, name); - - let isDir = false; - let size = 0; - let mtime = 0; - try { - const s = await fs.stat(entryPath); - isDir = s.isDirectory(); - size = isDir ? 0 : s.size; - mtime = s.mtimeMs; - } catch { - // stat may fail for broken symlinks; keep zeros and treat as file - } + for (const entry of page) { + const entryPath = path.join(canonical, entry.name); + const isDir = entry.isDirectory; entries.push({ - name, + name: entry.name, path: entryPath, - size, - mimeType: isDir ? "inode/directory" : mimeFromExtension(name), + size: isDir ? 0 : entry.size, + mimeType: isDir ? "inode/directory" : mimeFromExtension(entry.name), isDir, - mtime, + mtime: entry.mtimeMs, }); } diff --git a/extensions/file-transfer/src/node-host/file-fetch.ts b/extensions/file-transfer/src/node-host/file-fetch.ts index fd4151b1491..3edf53a589d 100644 --- a/extensions/file-transfer/src/node-host/file-fetch.ts +++ b/extensions/file-transfer/src/node-host/file-fetch.ts @@ -1,7 +1,11 @@ import { spawnSync } from "node:child_process"; import crypto from "node:crypto"; -import fs from "node:fs/promises"; import path from "node:path"; +import { + FsSafeError, + resolveAbsolutePathForRead, + root, +} from "openclaw/plugin-sdk/security-runtime"; import { EXTENSION_MIME } from "../shared/mime.js"; export const FILE_FETCH_HARD_MAX_BYTES = 16 * 1024 * 1024; @@ -70,6 +74,20 @@ function clampMaxBytes(input: unknown): number { } function classifyFsError(err: unknown): FileFetchErrCode { + if (err instanceof FsSafeError) { + if (err.code === "not-found") { + return "NOT_FOUND"; + } + if (err.code === "symlink") { + return "SYMLINK_REDIRECT"; + } + if (err.code === "invalid-path") { + return "INVALID_PATH"; + } + if (err.code === "not-file") { + return "IS_DIRECTORY"; + } + } const code = (err as { code?: string } | null)?.code; if (code === "ENOENT") { return "NOT_FOUND"; @@ -101,103 +119,102 @@ export async function handleFileFetch(params: FileFetchParams): Promise.followSymlinks=true to allow, or update allowReadPaths to the canonical path)" + : `realpath failed: ${String(err)}`, + ...(canonicalPath ? { canonicalPath } : {}), + }; + } + + let opened: Awaited>["open"]>>; + try { + const parentRoot = await root(path.dirname(canonical)); + opened = await parentRoot.open(path.basename(canonical)); } catch (err) { const code = classifyFsError(err); return { ok: false, code, - message: code === "NOT_FOUND" ? "file not found" : `realpath failed: ${String(err)}`, - }; - } - - // Refuse to follow symlinks anywhere in the path unless the operator - // has explicitly opted in. A symlink in user-controlled territory - // (e.g. ~/Downloads/evil → /etc) could redirect an allowed-looking - // request to a disallowed canonical target. The error includes the - // canonical path so the operator can either update their allowlist - // to the canonical form or set followSymlinks=true on this node. - if (!followSymlinks && canonical !== requestedPath) { - return { - ok: false, - code: "SYMLINK_REDIRECT", - message: `path traverses a symlink; refusing because followSymlinks=false (set plugins.entries.file-transfer.config.nodes..followSymlinks=true to allow, or update allowReadPaths to the canonical path)`, + message: code === "IS_DIRECTORY" ? "path is a directory" : `open failed: ${String(err)}`, canonicalPath: canonical, }; } - let stats: Awaited>; try { - stats = await fs.stat(canonical); - } catch (err) { - const code = classifyFsError(err); - return { ok: false, code, message: `stat failed: ${String(err)}`, canonicalPath: canonical }; - } + const stats = opened.stat; + if (stats.size > maxBytes) { + return { + ok: false, + code: "FILE_TOO_LARGE", + message: `file size ${stats.size} exceeds limit ${maxBytes}`, + canonicalPath: opened.realPath, + }; + } - if (stats.isDirectory()) { - return { - ok: false, - code: "IS_DIRECTORY", - message: "path is a directory", - canonicalPath: canonical, - }; - } - if (!stats.isFile()) { - return { - ok: false, - code: "READ_ERROR", - message: "path is not a regular file", - canonicalPath: canonical, - }; - } - if (stats.size > maxBytes) { - return { - ok: false, - code: "FILE_TOO_LARGE", - message: `file size ${stats.size} exceeds limit ${maxBytes}`, - canonicalPath: canonical, - }; - } + if (preflightOnly) { + return { + ok: true, + path: opened.realPath, + size: stats.size, + mimeType: "", + base64: "", + sha256: "", + preflightOnly: true, + }; + } + + const buffer = await opened.handle.readFile(); + if (buffer.byteLength > maxBytes) { + return { + ok: false, + code: "FILE_TOO_LARGE", + message: `read ${buffer.byteLength} bytes exceeds limit ${maxBytes}`, + canonicalPath: opened.realPath, + }; + } + + const sha256 = crypto.createHash("sha256").update(buffer).digest("hex"); + const base64 = buffer.toString("base64"); + const mimeType = detectMimeType(opened.realPath); - if (preflightOnly) { return { ok: true, - path: canonical, - size: stats.size, - mimeType: "", - base64: "", - sha256: "", - preflightOnly: true, + path: opened.realPath, + size: buffer.byteLength, + mimeType, + base64, + sha256, }; - } - - let buffer: Buffer; - try { - buffer = await fs.readFile(canonical); } catch (err) { const code = classifyFsError(err); - return { ok: false, code, message: `read failed: ${String(err)}`, canonicalPath: canonical }; - } - - if (buffer.byteLength > maxBytes) { return { ok: false, - code: "FILE_TOO_LARGE", - message: `read ${buffer.byteLength} bytes exceeds limit ${maxBytes}`, - canonicalPath: canonical, + code, + message: `read failed: ${String(err)}`, + canonicalPath: opened.realPath, }; + } finally { + await opened.handle.close().catch(() => undefined); } - - const sha256 = crypto.createHash("sha256").update(buffer).digest("hex"); - const base64 = buffer.toString("base64"); - const mimeType = detectMimeType(canonical); - - return { - ok: true, - path: canonical, - size: buffer.byteLength, - mimeType, - base64, - sha256, - }; } diff --git a/extensions/file-transfer/src/node-host/file-write.ts b/extensions/file-transfer/src/node-host/file-write.ts index 88e030c1b6b..3f5fb2fbec5 100644 --- a/extensions/file-transfer/src/node-host/file-write.ts +++ b/extensions/file-transfer/src/node-host/file-write.ts @@ -1,6 +1,12 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { + canonicalPathFromExistingAncestor, + FsSafeError, + resolveAbsolutePathForWrite, + root, +} from "openclaw/plugin-sdk/security-runtime"; const MAX_CONTENT_BYTES = 16 * 1024 * 1024; // 16 MB @@ -39,74 +45,37 @@ function err(code: string, message: string, canonicalPath?: string): FileWriteEr return { ok: false, code, message, ...(canonicalPath ? { canonicalPath } : {}) }; } -async function pathExists(p: string): Promise { - try { - await fs.access(p); - return true; - } catch { - return false; - } -} - -async function findExistingAncestor(p: string): Promise { - let current = p; - while (true) { - try { - await fs.lstat(current); - return current; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - const parent = path.dirname(current); - if (parent === current) { - return null; - } - current = parent; - } -} - -async function canonicalTargetFromExistingAncestor(targetPath: string): Promise { - const ancestor = await findExistingAncestor(targetPath); - if (!ancestor) { - return targetPath; - } - let canonicalAncestor: string; - try { - canonicalAncestor = await fs.realpath(ancestor); - } catch { - canonicalAncestor = ancestor; - } - const relative = path.relative(ancestor, targetPath); - return relative ? path.join(canonicalAncestor, relative) : canonicalAncestor; -} - -async function rejectParentSymlinkRedirect( - targetPath: string, - parentDir: string, -): Promise { - const ancestor = await findExistingAncestor(parentDir); - if (!ancestor) { - return null; - } - let canonicalAncestor: string; - try { - canonicalAncestor = await fs.realpath(ancestor); - } catch { - return null; - } - if (canonicalAncestor === ancestor) { - return null; - } - const canonicalTarget = path.join(canonicalAncestor, path.relative(ancestor, targetPath)); +function symlinkRedirectError(error: FsSafeError): FileWriteError { + const canonicalTarget = + error.cause && + typeof error.cause === "object" && + "canonicalPath" in error.cause && + typeof error.cause.canonicalPath === "string" + ? error.cause.canonicalPath + : undefined; return err( "SYMLINK_REDIRECT", - `parent ${ancestor} resolves through a symlink to ${canonicalAncestor}; refusing because followSymlinks=false (set plugins.entries.file-transfer.config.nodes..followSymlinks=true to allow, or update allowWritePaths to the canonical path)`, + "path traverses a symlink; refusing because followSymlinks=false (set plugins.entries.file-transfer.config.nodes..followSymlinks=true to allow, or update allowWritePaths to the canonical path)", canonicalTarget, ); } +function writeFsSafeError(error: FsSafeError, targetPath: string): FileWriteError { + if (error.code === "symlink") { + return err( + "SYMLINK_TARGET_DENIED", + `path is a symlink; refusing to write through it: ${targetPath}`, + ); + } + if (error.code === "not-file") { + return err("IS_DIRECTORY", `path resolves to a directory: ${targetPath}`); + } + if (error.code === "already-exists") { + return err("EXISTS_NO_OVERWRITE", `file already exists and overwrite is false: ${targetPath}`); + } + return err("WRITE_ERROR", error.message, targetPath); +} + export async function handleFileWrite( params: Partial & Record, ): Promise { @@ -158,20 +127,21 @@ export async function handleFileWrite( ); } - // 3. Resolve parent dir - const targetPath = path.normalize(rawPath); - const parentDir = path.dirname(targetPath); - - const parentExists = await pathExists(parentDir); - - // Refuse symlink traversal in the existing parent chain before creating - // missing directories. Recursive mkdir follows symlinked ancestors, so this - // has to run before mkdir can mutate the canonical target. - if (!followSymlinks) { - const redirect = await rejectParentSymlinkRedirect(targetPath, parentDir); - if (redirect) { - return redirect; + let targetPath: string; + let parentDir: string; + let parentExists: boolean; + try { + const resolved = await resolveAbsolutePathForWrite(rawPath, { + symlinks: followSymlinks ? "follow" : "reject", + }); + targetPath = resolved.path; + parentDir = resolved.parentDir; + parentExists = resolved.parentExists; + } catch (error) { + if (error instanceof FsSafeError && error.code === "symlink") { + return symlinkRedirectError(error); } + throw error; } if (!parentExists) { @@ -189,7 +159,7 @@ export async function handleFileWrite( } return { ok: true, - path: await canonicalTargetFromExistingAncestor(targetPath), + path: await canonicalPathFromExistingAncestor(targetPath), size: buf.length, sha256: computedSha256, overwritten: false, @@ -203,15 +173,19 @@ export async function handleFileWrite( } } - // Re-check after mkdir as a race-defense: if the parent chain changed - // between the first check and directory creation, fail before writing bytes. - if (!followSymlinks) { - const redirect = await rejectParentSymlinkRedirect(targetPath, parentDir); - if (redirect) { - return redirect; + try { + await resolveAbsolutePathForWrite(targetPath, { + symlinks: followSymlinks ? "follow" : "reject", + }); + } catch (error) { + if (error instanceof FsSafeError && error.code === "symlink") { + return symlinkRedirectError(error); } + throw error; } + const targetFileName = path.basename(targetPath); + const parentRoot = await root(parentDir); let overwritten = false; try { const existingLStat = await fs.lstat(targetPath); @@ -232,8 +206,9 @@ export async function handleFileWrite( } overwritten = true; } catch (statErr: unknown) { - // ENOENT is fine — file does not exist yet - if ((statErr as NodeJS.ErrnoException).code !== "ENOENT") { + const statErrorCode = + statErr instanceof FsSafeError ? statErr.code : (statErr as NodeJS.ErrnoException).code; + if (statErrorCode !== "not-found" && statErrorCode !== "ENOENT") { const message = statErr instanceof Error ? statErr.message : String(statErr); if (message.toLowerCase().includes("permission")) { return err("PERMISSION_DENIED", `permission denied: ${targetPath}`); @@ -259,55 +234,45 @@ export async function handleFileWrite( if (preflightOnly) { return { ok: true, - path: await canonicalTargetFromExistingAncestor(targetPath), + path: await canonicalPathFromExistingAncestor(targetPath), size: buf.length, sha256: computedSha256, overwritten, }; } - // 6. Atomic write: write to tmp, then rename - const tmpSuffix = crypto.randomBytes(8).toString("hex"); - const tmpPath = `${targetPath}.${tmpSuffix}.tmp`; - try { - await fs.writeFile(tmpPath, buf); + if (overwrite) { + await parentRoot.write(targetFileName, buf); + } else { + await parentRoot.create(targetFileName, buf); + } } catch (writeErr) { + if (writeErr instanceof FsSafeError) { + return writeFsSafeError(writeErr, targetPath); + } const message = writeErr instanceof Error ? writeErr.message : String(writeErr); - // Clean up tmp if possible - await fs.unlink(tmpPath).catch(() => {}); if (message.toLowerCase().includes("permission") || message.toLowerCase().includes("access")) { return err("PERMISSION_DENIED", `permission denied writing to: ${parentDir}`); } return err("WRITE_ERROR", `failed to write file: ${message}`); } - try { - await fs.rename(tmpPath, targetPath); - } catch (renameErr) { - const message = renameErr instanceof Error ? renameErr.message : String(renameErr); - await fs.unlink(tmpPath).catch(() => {}); - if (message.toLowerCase().includes("permission") || message.toLowerCase().includes("access")) { - return err("PERMISSION_DENIED", `permission denied renaming to: ${targetPath}`); - } - return err("WRITE_ERROR", `failed to rename tmp to target: ${message}`); - } - - const writtenBuf = buf; - - // 8. Re-realpath to resolve any symlinks in the final path let canonicalPath = targetPath; try { - canonicalPath = await fs.realpath(targetPath); - } catch { - // Best effort; use normalized path as fallback - canonicalPath = targetPath; + const opened = await parentRoot.open(targetFileName); + canonicalPath = opened.realPath; + await opened.handle.close().catch(() => undefined); + } catch (openErr) { + if (openErr instanceof FsSafeError) { + return writeFsSafeError(openErr, targetPath); + } } return { ok: true, path: canonicalPath, - size: writtenBuf.length, + size: buf.length, sha256: computedSha256, overwritten, }; diff --git a/extensions/file-transfer/src/shared/audit.ts b/extensions/file-transfer/src/shared/audit.ts index d30e6148548..9d0f57cbc65 100644 --- a/extensions/file-transfer/src/shared/audit.ts +++ b/extensions/file-transfer/src/shared/audit.ts @@ -11,6 +11,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; export type FileTransferAuditOp = "file.fetch" | "dir.list" | "dir.fetch" | "file.write"; @@ -86,7 +87,11 @@ export async function appendFileTransferAudit( timestamp: new Date().toISOString(), ...record, })}\n`; - await fs.appendFile(auditFilePath(dir), line, { mode: 0o600 }); + await appendRegularFile({ + filePath: auditFilePath(dir), + content: line, + rejectSymlinkParents: true, + }); } catch (e) { process.stderr.write(`[file-transfer:audit] append failed: ${String(e)}\n`); } diff --git a/extensions/file-transfer/src/tools/file-write-tool.ts b/extensions/file-transfer/src/tools/file-write-tool.ts index dd5dde87079..e900cdb430c 100644 --- a/extensions/file-transfer/src/tools/file-write-tool.ts +++ b/extensions/file-transfer/src/tools/file-write-tool.ts @@ -1,5 +1,4 @@ import crypto from "node:crypto"; -import fs from "node:fs/promises"; import { callGatewayTool, listNodes, @@ -7,7 +6,7 @@ import { type AnyAgentTool, type NodeListNode, } from "openclaw/plugin-sdk/agent-harness-runtime"; -import { resolveMediaBufferPath } from "openclaw/plugin-sdk/media-store"; +import { readMediaBuffer } from "openclaw/plugin-sdk/media-store"; import { appendFileTransferAudit } from "../shared/audit.js"; import { throwFromNodePayload } from "../shared/errors.js"; import { @@ -28,14 +27,11 @@ async function readSourceBytes(input: { }): Promise<{ buffer: Buffer; contentBase64: string; source: "inline" | "media" }> { const sourceMediaId = input.sourceMediaId?.trim(); if (sourceMediaId) { - const mediaPath = await resolveMediaBufferPath(sourceMediaId, FILE_TRANSFER_SUBDIR); - const stat = await fs.stat(mediaPath); - if (stat.size > FILE_WRITE_HARD_MAX_BYTES) { - throw new Error( - `sourceMediaId too large: ${stat.size} bytes; maximum is ${FILE_WRITE_HARD_MAX_BYTES} bytes`, - ); - } - const buffer = await fs.readFile(mediaPath); + const { buffer } = await readMediaBuffer( + sourceMediaId, + FILE_TRANSFER_SUBDIR, + FILE_WRITE_HARD_MAX_BYTES, + ); return { buffer, contentBase64: buffer.toString("base64"), source: "media" }; } if (input.contentBase64 === undefined) { diff --git a/extensions/google/video-generation-provider.ts b/extensions/google/video-generation-provider.ts index 423278ed7b6..eb8e98838e1 100644 --- a/extensions/google/video-generation-provider.ts +++ b/extensions/google/video-generation-provider.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import path from "node:path"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { @@ -7,7 +7,7 @@ import { waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { GeneratedVideoAsset, @@ -151,24 +151,22 @@ async function downloadGeneratedVideo(params: { file: unknown; index: number; }): Promise { - const tempDir = await mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-google-video-"), + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-google-video-" }, + async ({ dir: tempDir }) => { + const downloadPath = path.join(tempDir, `video-${params.index + 1}.mp4`); + await params.client.files.download({ + file: params.file as never, + downloadPath, + }); + const buffer = await readFile(downloadPath); + return { + buffer, + mimeType: "video/mp4", + fileName: `video-${params.index + 1}.mp4`, + }; + }, ); - const downloadPath = path.join(tempDir, `video-${params.index + 1}.mp4`); - try { - await params.client.files.download({ - file: params.file as never, - downloadPath, - }); - const buffer = await readFile(downloadPath); - return { - buffer, - mimeType: "video/mp4", - fileName: `video-${params.index + 1}.mp4`, - }; - } finally { - await rm(tempDir, { recursive: true, force: true }); - } } function resolveGoogleGeneratedVideoDownloadUrl(params: { diff --git a/extensions/irc/src/client.ts b/extensions/irc/src/client.ts index df0abf4df79..0f75be1dbf0 100644 --- a/extensions/irc/src/client.ts +++ b/extensions/irc/src/client.ts @@ -1,5 +1,6 @@ import net from "node:net"; import tls from "node:tls"; +import { withTimeout } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { parseIrcLine, @@ -64,24 +65,6 @@ function toError(err: unknown): Error { return new Error(typeof err === "string" ? err : JSON.stringify(err)); } -function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout( - () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), - timeoutMs, - ); - promise - .then((result) => { - clearTimeout(timer); - resolve(result); - }) - .catch((error) => { - clearTimeout(timer); - reject(error); - }); - }); -} - function buildFallbackNick(nick: string): string { const normalized = nick.replace(/\s+/g, ""); const safe = normalized.replace(/[^A-Za-z0-9_\-[\]\\`^{}|]/g, ""); diff --git a/extensions/llm-task/api.ts b/extensions/llm-task/api.ts index 43063ce557e..a1ad56d8cc7 100644 --- a/extensions/llm-task/api.ts +++ b/extensions/llm-task/api.ts @@ -1,4 +1,4 @@ -export { resolvePreferredOpenClawTmpDir } from "./src/runtime-api.js"; +export { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "./src/runtime-api.js"; export { definePluginEntry, type AnyAgentTool, diff --git a/extensions/llm-task/src/llm-task-tool.ts b/extensions/llm-task/src/llm-task-tool.ts index 6449fb3f0a4..d01fec079c2 100644 --- a/extensions/llm-task/src/llm-task-tool.ts +++ b/extensions/llm-task/src/llm-task-tool.ts @@ -1,9 +1,8 @@ -import fs from "node:fs/promises"; import path from "node:path"; import Ajv from "ajv"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import { Type } from "typebox"; -import { resolvePreferredOpenClawTmpDir } from "../api.js"; +import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "../api.js"; import type { OpenClawPluginApi } from "../api.js"; const AjvCtor = Ajv as unknown as typeof import("ajv").default; @@ -208,78 +207,69 @@ export function createLlmTaskTool(api: OpenClawPluginApi) { const fullPrompt = `${system}\n\nTASK:\n${prompt}\n\nINPUT_JSON:\n${inputJson}\n`; - let tmpDir: string | null = null; - try { - tmpDir = await fs.mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-llm-task-"), - ); - const sessionId = `llm-task-${Date.now()}`; - const sessionFile = path.join(tmpDir, "session.json"); + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-llm-task-" }, + async ({ dir: tmpDir }) => { + const sessionId = `llm-task-${Date.now()}`; + const sessionFile = path.join(tmpDir, "session.json"); - const result = await api.runtime.agent.runEmbeddedPiAgent({ - sessionId, - sessionFile, - workspaceDir: api.config?.agents?.defaults?.workspace ?? process.cwd(), - config: api.config, - prompt: fullPrompt, - timeoutMs, - runId: `llm-task-${Date.now()}`, - provider, - model, - authProfileId, - authProfileIdSource: authProfileId ? "user" : "auto", - thinkLevel, - streamParams, - disableTools: true, - }); + const result = await api.runtime.agent.runEmbeddedPiAgent({ + sessionId, + sessionFile, + workspaceDir: api.config?.agents?.defaults?.workspace ?? process.cwd(), + config: api.config, + prompt: fullPrompt, + timeoutMs, + runId: `llm-task-${Date.now()}`, + provider, + model, + authProfileId, + authProfileIdSource: authProfileId ? "user" : "auto", + thinkLevel, + streamParams, + disableTools: true, + }); - const text = collectText( - typeof result === "object" && result !== null && "payloads" in result - ? (result as { payloads?: Array<{ text?: string; isError?: boolean }> }).payloads - : undefined, - ); - if (!text) { - throw new Error("LLM returned empty output"); - } - - const raw = stripCodeFences(text); - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - throw new Error("LLM returned invalid JSON"); - } - - const schema = params.schema; - if (schema && typeof schema === "object" && !Array.isArray(schema)) { - const ajv = new AjvCtor({ allErrors: true, strict: false }); - const validate = ajv.compile(schema); - const ok = validate(parsed); - if (!ok) { - const msg = - validate.errors - ?.map( - (e: { instancePath?: string; message?: string }) => - `${e.instancePath || ""} ${e.message || "invalid"}`, - ) - .join("; ") ?? "invalid"; - throw new Error(`LLM JSON did not match schema: ${msg}`); + const text = collectText( + typeof result === "object" && result !== null && "payloads" in result + ? (result as { payloads?: Array<{ text?: string; isError?: boolean }> }).payloads + : undefined, + ); + if (!text) { + throw new Error("LLM returned empty output"); } - } - return { - content: [{ type: "text", text: JSON.stringify(parsed, null, 2) }], - details: { json: parsed, provider, model }, - }; - } finally { - if (tmpDir) { + const raw = stripCodeFences(text); + let parsed: unknown; try { - await fs.rm(tmpDir, { recursive: true, force: true }); + parsed = JSON.parse(raw); } catch { - // ignore + throw new Error("LLM returned invalid JSON"); } - } - } + + const schema = params.schema; + if (schema && typeof schema === "object" && !Array.isArray(schema)) { + const ajv = new AjvCtor({ allErrors: true, strict: false }); + const validate = ajv.compile(schema); + const ok = validate(parsed); + if (!ok) { + const msg = + validate.errors + ?.map( + (e: { instancePath?: string; message?: string }) => + `${e.instancePath || ""} ${e.message || "invalid"}`, + ) + .join("; ") ?? "invalid"; + throw new Error(`LLM JSON did not match schema: ${msg}`); + } + } + + return { + content: [{ type: "text", text: JSON.stringify(parsed, null, 2) }], + details: { json: parsed, provider, model }, + }; + }, + ); }, }; } diff --git a/extensions/llm-task/src/runtime-api.ts b/extensions/llm-task/src/runtime-api.ts index df7c6b16db6..cd2e8654f39 100644 --- a/extensions/llm-task/src/runtime-api.ts +++ b/extensions/llm-task/src/runtime-api.ts @@ -1 +1 @@ -export { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +export { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; diff --git a/extensions/memory-core/src/dreaming-narrative.ts b/extensions/memory-core/src/dreaming-narrative.ts index 39ec17dfa99..d0214ecc6d5 100644 --- a/extensions/memory-core/src/dreaming-narrative.ts +++ b/extensions/memory-core/src/dreaming-narrative.ts @@ -13,6 +13,7 @@ import { import { resolveGlobalMap } from "openclaw/plugin-sdk/global-singleton"; import { resolveStateDir } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; import { getRuntimeConfig } from "openclaw/plugin-sdk/runtime-config-snapshot"; +import { pathExists, replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { loadSessionStore, resolveStorePath, @@ -488,29 +489,14 @@ async function assertSafeDreamsPath(dreamsPath: string): Promise { async function writeDreamsFileAtomic(dreamsPath: string, content: string): Promise { await assertSafeDreamsPath(dreamsPath); - const existing = await fs.stat(dreamsPath).catch((err: NodeJS.ErrnoException) => { - if (err.code === "ENOENT") { - return null; - } - throw err; + await replaceFileAtomic({ + filePath: dreamsPath, + content, + mode: 0o600, + preserveExistingMode: true, + tempPrefix: `${path.basename(dreamsPath)}.dreams`, + throwOnCleanupError: true, }); - const mode = existing?.mode ?? 0o600; - const tempPath = `${dreamsPath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile(tempPath, content, { encoding: "utf-8", flag: "wx", mode }); - await fs.chmod(tempPath, mode).catch(() => undefined); - try { - await fs.rename(tempPath, dreamsPath); - await fs.chmod(dreamsPath, mode).catch(() => undefined); - } catch (err) { - const cleanupError = await fs.rm(tempPath, { force: true }).catch((rmErr) => rmErr); - if (cleanupError) { - throw new Error( - `Atomic DREAMS.md write failed (${formatErrorMessage(err)}); cleanup also failed (${formatErrorMessage(cleanupError)})`, - { cause: err }, - ); - } - throw err; - } } async function updateDreamsFile(params: { @@ -710,15 +696,6 @@ export async function appendNarrativeEntry(params: { // ── Orchestrator ─────────────────────────────────────────────────────── -async function safePathExists(pathname: string): Promise { - try { - await fs.stat(pathname); - return true; - } catch { - return false; - } -} - function normalizeComparablePath(pathname: string): string { return process.platform === "win32" ? pathname.toLowerCase() : pathname; } @@ -814,7 +791,7 @@ async function scrubDreamingNarrativeArtifacts(logger: Logger): Promise { if (!isDreamingSessionStoreKey(key)) { continue; } - if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) { + if (!normalizedSessionFile || !(await pathExists(normalizedSessionFile))) { needsStoreUpdate = true; } } @@ -834,7 +811,7 @@ async function scrubDreamingNarrativeArtifacts(logger: Logger): Promise { if (!isDreamingSessionStoreKey(key)) { continue; } - if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) { + if (!normalizedSessionFile || !(await pathExists(normalizedSessionFile))) { delete lockedStore[key]; prunedForAgent += 1; } diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 9e7d68aab83..7a9948c00ab 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -18,6 +18,7 @@ import { resolveMemoryRemDreamingConfig, } from "openclaw/plugin-sdk/memory-core-host-status"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { appendRegularFile, privateFileStore } from "openclaw/plugin-sdk/security-runtime"; import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js"; import { generateAndAppendDreamNarrative, @@ -443,11 +444,11 @@ function normalizeMemoryDay(value: unknown): string | undefined { async function readDailyIngestionState(workspaceDir: string): Promise { const statePath = resolveDailyIngestionStatePath(workspaceDir); try { - const raw = await fs.readFile(statePath, "utf-8"); - return normalizeDailyIngestionState(JSON.parse(raw) as unknown); + return normalizeDailyIngestionState( + await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, statePath)), + ); } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code; - if (code === "ENOENT" || err instanceof SyntaxError) { + if (err instanceof SyntaxError) { return { version: 1, files: {} }; } throw err; @@ -459,10 +460,9 @@ async function writeDailyIngestionState( state: DailyIngestionState, ): Promise { const statePath = resolveDailyIngestionStatePath(workspaceDir); - await fs.mkdir(path.dirname(statePath), { recursive: true }); - const tmpPath = `${statePath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, "utf-8"); - await fs.rename(tmpPath, statePath); + await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, statePath), state, { + trailingNewline: true, + }); } type SessionIngestionFileState = { @@ -556,11 +556,11 @@ function normalizeSessionIngestionState(raw: unknown): SessionIngestionState { async function readSessionIngestionState(workspaceDir: string): Promise { const statePath = resolveSessionIngestionStatePath(workspaceDir); try { - const raw = await fs.readFile(statePath, "utf-8"); - return normalizeSessionIngestionState(JSON.parse(raw) as unknown); + return normalizeSessionIngestionState( + await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, statePath)), + ); } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code; - if (code === "ENOENT" || err instanceof SyntaxError) { + if (err instanceof SyntaxError) { return { version: 3, files: {}, seenMessages: {} }; } throw err; @@ -572,10 +572,9 @@ async function writeSessionIngestionState( state: SessionIngestionState, ): Promise { const statePath = resolveSessionIngestionStatePath(workspaceDir); - await fs.mkdir(path.dirname(statePath), { recursive: true }); - const tmpPath = `${statePath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, "utf-8"); - await fs.rename(tmpPath, statePath); + await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, statePath), state, { + trailingNewline: true, + }); } function trimTrackedSessionScopes( @@ -714,7 +713,11 @@ async function appendSessionCorpusLines(params: { ? normalizedExisting.slice(0, -1).split("\n").length : normalizedExisting.split("\n").length; const payload = `${params.lines.map((entry) => entry.rendered).join("\n")}\n`; - await fs.appendFile(absolutePath, payload, "utf-8"); + await appendRegularFile({ + filePath: absolutePath, + content: payload, + rejectSymlinkParents: true, + }); return params.lines.map((entry, index) => { const lineNumber = existingLineCount + index + 1; return { diff --git a/extensions/memory-core/src/memory/qmd-manager.ts b/extensions/memory-core/src/memory/qmd-manager.ts index 864e6444e63..0af76e4428f 100644 --- a/extensions/memory-core/src/memory/qmd-manager.ts +++ b/extensions/memory-core/src/memory/qmd-manager.ts @@ -9,12 +9,13 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { withFileLock } from "openclaw/plugin-sdk/file-lock"; import { createSubsystemLogger, + isPathInside, + root, resolveAgentContextLimits, resolveMemorySearchSyncConfig, resolveAgentWorkspaceDir, resolveGlobalSingleton, resolveStateDir, - writeFileWithinRoot, type OpenClawConfig, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; import { @@ -1302,7 +1303,15 @@ export class QmdMemoryManager implements MemorySearchManager { if (!absPath.endsWith(".md")) { throw new Error("path required"); } - const statResult = await statRegularFile(absPath); + let statResult: Awaited>; + try { + statResult = await statRegularFile(absPath); + } catch (err) { + if (err instanceof Error && err.message === "path must be a regular file") { + throw new Error("path required", { cause: err }); + } + throw err; + } if (statResult.missing) { return { text: "", path: relPath }; } @@ -2203,6 +2212,7 @@ export class QmdMemoryManager implements MemorySearchManager { } const exportDir = this.sessionExporter.dir; await fs.mkdir(exportDir, { recursive: true }); + const exportRoot = await root(exportDir); const files = await listSessionFilesForAgent(this.agentId); const keep = new Set(); const tracked = new Set(); @@ -2222,10 +2232,7 @@ export class QmdMemoryManager implements MemorySearchManager { tracked.add(sessionFile); const state = this.exportedSessionState.get(sessionFile); if (!state || state.hash !== entry.hash || state.mtimeMs !== entry.mtimeMs) { - await writeFileWithinRoot({ - rootDir: exportDir, - relativePath: targetName, - data: this.renderSessionMarkdown(entry), + await exportRoot.write(targetName, this.renderSessionMarkdown(entry), { encoding: "utf-8", }); } @@ -2236,18 +2243,18 @@ export class QmdMemoryManager implements MemorySearchManager { }); keep.add(target); } - const exported = await fs.readdir(exportDir).catch(() => []); + const exported = await exportRoot.list(".").catch(() => []); for (const name of exported) { if (!name.endsWith(".md")) { continue; } const full = path.join(exportDir, name); if (!keep.has(full)) { - await fs.rm(full, { force: true }); + await exportRoot.remove(name).catch(() => undefined); } } for (const [sessionFile, state] of this.exportedSessionState) { - if (!tracked.has(sessionFile) || !state.target.startsWith(exportDir + path.sep)) { + if (!tracked.has(sessionFile) || !isPathInside(exportDir, state.target)) { this.exportedSessionState.delete(sessionFile); } } @@ -2788,23 +2795,11 @@ export class QmdMemoryManager implements MemorySearchManager { } private isWithinWorkspace(absPath: string): boolean { - const normalizedWorkspace = this.workspaceDir.endsWith(path.sep) - ? this.workspaceDir - : `${this.workspaceDir}${path.sep}`; - if (absPath === this.workspaceDir) { - return true; - } - const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`; - return candidate.startsWith(normalizedWorkspace); + return isPathInside(this.workspaceDir, absPath); } private isWithinRoot(root: string, candidate: string): boolean { - const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`; - if (candidate === root) { - return true; - } - const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`; - return next.startsWith(normalizedRoot); + return isPathInside(root, candidate); } private clampResultsByInjectedChars(results: MemorySearchResult[]): MemorySearchResult[] { diff --git a/extensions/memory-core/src/public-artifacts.ts b/extensions/memory-core/src/public-artifacts.ts index 9c155ffbbbd..22f4991f5a8 100644 --- a/extensions/memory-core/src/public-artifacts.ts +++ b/extensions/memory-core/src/public-artifacts.ts @@ -3,17 +3,9 @@ import path from "node:path"; import { resolveMemoryHostEventLogPath } from "openclaw/plugin-sdk/memory-core-host-events"; import { resolveMemoryDreamingWorkspaces } from "openclaw/plugin-sdk/memory-core-host-status"; import type { MemoryPluginPublicArtifact } from "openclaw/plugin-sdk/memory-host-core"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import type { OpenClawConfig } from "../api.js"; -async function pathExists(inputPath: string): Promise { - try { - await fs.access(inputPath); - return true; - } catch { - return false; - } -} - async function listMarkdownFilesRecursive(rootDir: string): Promise { const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []); const files: string[] = []; diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index ee96e5f4a1e..8dad1901422 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -1,9 +1,10 @@ -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status"; import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events"; +import { privateFileStore } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { deriveConceptTags, @@ -758,9 +759,10 @@ async function withShortTermLock(workspaceDir: string, task: () => Promise async function readStore(workspaceDir: string, nowIso: string): Promise { const storePath = resolveStorePath(workspaceDir); try { - const raw = await fs.readFile(storePath, "utf-8"); - const parsed = JSON.parse(raw) as unknown; - return normalizeStore(parsed, nowIso); + return normalizeStore( + await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, storePath)), + nowIso, + ); } catch (err) { if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { return emptyStore(nowIso); @@ -830,13 +832,13 @@ async function readPhaseSignalStore( ): Promise { const phaseSignalPath = resolvePhaseSignalPath(workspaceDir); try { - const raw = await fs.readFile(phaseSignalPath, "utf-8"); - return normalizePhaseSignalStore(JSON.parse(raw) as unknown, nowIso); - } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code; - if (code === "ENOENT" || err instanceof SyntaxError) { - return emptyPhaseSignalStore(nowIso); - } + return normalizePhaseSignalStore( + await privateFileStore(workspaceDir).readJsonIfExists( + path.relative(workspaceDir, phaseSignalPath), + ), + nowIso, + ); + } catch { return emptyPhaseSignalStore(nowIso); } } @@ -847,17 +849,21 @@ async function writePhaseSignalStore( ): Promise { const phaseSignalPath = resolvePhaseSignalPath(workspaceDir); await ensureShortTermArtifactsDir(workspaceDir); - const tmpPath = `${phaseSignalPath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await fs.writeFile(tmpPath, `${JSON.stringify(store, null, 2)}\n`, "utf-8"); - await fs.rename(tmpPath, phaseSignalPath); + await privateFileStore(workspaceDir).writeJson( + path.relative(workspaceDir, phaseSignalPath), + store, + { + trailingNewline: true, + }, + ); } async function writeStore(workspaceDir: string, store: ShortTermRecallStore): Promise { const storePath = resolveStorePath(workspaceDir); await ensureShortTermArtifactsDir(workspaceDir); - const tmpPath = `${storePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await fs.writeFile(tmpPath, `${JSON.stringify(store, null, 2)}\n`, "utf-8"); - await fs.rename(tmpPath, storePath); + await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, storePath), store, { + trailingNewline: true, + }); } export function isShortTermMemoryPath(filePath: string): boolean { diff --git a/extensions/memory-wiki/src/apply.ts b/extensions/memory-wiki/src/apply.ts index 6295db1718b..c47a82ff75f 100644 --- a/extensions/memory-wiki/src/apply.ts +++ b/extensions/memory-wiki/src/apply.ts @@ -1,9 +1,9 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { replaceManagedMarkdownBlock, withTrailingNewline, } from "openclaw/plugin-sdk/memory-host-markdown"; +import { root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; import { compileMemoryWikiVault, type CompileMemoryWikiResult } from "./compile.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { @@ -150,22 +150,23 @@ function buildSynthesisBody(params: { } async function writeWikiPage(params: { - absolutePath: string; + rootDir: string; + relativePath: string; frontmatter: Record; body: string; }): Promise { + const root = await fsRoot(params.rootDir); const rendered = withTrailingNewline( renderWikiMarkdown({ frontmatter: params.frontmatter, body: params.body, }), ); - const existing = await fs.readFile(params.absolutePath, "utf8").catch(() => ""); + const existing = await root.readText(params.relativePath).catch(() => ""); if (existing === rendered) { return false; } - await fs.mkdir(path.dirname(params.absolutePath), { recursive: true }); - await fs.writeFile(params.absolutePath, rendered, "utf8"); + await root.write(params.relativePath, rendered); return true; } @@ -183,14 +184,15 @@ async function applyCreateSynthesisMutation(params: { }): Promise<{ changed: boolean; pagePath: string; pageId: string }> { const slug = slugifyWikiSegment(params.mutation.title); const pagePath = path.join("syntheses", `${slug}.md`).replace(/\\/g, "/"); - const absolutePath = path.join(params.config.vault.path, pagePath); - const existing = await fs.readFile(absolutePath, "utf8").catch(() => ""); + const root = await fsRoot(params.config.vault.path); + const existing = await root.readText(pagePath).catch(() => ""); const parsed = parseWikiMarkdown(existing); const pageId = (typeof parsed.frontmatter.id === "string" && parsed.frontmatter.id.trim()) || `synthesis.${slug}`; const changed = await writeWikiPage({ - absolutePath, + rootDir: params.config.vault.path, + relativePath: pagePath, frontmatter: { ...parsed.frontmatter, pageType: "synthesis", @@ -278,7 +280,8 @@ async function applyUpdateMetadataMutation(params: { } const parsed = parseWikiMarkdown(page.raw); const changed = await writeWikiPage({ - absolutePath: page.absolutePath, + rootDir: params.config.vault.path, + relativePath: page.relativePath, frontmatter: buildUpdatedFrontmatter({ original: parsed.frontmatter, mutation: params.mutation, diff --git a/extensions/memory-wiki/src/compile.ts b/extensions/memory-wiki/src/compile.ts index 90e0fddb4e4..4d876d8a10c 100644 --- a/extensions/memory-wiki/src/compile.ts +++ b/extensions/memory-wiki/src/compile.ts @@ -4,6 +4,7 @@ import { replaceManagedMarkdownBlock, withTrailingNewline, } from "openclaw/plugin-sdk/memory-host-markdown"; +import { root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import { assessClaimFreshness, @@ -768,12 +769,13 @@ async function refreshPageRelatedBlocks(params: { if (!params.config.render.createBacklinks) { return []; } + const root = await fsRoot(params.config.vault.path); const updatedFiles: string[] = []; for (const page of params.pages) { if (page.kind === "report") { continue; } - const original = await fs.readFile(page.absolutePath, "utf8"); + const original = await root.readText(page.relativePath); const updated = withTrailingNewline( replaceManagedMarkdownBlock({ original, @@ -790,7 +792,7 @@ async function refreshPageRelatedBlocks(params: { if (updated === original) { continue; } - await fs.writeFile(page.absolutePath, updated, "utf8"); + await root.write(page.relativePath, updated); updatedFiles.push(page.absolutePath); } return updatedFiles; @@ -817,13 +819,15 @@ function renderSectionList(params: { } async function writeManagedMarkdownFile(params: { - filePath: string; + rootDir: string; + relativePath: string; title: string; startMarker: string; endMarker: string; body: string; }): Promise { - const original = await fs.readFile(params.filePath, "utf8").catch(() => `# ${params.title}\n`); + const root = await fsRoot(params.rootDir); + const original = await root.readText(params.relativePath).catch(() => `# ${params.title}\n`); const updated = replaceManagedMarkdownBlock({ original, heading: "## Generated", @@ -835,7 +839,7 @@ async function writeManagedMarkdownFile(params: { if (rendered === original) { return false; } - await fs.writeFile(params.filePath, rendered, "utf8"); + await root.write(params.relativePath, rendered); return true; } @@ -846,8 +850,8 @@ async function writeDashboardPage(params: { pages: WikiPageSummary[]; now: Date; }): Promise { - const filePath = path.join(params.rootDir, params.definition.relativePath); - const original = await fs.readFile(filePath, "utf8").catch(() => + const root = await fsRoot(params.rootDir); + const original = await root.readText(params.definition.relativePath).catch(() => renderWikiMarkdown({ frontmatter: { pageType: "report", @@ -911,7 +915,7 @@ async function writeDashboardPage(params: { body: updatedBody, }), ); - await fs.writeFile(filePath, rendered, "utf8"); + await root.write(params.definition.relativePath, rendered); return true; } @@ -1267,11 +1271,13 @@ async function writeAgentDigestArtifacts(params: { [agentDigestPath, agentDigest], [claimsDigestPath, claimsDigest], ] as const) { - const existing = await fs.readFile(filePath, "utf8").catch(() => ""); + const relativePath = path.relative(params.rootDir, filePath); + const root = await fsRoot(params.rootDir); + const existing = await root.readText(relativePath).catch(() => ""); if (existing === content) { continue; } - await fs.writeFile(filePath, content, "utf8"); + await root.write(relativePath, content); updatedFiles.push(filePath); } return updatedFiles; @@ -1303,7 +1309,8 @@ export async function compileMemoryWikiVault( const rootIndexPath = path.join(rootDir, "index.md"); if ( await writeManagedMarkdownFile({ - filePath: rootIndexPath, + rootDir, + relativePath: "index.md", title: "Wiki Index", startMarker: "", endMarker: "", @@ -1314,10 +1321,12 @@ export async function compileMemoryWikiVault( } for (const group of COMPILE_PAGE_GROUPS) { - const filePath = path.join(rootDir, group.dir, "index.md"); + const relativePath = path.join(group.dir, "index.md").replace(/\\/g, "/"); + const filePath = path.join(rootDir, relativePath); if ( await writeManagedMarkdownFile({ - filePath, + rootDir, + relativePath, title: group.heading, startMarker: ``, endMarker: ``, diff --git a/extensions/memory-wiki/src/ingest.ts b/extensions/memory-wiki/src/ingest.ts index e8e5bce0373..d8f3a19e6d3 100644 --- a/extensions/memory-wiki/src/ingest.ts +++ b/extensions/memory-wiki/src/ingest.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { compileMemoryWikiVault } from "./compile.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { appendMemoryWikiLog } from "./log.js"; @@ -16,13 +17,6 @@ type IngestMemoryWikiSourceResult = { indexUpdatedFiles: string[]; }; -function pathExists(filePath: string): Promise { - return fs - .access(filePath) - .then(() => true) - .catch(() => false); -} - function resolveSourceTitle(sourcePath: string, explicitTitle?: string): string { if (explicitTitle?.trim()) { return explicitTitle.trim(); diff --git a/extensions/memory-wiki/src/log.ts b/extensions/memory-wiki/src/log.ts index c23c5b1c4d2..86e388dc6c1 100644 --- a/extensions/memory-wiki/src/log.ts +++ b/extensions/memory-wiki/src/log.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; type MemoryWikiLogEntry = { type: "init" | "ingest" | "compile" | "lint"; @@ -13,5 +14,9 @@ export async function appendMemoryWikiLog( ): Promise { const logPath = path.join(vaultRoot, ".openclaw-wiki", "log.jsonl"); await fs.mkdir(path.dirname(logPath), { recursive: true }); - await fs.appendFile(logPath, `${JSON.stringify(entry)}\n`, "utf8"); + await appendRegularFile({ + filePath: logPath, + content: `${JSON.stringify(entry)}\n`, + rejectSymlinkParents: true, + }); } diff --git a/extensions/memory-wiki/src/source-page-shared.ts b/extensions/memory-wiki/src/source-page-shared.ts index e49bd06f8f4..be8e0759976 100644 --- a/extensions/memory-wiki/src/source-page-shared.ts +++ b/extensions/memory-wiki/src/source-page-shared.ts @@ -1,7 +1,5 @@ -import { randomUUID } from "node:crypto"; -import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; -import path from "node:path"; +import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; import { setImportedSourceEntry, shouldSkipImportedSourceWrite, @@ -9,123 +7,6 @@ import { } from "./source-sync-state.js"; type ImportedSourceState = Parameters[0]["state"]; -type FileStats = Awaited>; - -function isPathInside(parent: string, child: string): boolean { - const relative = path.relative(parent, child); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - -async function resolveWritableVaultPagePath(params: { - vaultRoot: string; - pagePath: string; -}): Promise<{ - pageAbsPath: string; - pageDir: string; - pageDirRealPath: string; - vaultRealPath: string; - existing: FileStats | null; -}> { - const vaultAbsPath = path.resolve(params.vaultRoot); - const pageAbsPath = path.resolve(vaultAbsPath, params.pagePath); - if (!isPathInside(vaultAbsPath, pageAbsPath)) { - throw new Error(`Refusing to write imported source page outside vault: ${params.pagePath}`); - } - - const vaultRealPath = await fs.realpath(vaultAbsPath); - const pageDir = path.dirname(pageAbsPath); - await fs.mkdir(pageDir, { recursive: true }); - const pageDirRealPath = await fs.realpath(pageDir); - if (!isPathInside(vaultRealPath, pageDirRealPath)) { - throw new Error(`Refusing to write imported source page outside vault: ${params.pagePath}`); - } - - const existing = await fs.lstat(pageAbsPath).catch((err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - return null; - } - throw err; - }); - if (existing?.isSymbolicLink()) { - throw new Error(`Refusing to write imported source page through symlink: ${params.pagePath}`); - } - if (existing && !existing.isFile()) { - throw new Error(`Refusing to write imported source page over non-file: ${params.pagePath}`); - } - return { pageAbsPath, pageDir, pageDirRealPath, vaultRealPath, existing }; -} - -async function assertWritablePageDir(params: { - pageDir: string; - pageDirRealPath: string; - vaultRealPath: string; - pagePath: string; -}): Promise { - const currentPageDirRealPath = await fs.realpath(params.pageDir); - if ( - currentPageDirRealPath !== params.pageDirRealPath || - !isPathInside(params.vaultRealPath, currentPageDirRealPath) - ) { - throw new Error(`Refusing to write imported source page outside vault: ${params.pagePath}`); - } -} - -async function validateDestinationForReplace(filePath: string, pagePath: string): Promise { - const existing = await fs.lstat(filePath).catch((err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - return null; - } - throw err; - }); - if (existing?.isSymbolicLink()) { - throw new Error(`Refusing to write imported source page through symlink: ${pagePath}`); - } - if (existing && !existing.isFile()) { - throw new Error(`Refusing to write imported source page over non-file: ${pagePath}`); - } -} - -async function writeFileAtomicInVault(params: { - filePath: string; - pageDir: string; - pageDirRealPath: string; - vaultRealPath: string; - pagePath: string; - content: string; -}): Promise { - const noFollow = fsConstants.O_NOFOLLOW ?? 0; - await assertWritablePageDir(params); - - const tempPath = path.join(params.pageDir, `.openclaw-wiki-${process.pid}-${randomUUID()}.tmp`); - let shouldRemoveTemp = true; - try { - const handle = await fs.open( - tempPath, - fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | noFollow, - 0o600, - ); - try { - const tempStat = await handle.stat(); - if (!tempStat.isFile() || tempStat.nlink !== 1) { - throw new Error( - `Refusing to write imported source page through unsafe temp file: ${params.pagePath}`, - ); - } - await handle.writeFile(params.content, "utf8"); - } finally { - await handle.close(); - } - await assertWritablePageDir(params); - await validateDestinationForReplace(params.filePath, params.pagePath); - await fs.rename(tempPath, params.filePath); - shouldRemoveTemp = false; - await assertWritablePageDir(params); - } finally { - if (shouldRemoveTemp) { - await fs.rm(tempPath, { force: true }); - } - } -} export async function writeImportedSourcePage(params: { vaultRoot: string; @@ -139,15 +20,15 @@ export async function writeImportedSourcePage(params: { state: ImportedSourceState; buildRendered: (raw: string, updatedAt: string) => string; }): Promise<{ pagePath: string; changed: boolean; created: boolean }> { - const { - pageAbsPath, - pageDir, - pageDirRealPath, - vaultRealPath, - existing: pageStat, - } = await resolveWritableVaultPagePath({ - vaultRoot: params.vaultRoot, - pagePath: params.pagePath, + const vault = await fsRoot(params.vaultRoot); + const pageStat = await vault.stat(params.pagePath).catch((error: unknown) => { + if ( + error instanceof FsSafeError && + (error.code === "not-found" || error.code === "path-alias") + ) { + return null; + } + throw error; }); const created = !pageStat; const updatedAt = new Date(params.sourceUpdatedAtMs).toISOString(); @@ -167,16 +48,22 @@ export async function writeImportedSourcePage(params: { const raw = await fs.readFile(params.sourcePath, "utf8"); const rendered = params.buildRendered(raw, updatedAt); - const existing = pageStat ? await fs.readFile(pageAbsPath, "utf8").catch(() => "") : ""; + const existing = pageStat ? await vault.readText(params.pagePath).catch(() => "") : ""; if (existing !== rendered) { - await writeFileAtomicInVault({ - filePath: pageAbsPath, - pageDir, - pageDirRealPath, - vaultRealPath, - pagePath: params.pagePath, - content: rendered, - }); + try { + if (pageStat && pageStat.nlink > 1) { + await vault.remove(params.pagePath); + } + await vault.write(params.pagePath, rendered); + } catch (error) { + if (error instanceof FsSafeError) { + throw new Error( + `Refusing to write imported source page through symlink: ${params.pagePath}`, + { cause: error }, + ); + } + throw error; + } } setImportedSourceEntry({ diff --git a/extensions/memory-wiki/src/status.ts b/extensions/memory-wiki/src/status.ts index a439900ab10..8454c94dcaa 100644 --- a/extensions/memory-wiki/src/status.ts +++ b/extensions/memory-wiki/src/status.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { listActiveMemoryPublicArtifacts } from "openclaw/plugin-sdk/memory-host-core"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import type { OpenClawConfig } from "../api.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { inferWikiPageKind, toWikiPageSummary, type WikiPageKind } from "./markdown.js"; @@ -65,15 +66,6 @@ type ResolveMemoryWikiStatusDeps = { resolveCommand?: (command: string) => Promise; }; -async function pathExists(inputPath: string): Promise { - try { - await fs.access(inputPath); - return true; - } catch { - return false; - } -} - async function collectVaultCounts(vaultPath: string): Promise<{ pageCounts: Record; sourceCounts: MemoryWikiStatus["sourceCounts"]; diff --git a/extensions/memory-wiki/src/vault.ts b/extensions/memory-wiki/src/vault.ts index 4818ebedcb9..71368fdd5ea 100644 --- a/extensions/memory-wiki/src/vault.ts +++ b/extensions/memory-wiki/src/vault.ts @@ -4,6 +4,7 @@ import { replaceManagedMarkdownBlock, withTrailingNewline, } from "openclaw/plugin-sdk/memory-host-markdown"; +import { FsSafeError, pathExists, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { appendMemoryWikiLog } from "./log.js"; @@ -72,25 +73,22 @@ This vault is maintained by the OpenClaw memory-wiki plugin. `); } -async function pathExists(inputPath: string): Promise { - try { - await fs.access(inputPath); - return true; - } catch { - return false; - } -} - async function writeFileIfMissing( - filePath: string, + rootDir: string, + relativePath: string, content: string, createdFiles: string[], ): Promise { - if (await pathExists(filePath)) { - return; + const root = await fsRoot(rootDir); + try { + await root.create(relativePath, content); + } catch (err) { + if (err instanceof FsSafeError && err.code === "already-exists") { + return; + } + throw err; } - await fs.writeFile(filePath, content, "utf8"); - createdFiles.push(filePath); + createdFiles.push(path.join(rootDir, relativePath)); } export async function initializeMemoryWikiVault( @@ -114,20 +112,18 @@ export async function initializeMemoryWikiVault( await fs.mkdir(fullPath, { recursive: true }); } - await writeFileIfMissing(path.join(rootDir, "AGENTS.md"), buildAgentsMarkdown(), createdFiles); + await writeFileIfMissing(rootDir, "AGENTS.md", buildAgentsMarkdown(), createdFiles); + await writeFileIfMissing(rootDir, "WIKI.md", buildWikiOverviewMarkdown(config), createdFiles); + await writeFileIfMissing(rootDir, "index.md", buildIndexMarkdown(), createdFiles); await writeFileIfMissing( - path.join(rootDir, "WIKI.md"), - buildWikiOverviewMarkdown(config), - createdFiles, - ); - await writeFileIfMissing(path.join(rootDir, "index.md"), buildIndexMarkdown(), createdFiles); - await writeFileIfMissing( - path.join(rootDir, "inbox.md"), + rootDir, + "inbox.md", withTrailingNewline("# Inbox\n\nDrop raw ideas, questions, and source links here.\n"), createdFiles, ); await writeFileIfMissing( - path.join(rootDir, ".openclaw-wiki", "state.json"), + rootDir, + ".openclaw-wiki/state.json", withTrailingNewline( JSON.stringify( { @@ -141,7 +137,7 @@ export async function initializeMemoryWikiVault( ), createdFiles, ); - await writeFileIfMissing(path.join(rootDir, ".openclaw-wiki", "log.jsonl"), "", createdFiles); + await writeFileIfMissing(rootDir, ".openclaw-wiki/log.jsonl", "", createdFiles); if (createdDirectories.length > 0 || createdFiles.length > 0) { await appendMemoryWikiLog(rootDir, { diff --git a/extensions/microsoft/speech-provider.ts b/extensions/microsoft/speech-provider.ts index 3c021648964..78023ec3b26 100644 --- a/extensions/microsoft/speech-provider.ts +++ b/extensions/microsoft/speech-provider.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { readFileSync } from "node:fs"; import path from "node:path"; import { CHROMIUM_FULL_VERSION, @@ -21,7 +21,7 @@ import { fetchWithSsrFGuard, ssrfPolicyFromHttpBaseUrlAllowedHostname, } from "openclaw/plugin-sdk/ssrf-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { edgeTTS, inferEdgeExtension } from "./tts.js"; const DEFAULT_EDGE_VOICE = "en-US-MichelleNeural"; @@ -236,9 +236,11 @@ export function buildMicrosoftSpeechProvider(): SpeechProviderPlugin { isConfigured: ({ providerConfig }) => readMicrosoftProviderConfig(providerConfig).enabled, synthesize: async (req) => { const config = readMicrosoftProviderConfig(req.providerConfig); - const tempRoot = resolvePreferredOpenClawTmpDir(); - mkdirSync(tempRoot, { recursive: true, mode: 0o700 }); - const tempDir = mkdtempSync(path.join(tempRoot, "tts-microsoft-")); + const temp = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "tts-microsoft-", + }); + const tempDir = temp.dir; const overrideVoice = trimToUndefined(req.providerOverrides?.voice); let voice = overrideVoice ?? config.voice; let lang = config.lang; @@ -286,7 +288,7 @@ export function buildMicrosoftSpeechProvider(): SpeechProviderPlugin { return await runEdge(outputFormat); } } finally { - rmSync(tempDir, { recursive: true, force: true }); + await temp.cleanup(); } }, }; diff --git a/extensions/migrate-claude/helpers.ts b/extensions/migrate-claude/helpers.ts index a93339f2617..32d9d7356aa 100644 --- a/extensions/migrate-claude/helpers.ts +++ b/extensions/migrate-claude/helpers.ts @@ -6,6 +6,7 @@ import { MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, } from "openclaw/plugin-sdk/migration"; import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { appendRegularFile, pathExists } from "openclaw/plugin-sdk/security-runtime"; export function resolveHomePath(input: string): string { if (input === "~") { @@ -18,12 +19,7 @@ export function resolveHomePath(input: string): string { } export async function exists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } + return await pathExists(filePath); } export async function isDirectory(dirPath: string): Promise { @@ -92,7 +88,11 @@ export async function appendItem(item: MigrationItem): Promise { : path.basename(item.source); const header = `\n\n\n\n`; await fs.mkdir(path.dirname(item.target), { recursive: true }); - await fs.appendFile(item.target, `${header}${content.trimEnd()}\n`, "utf8"); + await appendRegularFile({ + filePath: item.target, + content: `${header}${content.trimEnd()}\n`, + rejectSymlinkParents: true, + }); return { ...item, status: "migrated" }; } catch (err) { return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); diff --git a/extensions/migrate-hermes/helpers.ts b/extensions/migrate-hermes/helpers.ts index ad11ab6d7c0..401f400d7d8 100644 --- a/extensions/migrate-hermes/helpers.ts +++ b/extensions/migrate-hermes/helpers.ts @@ -6,6 +6,7 @@ import { MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, } from "openclaw/plugin-sdk/migration"; import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { appendRegularFile, pathExists } from "openclaw/plugin-sdk/security-runtime"; import { parse as parseYaml } from "yaml"; export function resolveHomePath(input: string): string { @@ -19,12 +20,7 @@ export function resolveHomePath(input: string): string { } export async function exists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } + return await pathExists(filePath); } export async function isDirectory(dirPath: string): Promise { @@ -126,7 +122,11 @@ export async function appendItem(item: MigrationItem): Promise { const content = await fs.readFile(item.source, "utf8"); const header = `\n\n\n\n`; await fs.mkdir(path.dirname(item.target), { recursive: true }); - await fs.appendFile(item.target, `${header}${content.trimEnd()}\n`, "utf8"); + await appendRegularFile({ + filePath: item.target, + content: `${header}${content.trimEnd()}\n`, + rejectSymlinkParents: true, + }); return { ...item, status: "migrated" }; } catch (err) { return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); diff --git a/extensions/msteams/src/feedback-reflection-store.ts b/extensions/msteams/src/feedback-reflection-store.ts index bc98e49d608..192b47f3c4a 100644 --- a/extensions/msteams/src/feedback-reflection-store.ts +++ b/extensions/msteams/src/feedback-reflection-store.ts @@ -1,5 +1,5 @@ import fs from "node:fs/promises"; -import path from "node:path"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; /** Default cooldown between reflections per session (5 minutes). */ export const DEFAULT_COOLDOWN_MS = 300_000; @@ -93,8 +93,11 @@ export async function storeSessionLearning(params: { learnings = learnings.slice(-10); } - await fs.mkdir(path.dirname(learningsFile), { recursive: true }); - await fs.writeFile(learningsFile, JSON.stringify(learnings, null, 2), "utf-8"); + await replaceFileAtomic({ + filePath: learningsFile, + content: JSON.stringify(learnings, null, 2), + tempPrefix: ".msteams-learnings", + }); if (!exists && legacyLearningsFile !== learningsFile) { await fs.rm(legacyLearningsFile, { force: true }).catch(() => undefined); } diff --git a/extensions/msteams/src/monitor-handler.ts b/extensions/msteams/src/monitor-handler.ts index 963281a5fe1..98874072e3b 100644 --- a/extensions/msteams/src/monitor-handler.ts +++ b/extensions/msteams/src/monitor-handler.ts @@ -1,6 +1,6 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { resolveThreadSessionKeys } from "openclaw/plugin-sdk/routing"; +import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; import { normalizeOptionalLowercaseString } from "openclaw/plugin-sdk/text-runtime"; import { formatUnknownError } from "./errors.js"; import { buildFeedbackEvent, runFeedbackReflection } from "./feedback-reflection.js"; @@ -256,7 +256,11 @@ async function handleFeedbackInvoke( }); const safeKey = route.sessionKey.replace(/[^a-zA-Z0-9_-]/g, "_"); const transcriptFile = path.join(storePath, `${safeKey}.jsonl`); - await fs.appendFile(transcriptFile, JSON.stringify(feedbackEvent) + "\n", "utf-8").catch(() => { + await appendRegularFile({ + filePath: transcriptFile, + content: `${JSON.stringify(feedbackEvent)}\n`, + rejectSymlinkParents: true, + }).catch(() => { // Best effort — transcript dir may not exist yet }); } catch { diff --git a/extensions/msteams/src/store-fs.ts b/extensions/msteams/src/store-fs.ts index d2d6767da59..facbb51c70a 100644 --- a/extensions/msteams/src/store-fs.ts +++ b/extensions/msteams/src/store-fs.ts @@ -1,6 +1,6 @@ -import fs from "node:fs"; import { withFileLock as withPathLock } from "openclaw/plugin-sdk/file-lock"; import { readJsonFileWithFallback, writeJsonFileAtomically } from "openclaw/plugin-sdk/json-store"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; const STORE_LOCK_OPTIONS = { retries: { @@ -25,9 +25,7 @@ export async function writeJsonFile(filePath: string, value: unknown): Promise { const filePath = resolveNostrStatePath(params.accountId, params.env); try { - const raw = await fs.readFile(filePath, "utf-8"); - return safeParseState(raw); - } catch (err) { - const code = (err as { code?: string }).code; - if (code === "ENOENT") { + const raw = await privateFileStore(path.dirname(filePath)).readTextIfExists( + path.basename(filePath), + ); + if (raw === null) { return null; } + return safeParseState(raw); + } catch { return null; } } @@ -133,20 +133,15 @@ export async function writeNostrBusState(params: { env?: NodeJS.ProcessEnv; }): Promise { const filePath = resolveNostrStatePath(params.accountId, params.env); - const dir = path.dirname(filePath); - await fs.mkdir(dir, { recursive: true, mode: 0o700 }); - const tmp = path.join(dir, `${path.basename(filePath)}.${crypto.randomUUID()}.tmp`); const payload: NostrBusState = { version: STORE_VERSION, lastProcessedAt: params.lastProcessedAt, gatewayStartedAt: params.gatewayStartedAt, recentEventIds: (params.recentEventIds ?? []).filter((x): x is string => typeof x === "string"), }; - await fs.writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, { - encoding: "utf-8", + await privateFileStore(path.dirname(filePath)).writeJson(path.basename(filePath), payload, { + trailingNewline: true, }); - await fs.chmod(tmp, 0o600); - await fs.rename(tmp, filePath); } /** @@ -187,13 +182,14 @@ export async function readNostrProfileState(params: { }): Promise { const filePath = resolveNostrProfileStatePath(params.accountId, params.env); try { - const raw = await fs.readFile(filePath, "utf-8"); - return safeParseProfileState(raw); - } catch (err) { - const code = (err as { code?: string }).code; - if (code === "ENOENT") { + const raw = await privateFileStore(path.dirname(filePath)).readTextIfExists( + path.basename(filePath), + ); + if (raw === null) { return null; } + return safeParseProfileState(raw); + } catch { return null; } } @@ -206,18 +202,13 @@ export async function writeNostrProfileState(params: { env?: NodeJS.ProcessEnv; }): Promise { const filePath = resolveNostrProfileStatePath(params.accountId, params.env); - const dir = path.dirname(filePath); - await fs.mkdir(dir, { recursive: true, mode: 0o700 }); - const tmp = path.join(dir, `${path.basename(filePath)}.${crypto.randomUUID()}.tmp`); const payload: NostrProfileState = { version: PROFILE_STATE_VERSION, lastPublishedAt: params.lastPublishedAt, lastPublishedEventId: params.lastPublishedEventId, lastPublishResults: params.lastPublishResults, }; - await fs.writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, { - encoding: "utf-8", + await privateFileStore(path.dirname(filePath)).writeJson(path.basename(filePath), payload, { + trailingNewline: true, }); - await fs.chmod(tmp, 0o600); - await fs.rename(tmp, filePath); } diff --git a/extensions/openshell/src/backend.ts b/extensions/openshell/src/backend.ts index a482a3ac525..fbd4633dd2b 100644 --- a/extensions/openshell/src/backend.ts +++ b/extensions/openshell/src/backend.ts @@ -15,6 +15,7 @@ import { resolvePreferredOpenClawTmpDir, runSshSandboxCommand, sanitizeEnvVars, + withTempWorkspace, } from "openclaw/plugin-sdk/sandbox"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; import type { OpenShellSandboxBackend } from "./backend.types.js"; @@ -411,65 +412,61 @@ class OpenShellSandboxBackendImpl { } private async syncWorkspaceFromRemote(): Promise { - const tmpDir = await fs.mkdtemp( - path.join(resolveOpenShellTmpRoot(), "openclaw-openshell-sync-"), + await withTempWorkspace( + { rootDir: resolveOpenShellTmpRoot(), prefix: "openclaw-openshell-sync-" }, + async ({ dir: tmpDir }) => { + const result = await runOpenShellCli({ + context: this.params.execContext, + args: [ + "sandbox", + "download", + this.params.execContext.sandboxName, + this.params.remoteWorkspaceDir, + tmpDir, + ], + cwd: this.params.createParams.workspaceDir, + }); + if (result.code !== 0) { + throw new Error(result.stderr.trim() || "openshell sandbox download failed"); + } + await replaceDirectoryContents({ + sourceDir: tmpDir, + targetDir: this.params.createParams.workspaceDir, + // Never sync trusted host hook directories or repository metadata from + // the remote sandbox. + excludeDirs: DEFAULT_OPEN_SHELL_MIRROR_EXCLUDE_DIRS, + }); + }, ); - try { - const result = await runOpenShellCli({ - context: this.params.execContext, - args: [ - "sandbox", - "download", - this.params.execContext.sandboxName, - this.params.remoteWorkspaceDir, - tmpDir, - ], - cwd: this.params.createParams.workspaceDir, - }); - if (result.code !== 0) { - throw new Error(result.stderr.trim() || "openshell sandbox download failed"); - } - await replaceDirectoryContents({ - sourceDir: tmpDir, - targetDir: this.params.createParams.workspaceDir, - // Never sync trusted host hook directories or repository metadata from - // the remote sandbox. - excludeDirs: DEFAULT_OPEN_SHELL_MIRROR_EXCLUDE_DIRS, - }); - } finally { - await fs.rm(tmpDir, { recursive: true, force: true }); - } } private async uploadPathToRemote(localPath: string, remotePath: string): Promise { - const tmpDir = await fs.mkdtemp( - path.join(resolveOpenShellTmpRoot(), "openclaw-openshell-upload-"), + await withTempWorkspace( + { rootDir: resolveOpenShellTmpRoot(), prefix: "openclaw-openshell-upload-" }, + async ({ dir: tmpDir }) => { + // Stage a symlink-free snapshot so upload never dereferences host paths + // outside the mirrored workspace tree. + await stageDirectoryContents({ + sourceDir: localPath, + targetDir: tmpDir, + }); + const result = await runOpenShellCli({ + context: this.params.execContext, + args: [ + "sandbox", + "upload", + "--no-git-ignore", + this.params.execContext.sandboxName, + tmpDir, + remotePath, + ], + cwd: this.params.createParams.workspaceDir, + }); + if (result.code !== 0) { + throw new Error(result.stderr.trim() || "openshell sandbox upload failed"); + } + }, ); - try { - // Stage a symlink-free snapshot so upload never dereferences host paths - // outside the mirrored workspace tree. - await stageDirectoryContents({ - sourceDir: localPath, - targetDir: tmpDir, - }); - const result = await runOpenShellCli({ - context: this.params.execContext, - args: [ - "sandbox", - "upload", - "--no-git-ignore", - this.params.execContext.sandboxName, - tmpDir, - remotePath, - ], - cwd: this.params.createParams.workspaceDir, - }); - if (result.code !== 0) { - throw new Error(result.stderr.trim() || "openshell sandbox upload failed"); - } - } finally { - await fs.rm(tmpDir, { recursive: true, force: true }); - } } private async maybeSeedRemoteWorkspace(): Promise { diff --git a/extensions/openshell/src/fs-bridge.ts b/extensions/openshell/src/fs-bridge.ts index d858b8c69b0..71333ae46b4 100644 --- a/extensions/openshell/src/fs-bridge.ts +++ b/extensions/openshell/src/fs-bridge.ts @@ -1,14 +1,13 @@ -import fs from "node:fs"; import fsPromises from "node:fs/promises"; -import type { FileHandle } from "node:fs/promises"; import path from "node:path"; -import { writeFileWithinRoot } from "openclaw/plugin-sdk/file-access-runtime"; +import { root as fsRoot } from "openclaw/plugin-sdk/file-access-runtime"; import type { SandboxFsBridge, SandboxFsStat, SandboxResolvedPath, } from "openclaw/plugin-sdk/sandbox"; import { createWritableRenameTargetResolver } from "openclaw/plugin-sdk/sandbox"; +import { isPathInside } from "openclaw/plugin-sdk/security-runtime"; import type { OpenShellFsBridgeContext, OpenShellSandboxBackend } from "./backend.types.js"; import { movePathWithCopyFallback } from "./mirror.js"; @@ -52,15 +51,28 @@ class OpenShellFsBridge implements SandboxFsBridge { }): Promise { const target = this.resolveTarget(params); const hostPath = this.requireHostPath(target); - const handle = await openPinnedReadableFile({ - absolutePath: hostPath, - rootPath: target.mountHostRoot, - containerPath: target.containerPath, - }); + let opened: Awaited>["open"]>>; try { - return (await handle.readFile()) as Buffer; - } finally { - await handle.close(); + await assertLocalPathSafety({ + target, + root: target.mountHostRoot, + allowMissingLeaf: false, + allowFinalSymlinkForUnlink: false, + }); + const root = await fsRoot(target.mountHostRoot); + opened = await root.open(path.relative(target.mountHostRoot, hostPath), { + hardlinks: "reject", + }); + try { + return (await opened.handle.readFile()) as Buffer; + } finally { + await opened.handle.close(); + } + } catch (err) { + throw new Error( + `Sandbox boundary checks failed; cannot read files: ${target.containerPath}`, + { cause: err }, + ); } } @@ -84,10 +96,8 @@ class OpenShellFsBridge implements SandboxFsBridge { const buffer = Buffer.isBuffer(params.data) ? params.data : Buffer.from(params.data, params.encoding ?? "utf8"); - await writeFileWithinRoot({ - rootDir: target.mountHostRoot, - relativePath: path.relative(target.mountHostRoot, hostPath), - data: buffer, + const root = await fsRoot(target.mountHostRoot); + await root.write(path.relative(target.mountHostRoot, hostPath), buffer, { mkdir: params.mkdir, }); await this.backend.syncLocalPathToRemote(hostPath, target.containerPath); @@ -291,11 +301,6 @@ class OpenShellFsBridge implements SandboxFsBridge { } } -function isPathInside(root: string, target: string): boolean { - const relative = path.relative(root, target); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - async function assertLocalPathSafety(params: { target: ResolvedMountPath; root: string; @@ -358,199 +363,8 @@ async function resolveCanonicalCandidate(targetPath: string): Promise { } } -async function openPinnedReadableFile(params: { - absolutePath: string; - rootPath: string; - containerPath: string; -}): Promise { - // The literal root is what `resolveTarget` joins caller-provided relative - // paths against, so pre-open containment must be checked in literal form. - // The canonical root is derived separately and used for the post-open - // path checks (fd-path readlink and realpath cross-check), so a workspace - // that is itself configured as a symlink still works. - const literalRoot = path.resolve(params.rootPath); - const canonicalRoot = await fsPromises.realpath(literalRoot).catch(() => literalRoot); - const literalPath = path.resolve(params.absolutePath); - // Cheap string-prefix check on the caller-provided absolute path; no - // filesystem state is read here, so there is no TOCTOU window. Deeper - // checks run after the fd is pinned. - if (!isPathInside(literalRoot, literalPath)) { - throw new Error(`Sandbox path escapes allowed mounts; cannot access: ${params.containerPath}`); - } - const { flags: openReadFlags, supportsNoFollow } = resolveOpenReadFlags(); - // Open first so every later check runs against an fd that is already pinned - // to one specific inode. `O_NOFOLLOW` prevents the final path component from - // being a symlink; the ancestor walk below handles parent-directory symlink - // swaps on platforms where fd-path readlink is not available. - const handle = await fsPromises.open(literalPath, openReadFlags); - try { - const openedStat = await handle.stat(); - if (!openedStat.isFile()) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${params.containerPath}`); - } - if (openedStat.nlink > 1) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${params.containerPath}`); - } - const resolvedPath = await resolveOpenedReadablePath(handle.fd); - if (resolvedPath !== null) { - // Primary guarantee on Linux: the fd's resolved path is derived from the - // kernel, so a parent-directory swap cannot make this return a stale path. - if (!isPathInside(canonicalRoot, resolvedPath)) { - throw new Error( - `Sandbox boundary checks failed; cannot read files: ${params.containerPath}`, - ); - } - return handle; - } - // Fallback for platforms where fd-path readlink is unavailable. On macOS, - // `/dev/fd/N` is a character device so readlink returns EINVAL; on Windows - // there is no `/proc` equivalent. With no kernel-backed path readback we - // must prove the pinned fd is in-root without trusting a separate - // `realpath` + `lstat` pair that would race between the two awaits. Walk - // every ancestor between `literalRoot` and `literalPath` — the actual - // on-disk chain — and reject if any ancestor is a symlink, then use a - // single `stat` call to confirm that the path still resolves to the - // same file the fd has pinned. `fs.promises.stat` resolves the path and - // returns the final file's identity in one syscall, so there is no - // between-await window for an attacker to race. - await assertAncestorChainHasNoSymlinks(literalRoot, literalPath, params.containerPath, { - // On platforms where `O_NOFOLLOW` is unavailable (Windows), the open - // call would have transparently followed a final-component symlink, so - // the ancestor walk has to lstat the leaf as well. - includeLeaf: !supportsNoFollow, - }); - const currentResolvedStat = await fsPromises.stat(literalPath); - if (!sameFileIdentity(currentResolvedStat, openedStat)) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${params.containerPath}`); - } - // Belt-and-suspenders: re-fstat the pinned fd after the identity check and - // confirm the file type and link count are still trustworthy. A hardlink - // that appeared between the initial fstat and here is not exploitable for - // the read (the fd is already pinned to the original inode), but failing - // closed here keeps the guarantee simple: the bytes we return always come - // from a file that was a single-linked regular file at verification time. - const postCheckStat = await handle.stat(); - if (!postCheckStat.isFile() || postCheckStat.nlink > 1) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${params.containerPath}`); - } - return handle; - } catch (error) { - await handle.close(); - throw error; - } -} - -// Walks each directory between canonicalRoot (exclusive) and -// targetAbsolutePath, `lstat`'ing each segment. Rejects if any intermediate -// segment is a symlink or a non-directory. By default the final component is -// not walked because `O_NOFOLLOW` already protects it on the open call. Pass -// `includeLeaf: true` on platforms where `O_NOFOLLOW` is unavailable -// (Windows) so a symlinked leaf cannot be followed silently by `open`. -async function assertAncestorChainHasNoSymlinks( - canonicalRoot: string, - targetAbsolutePath: string, - containerPath: string, - options: { includeLeaf?: boolean } = {}, -): Promise { - const relative = path.relative(canonicalRoot, targetAbsolutePath); - if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { - return; - } - const segments = relative.split(path.sep).filter((segment) => segment.length > 0); - const lastIndex = options.includeLeaf ? segments.length : segments.length - 1; - let cursor = canonicalRoot; - for (let i = 0; i < lastIndex; i += 1) { - cursor = path.join(cursor, segments[i]); - const stat = await fsPromises.lstat(cursor).catch(() => null); - if (!stat) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${containerPath}`); - } - const isLeaf = i === segments.length - 1; - if (stat.isSymbolicLink()) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${containerPath}`); - } - if (!isLeaf && !stat.isDirectory()) { - throw new Error(`Sandbox boundary checks failed; cannot read files: ${containerPath}`); - } - } -} - -type ReadOpenFlagsResolution = { flags: number; supportsNoFollow: boolean }; - -let readOpenFlagsResolverForTest: (() => ReadOpenFlagsResolution) | undefined; - -function resolveOpenReadFlags(): ReadOpenFlagsResolution { - if (readOpenFlagsResolverForTest) { - return readOpenFlagsResolverForTest(); - } - const closeOnExec = (fs.constants as Record).O_CLOEXEC ?? 0; - const supportsNoFollow = typeof fs.constants.O_NOFOLLOW === "number"; - const noFollow = supportsNoFollow ? fs.constants.O_NOFOLLOW : 0; - return { - flags: fs.constants.O_RDONLY | noFollow | closeOnExec, - supportsNoFollow, - }; -} - -/** - * Test-only seam for forcing the open-flag/`O_NOFOLLOW` resolution. Used to - * exercise the Windows-style fallback (no `O_NOFOLLOW`, ancestor walk - * includes the leaf) on platforms where `fs.constants.O_NOFOLLOW` is a - * non-configurable native data property and cannot be patched directly. - * - * @internal - */ export function setReadOpenFlagsResolverForTest( - resolver: (() => ReadOpenFlagsResolution) | undefined, + _resolver: (() => { flags: number; supportsNoFollow: boolean }) | undefined, ): void { - readOpenFlagsResolverForTest = resolver; -} - -// Resolves the absolute path associated with an open fd via the kernel-backed -// `/proc/self/fd/` (Linux) or `/dev/fd/` (some BSDs). Returns null -// when no fd-path endpoint is available. Note: on macOS `/dev/fd/N` is a -// character device rather than a symlink, so `readlink` fails with EINVAL -// there and the caller must use the ancestor-walk fallback instead. -async function resolveOpenedReadablePath(fd: number): Promise { - for (const fdPath of [`/proc/self/fd/${fd}`, `/dev/fd/${fd}`]) { - try { - const openedPath = await fsPromises.readlink(fdPath); - return normalizeOpenedReadablePath(openedPath); - } catch { - continue; - } - } - return null; -} - -function normalizeOpenedReadablePath(openedPath: string): string { - const deletedSuffix = " (deleted)"; - const withoutDeletedSuffix = openedPath.endsWith(deletedSuffix) - ? openedPath.slice(0, -deletedSuffix.length) - : openedPath; - return path.resolve(withoutDeletedSuffix); -} - -// File identity comparison with win32-aware `dev=0` handling, matching the -// shared `src/infra/file-identity.ts` contract. Kept local because extension -// production code is not allowed to reach into core `src/**` by relative -// import, and this helper is not yet part of the `openclaw/plugin-sdk/*` -// public surface. Stats here come from `FileHandle.stat()` / `fs.promises.stat()` -// with no `{ bigint: true }` option, so all fields are numbers. -function sameFileIdentity( - left: { dev: number; ino: number }, - right: { dev: number; ino: number }, - platform: NodeJS.Platform = process.platform, -): boolean { - if (left.ino !== right.ino) { - return false; - } - if (left.dev === right.dev) { - return true; - } - // On Windows, path-based stat can report `dev=0` while fd-based stat reports - // a real volume serial. Treat either side `dev=0` as "unknown device" - // rather than a mismatch so legitimate Windows fallback reads are not - // rejected. - return platform === "win32" && (left.dev === 0 || right.dev === 0); + // Retained for older OpenShell tests; pinned reads now delegate to fs-safe. } diff --git a/extensions/openshell/src/mirror.ts b/extensions/openshell/src/mirror.ts index 1884668f9fd..106db11566a 100644 --- a/extensions/openshell/src/mirror.ts +++ b/extensions/openshell/src/mirror.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { movePathWithCopyFallback } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime"; export const DEFAULT_OPEN_SHELL_MIRROR_EXCLUDE_DIRS = ["hooks", "git-hooks", ".git"] as const; @@ -137,23 +138,4 @@ export async function stageDirectoryContents(params: { } } -export async function movePathWithCopyFallback(params: { - from: string; - to: string; -}): Promise { - try { - await fs.rename(params.from, params.to); - return; - } catch (error) { - const code = (error as NodeJS.ErrnoException | null)?.code; - if (code !== "EXDEV") { - throw error; - } - } - await fs.cp(params.from, params.to, { - recursive: true, - force: true, - dereference: false, - }); - await fs.rm(params.from, { recursive: true, force: true }); -} +export { movePathWithCopyFallback }; diff --git a/extensions/openshell/src/openshell-core.test.ts b/extensions/openshell/src/openshell-core.test.ts index 3d3539ebb5d..342d39b3245 100644 --- a/extensions/openshell/src/openshell-core.test.ts +++ b/extensions/openshell/src/openshell-core.test.ts @@ -1,4 +1,3 @@ -import nodeFs from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -201,22 +200,6 @@ afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); -function cloneStatWithDev( - stat: T, - dev: number | bigint, -): T { - return Object.defineProperty( - Object.create(Object.getPrototypeOf(stat), Object.getOwnPropertyDescriptors(stat)), - "dev", - { - value: dev, - configurable: true, - enumerable: true, - writable: true, - }, - ) as T; -} - function createMirrorBackendMock(): OpenShellSandboxBackend { return { id: "openshell", @@ -324,12 +307,11 @@ describe("openshell fs bridges", () => { expect(backend.syncLocalPathToRemote).not.toHaveBeenCalled(); }); - it("rejects a parent symlink swap that lands outside the sandbox root", async () => { + it("rejects a parent symlink that lands outside the sandbox root", async () => { const workspaceDir = await makeTempDir("openclaw-openshell-fs-"); const outsideDir = await makeTempDir("openclaw-openshell-outside-"); - await fs.mkdir(path.join(workspaceDir, "subdir"), { recursive: true }); - await fs.writeFile(path.join(workspaceDir, "subdir", "secret.txt"), "inside", "utf8"); await fs.writeFile(path.join(outsideDir, "secret.txt"), "outside", "utf8"); + await fs.symlink(outsideDir, path.join(workspaceDir, "subdir")); const backend = createMirrorBackendMock(); const sandbox = createSandboxTestContext({ overrides: { @@ -342,30 +324,13 @@ describe("openshell fs bridges", () => { const { createOpenShellFsBridge } = await import("./fs-bridge.js"); const bridge = createOpenShellFsBridge({ sandbox, backend }); - const originalOpen = fs.open.bind(fs); - const targetPath = path.join(workspaceDir, "subdir", "secret.txt"); - let swapped = false; - const openSpy = vi.spyOn(fs, "open").mockImplementation((async (...args: unknown[]) => { - const filePath = args[0]; - if (!swapped && filePath === targetPath) { - swapped = true; - nodeFs.rmSync(path.join(workspaceDir, "subdir"), { recursive: true, force: true }); - nodeFs.symlinkSync(outsideDir, path.join(workspaceDir, "subdir")); - } - return await (originalOpen as (...delegated: unknown[]) => Promise)(...args); - }) as unknown as typeof fs.open); - try { - await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( - "Sandbox boundary checks failed", - ); - expect(openSpy).toHaveBeenCalled(); - } finally { - openSpy.mockRestore(); - } + await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( + "Sandbox boundary checks failed", + ); }); - it("falls back to inode checks when fd path resolution is unavailable", async () => { + it("reads regular files through the shared safe fs root", async () => { const workspaceDir = await makeTempDir("openclaw-openshell-fs-"); await fs.mkdir(path.join(workspaceDir, "subdir"), { recursive: true }); await fs.writeFile(path.join(workspaceDir, "subdir", "secret.txt"), "inside", "utf8"); @@ -382,127 +347,17 @@ describe("openshell fs bridges", () => { const { createOpenShellFsBridge } = await import("./fs-bridge.js"); const bridge = createOpenShellFsBridge({ sandbox, backend }); - const readlinkSpy = vi - .spyOn(fs, "readlink") - .mockRejectedValue(new Error("fd path unavailable")); - try { - await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).resolves.toEqual( - Buffer.from("inside"), - ); - expect(readlinkSpy).toHaveBeenCalled(); - } finally { - readlinkSpy.mockRestore(); - } + await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).resolves.toEqual( + Buffer.from("inside"), + ); }); - // The shared `sameFileIdentity` contract intentionally treats either-side - // `dev=0` as "unknown device" on win32 (path-based stat can legitimately - // report `dev=0` there) and only fails closed on other platforms. Skip the - // Linux/macOS rejection expectation on Windows runners. - it.skipIf(process.platform === "win32")( - "rejects fallback reads when path stats report an unknown device id", - async () => { - const workspaceDir = await makeTempDir("openclaw-openshell-fs-"); - const targetPath = path.join(workspaceDir, "subdir", "secret.txt"); - await fs.mkdir(path.join(workspaceDir, "subdir"), { recursive: true }); - await fs.writeFile(targetPath, "inside", "utf8"); - - const backend = createMirrorBackendMock(); - const sandbox = createSandboxTestContext({ - overrides: { - backendId: "openshell", - workspaceDir, - agentWorkspaceDir: workspaceDir, - containerWorkdir: "/sandbox", - }, - }); - - const { createOpenShellFsBridge } = await import("./fs-bridge.js"); - const bridge = createOpenShellFsBridge({ sandbox, backend }); - const readlinkSpy = vi - .spyOn(fs, "readlink") - .mockRejectedValue(new Error("fd path unavailable")); - const originalStat = fs.stat.bind(fs); - const statSpy = vi.spyOn(fs, "stat").mockImplementation(async (...args) => { - const stat = await originalStat(...args); - if (args[0] === targetPath) { - return cloneStatWithDev(stat, 0); - } - return stat; - }); - - try { - await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( - "Sandbox boundary checks failed", - ); - expect(readlinkSpy).toHaveBeenCalled(); - expect(statSpy).toHaveBeenCalledWith(targetPath); - } finally { - statSpy.mockRestore(); - readlinkSpy.mockRestore(); - } - }, - ); - - it("rejects fallback reads when an ancestor directory is swapped to a symlink", async () => { - const workspaceDir = await makeTempDir("openclaw-openshell-fs-"); - const outsideDir = await makeTempDir("openclaw-openshell-outside-"); - await fs.mkdir(path.join(workspaceDir, "subdir"), { recursive: true }); - await fs.writeFile(path.join(workspaceDir, "subdir", "secret.txt"), "inside", "utf8"); - await fs.writeFile(path.join(outsideDir, "secret.txt"), "outside", "utf8"); - - const backend = createMirrorBackendMock(); - const sandbox = createSandboxTestContext({ - overrides: { - backendId: "openshell", - workspaceDir, - agentWorkspaceDir: workspaceDir, - containerWorkdir: "/sandbox", - }, - }); - - const { createOpenShellFsBridge } = await import("./fs-bridge.js"); - const bridge = createOpenShellFsBridge({ sandbox, backend }); - const originalOpen = fs.open.bind(fs); - const targetPath = path.join(workspaceDir, "subdir", "secret.txt"); - let swapped = false; - const openSpy = vi.spyOn(fs, "open").mockImplementation((async (...args: unknown[]) => { - const filePath = args[0]; - if (!swapped && filePath === targetPath) { - swapped = true; - nodeFs.rmSync(path.join(workspaceDir, "subdir"), { recursive: true, force: true }); - nodeFs.symlinkSync(outsideDir, path.join(workspaceDir, "subdir")); - } - return await (originalOpen as (...delegated: unknown[]) => Promise)(...args); - }) as unknown as typeof fs.open); - // Force the fallback verification path even on Linux so the ancestor-walk - // guard is exercised directly. - const readlinkSpy = vi - .spyOn(fs, "readlink") - .mockRejectedValue(new Error("fd path unavailable")); - - try { - await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( - "Sandbox boundary checks failed", - ); - expect(openSpy).toHaveBeenCalled(); - expect(readlinkSpy).toHaveBeenCalled(); - } finally { - readlinkSpy.mockRestore(); - openSpy.mockRestore(); - } - }); - - it("rejects fallback reads of a symlinked leaf when O_NOFOLLOW is unavailable", async () => { + it("rejects reads of a symlinked leaf", async () => { const workspaceDir = await makeTempDir("openclaw-openshell-fs-"); const outsideDir = await makeTempDir("openclaw-openshell-outside-"); await fs.mkdir(path.join(workspaceDir, "subdir"), { recursive: true }); await fs.writeFile(path.join(outsideDir, "secret.txt"), "outside", "utf8"); - // The workspace contains a symlink as the FINAL path component pointing - // out-of-root. On Windows `O_NOFOLLOW` is `undefined`, so `open` would - // silently traverse the symlink to the outside file; the ancestor walk - // must lstat the leaf in that case to fail closed. await fs.symlink( path.join(outsideDir, "secret.txt"), path.join(workspaceDir, "subdir", "secret.txt"), @@ -518,30 +373,12 @@ describe("openshell fs bridges", () => { }, }); - const { createOpenShellFsBridge, setReadOpenFlagsResolverForTest } = - await import("./fs-bridge.js"); + const { createOpenShellFsBridge } = await import("./fs-bridge.js"); const bridge = createOpenShellFsBridge({ sandbox, backend }); - // Force the fallback path so the leaf-lstat guard is exercised. - const readlinkSpy = vi - .spyOn(fs, "readlink") - .mockRejectedValue(new Error("fd path unavailable")); - // Simulate a host that lacks `O_NOFOLLOW` (e.g. Windows) without touching - // the non-configurable native `fs.constants` data property. The bridge - // exposes a test-only seam for exactly this case. - setReadOpenFlagsResolverForTest(() => ({ - flags: nodeFs.constants.O_RDONLY, - supportsNoFollow: false, - })); - try { - await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( - "Sandbox boundary checks failed", - ); - expect(readlinkSpy).toHaveBeenCalled(); - } finally { - setReadOpenFlagsResolverForTest(undefined); - readlinkSpy.mockRestore(); - } + await expect(bridge.readFile({ filePath: "subdir/secret.txt" })).rejects.toThrow( + "Sandbox boundary checks failed", + ); }); it("rejects hardlinked files inside the sandbox root", async () => { diff --git a/extensions/phone-control/index.ts b/extensions/phone-control/index.ts index e94b1cd0883..1b69ec1438b 100644 --- a/extensions/phone-control/index.ts +++ b/extensions/phone-control/index.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, @@ -150,7 +151,6 @@ async function readArmState(statePath: string): Promise { } async function writeArmState(statePath: string, state: ArmStateFile | null): Promise { - await fs.mkdir(path.dirname(statePath), { recursive: true }); if (!state) { try { await fs.unlink(statePath); @@ -159,7 +159,11 @@ async function writeArmState(statePath: string, state: ArmStateFile | null): Pro } return; } - await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + await replaceFileAtomic({ + filePath: statePath, + content: `${JSON.stringify(state, null, 2)}\n`, + tempPrefix: ".phone-control-arm", + }); } function normalizeDenyList(cfg: OpenClawPluginApi["config"]): string[] { diff --git a/extensions/qa-lab/src/cli-paths.ts b/extensions/qa-lab/src/cli-paths.ts index 392ae9f75b4..bacf0ebf541 100644 --- a/extensions/qa-lab/src/cli-paths.ts +++ b/extensions/qa-lab/src/cli-paths.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { assertNoSymlinkParents, pathScope } from "openclaw/plugin-sdk/security-runtime"; export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: string) { if (!outputDir) { @@ -8,12 +9,11 @@ export function resolveRepoRelativeOutputDir(repoRoot: string, outputDir?: strin if (path.isAbsolute(outputDir)) { throw new Error("--output-dir must be a relative path inside the repo root."); } - const resolved = path.resolve(repoRoot, outputDir); - const relative = path.relative(repoRoot, resolved); - if (relative.startsWith("..") || path.isAbsolute(relative)) { + const resolved = pathScope(repoRoot, { label: "repo root" }).resolve(outputDir); + if (!resolved.ok) { throw new Error("--output-dir must stay within the repo root."); } - return resolved; + return resolved.path; } async function resolveNearestExistingPath(targetPath: string) { @@ -44,22 +44,18 @@ function assertRepoRelativePath(repoRoot: string, targetPath: string, label: str } async function assertNoSymlinkSegments(repoRoot: string, targetPath: string, label: string) { - const relative = assertRepoRelativePath(repoRoot, targetPath, label); - let current = repoRoot; - for (const segment of relative.split(path.sep).filter((entry) => entry.length > 0)) { - current = path.join(current, segment); - let stats: Awaited> | null = null; - try { - stats = await fs.lstat(current); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - break; - } - throw error; - } - if (stats.isSymbolicLink()) { - throw new Error(`${label} must not traverse symlinks.`); + assertRepoRelativePath(repoRoot, targetPath, label); + try { + await assertNoSymlinkParents({ + rootDir: repoRoot, + targetPath, + messagePrefix: label, + }); + } catch (error) { + if (error instanceof Error && error.message.includes("symlink")) { + throw new Error(`${label} must not traverse symlinks.`, { cause: error }); } + throw error; } } @@ -81,40 +77,10 @@ export async function ensureRepoBoundDirectory( label: string, opts?: { mode?: number }, ) { - const repoRootResolved = path.resolve(repoRoot); - const targetResolved = path.resolve(targetDir); - const relative = assertRepoRelativePath(repoRootResolved, targetResolved, label); - const repoRootReal = await fs.realpath(repoRootResolved); - let current = repoRootResolved; - for (const segment of relative.split(path.sep).filter((entry) => entry.length > 0)) { - current = path.join(current, segment); - while (true) { - try { - const stats = await fs.lstat(current); - if (stats.isSymbolicLink()) { - throw new Error(`${label} must not traverse symlinks.`); - } - if (!stats.isDirectory()) { - throw new Error(`${label} must point to a directory.`); - } - break; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "ENOENT") { - throw error; - } - try { - await fs.mkdir(current, { recursive: false, mode: opts?.mode }); - } catch (mkdirError) { - if ((mkdirError as NodeJS.ErrnoException).code === "EEXIST") { - continue; - } - throw mkdirError; - } - } - } + await assertNoSymlinkSegments(path.resolve(repoRoot), path.resolve(targetDir), label); + const result = await pathScope(repoRoot, { label }).ensureDir(targetDir, { mode: opts?.mode }); + if (!result.ok) { + throw new Error(`${label} must stay within the repo root.`); } - const targetReal = await fs.realpath(targetResolved); - assertRepoRelativePath(repoRootReal, targetReal, label); - return targetResolved; + return result.path; } diff --git a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts index fb01a37648e..4b2054a396b 100644 --- a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js"; export type MantisDesktopBrowserSmokeOptions = { @@ -146,15 +147,6 @@ async function defaultCommandRunner( }); } -async function pathExists(filePath: string) { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function resolveCrabboxBin(params: { env: NodeJS.ProcessEnv; explicit?: string; diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index fc10f5bdbfa..5d523b9983f 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -2,6 +2,7 @@ import { spawn, type SpawnOptions } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js"; import { acquireQaCredentialLease, @@ -255,15 +256,6 @@ async function defaultCommandRunner( }); } -async function pathExists(filePath: string) { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function readRemoteMetadata( outputDir: string, ): Promise { @@ -289,7 +281,6 @@ async function readRemoteMetadata( return undefined; } } - async function resolveCrabboxBin(params: { env: NodeJS.ProcessEnv; explicit?: string; diff --git a/extensions/qa-lab/src/mantis/visual-task.runtime.ts b/extensions/qa-lab/src/mantis/visual-task.runtime.ts index 68464cc8c0f..57d0e272bcf 100644 --- a/extensions/qa-lab/src/mantis/visual-task.runtime.ts +++ b/extensions/qa-lab/src/mantis/visual-task.runtime.ts @@ -2,6 +2,7 @@ import { spawn, type SpawnOptions } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { pathExists } from "openclaw/plugin-sdk/security-runtime"; import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js"; export type MantisVisualTaskVisionMode = "image-describe" | "metadata"; @@ -211,15 +212,6 @@ async function defaultCommandRunner( }); } -async function pathExists(filePath: string) { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function nonEmptyFileExists(filePath: string) { try { const stat = await fs.stat(filePath); diff --git a/extensions/qa-lab/src/multipass.runtime.ts b/extensions/qa-lab/src/multipass.runtime.ts index ec4f5ba0e98..a7a94815024 100644 --- a/extensions/qa-lab/src/multipass.runtime.ts +++ b/extensions/qa-lab/src/multipass.runtime.ts @@ -1,8 +1,9 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; -import { access, appendFile, mkdir, writeFile } from "node:fs/promises"; +import { access, mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; +import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import type { QaProviderMode } from "./model-selection.js"; import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js"; @@ -432,7 +433,7 @@ export function renderQaMultipassGuestScript( } async function appendMultipassLog(logPath: string, message: string) { - await appendFile(logPath, message, "utf8"); + await appendRegularFile({ filePath: logPath, content: message }); } async function runMultipassCommand(logPath: string, args: string[], options: ExecFileOptions = {}) { diff --git a/extensions/qa-lab/src/temp-dir.test-helper.ts b/extensions/qa-lab/src/temp-dir.test-helper.ts index 611268ea1aa..983613f2b36 100644 --- a/extensions/qa-lab/src/temp-dir.test-helper.ts +++ b/extensions/qa-lab/src/temp-dir.test-helper.ts @@ -1,20 +1,23 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { + tempWorkspace, + resolvePreferredOpenClawTmpDir, + type TempWorkspace, +} from "openclaw/plugin-sdk/temp-path"; export function createTempDirHarness() { - const tempDirs: string[] = []; + const tempDirs: TempWorkspace[] = []; return { async cleanup() { - await Promise.all( - tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })), - ); + await Promise.all(tempDirs.splice(0).map((dir) => dir.cleanup())); }, async makeTempDir(prefix: string) { - const dir = await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), prefix)); + const dir = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix, + }); tempDirs.push(dir); - return dir; + return dir.dir; }, }; } diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-config.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-config.ts index 14346325739..660e888f515 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-config.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-config.ts @@ -1,5 +1,5 @@ -import { randomUUID } from "node:crypto"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; export function isMatrixQaPlainRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); @@ -19,9 +19,12 @@ async function readMatrixQaGatewayConfigFile(configPath: string) { } async function writeMatrixQaGatewayConfigFile(configPath: string, config: unknown) { - const tempPath = `${configPath}.${randomUUID()}.tmp`; - await writeFile(tempPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); - await rename(tempPath, configPath); + await replaceFileAtomic({ + filePath: configPath, + content: `${JSON.stringify(config, null, 2)}\n`, + mode: 0o600, + tempPrefix: ".matrix-qa-config", + }); } export async function readMatrixQaGatewayMatrixAccount(params: { diff --git a/extensions/qqbot/src/engine/api/media-chunked.test.ts b/extensions/qqbot/src/engine/api/media-chunked.test.ts index beb1acbcdbd..1b2f764af88 100644 --- a/extensions/qqbot/src/engine/api/media-chunked.test.ts +++ b/extensions/qqbot/src/engine/api/media-chunked.test.ts @@ -3,6 +3,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { normalizeSource } from "../messaging/media-source.js"; import { ApiError, MediaFileType, @@ -333,4 +334,51 @@ describe("media-chunked: ChunkedMediaApi.uploadChunked", () => { await fs.promises.rm(tmp, { recursive: true, force: true }); } }); + + it("uses the verified localPath handle if the path is replaced before chunked upload", async () => { + const tmp = await fs.promises.mkdtemp(path.join(os.tmpdir(), "chunked-verified-")); + const filePath = path.join(tmp, "fixture.bin"); + await fs.promises.writeFile(filePath, FIXTURE_BUFFER); + const source = await normalizeSource({ localPath: filePath }, { maxSize: 1_000_000 }); + await fs.promises.rm(filePath); + await fs.promises.writeFile(filePath, Buffer.from("replacement bytes")); + try { + const client = mockApiClient(); + const tm = mockTokenManager(); + stubFetchOk(); + + client.request.mockImplementation(async (_t, _m, p) => { + if (p.endsWith("/upload_prepare")) { + return makePrepareResponse("uid-verified", 3); + } + if (p.endsWith("/upload_part_finish")) { + return {}; + } + if (p.endsWith("/files")) { + return { file_uuid: "u", file_info: "fi", ttl: 10 } satisfies UploadMediaResponse; + } + throw new Error(`unexpected ${p}`); + }); + + const api = new ChunkedMediaApi(client, tm); + await api.uploadChunked({ + scope: "c2c", + targetId: "u1", + fileType: MediaFileType.VIDEO, + source, + creds: { appId: "a", clientSecret: "s" }, + }); + + const prepareCall = client.request.mock.calls.find((c) => + String(c[2]).endsWith("/upload_prepare"), + )!; + const prepareBody = prepareCall[3] as { md5: string }; + expect(prepareBody.md5).toBe(crypto.createHash("md5").update(FIXTURE_BUFFER).digest("hex")); + } finally { + if (source.kind === "localPath") { + await source.opened?.close().catch(() => undefined); + } + await fs.promises.rm(tmp, { recursive: true, force: true }); + } + }); }); diff --git a/extensions/qqbot/src/engine/api/media-chunked.ts b/extensions/qqbot/src/engine/api/media-chunked.ts index ee5aeebc484..f645761aee9 100644 --- a/extensions/qqbot/src/engine/api/media-chunked.ts +++ b/extensions/qqbot/src/engine/api/media-chunked.ts @@ -35,8 +35,9 @@ */ import * as crypto from "node:crypto"; -import * as fs from "node:fs"; -import type { MediaSource } from "../messaging/media-source.js"; +import type { FileHandle } from "node:fs/promises"; +import type { MediaSource, OpenedLocalFile } from "../messaging/media-source.js"; +import { openLocalFile } from "../messaging/media-source.js"; import { ApiError, MediaFileType, @@ -178,138 +179,137 @@ export class ChunkedMediaApi { async uploadChunked(opts: UploadChunkedOptions): Promise { const prefix = opts.logPrefix ?? "[qqbot:chunked-upload]"; - // 1. Resolve input: size + local path (or temp buffer handle). - const input = resolveSource(opts.source, opts.fileName); - - const displayName = input.fileName; - const fileSize = input.size; - const pathLabel = input.kind === "localPath" ? input.path : ""; - - this.logger?.info?.( - `${prefix} Start: file=${displayName} size=${formatFileSize(fileSize)} type=${opts.fileType}`, - ); - - // 2. Compute md5 / sha1 / md5_10m. Identical for buffer and localPath, - // but the localPath path streams so it never has to materialize the - // whole file twice. - const hashes = await computeHashes(input); - this.logger?.debug?.( - `${prefix} hashes: md5=${hashes.md5} sha1=${hashes.sha1} md5_10m=${hashes.md5_10m}`, - ); - - // 3. Upload-cache fast path: the md5 hash is already a strong content - // identifier, so we can short-circuit before even calling upload_prepare. - if (this.cache) { - const cached = this.cache.get(hashes.md5, opts.scope, opts.targetId, opts.fileType); - if (cached) { - this.logger?.info?.( - `${prefix} cache HIT (md5=${hashes.md5.slice(0, 8)}) — skipping chunked upload`, - ); - return { file_uuid: "", file_info: cached, ttl: 0 }; - } - } - - // 4. upload_prepare. - const fileNameForPrepare = - opts.fileType === MediaFileType.FILE ? this.sanitize(displayName) : displayName; - const prepareResp = await this.callUploadPrepare( - opts, - fileNameForPrepare, - fileSize, - hashes, - pathLabel, - ); - - const { upload_id, parts } = prepareResp; - const block_size = prepareResp.block_size; - const maxConcurrent = Math.min( - prepareResp.concurrency ? prepareResp.concurrency : DEFAULT_CONCURRENT_PARTS, - MAX_CONCURRENT_PARTS, - ); - const retryTimeoutMs = prepareResp.retry_timeout - ? Math.min(prepareResp.retry_timeout * 1000, MAX_PART_FINISH_RETRY_TIMEOUT_MS) - : undefined; - - this.logger?.info?.( - `${prefix} prepared: upload_id=${upload_id} block=${formatFileSize(block_size)} parts=${parts.length} concurrency=${maxConcurrent}`, - ); - - // 5. Upload every part. Concurrency is per-upload, not global. - let completedParts = 0; - let uploadedBytes = 0; - - const uploadPart = async (part: UploadPart): Promise => { - const partIndex = part.index; // 1-based. - const offset = (partIndex - 1) * block_size; - const length = Math.min(block_size, fileSize - offset); - - const partBuffer = await readPart(input, offset, length); - const md5Hex = crypto.createHash("md5").update(partBuffer).digest("hex"); - - this.logger?.debug?.( - `${prefix} part ${partIndex}/${parts.length}: ${formatFileSize(length)} offset=${offset} md5=${md5Hex}`, - ); - - // 5a. PUT to pre-signed COS URL. - await putToPresignedUrl( - part.presigned_url, - partBuffer, - partIndex, - parts.length, - this.logger, - prefix, - ); - - // 5b. upload_part_finish — fetch a fresh token each time to defend - // against long uploads exceeding the token TTL. - await this.callUploadPartFinish(opts, upload_id, partIndex, length, md5Hex, retryTimeoutMs); - - completedParts++; - uploadedBytes += length; - this.logger?.info?.( - `${prefix} part ${partIndex}/${parts.length} done (${completedParts}/${parts.length})`, - ); - - opts.onProgress?.({ - completedParts, - totalParts: parts.length, - uploadedBytes, - totalBytes: fileSize, - }); - }; + // 1. Resolve input: size + verified local file descriptor (or buffer). + const input = await resolveSource(opts.source, opts.fileName); try { + const displayName = input.fileName; + const fileSize = input.size; + const pathLabel = input.kind === "localPath" ? input.path : ""; + + this.logger?.info?.( + `${prefix} Start: file=${displayName} size=${formatFileSize(fileSize)} type=${opts.fileType}`, + ); + + // 2. Compute md5 / sha1 / md5_10m. Identical for buffer and localPath, + // but the localPath descriptor streams so it never has to materialize the + // whole file twice or reopen a path after validation. + const hashes = await computeHashes(input); + this.logger?.debug?.( + `${prefix} hashes: md5=${hashes.md5} sha1=${hashes.sha1} md5_10m=${hashes.md5_10m}`, + ); + + // 3. Upload-cache fast path: the md5 hash is already a strong content + // identifier, so we can short-circuit before even calling upload_prepare. + if (this.cache) { + const cached = this.cache.get(hashes.md5, opts.scope, opts.targetId, opts.fileType); + if (cached) { + this.logger?.info?.( + `${prefix} cache HIT (md5=${hashes.md5.slice(0, 8)}) — skipping chunked upload`, + ); + return { file_uuid: "", file_info: cached, ttl: 0 }; + } + } + + // 4. upload_prepare. + const fileNameForPrepare = + opts.fileType === MediaFileType.FILE ? this.sanitize(displayName) : displayName; + const prepareResp = await this.callUploadPrepare( + opts, + fileNameForPrepare, + fileSize, + hashes, + pathLabel, + ); + + const { upload_id, parts } = prepareResp; + const block_size = prepareResp.block_size; + const maxConcurrent = Math.min( + prepareResp.concurrency ? prepareResp.concurrency : DEFAULT_CONCURRENT_PARTS, + MAX_CONCURRENT_PARTS, + ); + const retryTimeoutMs = prepareResp.retry_timeout + ? Math.min(prepareResp.retry_timeout * 1000, MAX_PART_FINISH_RETRY_TIMEOUT_MS) + : undefined; + + this.logger?.info?.( + `${prefix} prepared: upload_id=${upload_id} block=${formatFileSize(block_size)} parts=${parts.length} concurrency=${maxConcurrent}`, + ); + + // 5. Upload every part. Concurrency is per-upload, not global. + let completedParts = 0; + let uploadedBytes = 0; + + const uploadPart = async (part: UploadPart): Promise => { + const partIndex = part.index; // 1-based. + const offset = (partIndex - 1) * block_size; + const length = Math.min(block_size, fileSize - offset); + + const partBuffer = await readPart(input, offset, length); + const md5Hex = crypto.createHash("md5").update(partBuffer).digest("hex"); + + this.logger?.debug?.( + `${prefix} part ${partIndex}/${parts.length}: ${formatFileSize(length)} offset=${offset} md5=${md5Hex}`, + ); + + // 5a. PUT to pre-signed COS URL. + await putToPresignedUrl( + part.presigned_url, + partBuffer, + partIndex, + parts.length, + this.logger, + prefix, + ); + + // 5b. upload_part_finish — fetch a fresh token each time to defend + // against long uploads exceeding the token TTL. + await this.callUploadPartFinish(opts, upload_id, partIndex, length, md5Hex, retryTimeoutMs); + + completedParts++; + uploadedBytes += length; + this.logger?.info?.( + `${prefix} part ${partIndex}/${parts.length} done (${completedParts}/${parts.length})`, + ); + + opts.onProgress?.({ + completedParts, + totalParts: parts.length, + uploadedBytes, + totalBytes: fileSize, + }); + }; + await runWithConcurrency( parts.map((part) => () => uploadPart(part)), maxConcurrent, ); + + this.logger?.info?.(`${prefix} all parts uploaded, completing...`); + + // 6. complete_upload. + const result = await this.callCompleteUpload(opts, upload_id); + this.logger?.info?.(`${prefix} completed: file_uuid=${result.file_uuid} ttl=${result.ttl}s`); + + // 7. Populate the shared upload cache so subsequent sends skip re-uploading. + if (this.cache && result.file_info && result.ttl > 0) { + this.cache.set( + hashes.md5, + opts.scope, + opts.targetId, + opts.fileType, + result.file_info, + result.file_uuid, + result.ttl, + ); + } + + return result; } finally { - // If the input opened a buffered read stream we don't keep state, - // but localPath readers open / close the file per-part so there - // is nothing to unwind here. Kept as a seam for future streaming - // optimizations. + if (input.kind === "localPath" && input.closeWhenDone) { + await input.opened.close().catch(() => undefined); + } } - - this.logger?.info?.(`${prefix} all parts uploaded, completing...`); - - // 6. complete_upload. - const result = await this.callCompleteUpload(opts, upload_id); - this.logger?.info?.(`${prefix} completed: file_uuid=${result.file_uuid} ttl=${result.ttl}s`); - - // 7. Populate the shared upload cache so subsequent sends skip re-uploading. - if (this.cache && result.file_info && result.ttl > 0) { - this.cache.set( - hashes.md5, - opts.scope, - opts.targetId, - opts.fileType, - result.file_info, - result.file_uuid, - result.ttl, - ); - } - - return result; } // -------- Internal call wrappers -------- @@ -429,17 +429,31 @@ export function isChunkedUploadImplemented(): boolean { * the bytes plus the metadata required by `upload_prepare`. */ type ChunkedInput = - | { kind: "localPath"; path: string; size: number; fileName: string } + | { + kind: "localPath"; + path: string; + size: number; + fileName: string; + opened: OpenedLocalFile; + closeWhenDone: boolean; + } | { kind: "buffer"; buffer: Buffer; size: number; fileName: string }; -function resolveSource(source: MediaSource, fileNameOverride?: string): ChunkedInput { +async function resolveSource( + source: MediaSource, + fileNameOverride?: string, +): Promise { if (source.kind === "localPath") { const inferredName = source.path.split(/[/\\]/).pop() || "file"; + const opened = + source.opened ?? (await openLocalFile(source.path, { maxSize: Number.MAX_SAFE_INTEGER })); return { kind: "localPath", path: source.path, - size: source.size, + size: opened.size, fileName: fileNameOverride ?? inferredName, + opened, + closeWhenDone: source.opened === undefined, }; } if (source.kind === "buffer") { @@ -460,14 +474,9 @@ async function readPart(input: ChunkedInput, offset: number, length: number): Pr if (input.kind === "buffer") { return input.buffer.subarray(offset, offset + length); } - const handle = await fs.promises.open(input.path, "r"); - try { - const buf = Buffer.alloc(length); - const { bytesRead } = await handle.read(buf, 0, length, offset); - return bytesRead < length ? buf.subarray(0, bytesRead) : buf; - } finally { - await handle.close(); - } + const buf = Buffer.alloc(length); + const { bytesRead } = await input.opened.handle.read(buf, 0, length, offset); + return bytesRead < length ? buf.subarray(0, bytesRead) : buf; } // ============ Hash computation ============ @@ -476,8 +485,8 @@ async function readPart(input: ChunkedInput, offset: number, length: number): Pr * Stream the source once to compute md5 + sha1 + md5_10m. * * For buffer inputs the three hashes are computed in a single pass over - * the existing memory. For localPath inputs a ReadStream drives the - * hashers so memory use stays constant. + * the existing memory. For localPath inputs the verified descriptor drives + * the hashers so memory use stays constant. */ async function computeHashes(input: ChunkedInput): Promise { if (input.kind === "buffer") { @@ -497,7 +506,7 @@ async function computeHashes(input: ChunkedInput): Promise let consumed = 0; const needsMd5_10m = input.size > MD5_10M_SIZE; - const stream = fs.createReadStream(input.path); + const stream = createReadStreamFromHandle(input.opened.handle); stream.on("data", (chunk: Buffer | string) => { const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); md5.update(buf); @@ -523,6 +532,10 @@ async function computeHashes(input: ChunkedInput): Promise }); } +function createReadStreamFromHandle(handle: FileHandle): NodeJS.ReadableStream { + return handle.createReadStream({ autoClose: false, start: 0 }); +} + // ============ COS PUT ============ /** Per-part retry budget for the COS PUT call (exponential backoff). */ diff --git a/extensions/qqbot/src/engine/config/credential-backup.ts b/extensions/qqbot/src/engine/config/credential-backup.ts index b8bab5e723b..0279818c8c4 100644 --- a/extensions/qqbot/src/engine/config/credential-backup.ts +++ b/extensions/qqbot/src/engine/config/credential-backup.ts @@ -26,7 +26,7 @@ */ import fs from "node:fs"; -import path from "node:path"; +import { replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; import { getCredentialBackupFile, getLegacyCredentialBackupFile } from "../utils/data-paths.js"; interface CredentialBackup { @@ -43,16 +43,17 @@ export function saveCredentialBackup(accountId: string, appId: string, clientSec } try { const backupPath = getCredentialBackupFile(accountId); - fs.mkdirSync(path.dirname(backupPath), { recursive: true }); const data: CredentialBackup = { accountId, appId, clientSecret, savedAt: new Date().toISOString(), }; - const tmpPath = `${backupPath}.tmp`; - fs.writeFileSync(tmpPath, `${JSON.stringify(data, null, 2)}\n`, "utf8"); - fs.renameSync(tmpPath, backupPath); + replaceFileAtomicSync({ + filePath: backupPath, + content: `${JSON.stringify(data, null, 2)}\n`, + tempPrefix: ".qqbot-credential-backup", + }); } catch { /* best-effort — ignore */ } @@ -89,10 +90,11 @@ export function loadCredentialBackup(accountId?: string): CredentialBackup | nul if (data.accountId) { try { const backupPath = getCredentialBackupFile(data.accountId); - fs.mkdirSync(path.dirname(backupPath), { recursive: true }); - const tmpPath = `${backupPath}.tmp`; - fs.writeFileSync(tmpPath, `${JSON.stringify(data, null, 2)}\n`, "utf8"); - fs.renameSync(tmpPath, backupPath); + replaceFileAtomicSync({ + filePath: backupPath, + content: `${JSON.stringify(data, null, 2)}\n`, + tempPrefix: ".qqbot-credential-backup", + }); fs.unlinkSync(legacy); } catch { /* ignore migration errors */ diff --git a/extensions/qqbot/src/engine/messaging/media-source.ts b/extensions/qqbot/src/engine/messaging/media-source.ts index cdae52dc4f6..8e2c5788d70 100644 --- a/extensions/qqbot/src/engine/messaging/media-source.ts +++ b/extensions/qqbot/src/engine/messaging/media-source.ts @@ -9,9 +9,8 @@ * * - `url` — remote http(s) URL that the QQ server can fetch directly. * - `base64` — in-memory base64 string (typically from a `data:` URL). - * - `localPath` — on-disk file; kept as a path so a future chunked-upload - * implementation can stream it via `fs.createReadStream` without the 4/3× - * base64 memory overhead. + * - `localPath` — on-disk file; kept as a path plus an optional verified + * descriptor so uploaders can avoid reopening a path after validation. * - `buffer` — in-memory raw bytes (e.g. TTS output, downloaded url-fallback). * * ## Security baseline (localPath branch) @@ -29,7 +28,8 @@ * reading the whole file first. */ -import * as fs from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import { FsSafeError, openLocalFileSafely } from "openclaw/plugin-sdk/security-runtime"; import { MAX_UPLOAD_SIZE, formatFileSize, getMimeType } from "../utils/file-utils.js"; // ============ Types ============ @@ -39,14 +39,14 @@ import { MAX_UPLOAD_SIZE, formatFileSize, getMimeType } from "../utils/file-util * * - `url`: remote URL — upload via `file_data=null; url=...`. * - `base64`: already-encoded base64 — upload via `file_data=...`. - * - `localPath`: on-disk file — one-shot path reads it into a buffer; - * chunked path (future) streams it via `fs.createReadStream`. + * - `localPath`: on-disk file — uploaders should prefer `opened` when present + * and only reopen `path` for direct, already-normalized test/helper calls. * - `buffer`: raw bytes in memory — same as above minus disk I/O. */ export type MediaSource = | { kind: "url"; url: string } | { kind: "base64"; data: string; mime?: string } - | { kind: "localPath"; path: string; size: number; mime?: string } + | { kind: "localPath"; path: string; size: number; mime?: string; opened?: OpenedLocalFile } | { kind: "buffer"; buffer: Buffer; fileName?: string; mime?: string }; /** @@ -92,8 +92,8 @@ function tryParseDataUrl(value: string): { mime: string; data: string } | null { * * Callers MUST call {@link OpenedLocalFile.close} (typically in a `finally`). */ -interface OpenedLocalFile { - handle: fs.promises.FileHandle; +export interface OpenedLocalFile { + handle: FileHandle; size: number; close(): Promise; } @@ -120,27 +120,26 @@ export async function openLocalFile( opts: { maxSize?: number } = {}, ): Promise { const maxSize = opts.maxSize ?? MAX_UPLOAD_SIZE; - const openFlags = - fs.constants.O_RDONLY | ("O_NOFOLLOW" in fs.constants ? fs.constants.O_NOFOLLOW : 0); - const handle = await fs.promises.open(filePath, openFlags); - try { - const stat = await handle.stat(); - if (!stat.isFile()) { - throw new Error("Path is not a regular file"); + const opened = await openLocalFileSafely({ filePath }).catch((err: unknown) => { + if (err instanceof FsSafeError && err.code === "not-file") { + throw new Error("Path is not a regular file", { cause: err }); } - if (stat.size > maxSize) { + throw err; + }); + try { + if (opened.stat.size > maxSize) { throw new Error( - `File is too large (${formatFileSize(stat.size)}); QQ Bot API limit is ${formatFileSize(maxSize)}`, + `File is too large (${formatFileSize(opened.stat.size)}); QQ Bot API limit is ${formatFileSize(maxSize)}`, ); } return { - handle, - size: stat.size, - close: () => handle.close(), + handle: opened.handle, + size: opened.stat.size, + close: () => opened.handle.close(), }; } catch (err) { // Close the handle on any validation failure to avoid fd leaks. - await handle.close().catch(() => undefined); + await opened.handle.close().catch(() => undefined); throw err; } } @@ -153,10 +152,9 @@ export async function openLocalFile( * - Strings passed via `{ url }` that start with `data:` are auto-resolved * to a `base64` branch (this is the unified `data:` URL support that was * previously only implemented in `sendImage`). - * - `localPath` branches open the file with {@link openLocalFile} solely to - * validate size / regular-file / O_NOFOLLOW invariants. The handle is - * closed immediately — actual reading is deferred to the uploader so - * the chunked path can stream without double-reading. + * - `localPath` branches open the file with {@link openLocalFile} and carry + * that descriptor to the uploader, so later reads use the exact file that + * passed regular-file / O_NOFOLLOW / size validation. * - `buffer` branches enforce the same ceiling inline. * * `maxSize` defaults to {@link MAX_UPLOAD_SIZE} (20MB, one-shot upload limit). @@ -188,16 +186,13 @@ export async function normalizeSource( if ("localPath" in raw) { const opened = await openLocalFile(raw.localPath, { maxSize }); - try { - return { - kind: "localPath", - path: raw.localPath, - size: opened.size, - mime: getMimeType(raw.localPath), - }; - } finally { - await opened.close(); - } + return { + kind: "localPath", + path: raw.localPath, + size: opened.size, + mime: getMimeType(raw.localPath), + opened, + }; } // buffer branch diff --git a/extensions/qqbot/src/engine/messaging/outbound-media-send.ts b/extensions/qqbot/src/engine/messaging/outbound-media-send.ts index ebc51818795..89068eb04d2 100644 --- a/extensions/qqbot/src/engine/messaging/outbound-media-send.ts +++ b/extensions/qqbot/src/engine/messaging/outbound-media-send.ts @@ -2,8 +2,11 @@ * Low-level outbound media sends (photo, voice, video, document) and path resolution. */ -import fs from "node:fs"; import path from "node:path"; +import { + pathExistsSync, + resolveLocalPathFromRootsSync, +} from "openclaw/plugin-sdk/security-runtime"; import type { GatewayAccount } from "../types.js"; import { MediaFileType } from "../types.js"; import { @@ -98,79 +101,32 @@ function isHttpOrDataSource(pathValue: string): boolean { ); } -function isPathWithinRoot(candidate: string, root: string): boolean { - const relative = path.relative(root, candidate); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - function resolveMissingPathWithinMediaRoot(normalizedPath: string): string | null { const resolvedCandidate = path.resolve(normalizedPath); - if (fs.existsSync(resolvedCandidate)) { + if (pathExistsSync(resolvedCandidate)) { return null; } - - const allowedRoot = path.resolve(getQQBotMediaDir()); - let canonicalAllowedRoot: string; - try { - canonicalAllowedRoot = fs.realpathSync(allowedRoot); - } catch { - return null; - } - - const missingSegments: string[] = []; - let cursor = resolvedCandidate; - while (!fs.existsSync(cursor)) { - const parent = path.dirname(cursor); - if (parent === cursor) { - break; - } - missingSegments.unshift(path.basename(cursor)); - cursor = parent; - } - - if (!fs.existsSync(cursor)) { - return null; - } - - let canonicalCursor: string; - try { - canonicalCursor = fs.realpathSync(cursor); - } catch { - return null; - } - const canonicalCandidate = - missingSegments.length > 0 ? path.join(canonicalCursor, ...missingSegments) : canonicalCursor; - - return isPathWithinRoot(canonicalCandidate, canonicalAllowedRoot) ? canonicalCandidate : null; + return ( + resolveLocalPathFromRootsSync({ + filePath: resolvedCandidate, + roots: [getQQBotMediaDir()], + label: "QQ Bot media storage", + allowMissing: true, + })?.path ?? null + ); } function resolveExistingPathWithinRoots( normalizedPath: string, allowedRoots: readonly string[], ): string | null { - const resolvedCandidate = path.resolve(normalizedPath); - if (!fs.existsSync(resolvedCandidate)) { - return null; - } - - let canonicalCandidate: string; - try { - canonicalCandidate = fs.realpathSync(resolvedCandidate); - } catch { - return null; - } - - for (const root of allowedRoots) { - const resolvedRoot = path.resolve(root); - const canonicalRoot = fs.existsSync(resolvedRoot) - ? fs.realpathSync(resolvedRoot) - : resolvedRoot; - if (isPathWithinRoot(canonicalCandidate, canonicalRoot)) { - return canonicalCandidate; - } - } - - return null; + return ( + resolveLocalPathFromRootsSync({ + filePath: normalizedPath, + roots: allowedRoots, + label: "QQ Bot local roots", + })?.path ?? null + ); } export function resolveOutboundMediaPath( diff --git a/extensions/qqbot/src/engine/messaging/sender.ts b/extensions/qqbot/src/engine/messaging/sender.ts index 6ac94cae63e..e831c0ad012 100644 --- a/extensions/qqbot/src/engine/messaging/sender.ts +++ b/extensions/qqbot/src/engine/messaging/sender.ts @@ -596,33 +596,39 @@ async function sendMediaInternal( maxSize: Number.MAX_SAFE_INTEGER, }); - const uploadResult = await dispatchUpload( - ctx, - scope, - opts.target.id, - KIND_TO_FILE_TYPE[opts.kind], - source, - c, - opts.fileName, - ); + try { + const uploadResult = await dispatchUpload( + ctx, + scope, + opts.target.id, + KIND_TO_FILE_TYPE[opts.kind], + source, + c, + opts.fileName, + ); - // Content is semantically meaningful only for image / video — the voice - // and file APIs ignore it. - const msgContent = opts.kind === "image" || opts.kind === "video" ? opts.content : undefined; + // Content is semantically meaningful only for image / video — the voice + // and file APIs ignore it. + const msgContent = opts.kind === "image" || opts.kind === "video" ? opts.content : undefined; - const result = await ctx.mediaApi.sendMediaMessage( - scope, - opts.target.id, - uploadResult.file_info, - c, - { - msgId: opts.msgId, - content: msgContent, - }, - ); + const result = await ctx.mediaApi.sendMediaMessage( + scope, + opts.target.id, + uploadResult.file_info, + c, + { + msgId: opts.msgId, + content: msgContent, + }, + ); - notifyMediaHook(opts.creds.appId, result, buildOutboundMeta(opts, source)); - return result; + notifyMediaHook(opts.creds.appId, result, buildOutboundMeta(opts, source)); + return result; + } finally { + if (source.kind === "localPath") { + await source.opened?.close().catch(() => undefined); + } + } } /** @@ -668,6 +674,12 @@ async function dispatchUpload( fileName, }); } + if (source.opened) { + return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, { + buffer: await source.opened.handle.readFile(), + fileName, + }); + } return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, { localPath: source.path, fileName, diff --git a/extensions/qqbot/src/engine/ref/store.ts b/extensions/qqbot/src/engine/ref/store.ts index f13548a5722..460263368b3 100644 --- a/extensions/qqbot/src/engine/ref/store.ts +++ b/extensions/qqbot/src/engine/ref/store.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import path from "node:path"; +import { appendRegularFileSync, replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js"; @@ -88,7 +89,7 @@ function ensureDir(): void { function appendLine(line: RefIndexLine): void { try { ensureDir(); - fs.appendFileSync(getRefIndexFile(), JSON.stringify(line) + "\n", "utf-8"); + appendRegularFileSync({ filePath: getRefIndexFile(), content: JSON.stringify(line) + "\n" }); totalLinesOnDisk++; } catch (err) { debugError(`[ref-index-store] Failed to append: ${formatErrorMessage(err)}`); @@ -109,7 +110,6 @@ function compactFile(): void { try { ensureDir(); const refIndexFile = getRefIndexFile(); - const tmpPath = refIndexFile + ".tmp"; const lines: string[] = []; for (const [key, entry] of cache) { lines.push( @@ -127,8 +127,11 @@ function compactFile(): void { }), ); } - fs.writeFileSync(tmpPath, lines.join("\n") + "\n", "utf-8"); - fs.renameSync(tmpPath, refIndexFile); + replaceFileAtomicSync({ + filePath: refIndexFile, + content: `${lines.join("\n")}\n`, + tempPrefix: ".qqbot-ref-index", + }); totalLinesOnDisk = cache.size; debugLog(`[ref-index-store] Compacted: ${before} lines → ${totalLinesOnDisk} lines`); } catch (err) { diff --git a/extensions/qqbot/src/engine/session/known-users.ts b/extensions/qqbot/src/engine/session/known-users.ts index 7fdb35264aa..0b94dcf14ac 100644 --- a/extensions/qqbot/src/engine/session/known-users.ts +++ b/extensions/qqbot/src/engine/session/known-users.ts @@ -5,8 +5,8 @@ * built-ins + log + platform (both zero plugin-sdk). */ -import fs from "node:fs"; import path from "node:path"; +import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import type { ChatScope } from "../types.js"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; @@ -49,9 +49,10 @@ function loadUsersFromFile(): Map { usersCache = new Map(); try { const knownUsersFile = getKnownUsersFile(); - if (fs.existsSync(knownUsersFile)) { - const data = fs.readFileSync(knownUsersFile, "utf-8"); - const users = JSON.parse(data) as KnownUser[]; + const users = privateFileStoreSync(path.dirname(knownUsersFile)).readJsonIfExists( + path.basename(knownUsersFile), + ); + if (users) { for (const user of users) { usersCache.set(makeUserKey(user), user); } @@ -80,10 +81,10 @@ function doSaveUsersToFile(): void { } try { ensureDir(); - fs.writeFileSync( - getKnownUsersFile(), - JSON.stringify(Array.from(usersCache.values()), null, 2), - "utf-8", + const filePath = getKnownUsersFile(); + privateFileStoreSync(path.dirname(filePath)).writeJson( + path.basename(filePath), + Array.from(usersCache.values()), ); isDirty = false; } catch (err) { diff --git a/extensions/qqbot/src/engine/session/session-store.ts b/extensions/qqbot/src/engine/session/session-store.ts index d4cb4e48345..f0798366a2e 100644 --- a/extensions/qqbot/src/engine/session/session-store.ts +++ b/extensions/qqbot/src/engine/session/session-store.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import path from "node:path"; +import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError } from "../utils/log.js"; import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js"; @@ -66,18 +67,20 @@ function getCandidateSessionPaths(accountId: string): string[] { export function loadSession(accountId: string, expectedAppId?: string): SessionState | null { try { let filePath: string | null = null; + let state: SessionState | null = null; for (const candidatePath of getCandidateSessionPaths(accountId)) { - if (fs.existsSync(candidatePath)) { + state = privateFileStoreSync(path.dirname(candidatePath)).readJsonIfExists( + path.basename(candidatePath), + ); + if (state) { filePath = candidatePath; break; } } - if (!filePath) { + if (!filePath || !state) { return null; } - const data = fs.readFileSync(filePath, "utf-8"); - const state = JSON.parse(data) as SessionState; const now = Date.now(); if (now - state.savedAt > SESSION_EXPIRE_TIME) { @@ -162,7 +165,7 @@ function doSaveSession(state: SessionState): void { try { ensureDir(); const stateToSave: SessionState = { ...state, savedAt: Date.now() }; - fs.writeFileSync(filePath, JSON.stringify(stateToSave, null, 2), "utf-8"); + privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), stateToSave); if (legacyPath !== filePath && fs.existsSync(legacyPath)) { fs.unlinkSync(legacyPath); } diff --git a/extensions/qqbot/src/engine/utils/audio.ts b/extensions/qqbot/src/engine/utils/audio.ts index ead30c35418..4a5ce82261c 100644 --- a/extensions/qqbot/src/engine/utils/audio.ts +++ b/extensions/qqbot/src/engine/utils/audio.ts @@ -11,6 +11,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; +import { readRegularFileSync } from "openclaw/plugin-sdk/security-runtime"; import { formatErrorMessage } from "./format.js"; import { debugLog, debugError, debugWarn } from "./log.js"; import { normalizeLowercaseStringOrEmpty as normalizeLowercase } from "./string-normalize.js"; @@ -81,11 +82,13 @@ export async function convertSilkToWav( inputPath: string, outputDir?: string, ): Promise<{ wavPath: string; duration: number } | null> { - if (!fs.existsSync(inputPath)) { + let fileBuf: Buffer; + try { + fileBuf = readRegularFileSync({ filePath: inputPath }).buffer; + } catch { return null; } - const fileBuf = fs.readFileSync(inputPath); const strippedBuf = stripAmrHeader(fileBuf); const rawData = new Uint8Array( strippedBuf.buffer, @@ -188,11 +191,13 @@ export async function audioFileToSilkBase64( filePath: string, directUploadFormats?: string[], ): Promise { - if (!fs.existsSync(filePath)) { + let buf: Buffer; + try { + buf = readRegularFileSync({ filePath }).buffer; + } catch { return null; } - const buf = fs.readFileSync(filePath); if (buf.length === 0) { debugError(`[audio-convert] file is empty: ${filePath}`); return null; diff --git a/extensions/qqbot/src/engine/utils/file-utils.test.ts b/extensions/qqbot/src/engine/utils/file-utils.test.ts index ed3be3005a7..42cee05d06a 100644 --- a/extensions/qqbot/src/engine/utils/file-utils.test.ts +++ b/extensions/qqbot/src/engine/utils/file-utils.test.ts @@ -13,7 +13,13 @@ vi.mock("../adapter/index.js", () => ({ }), })); -import { QQBOT_MEDIA_SSRF_POLICY, downloadFile } from "./file-utils.js"; +import { + QQBOT_MEDIA_SSRF_POLICY, + checkFileSize, + downloadFile, + fileExistsAsync, + readFileAsync, +} from "./file-utils.js"; describe("qqbot file-utils downloadFile", () => { let tempDir: string; @@ -69,4 +75,15 @@ describe("qqbot file-utils downloadFile", () => { expect(savedPath).toBeNull(); expect(adapterMocks.fetchMedia).not.toHaveBeenCalled(); }); + + it.skipIf(process.platform === "win32")("rejects symlinked local media helpers", async () => { + const targetPath = path.join(tempDir, "target.png"); + const linkPath = path.join(tempDir, "link.png"); + await fs.promises.writeFile(targetPath, "image-bytes"); + await fs.promises.symlink(targetPath, linkPath); + + expect(checkFileSize(linkPath).ok).toBe(false); + await expect(readFileAsync(linkPath)).rejects.toThrow(); + await expect(fileExistsAsync(linkPath)).resolves.toBe(false); + }); }); diff --git a/extensions/qqbot/src/engine/utils/file-utils.ts b/extensions/qqbot/src/engine/utils/file-utils.ts index 2ce1dacea3c..36d2eb0dd51 100644 --- a/extensions/qqbot/src/engine/utils/file-utils.ts +++ b/extensions/qqbot/src/engine/utils/file-utils.ts @@ -1,6 +1,11 @@ import crypto from "node:crypto"; import * as fs from "node:fs"; import * as path from "node:path"; +import { + openLocalFileSafely, + readRegularFile, + statRegularFileSync, +} from "openclaw/plugin-sdk/security-runtime"; import { getPlatformAdapter } from "../adapter/index.js"; import type { SsrfPolicyConfig } from "../adapter/types.js"; import { MediaFileType } from "../types.js"; @@ -72,17 +77,20 @@ interface FileSizeCheckResult { /** Validate that a file is within the allowed upload size. */ export function checkFileSize(filePath: string, maxSize = MAX_UPLOAD_SIZE): FileSizeCheckResult { try { - const stat = fs.statSync(filePath); - if (stat.size > maxSize) { - const sizeMB = (stat.size / (1024 * 1024)).toFixed(1); + const result = statRegularFileSync(filePath); + if (result.missing) { + throw Object.assign(new Error(`File not found: ${filePath}`), { code: "ENOENT" }); + } + if (result.stat.size > maxSize) { + const sizeMB = (result.stat.size / (1024 * 1024)).toFixed(1); const limitMB = (maxSize / (1024 * 1024)).toFixed(0); return { ok: false, - size: stat.size, + size: result.stat.size, error: `File is too large (${sizeMB}MB); QQ Bot API limit is ${limitMB}MB`, }; } - return { ok: true, size: stat.size }; + return { ok: true, size: result.stat.size }; } catch (err) { return { ok: false, @@ -94,16 +102,21 @@ export function checkFileSize(filePath: string, maxSize = MAX_UPLOAD_SIZE): File /** Read file contents asynchronously. */ export async function readFileAsync(filePath: string): Promise { - return fs.promises.readFile(filePath); + return (await readRegularFile({ filePath })).buffer; } /** Check file readability asynchronously. */ export async function fileExistsAsync(filePath: string): Promise { + const opened = await openLocalFileSafely({ filePath }).catch(() => null); + if (!opened) { + return false; + } try { - await fs.promises.access(filePath, fs.constants.R_OK); return true; } catch { return false; + } finally { + await opened.handle.close().catch(() => undefined); } } diff --git a/extensions/skill-workshop/src/skills.ts b/extensions/skill-workshop/src/skills.ts index 5adc99a4497..8170c25bf77 100644 --- a/extensions/skill-workshop/src/skills.ts +++ b/extensions/skill-workshop/src/skills.ts @@ -1,6 +1,10 @@ -import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { + pathExists, + replaceFileAtomic, + resolvePathWithinRoot, +} from "openclaw/plugin-sdk/security-runtime"; import { bumpSkillsSnapshotVersion } from "../api.js"; import { assertSkillContentSafe, scanSkillContent } from "./scanner.js"; import type { SkillProposal, SkillScanFinding } from "./types.js"; @@ -38,31 +42,27 @@ function assertValidSection(section: string): string { function skillDir(workspaceDir: string, skillName: string): string { const safeName = assertValidSkillName(skillName); const root = path.resolve(workspaceDir, "skills"); - const dir = path.resolve(root, safeName); - if (!dir.startsWith(`${root}${path.sep}`)) { + const dir = resolvePathWithinRoot({ + rootDir: root, + requestedPath: safeName, + scopeLabel: "workspace skills directory", + }); + if (!dir.ok) { throw new Error("skill path escapes workspace skills directory"); } - return dir; + return dir.path; } function skillPath(workspaceDir: string, skillName: string): string { return path.join(skillDir(workspaceDir, skillName), "SKILL.md"); } -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function atomicWrite(filePath: string, content: string): Promise { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - const tempPath = `${filePath}.tmp-${process.pid}-${Date.now().toString(36)}-${randomUUID()}`; - await fs.writeFile(tempPath, content, "utf8"); - await fs.rename(tempPath, filePath); + await replaceFileAtomic({ + filePath, + content, + tempPrefix: ".skill-workshop", + }); } function formatSkillMarkdown(params: { name: string; description: string; body: string }): string { @@ -173,10 +173,14 @@ export async function writeSupportFile(params: { } assertSkillContentSafe(params.content); const root = skillDir(params.workspaceDir, name); - const target = path.resolve(root, ...parts); - if (!target.startsWith(`${root}${path.sep}`)) { + const target = resolvePathWithinRoot({ + rootDir: root, + requestedPath: path.join(...parts), + scopeLabel: "skill directory", + }); + if (!target.ok) { throw new Error("support file path escapes skill directory"); } - await atomicWrite(target, `${params.content.trimEnd()}\n`); - return target; + await atomicWrite(target.path, `${params.content.trimEnd()}\n`); + return target.path; } diff --git a/extensions/skill-workshop/src/store.ts b/extensions/skill-workshop/src/store.ts index 8656d4edf3b..c58c163ad2c 100644 --- a/extensions/skill-workshop/src/store.ts +++ b/extensions/skill-workshop/src/store.ts @@ -1,6 +1,6 @@ -import { createHash, randomUUID } from "node:crypto"; -import fs from "node:fs/promises"; +import { createHash } from "node:crypto"; import path from "node:path"; +import { privateFileStore } from "openclaw/plugin-sdk/security-runtime"; import type { SkillProposal, SkillWorkshopStatus } from "./types.js"; type StoreFile = { @@ -42,24 +42,21 @@ async function withLock(key: string, task: () => Promise): Promise { } } -async function readJson(filePath: string): Promise { - try { - const raw = await fs.readFile(filePath, "utf8"); - const parsed = JSON.parse(raw) as StoreFile; - return { - version: 1, - proposals: Array.isArray(parsed.proposals) ? parsed.proposals : [], - review: - parsed.review && typeof parsed.review === "object" - ? normalizeReviewState(parsed.review as Partial) - : undefined, - }; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return { version: 1, proposals: [] }; - } - throw error; +async function readJson(rootDir: string, filePath: string): Promise { + const parsed = await privateFileStore(rootDir).readJsonIfExists( + path.relative(rootDir, filePath), + ); + if (!parsed) { + return { version: 1, proposals: [] }; } + return { + version: 1, + proposals: Array.isArray(parsed.proposals) ? parsed.proposals : [], + review: + parsed.review && typeof parsed.review === "object" + ? normalizeReviewState(parsed.review as Partial) + : undefined, + }; } function normalizeReviewState( @@ -80,26 +77,27 @@ function normalizeReviewState( }; } -async function atomicWriteJson(filePath: string, data: StoreFile): Promise { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - const tempPath = `${filePath}.tmp-${process.pid}-${Date.now().toString(36)}-${randomUUID()}`; - await fs.writeFile(tempPath, `${JSON.stringify(data, null, 2)}\n`, "utf8"); - await fs.rename(tempPath, filePath); +async function atomicWriteJson(rootDir: string, filePath: string, data: StoreFile): Promise { + await privateFileStore(rootDir).writeJson(path.relative(rootDir, filePath), data, { + trailingNewline: true, + }); } export class SkillWorkshopStore { + readonly stateDir: string; readonly filePath: string; constructor(params: { stateDir: string; workspaceDir: string }) { + this.stateDir = path.resolve(params.stateDir); this.filePath = path.join( - params.stateDir, + this.stateDir, "skill-workshop", `${workspaceKey(params.workspaceDir)}.json`, ); } async list(status?: SkillWorkshopStatus): Promise { - const file = await readJson(this.filePath); + const file = await readJson(this.stateDir, this.filePath); const proposals = status ? file.proposals.filter((proposal) => proposal.status === status) : file.proposals; @@ -112,7 +110,7 @@ export class SkillWorkshopStore { async add(proposal: SkillProposal, maxPending: number): Promise { return await withLock(this.filePath, async () => { - const file = await readJson(this.filePath); + const file = await readJson(this.stateDir, this.filePath); const duplicate = file.proposals.find( (item) => (item.status === "pending" || item.status === "quarantined") && @@ -134,48 +132,52 @@ export class SkillWorkshopStore { ).length <= maxPending ); }); - await atomicWriteJson(this.filePath, { ...file, version: 1, proposals: nextProposals }); + await atomicWriteJson(this.stateDir, this.filePath, { + ...file, + version: 1, + proposals: nextProposals, + }); return proposal; }); } async updateStatus(id: string, status: SkillWorkshopStatus): Promise { return await withLock(this.filePath, async () => { - const file = await readJson(this.filePath); + const file = await readJson(this.stateDir, this.filePath); const index = file.proposals.findIndex((proposal) => proposal.id === id); if (index < 0) { throw new Error(`proposal not found: ${id}`); } const updated = { ...file.proposals[index], status, updatedAt: Date.now() }; file.proposals[index] = updated; - await atomicWriteJson(this.filePath, file); + await atomicWriteJson(this.stateDir, this.filePath, file); return updated; }); } async recordReviewTurn(toolCalls: number): Promise { return await withLock(this.filePath, async () => { - const file = await readJson(this.filePath); + const file = await readJson(this.stateDir, this.filePath); const current = normalizeReviewState(file.review); const next = { ...current, turnsSinceReview: current.turnsSinceReview + 1, toolCallsSinceReview: current.toolCallsSinceReview + Math.max(0, Math.trunc(toolCalls)), }; - await atomicWriteJson(this.filePath, { ...file, review: next }); + await atomicWriteJson(this.stateDir, this.filePath, { ...file, review: next }); return next; }); } async markReviewed(): Promise { return await withLock(this.filePath, async () => { - const file = await readJson(this.filePath); + const file = await readJson(this.stateDir, this.filePath); const next = { turnsSinceReview: 0, toolCallsSinceReview: 0, lastReviewAt: Date.now(), }; - await atomicWriteJson(this.filePath, { ...file, review: next }); + await atomicWriteJson(this.stateDir, this.filePath, { ...file, review: next }); return next; }); } diff --git a/extensions/speech-core/src/audio-transcode.ts b/extensions/speech-core/src/audio-transcode.ts index 85fe8426cbf..b1719c9e4bb 100644 --- a/extensions/speech-core/src/audio-transcode.ts +++ b/extensions/speech-core/src/audio-transcode.ts @@ -1,7 +1,5 @@ import { spawn } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; +import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; type TranscodeOutcome = | { ok: true; buffer: Buffer } @@ -54,13 +52,13 @@ export async function transcodeAudioBuffer(params: { return { ok: false, reason: "platform-unsupported" }; } - const tmpRoot = resolvePreferredOpenClawTmpDir(); - mkdirSync(tmpRoot, { recursive: true, mode: 0o700 }); - const tmpDir = mkdtempSync(join(tmpRoot, "tts-transcode-")); - const inPath = join(tmpDir, `in.${source}`); - const outPath = join(tmpDir, `out.${target}`); + const tmp = tempWorkspaceSync({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "tts-transcode-", + }); + const inPath = tmp.write(`in.${source}`, params.audioBuffer); + const outPath = tmp.path(`out.${target}`); try { - writeFileSync(inPath, params.audioBuffer, { mode: 0o600 }); const result = await runAfconvert({ args: [...recipe, inPath, outPath], timeoutMs: params.timeoutMs ?? 5000, @@ -68,15 +66,11 @@ export async function transcodeAudioBuffer(params: { if (!result.ok) { return { ok: false, reason: "transcoder-failed", detail: result.detail }; } - return { ok: true, buffer: readFileSync(outPath) }; + return { ok: true, buffer: tmp.read(`out.${target}`) }; } catch (err) { return { ok: false, reason: "transcoder-failed", detail: (err as Error).message }; } finally { - try { - rmSync(tmpDir, { recursive: true, force: true }); - } catch { - // best-effort cleanup - } + tmp.cleanup(); } } diff --git a/extensions/speech-core/src/tts.ts b/extensions/speech-core/src/tts.ts index 332c59fb4ae..53ff0f13df1 100644 --- a/extensions/speech-core/src/tts.ts +++ b/extensions/speech-core/src/tts.ts @@ -1,13 +1,4 @@ -import { randomBytes } from "node:crypto"; -import { - existsSync, - mkdirSync, - readFileSync, - writeFileSync, - mkdtempSync, - renameSync, - unlinkSync, -} from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { resolveChannelTtsVoiceDelivery } from "openclaw/plugin-sdk/channel-targets"; import type { @@ -30,7 +21,8 @@ import { selectApplicableRuntimeConfig, } from "openclaw/plugin-sdk/runtime-config-snapshot"; import { isVerbose, logVerbose } from "openclaw/plugin-sdk/runtime-env"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; +import { tempWorkspaceSync, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox"; +import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, @@ -566,24 +558,12 @@ function readPrefs(prefsPath: string): TtsUserPrefs { } function atomicWriteFileSync(filePath: string, content: string): void { - const tmpPath = `${filePath}.tmp.${Date.now()}.${randomBytes(8).toString("hex")}`; - writeFileSync(tmpPath, content, { mode: 0o600 }); - try { - renameSync(tmpPath, filePath); - } catch (err) { - try { - unlinkSync(tmpPath); - } catch { - // ignore - } - throw err; - } + privateFileStoreSync(path.dirname(filePath)).writeText(path.basename(filePath), content); } function updatePrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void { const prefs = readPrefs(prefsPath); update(prefs); - mkdirSync(path.dirname(prefsPath), { recursive: true }); atomicWriteFileSync(prefsPath, JSON.stringify(prefs, null, 2)); } @@ -1136,12 +1116,12 @@ export async function textToSpeech(params: { outputFormat = transcoded.outputFormat; } - const tempRoot = resolvePreferredOpenClawTmpDir(); - mkdirSync(tempRoot, { recursive: true, mode: 0o700 }); - const tempDir = mkdtempSync(path.join(tempRoot, "tts-")); - const audioPath = path.join(tempDir, `voice-${Date.now()}${fileExtension}`); - writeFileSync(audioPath, audioBuffer); - scheduleCleanup(tempDir); + const temp = tempWorkspaceSync({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "tts-", + }); + const audioPath = temp.write(`voice-${Date.now()}${fileExtension}`, audioBuffer); + scheduleCleanup(temp.dir); return { success: true, diff --git a/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts b/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts index 6e0b9ebfe77..b6a9da89908 100644 --- a/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts +++ b/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts @@ -6,10 +6,17 @@ import type { TelegramContext } from "./types.js"; const saveMediaBuffer = vi.fn(); const fetchRemoteMedia = vi.fn(); -const readFileWithinRoot = vi.fn(); +const rootRead = vi.fn(); vi.mock("openclaw/plugin-sdk/file-access-runtime", () => ({ - readFileWithinRoot: (...args: unknown[]) => readFileWithinRoot(...args), + root: async (rootDir: string) => ({ + read: async (relativePath: string, options?: { maxBytes?: number }) => + await rootRead({ + rootDir, + relativePath, + maxBytes: options?.maxBytes, + }), + }), })); vi.mock("./delivery.resolve-media.runtime.js", () => { @@ -201,7 +208,7 @@ describe("resolveMedia getFile retry", () => { vi.useFakeTimers(); fetchRemoteMedia.mockReset(); saveMediaBuffer.mockReset(); - readFileWithinRoot.mockReset(); + rootRead.mockReset(); }); afterEach(() => { @@ -435,7 +442,7 @@ describe("resolveMedia getFile retry", () => { it("copies trusted local absolute file paths into inbound media storage for media downloads", async () => { const getFile = vi.fn().mockResolvedValue({ file_path: "/var/lib/telegram-bot-api/file.pdf" }); - readFileWithinRoot.mockResolvedValueOnce({ + rootRead.mockResolvedValueOnce({ buffer: Buffer.from("pdf-data"), realPath: "/var/lib/telegram-bot-api/file.pdf", stat: { size: 8 }, @@ -451,7 +458,7 @@ describe("resolveMedia getFile retry", () => { ); expect(fetchRemoteMedia).not.toHaveBeenCalled(); - expect(readFileWithinRoot).toHaveBeenCalledWith({ + expect(rootRead).toHaveBeenCalledWith({ rootDir: "/var/lib/telegram-bot-api", relativePath: "file.pdf", maxBytes: MAX_MEDIA_BYTES, @@ -476,7 +483,7 @@ describe("resolveMedia getFile retry", () => { const getFile = vi .fn() .mockResolvedValue({ file_path: "/var/lib/telegram-bot-api/sticker.webp" }); - readFileWithinRoot.mockResolvedValueOnce({ + rootRead.mockResolvedValueOnce({ buffer: Buffer.from("sticker-data"), realPath: "/var/lib/telegram-bot-api/sticker.webp", stat: { size: 12 }, @@ -491,7 +498,7 @@ describe("resolveMedia getFile retry", () => { }); expect(fetchRemoteMedia).not.toHaveBeenCalled(); - expect(readFileWithinRoot).toHaveBeenCalledWith({ + expect(rootRead).toHaveBeenCalledWith({ rootDir: "/var/lib/telegram-bot-api", relativePath: "sticker.webp", maxBytes: MAX_MEDIA_BYTES, @@ -513,7 +520,7 @@ describe("resolveMedia getFile retry", () => { it("maps trusted local absolute path read failures to MediaFetchError", async () => { const getFile = vi.fn().mockResolvedValue({ file_path: "/var/lib/telegram-bot-api/file.pdf" }); - readFileWithinRoot.mockRejectedValueOnce(new Error("file not found")); + rootRead.mockRejectedValueOnce(new Error("file not found")); await expect( resolveMediaWithDefaults(makeCtx("document", getFile, { mime_type: "application/pdf" }), { @@ -530,7 +537,7 @@ describe("resolveMedia getFile retry", () => { it("maps oversized trusted local absolute path reads to MediaFetchError", async () => { const getFile = vi.fn().mockResolvedValue({ file_path: "/var/lib/telegram-bot-api/file.pdf" }); - readFileWithinRoot.mockRejectedValueOnce(new Error("file exceeds limit")); + rootRead.mockRejectedValueOnce(new Error("file exceeds limit")); await expect( resolveMediaWithDefaults(makeCtx("document", getFile, { mime_type: "application/pdf" }), { @@ -558,7 +565,7 @@ describe("resolveMedia getFile retry", () => { }), ); - expect(readFileWithinRoot).not.toHaveBeenCalled(); + expect(rootRead).not.toHaveBeenCalled(); expect(fetchRemoteMedia).not.toHaveBeenCalled(); }); }); diff --git a/extensions/telegram/src/bot/delivery.resolve-media.ts b/extensions/telegram/src/bot/delivery.resolve-media.ts index 84b54d9a1f5..4895c5c1aa8 100644 --- a/extensions/telegram/src/bot/delivery.resolve-media.ts +++ b/extensions/telegram/src/bot/delivery.resolve-media.ts @@ -1,6 +1,6 @@ import path from "node:path"; import { GrammyError } from "grammy"; -import { readFileWithinRoot } from "openclaw/plugin-sdk/file-access-runtime"; +import { root as fsRoot } from "openclaw/plugin-sdk/file-access-runtime"; import type { TelegramTransport } from "../fetch.js"; import { cacheSticker, getCachedSticker } from "../sticker-cache.js"; import { @@ -203,9 +203,8 @@ async function downloadAndSaveTelegramFile(params: { if (trustedLocalFile) { let localFile; try { - localFile = await readFileWithinRoot({ - rootDir: trustedLocalFile.rootDir, - relativePath: trustedLocalFile.relativePath, + const root = await fsRoot(trustedLocalFile.rootDir); + localFile = await root.read(trustedLocalFile.relativePath, { maxBytes: params.maxBytes, }); } catch (err) { diff --git a/extensions/telegram/src/sent-message-cache.ts b/extensions/telegram/src/sent-message-cache.ts index 46a2fefcea6..eed2bf4df31 100644 --- a/extensions/telegram/src/sent-message-cache.ts +++ b/extensions/telegram/src/sent-message-cache.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; -import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import { replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; import { resolveStorePath } from "openclaw/plugin-sdk/session-store-runtime"; const TTL_MS = 24 * 60 * 60 * 1000; @@ -119,10 +119,11 @@ function persistSentMessages(bucket: SentMessageBucket): void { fs.rmSync(persistedPath, { force: true }); return; } - fs.mkdirSync(path.dirname(persistedPath), { recursive: true }); - const tempPath = `${persistedPath}.${process.pid}.tmp`; - fs.writeFileSync(tempPath, JSON.stringify(serialized), "utf-8"); - fs.renameSync(tempPath, persistedPath); + replaceFileAtomicSync({ + filePath: persistedPath, + content: JSON.stringify(serialized), + tempPrefix: ".telegram-sent-message-cache", + }); } export function recordSentMessage( diff --git a/extensions/telegram/src/state-migrations.ts b/extensions/telegram/src/state-migrations.ts index 238fdc0230e..210374a9f51 100644 --- a/extensions/telegram/src/state-migrations.ts +++ b/extensions/telegram/src/state-migrations.ts @@ -1,12 +1,12 @@ -import fs from "node:fs"; import type { ChannelLegacyStateMigrationPlan } from "openclaw/plugin-sdk/channel-contract"; import { resolveChannelAllowFromPath } from "openclaw/plugin-sdk/channel-pairing-paths"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; +import { statRegularFileSync } from "openclaw/plugin-sdk/security-runtime"; import { resolveDefaultTelegramAccountId } from "./account-selection.js"; function fileExists(pathValue: string): boolean { try { - return fs.existsSync(pathValue) && fs.statSync(pathValue).isFile(); + return !statRegularFileSync(pathValue).missing; } catch { return false; } diff --git a/extensions/telegram/src/topic-name-cache.ts b/extensions/telegram/src/topic-name-cache.ts index 319fe1e3df7..1b7cc867b4f 100644 --- a/extensions/telegram/src/topic-name-cache.ts +++ b/extensions/telegram/src/topic-name-cache.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; -import path from "node:path"; import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import { replaceFileAtomicSync } from "openclaw/plugin-sdk/security-runtime"; const MAX_ENTRIES = 2_048; const TOPIC_NAME_CACHE_STATE_KEY = Symbol.for("openclaw.telegramTopicNameCacheState"); @@ -146,10 +146,11 @@ function persistTopicStore(persistedPath: string, store: TopicNameStore): void { fs.rmSync(persistedPath, { force: true }); return; } - fs.mkdirSync(path.dirname(persistedPath), { recursive: true }); - const tempPath = `${persistedPath}.${process.pid}.tmp`; - fs.writeFileSync(tempPath, JSON.stringify(Object.fromEntries(store)), "utf-8"); - fs.renameSync(tempPath, persistedPath); + replaceFileAtomicSync({ + filePath: persistedPath, + content: JSON.stringify(Object.fromEntries(store)), + tempPrefix: ".telegram-topic-name-cache", + }); } export function updateTopicName( diff --git a/extensions/tts-local-cli/speech-provider.ts b/extensions/tts-local-cli/speech-provider.ts index 6bbd3c99e1d..432e109ebe4 100644 --- a/extensions/tts-local-cli/speech-provider.ts +++ b/extensions/tts-local-cli/speech-provider.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import path from "node:path"; import { runFfmpeg } from "openclaw/plugin-sdk/media-runtime"; import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env"; @@ -9,7 +9,7 @@ import type { SpeechSynthesisRequest, SpeechTelephonySynthesisRequest, } from "openclaw/plugin-sdk/speech-core"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { tempWorkspace, resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; const log = createSubsystemLogger("tts-local-cli"); @@ -326,7 +326,11 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { log.debug(`synthesize: text=${req.text.slice(0, 50)}...`); - const tempDir = mkdtempSync(path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-tts-")); + const temp = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "openclaw-cli-tts-", + }); + const tempDir = temp.dir; try { const result = await runCli({ @@ -351,7 +355,7 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { const inputFile = result.audioPath ?? path.join(tempDir, `input${getFileExt(result.actualFormat)}`); if (!result.audioPath) { - writeFileSync(inputFile, result.buffer); + await temp.write(`input${getFileExt(result.actualFormat)}`, result.buffer); } buffer = await convertAudio(inputFile, tempDir, "opus"); format = "opus"; @@ -365,7 +369,7 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { const inputFile = result.audioPath ?? path.join(tempDir, `input${getFileExt(result.actualFormat)}`); if (!result.audioPath) { - writeFileSync(inputFile, result.buffer); + await temp.write(`input${getFileExt(result.actualFormat)}`, result.buffer); } buffer = await convertAudio(inputFile, tempDir, desired); format = desired; @@ -383,9 +387,7 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { voiceCompatible: req.target === "voice-note" && format === "opus", }; } finally { - try { - rmSync(tempDir, { recursive: true, force: true }); - } catch {} + await temp.cleanup(); } }, @@ -397,7 +399,11 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { log.debug(`synthesizeTelephony: text=${req.text.slice(0, 50)}...`); - const tempDir = mkdtempSync(path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-tts-")); + const temp = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "openclaw-cli-tts-", + }); + const tempDir = temp.dir; try { const result = await runCli({ @@ -415,7 +421,7 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { const inputFile = result.audioPath ?? path.join(tempDir, `input${getFileExt(result.actualFormat)}`); if (!result.audioPath) { - writeFileSync(inputFile, result.buffer); + await temp.write(`input${getFileExt(result.actualFormat)}`, result.buffer); } // Convert to raw 16kHz mono PCM for telephony (no WAV headers) @@ -427,9 +433,7 @@ export function buildCliSpeechProvider(): SpeechProviderPlugin { sampleRate: 16000, }; } finally { - try { - rmSync(tempDir, { recursive: true, force: true }); - } catch {} + await temp.cleanup(); } }, }; diff --git a/extensions/voice-call/src/manager/store.ts b/extensions/voice-call/src/manager/store.ts index 37b9f6976f0..004325f5a47 100644 --- a/extensions/voice-call/src/manager/store.ts +++ b/extensions/voice-call/src/manager/store.ts @@ -1,6 +1,9 @@ -import fs from "node:fs"; -import fsp from "node:fs/promises"; import path from "node:path"; +import { + appendRegularFile, + privateFileStore, + privateFileStoreSync, +} from "openclaw/plugin-sdk/security-runtime"; import { CallRecordSchema, TerminalStates, type CallId, type CallRecord } from "../types.js"; const pendingPersistWrites = new Set>(); @@ -9,8 +12,11 @@ export function persistCallRecord(storePath: string, call: CallRecord): void { const logPath = path.join(storePath, "calls.jsonl"); const line = `${JSON.stringify(call)}\n`; // Fire-and-forget async write to avoid blocking event loop. - const write = fsp - .appendFile(logPath, line) + const write = appendRegularFile({ + filePath: logPath, + content: line, + rejectSymlinkParents: true, + }) .catch((err) => { console.error("[voice-call] Failed to persist call record:", err); }) @@ -31,7 +37,8 @@ export function loadActiveCallsFromStore(storePath: string): { rejectedProviderCallIds: Set; } { const logPath = path.join(storePath, "calls.jsonl"); - if (!fs.existsSync(logPath)) { + const content = privateFileStoreSync(storePath).readTextIfExists(path.basename(logPath)); + if (content === null) { return { activeCalls: new Map(), providerCallIdMap: new Map(), @@ -39,8 +46,6 @@ export function loadActiveCallsFromStore(storePath: string): { rejectedProviderCallIds: new Set(), }; } - - const content = fs.readFileSync(logPath, "utf-8"); const lines = content.split("\n"); const callMap = new Map(); @@ -82,14 +87,10 @@ export async function getCallHistoryFromStore( limit = 50, ): Promise { const logPath = path.join(storePath, "calls.jsonl"); - - try { - await fsp.access(logPath); - } catch { + const content = await privateFileStore(storePath).readTextIfExists(path.basename(logPath)); + if (content === null) { return []; } - - const content = await fsp.readFile(logPath, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); const calls: CallRecord[] = []; diff --git a/extensions/voice-call/src/realtime-fast-context.ts b/extensions/voice-call/src/realtime-fast-context.ts index 3e95b3dfd8e..d5c0fb1cdde 100644 --- a/extensions/voice-call/src/realtime-fast-context.ts +++ b/extensions/voice-call/src/realtime-fast-context.ts @@ -5,6 +5,7 @@ import { parseRealtimeVoiceAgentConsultArgs, type RealtimeVoiceAgentConsultResult, } from "openclaw/plugin-sdk/realtime-voice"; +import { withTimeout } from "openclaw/plugin-sdk/security-runtime"; import type { VoiceCallRealtimeFastContextConfig } from "./config.js"; type Logger = { @@ -74,22 +75,6 @@ function buildMissText(query: string): string { ].join("\n\n"); } -async function withTimeout(promise: Promise, timeoutMs: number): Promise { - let timer: ReturnType | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new RealtimeFastContextTimeoutError(timeoutMs)), timeoutMs); - }), - ]); - } finally { - if (timer) { - clearTimeout(timer); - } - } -} - async function lookupFastContext(params: { cfg: OpenClawConfig; agentId: string; @@ -138,6 +123,7 @@ export async function resolveRealtimeFastContextConsult(params: { query, }), params.config.timeoutMs, + { createError: () => new RealtimeFastContextTimeoutError(params.config.timeoutMs) }, ); if (lookup.status === "unavailable") { params.logger.debug?.(`[voice-call] realtime fast context unavailable: ${lookup.error}`); diff --git a/extensions/whatsapp/src/auth-store.ts b/extensions/whatsapp/src/auth-store.ts index 77c676bb5b5..34fdc986fa9 100644 --- a/extensions/whatsapp/src/auth-store.ts +++ b/extensions/whatsapp/src/auth-store.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; @@ -7,6 +6,7 @@ import { DEFAULT_ACCOUNT_ID } from "openclaw/plugin-sdk/routing"; import { info, success } from "openclaw/plugin-sdk/runtime-env"; import { getChildLogger } from "openclaw/plugin-sdk/runtime-env"; import { defaultRuntime, type RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { resolveOAuthDir } from "./auth-store.runtime.js"; import { hasWebCredsSync, resolveWebCredsBackupPath, resolveWebCredsPath } from "./creds-files.js"; import { @@ -72,7 +72,6 @@ async function waitForWebAuthBarrier( export async function restoreCredsFromBackupIfNeeded(authDir: string): Promise { const logger = getChildLogger({ module: "web-session" }); - let tempRestorePath: string | null = null; try { const credsPath = resolveWebCredsPath(authDir); const backupPath = resolveWebCredsBackupPath(authDir); @@ -94,24 +93,17 @@ export async function restoreCredsFromBackupIfNeeded(authDir: string): Promise { - // best-effort temp cleanup - }); - } } return false; } diff --git a/extensions/whatsapp/src/creds-persistence.ts b/extensions/whatsapp/src/creds-persistence.ts index 7930dc8ffbb..5779b336a6e 100644 --- a/extensions/whatsapp/src/creds-persistence.ts +++ b/extensions/whatsapp/src/creds-persistence.ts @@ -1,6 +1,4 @@ -import { randomUUID } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; import { resolveWebCredsPath } from "./creds-files.js"; const CREDS_FILE_MODE = 0o600; @@ -15,47 +13,18 @@ async function stringifyCreds(creds: unknown): Promise { return JSON.stringify(creds, BufferJSON.replacer); } -async function syncDirectory(dirPath: string): Promise { - let handle: Awaited> | undefined; - try { - handle = await fs.open(dirPath, "r"); - await handle.sync(); - } catch { - // best-effort on platforms that do not support directory fsync - } finally { - await handle?.close().catch(() => { - // best-effort close - }); - } -} - export async function writeCredsJsonAtomically(authDir: string, creds: unknown): Promise { const credsPath = resolveWebCredsPath(authDir); - const tempPath = path.join(authDir, `.creds.${process.pid}.${randomUUID()}.tmp`); const json = await stringifyCreds(creds); - - let handle: Awaited> | undefined; - try { - handle = await fs.open(tempPath, "w", CREDS_FILE_MODE); - await handle.writeFile(json, { encoding: "utf-8" }); - await handle.sync(); - await handle.close(); - handle = undefined; - - await fs.rename(tempPath, credsPath); - await fs.chmod(credsPath, CREDS_FILE_MODE).catch(() => { - // best-effort on platforms that support it - }); - await syncDirectory(path.dirname(credsPath)); - } catch (error) { - await handle?.close().catch(() => { - // best-effort close - }); - await fs.rm(tempPath, { force: true }).catch(() => { - // best-effort cleanup - }); - throw error; - } + await replaceFileAtomic({ + filePath: credsPath, + content: json, + dirMode: 0o700, + mode: CREDS_FILE_MODE, + tempPrefix: ".creds", + syncTempFile: true, + syncParentDir: true, + }); } export function enqueueCredsSave( diff --git a/extensions/whatsapp/src/outbound-media-contract.ts b/extensions/whatsapp/src/outbound-media-contract.ts index 8db8e3883ac..7ed5d83529d 100644 --- a/extensions/whatsapp/src/outbound-media-contract.ts +++ b/extensions/whatsapp/src/outbound-media-contract.ts @@ -1,8 +1,7 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, runFfmpeg } from "openclaw/plugin-sdk/media-runtime"; import { sanitizeForPlainText } from "openclaw/plugin-sdk/outbound-runtime"; -import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; +import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; import { formatError } from "./session-errors.js"; import { sanitizeAssistantVisibleText, @@ -184,41 +183,38 @@ async function transcodeToWhatsAppVoiceOpus(params: { buffer: Buffer; fileName: string; }): Promise { - const tempRoot = resolvePreferredOpenClawTmpDir(); - await fs.mkdir(tempRoot, { recursive: true, mode: 0o700 }); - const tempDir = await fs.mkdtemp(path.join(tempRoot, "whatsapp-voice-")); - try { - const ext = path.extname(params.fileName).toLowerCase(); - const inputExt = ext && ext.length <= 12 ? ext : ".audio"; - const inputPath = path.join(tempDir, `input${inputExt}`); - const outputPath = path.join(tempDir, WHATSAPP_VOICE_FILE_NAME); - await fs.writeFile(inputPath, params.buffer, { mode: 0o600 }); - await runFfmpeg([ - "-hide_banner", - "-loglevel", - "error", - "-y", - "-i", - inputPath, - "-vn", - "-sn", - "-dn", - "-t", - String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), - "-ar", - String(WHATSAPP_VOICE_SAMPLE_RATE_HZ), - "-ac", - "1", - "-c:a", - "libopus", - "-b:a", - WHATSAPP_VOICE_BITRATE, - outputPath, - ]); - return await fs.readFile(outputPath); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "whatsapp-voice-" }, + async (workspace) => { + const ext = path.extname(params.fileName).toLowerCase(); + const inputExt = ext && ext.length <= 12 ? ext : ".audio"; + const inputPath = await workspace.write(`input${inputExt}`, params.buffer); + const outputPath = workspace.path(WHATSAPP_VOICE_FILE_NAME); + await runFfmpeg([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + inputPath, + "-vn", + "-sn", + "-dn", + "-t", + String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), + "-ar", + String(WHATSAPP_VOICE_SAMPLE_RATE_HZ), + "-ac", + "1", + "-c:a", + "libopus", + "-b:a", + WHATSAPP_VOICE_BITRATE, + outputPath, + ]); + return await workspace.read(WHATSAPP_VOICE_FILE_NAME); + }, + ); } function deriveWhatsAppDocumentFileName(mediaUrl: string | undefined): string | undefined { diff --git a/extensions/whatsapp/src/session.test.ts b/extensions/whatsapp/src/session.test.ts index 0ca9b11dc22..f67d7ece2d8 100644 --- a/extensions/whatsapp/src/session.test.ts +++ b/extensions/whatsapp/src/session.test.ts @@ -39,10 +39,9 @@ function createTempAuthDir(prefix: string) { function mockFsOpenForCredsWrites(params?: { onTempWrite?: (filePath: string) => Promise | void; }) { - const open = fs.open.bind(fs); + const writeFile = fs.writeFile.bind(fs); const tempHandles: Array<{ filePath: string; - writeFile: ReturnType; sync: ReturnType; close: ReturnType; }> = []; @@ -51,13 +50,20 @@ function mockFsOpenForCredsWrites(params?: { sync: ReturnType; close: ReturnType; }> = []; + const tempWrites: string[] = []; + const writeFileSpy = vi + .spyOn(fs, "writeFile") + .mockImplementation(async (filePath, data, opts) => { + if (typeof filePath === "string" && filePath.includes(".creds.")) { + tempWrites.push(filePath); + await params?.onTempWrite?.(filePath); + } + return await writeFile(filePath as never, data as never, opts as never); + }); const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { - if (typeof filePath === "string" && flags === "w" && filePath.includes(".creds.")) { + if (typeof filePath === "string" && flags === "r+" && filePath.includes(".creds.")) { const handle = { filePath, - writeFile: vi.fn(async () => { - await params?.onTempWrite?.(filePath); - }), sync: vi.fn(async () => {}), close: vi.fn(async () => {}), }; @@ -73,13 +79,18 @@ function mockFsOpenForCredsWrites(params?: { dirHandles.push(handle); return handle as never; } - return open(filePath as never, flags as never, mode as never); + throw new Error( + `unexpected fs.open call: ${String(filePath)} ${String(flags)} ${String(mode)}`, + ); }); return { openSpy, + writeFileSpy, + tempWrites, tempHandles, dirHandles, restore() { + writeFileSpy.mockRestore(); openSpy.mockRestore(); }, }; @@ -184,10 +195,10 @@ describe("web session", () => { expect(typeof passedLogger?.trace).toBe("function"); await emitCredsUpdate(authDir); - expect(openMock.openSpy).toHaveBeenCalledWith( + expect(openMock.writeFileSpy).toHaveBeenCalledWith( expect.stringContaining(path.join(authDir, ".creds.")), - "w", - 0o600, + expect.any(String), + expect.objectContaining({ mode: 0o600, flag: "wx" }), ); openMock.restore(); }); @@ -355,6 +366,7 @@ describe("web session", () => { await createWaSocket(false, false); await emitCredsUpdate(); + await waitForCredsSaveQueue(); expect(creds.copySpy).not.toHaveBeenCalled(); expect(openMock.tempHandles).toHaveLength(1); @@ -470,6 +482,7 @@ describe("web session", () => { await createWaSocket(false, false); await emitCredsUpdate(); + await waitForCredsSaveQueue(); expect(creds.copySpy).toHaveBeenCalledTimes(1); const args = creds.copySpy.mock.calls[0] ?? []; @@ -487,31 +500,42 @@ describe("web session", () => { const rmSpy = vi.spyOn(fs, "rm").mockResolvedValue(undefined); const chmodSpy = vi.spyOn(fs, "chmod").mockResolvedValue(undefined); - await writeCredsJsonAtomically("/tmp/openclaw-oauth/whatsapp/default", { - me: { id: "123@s.whatsapp.net" }, - }); + try { + await writeCredsJsonAtomically("/tmp/openclaw-oauth/whatsapp/default", { + me: { id: "123@s.whatsapp.net" }, + }); - expect(openMock.tempHandles).toHaveLength(1); - expect(openMock.tempHandles[0]?.writeFile).toHaveBeenCalledTimes(1); - expect(openMock.tempHandles[0]?.sync).toHaveBeenCalledTimes(1); - expect(openMock.tempHandles[0]?.close).toHaveBeenCalledTimes(1); - expect(renameSpy).toHaveBeenCalledTimes(1); - expect(rmSpy).not.toHaveBeenCalled(); - expect(chmodSpy).toHaveBeenCalledOnce(); - expect(openMock.dirHandles).toHaveLength(1); - expect(openMock.dirHandles[0]?.sync).toHaveBeenCalledTimes(1); - const writePath = openMock.tempHandles[0]?.filePath; - const renameArgs = renameSpy.mock.calls[0] ?? []; - expect(typeof writePath).toBe("string"); - expect(writePath).toContain(".creds."); - expect(String(renameArgs[1] ?? "")).toContain( - path.join("/tmp", "openclaw-oauth", "whatsapp", "default", "creds.json"), - ); - - openMock.restore(); - renameSpy.mockRestore(); - rmSpy.mockRestore(); - chmodSpy.mockRestore(); + expect(openMock.writeFileSpy).toHaveBeenCalledWith( + expect.stringContaining( + path.join("/tmp", "openclaw-oauth", "whatsapp", "default", ".creds."), + ), + expect.any(String), + expect.objectContaining({ mode: 0o600, flag: "wx" }), + ); + expect(openMock.tempHandles).toHaveLength(1); + expect(openMock.tempHandles[0]?.sync).toHaveBeenCalledTimes(1); + expect(openMock.tempHandles[0]?.close).toHaveBeenCalledTimes(1); + expect(renameSpy).toHaveBeenCalledTimes(1); + expect(rmSpy).not.toHaveBeenCalled(); + expect(chmodSpy).toHaveBeenCalledWith( + path.join("/tmp", "openclaw-oauth", "whatsapp", "default", "creds.json"), + 0o600, + ); + expect(openMock.dirHandles).toHaveLength(1); + expect(openMock.dirHandles[0]?.sync).toHaveBeenCalledTimes(1); + const writePath = openMock.tempHandles[0]?.filePath; + const renameArgs = renameSpy.mock.calls[0] ?? []; + expect(typeof writePath).toBe("string"); + expect(writePath).toContain(".creds."); + expect(String(renameArgs[1] ?? "")).toContain( + path.join("/tmp", "openclaw-oauth", "whatsapp", "default", "creds.json"), + ); + } finally { + openMock.restore(); + renameSpy.mockRestore(); + rmSpy.mockRestore(); + chmodSpy.mockRestore(); + } }); it("keeps the previous creds.json valid if the atomic rename fails", async () => { diff --git a/extensions/whatsapp/src/state-migrations.ts b/extensions/whatsapp/src/state-migrations.ts index d29791dd2e1..3c869bc4fd1 100644 --- a/extensions/whatsapp/src/state-migrations.ts +++ b/extensions/whatsapp/src/state-migrations.ts @@ -2,10 +2,11 @@ import fs from "node:fs"; import path from "node:path"; import { DEFAULT_ACCOUNT_ID } from "openclaw/plugin-sdk/account-id"; import type { ChannelLegacyStateMigrationPlan } from "openclaw/plugin-sdk/channel-contract"; +import { statRegularFileSync } from "openclaw/plugin-sdk/security-runtime"; function fileExists(pathValue: string): boolean { try { - return fs.existsSync(pathValue) && fs.statSync(pathValue).isFile(); + return !statRegularFileSync(pathValue).missing; } catch { return false; } diff --git a/extensions/zalo/src/outbound-media.ts b/extensions/zalo/src/outbound-media.ts index 4e78304de3b..db4a401dbc7 100644 --- a/extensions/zalo/src/outbound-media.ts +++ b/extensions/zalo/src/outbound-media.ts @@ -1,9 +1,10 @@ import { randomBytes } from "node:crypto"; import { rmSync } from "node:fs"; -import { chmod, mkdir, readdir, readFile, stat, unlink, writeFile } from "node:fs/promises"; +import { readdir, readFile, stat, unlink } from "node:fs/promises"; import type { IncomingMessage, ServerResponse } from "node:http"; import { join } from "node:path"; import { loadOutboundMediaFromUrl } from "openclaw/plugin-sdk/outbound-media"; +import { privateFileStore } from "openclaw/plugin-sdk/security-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { resolveWebhookPath } from "openclaw/plugin-sdk/webhook-ingress"; @@ -40,8 +41,8 @@ function createHostedZaloMediaToken(): string { } async function ensureHostedZaloMediaDir(): Promise { - await mkdir(ZALO_OUTBOUND_MEDIA_DIR, { recursive: true, mode: 0o700 }); - await chmod(ZALO_OUTBOUND_MEDIA_DIR, 0o700).catch(() => undefined); + await privateFileStore(ZALO_OUTBOUND_MEDIA_DIR).writeText(".ready", ""); + await unlink(join(ZALO_OUTBOUND_MEDIA_DIR, ".ready")).catch(() => undefined); } async function deleteHostedZaloMediaEntry(id: string): Promise { @@ -142,18 +143,15 @@ export async function prepareHostedZaloMediaUrl(params: { const token = createHostedZaloMediaToken(); const publicBaseUrl = new URL(params.webhookUrl).origin; - await writeFile(resolveHostedZaloMediaBufferPath(id), media.buffer, { mode: 0o600 }); + const store = privateFileStore(ZALO_OUTBOUND_MEDIA_DIR); + await store.writeText(`${id}.bin`, media.buffer); try { - await writeFile( - resolveHostedZaloMediaMetadataPath(id), - JSON.stringify({ - routePath, - token, - contentType: media.contentType, - expiresAt: Date.now() + ZALO_OUTBOUND_MEDIA_TTL_MS, - } satisfies HostedZaloMediaMetadata), - { encoding: "utf8", mode: 0o600 }, - ); + await store.writeJson(`${id}.json`, { + routePath, + token, + contentType: media.contentType, + expiresAt: Date.now() + ZALO_OUTBOUND_MEDIA_TTL_MS, + } satisfies HostedZaloMediaMetadata); } catch (error) { await deleteHostedZaloMediaEntry(id); throw error; diff --git a/extensions/zalouser/src/zalo-js.ts b/extensions/zalouser/src/zalo-js.ts index 90cdee92951..3859e8b66dc 100644 --- a/extensions/zalouser/src/zalo-js.ts +++ b/extensions/zalouser/src/zalo-js.ts @@ -3,6 +3,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { loadOutboundMediaFromUrl } from "openclaw/plugin-sdk/outbound-media"; +import { + privateFileStoreSync, + readRegularFileSync, + statRegularFileSync, + withTimeout, +} from "openclaw/plugin-sdk/security-runtime"; import { resolveStateDir as resolvePluginStateDir } from "openclaw/plugin-sdk/state-paths"; import { normalizeLowercaseStringOrEmpty, @@ -118,25 +124,9 @@ function isNodeErrorCode(error: unknown, code: string): boolean { ); } -function ensureCredentialsDir(): string { - const dir = resolveCredentialsDir(); - fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); - const stat = fs.lstatSync(dir); - if (!stat.isDirectory() || stat.isSymbolicLink()) { - throw new Error("Refusing to use non-directory Zalo credentials path"); - } - try { - fs.chmodSync(dir, 0o700); - } catch { - // Best-effort on platforms that support POSIX permissions. - } - return dir; -} - function isReadableCredentialFile(filePath: string): boolean { try { - const stat = fs.lstatSync(filePath); - return stat.isFile() && !stat.isSymbolicLink(); + return !statRegularFileSync(filePath).missing; } catch (error) { if (isNodeErrorCode(error, "ENOENT")) { return false; @@ -145,61 +135,8 @@ function isReadableCredentialFile(filePath: string): boolean { } } -function assertWritableCredentialTarget(filePath: string): void { - try { - const stat = fs.lstatSync(filePath); - if (!stat.isFile() || stat.isSymbolicLink()) { - throw new Error("Refusing to write Zalo credentials to symlinked path"); - } - } catch (error) { - if (isNodeErrorCode(error, "ENOENT")) { - return; - } - throw error; - } -} - function writeCredentialFileAtomic(filePath: string, payload: string): void { - const dir = ensureCredentialsDir(); - assertWritableCredentialTarget(filePath); - const tempPath = path.join(dir, `.${path.basename(filePath)}.tmp-${process.pid}-${randomUUID()}`); - try { - fs.writeFileSync(tempPath, payload, { encoding: "utf-8", mode: 0o600, flag: "wx" }); - try { - fs.chmodSync(tempPath, 0o600); - } catch { - // Best-effort on platforms that support POSIX permissions. - } - fs.renameSync(tempPath, filePath); - try { - fs.chmodSync(filePath, 0o600); - } catch { - // Best-effort on platforms that support POSIX permissions. - } - } finally { - try { - fs.unlinkSync(tempPath); - } catch { - // The temp file is normally moved by renameSync. - } - } -} - -function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - reject(new Error(label)); - }, timeoutMs); - void promise - .then((result) => { - clearTimeout(timer); - resolve(result); - }) - .catch((err) => { - clearTimeout(timer); - reject(err); - }); - }); + privateFileStoreSync(resolveCredentialsDir()).writeText(path.basename(filePath), payload); } function delay(ms: number): Promise { @@ -620,7 +557,7 @@ function readCredentials(profile: string): StoredZaloCredentials | null { if (!isReadableCredentialFile(filePath)) { return null; } - const raw = fs.readFileSync(filePath, "utf-8"); + const raw = readRegularFileSync({ filePath }).buffer.toString("utf-8"); const parsed = JSON.parse(raw) as Partial; if ( typeof parsed.imei !== "string" || @@ -814,7 +751,7 @@ async function ensureApi( language: stored.language, }), timeoutMs, - `Timed out restoring Zalo session for profile "${profile}"`, + { message: `Timed out restoring Zalo session for profile "${profile}"` }, ); apiByProfile.set(profile, api); writeApiCredentials(profile, api, stored); @@ -1040,7 +977,9 @@ export async function checkZaloAuthenticated(profileInput?: string | null): Prom await withZaloApi( profile, async (api) => - await withTimeout(api.fetchAccountInfo(), 12_000, "Timed out checking Zalo session"), + await withTimeout(api.fetchAccountInfo(), 12_000, { + message: "Timed out checking Zalo session", + }), { timeoutMs: 12_000 }, ); return true; diff --git a/package.json b/package.json index fc8ef3bc54a..2e8e8370562 100644 --- a/package.json +++ b/package.json @@ -1695,6 +1695,7 @@ "@mariozechner/pi-tui": "0.73.0", "@modelcontextprotocol/sdk": "1.29.0", "@mozilla/readability": "^0.6.0", + "@openclaw/fs-safe": "^0.1.0", "@slack/bolt": "^4.7.2", "@slack/types": "^2.21.0", "@slack/web-api": "^7.15.2", diff --git a/packages/memory-host-sdk/src/engine-foundation.ts b/packages/memory-host-sdk/src/engine-foundation.ts index bbd64312047..0c8400ed7f3 100644 --- a/packages/memory-host-sdk/src/engine-foundation.ts +++ b/packages/memory-host-sdk/src/engine-foundation.ts @@ -21,7 +21,8 @@ export { hasConfiguredSecretInput, normalizeResolvedSecretInputString, } from "./host/openclaw-runtime-config.js"; -export { writeFileWithinRoot } from "./host/openclaw-runtime-io.js"; +export { root } from "./host/openclaw-runtime-io.js"; +export { isPathInside } from "./host/fs-utils.js"; export { createSubsystemLogger } from "./host/openclaw-runtime-io.js"; export { detectMime } from "./host/openclaw-runtime-io.js"; export { resolveGlobalSingleton } from "./host/openclaw-runtime-io.js"; diff --git a/packages/memory-host-sdk/src/host/backend-config.ts b/packages/memory-host-sdk/src/host/backend-config.ts index cda2e9744b2..61e0bd7efdb 100644 --- a/packages/memory-host-sdk/src/host/backend-config.ts +++ b/packages/memory-host-sdk/src/host/backend-config.ts @@ -17,6 +17,7 @@ import { type SessionSendPolicyConfig, splitShellArgs, } from "./config-utils.js"; +import { isPathInside } from "./fs-utils.js"; import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; export type ResolvedMemoryBackendConfig = { @@ -143,11 +144,10 @@ function canonicalizePathForContainment(rawPath: string): string { } function isPathInsideRoot(candidatePath: string, rootPath: string): boolean { - const relative = path.relative( + return isPathInside( canonicalizePathForContainment(rootPath), canonicalizePathForContainment(candidatePath), ); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); } function ensureUniqueName(base: string, existing: Set): string { diff --git a/packages/memory-host-sdk/src/host/fs-utils.ts b/packages/memory-host-sdk/src/host/fs-utils.ts index 81107c7ef3d..2461dba8285 100644 --- a/packages/memory-host-sdk/src/host/fs-utils.ts +++ b/packages/memory-host-sdk/src/host/fs-utils.ts @@ -1,7 +1,11 @@ -import type { Stats } from "node:fs"; -import fs from "node:fs/promises"; - -export type RegularFileStatResult = { missing: true } | { missing: false; stat: Stats }; +import "../../../../src/infra/fs-safe-defaults.js"; +export { isPathInside } from "@openclaw/fs-safe/path"; +export { + readRegularFile, + statRegularFile, + type RegularFileStatResult, +} from "@openclaw/fs-safe/advanced"; +export { walkDirectory, type WalkDirectoryEntry } from "@openclaw/fs-safe/walk"; export function isFileMissingError( err: unknown, @@ -13,19 +17,3 @@ export function isFileMissingError( (err as Partial).code === "ENOENT", ); } - -export async function statRegularFile(absPath: string): Promise { - let stat: Stats; - try { - stat = await fs.lstat(absPath); - } catch (err) { - if (isFileMissingError(err)) { - return { missing: true }; - } - throw err; - } - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error("path required"); - } - return { missing: false, stat }; -} diff --git a/packages/memory-host-sdk/src/host/internal.ts b/packages/memory-host-sdk/src/host/internal.ts index 162cac93d64..70f6af914cd 100644 --- a/packages/memory-host-sdk/src/host/internal.ts +++ b/packages/memory-host-sdk/src/host/internal.ts @@ -5,7 +5,13 @@ import path from "node:path"; import { CANONICAL_ROOT_MEMORY_FILENAME } from "./config-utils.js"; import { estimateStructuredEmbeddingInputBytes } from "./embedding-input-limits.js"; import { buildTextEmbeddingInput, type EmbeddingInput } from "./embedding-inputs.js"; -import { isFileMissingError } from "./fs-utils.js"; +import { + isFileMissingError, + readRegularFile, + statRegularFile, + walkDirectory, + type WalkDirectoryEntry, +} from "./fs-utils.js"; import { buildMemoryMultimodalLabel, classifyMemoryMultimodalPath, @@ -103,36 +109,31 @@ function isAllowedMemoryFilePath(filePath: string, multimodal?: MemoryMultimodal ); } -async function walkDir( +function shouldDescendMemoryEntry( + entry: WalkDirectoryEntry, + shouldSkipPath?: (absPath: string) => boolean, +): boolean { + if (shouldSkipPath?.(entry.path)) { + return false; + } + return entry.kind === "directory" && entry.name !== ".openclaw-repair"; +} + +async function collectMemoryFilesFromDir( dir: string, files: string[], multimodal?: MemoryMultimodalSettings, shouldSkipPath?: (absPath: string) => boolean, -) { - const entries = await fs.readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const full = path.join(dir, entry.name); - if (shouldSkipPath?.(full)) { - continue; - } - if (entry.isSymbolicLink()) { - continue; - } - if (entry.isDirectory()) { - if (entry.name === ".openclaw-repair") { - continue; - } - await walkDir(full, files, multimodal, shouldSkipPath); - continue; - } - if (!entry.isFile()) { - continue; - } - if (!isAllowedMemoryFilePath(full, multimodal)) { - continue; - } - files.push(full); - } +): Promise { + const scan = await walkDirectory(dir, { + symlinks: "skip", + descend: (entry) => shouldDescendMemoryEntry(entry, shouldSkipPath), + include: (entry) => + !shouldSkipPath?.(entry.path) && + entry.kind === "file" && + isAllowedMemoryFilePath(entry.path, multimodal), + }); + files.push(...scan.entries.map((entry) => entry.path)); } export async function listMemoryFiles( @@ -148,8 +149,8 @@ export async function listMemoryFiles( const addMarkdownFile = async (absPath: string) => { try { - const stat = await fs.lstat(absPath); - if (stat.isSymbolicLink() || !stat.isFile()) { + const stat = await statRegularFile(absPath); + if (stat.missing) { return; } if (!absPath.endsWith(".md")) { @@ -166,7 +167,7 @@ export async function listMemoryFiles( try { const dirStat = await fs.lstat(memoryDir); if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) { - await walkDir(memoryDir, result, multimodal, shouldSkipWorkspaceMemoryPath); + await collectMemoryFilesFromDir(memoryDir, result, multimodal, shouldSkipWorkspaceMemoryPath); } } catch {} @@ -182,7 +183,12 @@ export async function listMemoryFiles( continue; } if (stat.isDirectory()) { - await walkDir(inputPath, result, multimodal, shouldSkipWorkspaceMemoryPath); + await collectMemoryFilesFromDir( + inputPath, + result, + multimodal, + shouldSkipWorkspaceMemoryPath, + ); continue; } if (stat.isFile() && isAllowedMemoryFilePath(inputPath, multimodal)) { @@ -215,15 +221,11 @@ export async function buildFileEntry( workspaceDir: string, multimodal?: MemoryMultimodalSettings, ): Promise { - let stat; - try { - stat = await fs.stat(absPath); - } catch (err) { - if (isFileMissingError(err)) { - return null; - } - throw err; + const regularFile = await statRegularFile(absPath); + if (regularFile.missing) { + return null; } + const stat = regularFile.stat; const normalizedPath = path.relative(workspaceDir, absPath).replace(/\\/g, "/"); const multimodalSettings = multimodal ?? DISABLED_MULTIMODAL_SETTINGS; const modality = classifyMemoryMultimodalPath(absPath, multimodalSettings); @@ -233,7 +235,12 @@ export async function buildFileEntry( } let buffer: Buffer; try { - buffer = await fs.readFile(absPath); + buffer = ( + await readRegularFile({ + filePath: absPath, + maxBytes: multimodalSettings.maxFileBytes, + }) + ).buffer; } catch (err) { if (isFileMissingError(err)) { return null; @@ -269,7 +276,7 @@ export async function buildFileEntry( } let content: string; try { - content = await fs.readFile(absPath, "utf-8"); + content = (await readRegularFile({ filePath: absPath })).buffer.toString("utf-8"); } catch (err) { if (isFileMissingError(err)) { return null; @@ -296,21 +303,17 @@ async function loadMultimodalEmbeddingInput( if (entry.kind !== "multimodal" || !entry.contentText || !entry.mimeType) { return null; } - let stat; - try { - stat = await fs.stat(entry.absPath); - } catch (err) { - if (isFileMissingError(err)) { - return null; - } - throw err; + const regularFile = await statRegularFile(entry.absPath); + if (regularFile.missing) { + return null; } + const stat = regularFile.stat; if (stat.size !== entry.size) { return null; } let buffer: Buffer; try { - buffer = await fs.readFile(entry.absPath); + buffer = (await readRegularFile({ filePath: entry.absPath, maxBytes: entry.size })).buffer; } catch (err) { if (isFileMissingError(err)) { return null; diff --git a/packages/memory-host-sdk/src/host/openclaw-runtime-io.ts b/packages/memory-host-sdk/src/host/openclaw-runtime-io.ts index cc24b099462..3d7567acc1d 100644 --- a/packages/memory-host-sdk/src/host/openclaw-runtime-io.ts +++ b/packages/memory-host-sdk/src/host/openclaw-runtime-io.ts @@ -4,6 +4,7 @@ export { DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS, applyWindowsSpawnProgramPolicy, configureSqliteWalMaintenance, + root, createSubsystemLogger, detectMime, estimateStringChars, @@ -21,7 +22,6 @@ export { shouldIgnoreWarning, splitShellArgs, truncateUtf16Safe, - writeFileWithinRoot, } from "./openclaw-runtime.js"; export type { diff --git a/packages/memory-host-sdk/src/host/openclaw-runtime.ts b/packages/memory-host-sdk/src/host/openclaw-runtime.ts index 4449b1814d0..e4e649a501c 100644 --- a/packages/memory-host-sdk/src/host/openclaw-runtime.ts +++ b/packages/memory-host-sdk/src/host/openclaw-runtime.ts @@ -74,7 +74,7 @@ export { isVerbose, setVerbose } from "../../../../src/globals.js"; // IO, network, and logging helpers. export { isExecCompletionEvent } from "../../../../src/infra/heartbeat-events-filter.js"; -export { writeFileWithinRoot } from "../../../../src/infra/fs-safe.js"; +export { root } from "../../../../src/infra/fs-safe.js"; export { fetchWithSsrFGuard } from "../../../../src/infra/net/fetch-guard.js"; export { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js"; export { ssrfPolicyFromHttpBaseUrlAllowedHostname } from "../../../../src/infra/net/ssrf.js"; diff --git a/packages/memory-host-sdk/src/host/read-file.ts b/packages/memory-host-sdk/src/host/read-file.ts index 4cb60a06538..a8a405b3ff8 100644 --- a/packages/memory-host-sdk/src/host/read-file.ts +++ b/packages/memory-host-sdk/src/host/read-file.ts @@ -6,7 +6,7 @@ import { resolveMemorySearchConfig, type OpenClawConfig, } from "./config-utils.js"; -import { isFileMissingError, statRegularFile } from "./fs-utils.js"; +import { isFileMissingError, isPathInside, readRegularFile, statRegularFile } from "./fs-utils.js"; import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js"; import { buildMemoryReadResult, @@ -43,7 +43,11 @@ export async function readMemoryFile(params: { continue; } if (stat.isDirectory()) { - if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) { + if (isPathInside(additionalPath, absPath)) { + const candidateStat = await fs.lstat(absPath).catch(() => null); + if (candidateStat?.isSymbolicLink()) { + continue; + } allowedAdditional = true; break; } @@ -68,7 +72,7 @@ export async function readMemoryFile(params: { } let content: string; try { - content = await fs.readFile(absPath, "utf-8"); + content = (await readRegularFile({ filePath: absPath })).buffer.toString("utf-8"); } catch (err) { if (isFileMissingError(err)) { return { text: "", path: relPath }; diff --git a/packages/memory-host-sdk/src/host/session-files.ts b/packages/memory-host-sdk/src/host/session-files.ts index 6010bb9f94a..824cebfe522 100644 --- a/packages/memory-host-sdk/src/host/session-files.ts +++ b/packages/memory-host-sdk/src/host/session-files.ts @@ -1,6 +1,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { readRegularFile, statRegularFile } from "./fs-utils.js"; import { hashText } from "./hash.js"; import { createSubsystemLogger, redactSensitiveText } from "./openclaw-runtime-io.js"; import { @@ -524,7 +525,11 @@ export async function buildSessionEntry( opts: BuildSessionEntryOptions = {}, ): Promise { try { - const stat = await fs.stat(absPath); + const regularFile = await statRegularFile(absPath); + if (regularFile.missing) { + return null; + } + const stat = regularFile.stat; if (shouldSkipTranscriptFileForDreaming(absPath)) { return { path: sessionPathForFile(absPath), @@ -537,7 +542,7 @@ export async function buildSessionEntry( messageTimestampsMs: [], }; } - const raw = await fs.readFile(absPath, "utf-8"); + const raw = (await readRegularFile({ filePath: absPath })).buffer.toString("utf-8"); const lines = raw.split("\n"); const collected: string[] = []; const lineMap: number[] = []; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c76fc38871..e9b8fc0f4cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,6 +103,9 @@ importers: '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 + '@openclaw/fs-safe': + specifier: ^0.1.0 + version: 0.1.0 '@slack/bolt': specifier: ^4.7.2 version: 4.7.2(@types/express@5.0.6) @@ -3230,6 +3233,10 @@ packages: cpu: [x64] os: [win32] + '@openclaw/fs-safe@0.1.0': + resolution: {integrity: sha512-ByFeXAA+7EXje15eaI3mqee9STFcglTkfdhenZQ2V4OGAkhkHzb2yxe3hInuWXYImG9gDolbccNEnwlsoTayaA==} + engines: {node: '>=20.11'} + '@opentelemetry/api-logs@0.216.0': resolution: {integrity: sha512-KmGTgvxTJ0J01d4mOeX1wMV5NUTNf9HebIuOOGDfIn0a/IrnXIQbOnlylDyl9tkDv4h0DUpdI/GqCdLzfTkUXg==} engines: {node: '>=8.0.0'} @@ -9998,6 +10005,11 @@ snapshots: '@openai/codex@0.128.0-win32-x64': optional: true + '@openclaw/fs-safe@0.1.0': + optionalDependencies: + jszip: 3.10.1 + tar: 7.5.13 + '@opentelemetry/api-logs@0.216.0': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dcfe6f1e47c..50933fa10fb 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -33,6 +33,7 @@ minimumReleaseAgeExclude: - "sqlite-vec-*" onlyBuiltDependencies: + - "@openclaw/fs-safe" - "@discordjs/opus" - "@google/genai" - "@lydell/node-pty" diff --git a/scripts/openclaw-npm-release-check.ts b/scripts/openclaw-npm-release-check.ts index b32574be439..fb8a9777a1d 100644 --- a/scripts/openclaw-npm-release-check.ts +++ b/scripts/openclaw-npm-release-check.ts @@ -63,6 +63,7 @@ export type NpmDistTagMirrorAuth = { }; const EXPECTED_REPOSITORY_URL = "https://github.com/openclaw/openclaw"; const OPTIONAL_LOCAL_EMBEDDING_RUNTIME_PACKAGE = "node-llama-cpp"; +const FS_SAFE_PACKAGE = "@openclaw/fs-safe"; const MAX_CALVER_DISTANCE_DAYS = 2; const REQUIRED_PACKED_PATHS = [ PACKAGE_DIST_INVENTORY_RELATIVE_PATH, @@ -183,6 +184,10 @@ function normalizeRepoUrl(value: unknown): string { .replace(/\/+$/, ""); } +function isLocalDependencySpec(value: string | undefined): boolean { + return /^(?:file|link|workspace):/u.test(value ?? ""); +} + export function parseReleaseVersion(version: string): ParsedReleaseVersion | null { return parseReleaseVersionBase(version) as ParsedReleaseVersion | null; } @@ -319,6 +324,11 @@ export function collectReleasePackageMetadataErrors(pkg: PackageJson): string[] `package.json dependencies["${OPTIONAL_LOCAL_EMBEDDING_RUNTIME_PACKAGE}"] must be omitted; keep it optional.`, ); } + if (isLocalDependencySpec(pkg.dependencies?.[FS_SAFE_PACKAGE])) { + errors.push( + `package.json dependencies["${FS_SAFE_PACKAGE}"] must use a published semver range before npm release; found "${pkg.dependencies?.[FS_SAFE_PACKAGE]}".`, + ); + } if (pkg.optionalDependencies?.[OPTIONAL_LOCAL_EMBEDDING_RUNTIME_PACKAGE]) { errors.push( `package.json optionalDependencies["${OPTIONAL_LOCAL_EMBEDDING_RUNTIME_PACKAGE}"] must be omitted; keep it operator-installed.`, diff --git a/src/acp/approval-classifier.ts b/src/acp/approval-classifier.ts index 4b6de57261d..0cb1e8a91fb 100644 --- a/src/acp/approval-classifier.ts +++ b/src/acp/approval-classifier.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { isKnownCoreToolId } from "../agents/tool-catalog.js"; import { isMutatingToolCall } from "../agents/tool-mutation.js"; import { resolveOwnerOnlyToolApprovalClass } from "../agents/tool-policy.js"; +import { isPathInside } from "../infra/path-guards.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, @@ -175,9 +176,7 @@ function isReadToolCallScopedToCwd( if (!absolutePath) { return false; } - const root = path.resolve(cwd); - const relative = path.relative(root, absolutePath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); + return isPathInside(path.resolve(cwd), absolutePath); } export function classifyAcpToolApproval(params: { diff --git a/src/agents/acp-spawn-parent-stream.ts b/src/agents/acp-spawn-parent-stream.ts index 6c380caa3a3..d5eb0b125a3 100644 --- a/src/agents/acp-spawn-parent-stream.ts +++ b/src/agents/acp-spawn-parent-stream.ts @@ -1,9 +1,10 @@ -import { appendFile, mkdir } from "node:fs/promises"; +import { mkdir } from "node:fs/promises"; import path from "node:path"; import { readAcpSessionEntry } from "../acp/runtime/session-meta.js"; import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js"; import { onAgentEvent } from "../infra/agent-events.js"; import { requestHeartbeat } from "../infra/heartbeat-wake.js"; +import { appendRegularFile } from "../infra/regular-file.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { scopedHeartbeatWakeOptions } from "../routing/session-key.js"; import { normalizeAssistantPhase } from "../shared/chat-message-content.js"; @@ -130,10 +131,7 @@ export function startAcpSpawnParentStreamRelay(params: { }); logDirReady = true; } - await appendFile(logPath, chunk, { - encoding: "utf-8", - mode: 0o600, - }); + await appendRegularFile({ filePath: logPath, content: chunk }); }) .catch(() => { // Best-effort diagnostics; never break relay flow. diff --git a/src/agents/agent-delete-safety.ts b/src/agents/agent-delete-safety.ts index 3de2e6d4067..29cda1f4720 100644 --- a/src/agents/agent-delete-safety.ts +++ b/src/agents/agent-delete-safety.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isPathInside } from "../infra/path-guards.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { lowercasePreservingWhitespace } from "../shared/string-coerce.js"; import { listAgentEntries, resolveAgentWorkspaceDir } from "./agent-scope.js"; @@ -19,17 +20,11 @@ function normalizeWorkspacePathForComparison(input: string): string { return normalized; } -function isPathWithinRoot(candidatePath: string, rootPath: string): boolean { - const relative = path.relative(rootPath, candidatePath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - function workspacePathsOverlap(left: string, right: string): boolean { const normalizedLeft = normalizeWorkspacePathForComparison(left); const normalizedRight = normalizeWorkspacePathForComparison(right); return ( - isPathWithinRoot(normalizedLeft, normalizedRight) || - isPathWithinRoot(normalizedRight, normalizedLeft) + isPathInside(normalizedRight, normalizedLeft) || isPathInside(normalizedLeft, normalizedRight) ); } diff --git a/src/agents/agent-scope.ts b/src/agents/agent-scope.ts index e992816745c..ceaaa8584a0 100644 --- a/src/agents/agent-scope.ts +++ b/src/agents/agent-scope.ts @@ -5,6 +5,7 @@ import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js"; import type { AgentModelConfig } from "../config/types.agents-shared.js"; import type { AgentConfig } from "../config/types.agents.js"; import type { OpenClawConfig } from "../config/types.js"; +import { isPathInside } from "../infra/path-guards.js"; import { normalizeAgentId, parseAgentSessionKey, @@ -239,11 +240,6 @@ function normalizePathForComparison(input: string): string { return normalized; } -function isPathWithinRoot(candidatePath: string, rootPath: string): boolean { - const relative = path.relative(rootPath, candidatePath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - export function resolveAgentIdsByWorkspacePath( cfg: OpenClawConfig, workspacePath: string, @@ -255,7 +251,7 @@ export function resolveAgentIdsByWorkspacePath( for (let index = 0; index < ids.length; index += 1) { const id = ids[index]; const workspaceDir = normalizePathForComparison(resolveAgentWorkspaceDir(cfg, id)); - if (!isPathWithinRoot(normalizedWorkspacePath, workspaceDir)) { + if (!isPathInside(workspaceDir, normalizedWorkspacePath)) { continue; } matches.push({ id, workspaceDir, order: index }); diff --git a/src/agents/apply-patch.test.ts b/src/agents/apply-patch.test.ts index eff818af141..5cb2fa9bcf3 100644 --- a/src/agents/apply-patch.test.ts +++ b/src/agents/apply-patch.test.ts @@ -9,111 +9,6 @@ import { import { applyPatch } from "./apply-patch.js"; import type { SandboxFsBridge } from "./sandbox/fs-bridge.js"; -const pinnedPathHelper = vi.hoisted(() => { - const fs = require("node:fs/promises") as typeof import("node:fs/promises"); - const path = require("node:path") as typeof import("node:path"); - const { pipeline } = require("node:stream/promises") as typeof import("node:stream/promises"); - - async function resolvePinnedParent(params: { - rootPath: string; - relativeParentPath?: string; - mkdir?: boolean; - }): Promise { - let current = params.rootPath; - for (const segment of (params.relativeParentPath ?? "").split("/").filter(Boolean)) { - const next = path.join(current, segment); - try { - const stat = await fs.lstat(next); - if (stat.isSymbolicLink() || !stat.isDirectory()) { - throw new Error("symbolic link or non-directory path segment"); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT" || !params.mkdir) { - throw error; - } - await fs.mkdir(next); - } - current = next; - } - return current; - } - - return { - runPinnedPathHelper: vi.fn( - async (params: { - operation: "mkdirp" | "remove"; - rootPath: string; - relativePath: string; - }) => { - const segments = params.relativePath.split("/").filter(Boolean); - const targetPath = path.join(params.rootPath, ...segments); - if (params.operation === "mkdirp") { - await resolvePinnedParent({ - rootPath: params.rootPath, - relativeParentPath: params.relativePath, - mkdir: true, - }); - return; - } - await resolvePinnedParent({ - rootPath: params.rootPath, - relativeParentPath: segments.slice(0, -1).join("/"), - mkdir: false, - }); - const stat = await fs.lstat(targetPath); - if (stat.isDirectory() && !stat.isSymbolicLink()) { - await fs.rmdir(targetPath); - return; - } - await fs.unlink(targetPath); - }, - ), - runPinnedWriteHelper: vi.fn( - async (params: { - rootPath: string; - relativeParentPath: string; - basename: string; - mkdir: boolean; - mode: number; - input: - | { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding } - | { kind: "stream"; stream: NodeJS.ReadableStream }; - }) => { - const parentPath = await resolvePinnedParent({ - rootPath: params.rootPath, - relativeParentPath: params.relativeParentPath, - mkdir: params.mkdir, - }); - const targetPath = path.join(parentPath, params.basename); - if (params.input.kind === "buffer") { - await fs.writeFile(targetPath, params.input.data, { - encoding: params.input.encoding, - mode: params.mode, - }); - } else { - const handle = await fs.open(targetPath, "w", params.mode); - try { - await pipeline(params.input.stream, handle.createWriteStream()); - } finally { - await handle.close().catch(() => undefined); - } - } - const stat = await fs.stat(targetPath); - return { dev: stat.dev, ino: stat.ino }; - }, - ), - }; -}); - -vi.mock("../infra/fs-pinned-path-helper.js", () => ({ - isPinnedPathHelperSpawnError: () => false, - runPinnedPathHelper: pinnedPathHelper.runPinnedPathHelper, -})); - -vi.mock("../infra/fs-pinned-write-helper.js", () => ({ - runPinnedWriteHelper: pinnedPathHelper.runPinnedWriteHelper, -})); - async function withTempDir(fn: (dir: string) => Promise) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-patch-")); try { diff --git a/src/agents/apply-patch.ts b/src/agents/apply-patch.ts index a16a987590b..64bfed640ec 100644 --- a/src/agents/apply-patch.ts +++ b/src/agents/apply-patch.ts @@ -3,12 +3,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { AgentTool } from "@mariozechner/pi-agent-core"; import { Type } from "typebox"; -import { openBoundaryFile, type BoundaryFileOpenResult } from "../infra/boundary-file-read.js"; -import { - mkdirPathWithinRoot, - removePathWithinRoot, - writeFileWithinRoot, -} from "../infra/fs-safe.js"; +import { openRootFile, type RootFileOpenResult } from "../infra/boundary-file-read.js"; +import { root as fsRoot } from "../infra/fs-safe.js"; import { PATH_ALIAS_POLICIES, type PathAliasPolicy } from "../infra/path-alias-guards.js"; import { applyUpdateHunk } from "./apply-patch-update.js"; import { toRelativeSandboxPath, resolvePathFromInput } from "./path-policy.js"; @@ -244,12 +240,13 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps { }; } const workspaceOnly = options.workspaceOnly !== false; + const rootPromise = workspaceOnly ? fsRoot(options.cwd) : undefined; return { readFile: async (filePath) => { if (!workspaceOnly) { return await fs.readFile(filePath, "utf8"); } - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: filePath, rootPath: options.cwd, boundaryLabel: "workspace root", @@ -267,12 +264,7 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps { return; } const relative = toRelativeSandboxPath(options.cwd, filePath); - await writeFileWithinRoot({ - rootDir: options.cwd, - relativePath: relative, - data: content, - encoding: "utf8", - }); + await (await rootPromise)?.write(relative, content, { encoding: "utf8" }); }, remove: async (filePath) => { if (!workspaceOnly) { @@ -280,10 +272,7 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps { return; } const relative = toRelativeSandboxPath(options.cwd, filePath); - await removePathWithinRoot({ - rootDir: options.cwd, - relativePath: relative, - }); + await (await rootPromise)?.remove(relative); }, mkdirp: async (dir) => { if (!workspaceOnly) { @@ -291,11 +280,15 @@ function resolvePatchFileOps(options: ApplyPatchOptions): PatchFileOps { return; } const relative = toRelativeSandboxPath(options.cwd, dir, { allowRoot: true }); - await mkdirPathWithinRoot({ - rootDir: options.cwd, - relativePath: relative, - allowRoot: true, - }); + const root = await rootPromise; + if (!root) { + return; + } + if (relative === "" || relative === ".") { + await root.ensureRoot(); + return; + } + await root.mkdir(relative); }, }; } @@ -352,9 +345,9 @@ async function resolvePatchPath( } function assertBoundaryRead( - opened: BoundaryFileOpenResult, + opened: RootFileOpenResult, targetPath: string, -): asserts opened is Extract { +): asserts opened is Extract { if (opened.ok) { return; } diff --git a/src/agents/bash-tools.exec.script-preflight.test.ts b/src/agents/bash-tools.exec.script-preflight.test.ts index d55ce752fbb..c21bae838a9 100644 --- a/src/agents/bash-tools.exec.script-preflight.test.ts +++ b/src/agents/bash-tools.exec.script-preflight.test.ts @@ -2,7 +2,7 @@ import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { __setFsSafeTestHooksForTest } from "../infra/fs-safe.js"; +import { __setFsSafeTestHooksForTest } from "../infra/fs-safe-test-hooks.js"; import { withTempDir } from "../test-utils/temp-dir.js"; import { __testing, createExecTool } from "./bash-tools.exec.js"; diff --git a/src/agents/bash-tools.exec.ts b/src/agents/bash-tools.exec.ts index a3403869180..7b7eb058ae1 100644 --- a/src/agents/bash-tools.exec.ts +++ b/src/agents/bash-tools.exec.ts @@ -139,9 +139,9 @@ async function loadFsSafeModule(): Promise { function shouldSkipScriptPreflightPathError( error: unknown, - SafeOpenError: FsSafeModule["SafeOpenError"], + FsSafeError: FsSafeModule["FsSafeError"], ): boolean { - if (error instanceof SafeOpenError) { + if (error instanceof FsSafeError) { return true; } const errorCode = getNodeErrorCode(error); @@ -155,8 +155,8 @@ function resolvePreflightRelativePath(params: { rootDir: string; absPath: string if (/^\.\.(?:[\\/]|$)/u.test(relative) || path.isAbsolute(relative)) { return null; } - // Preserve literal "~" path segments under the workdir. `readFileWithinRoot` - // expands home prefixes for relative paths, so normalize `~/...` to `./~/...`. + // Preserve literal "~" path segments under the workdir. Root reads + // expand home prefixes for relative paths, so normalize `~/...` to `./~/...`. return /^~(?:$|[\\/])/u.test(relative) ? `.${path.sep}${relative}` : relative; } @@ -973,7 +973,8 @@ async function validateScriptFileForShellBleed(params: { return; } - const { SafeOpenError, readFileWithinRoot } = await loadFsSafeModule(); + const { FsSafeError, root: fsRoot } = await loadFsSafeModule(); + const workspaceRoot = await fsRoot(params.workdir); for (const relOrAbsPath of target.relOrAbsPaths) { const absPath = path.isAbsolute(relOrAbsPath) ? path.resolve(relOrAbsPath) @@ -992,16 +993,14 @@ async function validateScriptFileForShellBleed(params: { // Use non-blocking open to avoid stalls if a path is swapped to a FIFO. let content: string; try { - const safeRead = await readFileWithinRoot({ - rootDir: params.workdir, - relativePath, + const safeRead = await workspaceRoot.read(relativePath, { nonBlockingRead: true, - allowSymlinkTargetWithinRoot: true, + symlinks: "follow-within-root", maxBytes: 512 * 1024, }); content = safeRead.buffer.toString("utf-8"); } catch (error) { - if (shouldSkipScriptPreflightPathError(error, SafeOpenError)) { + if (shouldSkipScriptPreflightPathError(error, FsSafeError)) { // Preflight validation is best-effort: skip path/read failures and // continue to execute the command normally. continue; diff --git a/src/agents/cli-runner/helpers.ts b/src/agents/cli-runner/helpers.ts index 730cf771277..1293e9613de 100644 --- a/src/agents/cli-runner/helpers.ts +++ b/src/agents/cli-runner/helpers.ts @@ -10,6 +10,8 @@ import type { SourceReplyDeliveryMode } from "../../auto-reply/get-reply-options import type { ThinkLevel } from "../../auto-reply/thinking.js"; import type { CliBackendConfig } from "../../config/types.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { privateFileStore } from "../../infra/private-file-store.js"; +import { tempWorkspace } from "../../infra/private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import { MAX_IMAGE_BYTES } from "../../media/constants.js"; import { extensionForMime } from "../../media/mime.js"; @@ -283,14 +285,14 @@ export async function writeCliImages(params: { workspaceDir: params.workspaceDir, }); await fs.mkdir(imageRoot, { recursive: true, mode: 0o700 }); + const store = privateFileStore(imageRoot); const paths: string[] = []; for (let i = 0; i < params.images.length; i += 1) { const image = params.images[i]; const fileName = path.basename(resolveCliImagePath(image)); - const filePath = path.join(imageRoot, fileName); const buffer = Buffer.from(image.data, "base64"); - await fs.writeFile(filePath, buffer, { mode: 0o600 }); - paths.push(filePath); + await store.writeText(fileName, buffer); + paths.push(store.path(fileName)); } // Keep content-addressed image paths stable across Claude CLI runs so prompt // text and argv don't churn on every turn with fresh temp-dir suffixes. @@ -308,19 +310,17 @@ export async function writeCliSystemPromptFile(params: { ) { return { cleanup: async () => {} }; } - const tempDir = await fs.mkdtemp( - path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-system-prompt-"), - ); - const filePath = path.join(tempDir, "system-prompt.md"); - await fs.writeFile(filePath, stripSystemPromptCacheBoundary(params.systemPrompt), { - encoding: "utf-8", - mode: 0o600, + const workspace = await tempWorkspace({ + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: "openclaw-cli-system-prompt-", }); + const filePath = await workspace.write( + "system-prompt.md", + stripSystemPromptCacheBoundary(params.systemPrompt), + ); return { filePath, - cleanup: async () => { - await fs.rm(tempDir, { recursive: true, force: true }); - }, + cleanup: async () => await workspace.cleanup(), }; } diff --git a/src/agents/cli-runner/session-history.ts b/src/agents/cli-runner/session-history.ts index 657b53d2359..dca82e48b6b 100644 --- a/src/agents/cli-runner/session-history.ts +++ b/src/agents/cli-runner/session-history.ts @@ -6,6 +6,7 @@ import { resolveSessionFilePathOptions, } from "../../config/sessions/paths.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { isPathInside } from "../../infra/path-guards.js"; import { resolveSessionAgentIds } from "../agent-scope.js"; import { limitAgentHookHistoryMessages, @@ -108,11 +109,6 @@ async function safeRealpath(filePath: string): Promise { } } -function isPathWithinBase(basePath: string, targetPath: string): boolean { - const relative = path.relative(basePath, targetPath); - return Boolean(relative) && !relative.startsWith("..") && !path.isAbsolute(relative); -} - function resolveSafeCliSessionFile(params: { sessionId: string; sessionFile: string; @@ -155,7 +151,11 @@ async function loadCliSessionEntries(params: { } const realSessionsDir = (await safeRealpath(sessionsDir)) ?? path.resolve(sessionsDir); const realSessionFile = await safeRealpath(sessionFile); - if (!realSessionFile || !isPathWithinBase(realSessionsDir, realSessionFile)) { + if ( + !realSessionFile || + realSessionFile === realSessionsDir || + !isPathInside(realSessionsDir, realSessionFile) + ) { return []; } const stat = await fsp.stat(realSessionFile); diff --git a/src/agents/harness/native-hook-relay.ts b/src/agents/harness/native-hook-relay.ts index dd78198f9d8..2169c081892 100644 --- a/src/agents/harness/native-hook-relay.ts +++ b/src/agents/harness/native-hook-relay.ts @@ -1,14 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { - chmodSync, - existsSync, - lstatSync, - mkdirSync, - readFileSync, - renameSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from "node:fs"; import { createServer, request as httpRequest, @@ -20,6 +11,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { resolveOpenClawPackageRootSync } from "../../infra/openclaw-root.js"; +import { privateFileStoreSync } from "../../infra/private-file-store.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { PluginApprovalResolutions } from "../../plugins/types.js"; import { runBeforeToolCallHook } from "../pi-tools.before-tool-call.js"; @@ -823,18 +815,10 @@ function writeNativeHookRelayBridgeRecord( registryPath: string, record: NativeHookRelayBridgeRecord, ): void { - const tempPath = path.join( - path.dirname(registryPath), - `.${path.basename(registryPath)}.${process.pid}.${randomUUID()}.tmp`, + privateFileStoreSync(path.dirname(registryPath)).writeText( + path.basename(registryPath), + `${JSON.stringify(record)}\n`, ); - try { - writeFileSync(tempPath, `${JSON.stringify(record)}\n`, { mode: 0o600, flag: "wx" }); - renameSync(tempPath, registryPath); - chmodSync(registryPath, 0o600); - } catch (error) { - rmSync(tempPath, { force: true }); - throw error; - } } function nativeHookRelayBridgeRegistryPath(relayId: string): string { diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index 7cb061c1f5e..dfa45081535 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -7,6 +7,7 @@ import { type OpenClawConfig, } from "../config/config.js"; import { createConfigRuntimeEnv } from "../config/env-vars.js"; +import { privateFileStore } from "../infra/private-file-store.js"; import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; import { resolveInstalledManifestRegistryIndexFingerprint } from "../plugins/manifest-registry-installed.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; @@ -85,7 +86,15 @@ async function readExistingModelsFile(pathname: string): Promise<{ parsed: unknown; }> { try { - const raw = await fs.readFile(pathname, "utf8"); + const raw = await privateFileStore(path.dirname(pathname)).readTextIfExists( + path.basename(pathname), + ); + if (raw === null) { + return { + raw: "", + parsed: null, + }; + } return { raw, parsed: JSON.parse(raw) as unknown, @@ -108,9 +117,7 @@ export async function writeModelsFileAtomicForModelsJson( targetPath: string, contents: string, ): Promise { - const tempPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`; - await fs.writeFile(tempPath, contents, { mode: 0o600 }); - await fs.rename(tempPath, targetPath); + await privateFileStore(path.dirname(targetPath)).writeText(path.basename(targetPath), contents); } function resolveModelsConfigInput(config?: OpenClawConfig): { diff --git a/src/agents/models-config.write-serialization.test.ts b/src/agents/models-config.write-serialization.test.ts index 4323bab02b4..9455ff52bae 100644 --- a/src/agents/models-config.write-serialization.test.ts +++ b/src/agents/models-config.write-serialization.test.ts @@ -12,6 +12,10 @@ import { import { readGeneratedModelsJson } from "./models-config.test-utils.js"; const planOpenClawModelsJsonMock = vi.fn(); +const writePrivateStoreTextWriteMock = vi.fn(); +let actualPrivateFileStore: + | typeof import("../infra/private-file-store.js").privateFileStore + | undefined; installModelsConfigTestHooks(); @@ -66,6 +70,27 @@ beforeAll(async () => { vi.doMock("./models-config.plan.js", () => ({ planOpenClawModelsJson: (...args: unknown[]) => planOpenClawModelsJsonMock(...args), })); + vi.doMock("../infra/private-file-store.js", async () => { + const actual = await vi.importActual( + "../infra/private-file-store.js", + ); + actualPrivateFileStore = actual.privateFileStore; + return { + ...actual, + privateFileStore: (rootDir: string) => { + const store = actual.privateFileStore(rootDir); + return { + ...store, + writeText: (relativePath: string, content: string | Uint8Array) => + writePrivateStoreTextWriteMock({ + rootDir, + filePath: path.join(rootDir, relativePath), + content, + }), + }; + }, + }; + }); ({ ensureOpenClawModelsJson } = await import("./models-config.js")); ({ clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot } = await import("../plugins/current-plugin-metadata-snapshot.js")); @@ -73,6 +98,19 @@ beforeAll(async () => { beforeEach(() => { clearCurrentPluginMetadataSnapshot(); + writePrivateStoreTextWriteMock + .mockReset() + .mockImplementation( + async (params: { filePath: string; rootDir: string; content: string | Uint8Array }) => { + if (!actualPrivateFileStore) { + throw new Error("private file store mock not initialized"); + } + return await actualPrivateFileStore(params.rootDir).writeText( + path.basename(params.filePath), + params.content, + ); + }, + ); planOpenClawModelsJsonMock .mockReset() .mockImplementation(async (params: { cfg?: typeof CUSTOM_PROXY_MODELS_CONFIG }) => ({ @@ -207,42 +245,35 @@ describe("models-config write serialization", () => { firstModel.name = "Proxy A"; secondModel.name = "Proxy B with longer name"; - const originalWriteFile = fs.writeFile.bind(fs); let inFlightWrites = 0; let maxInFlightWrites = 0; - const writeSpy = vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => { - const targetArg = args[0]; - const targetPath = - typeof targetArg === "string" - ? targetArg - : targetArg instanceof URL - ? targetArg.pathname - : undefined; - const isModelsTempWrite = - typeof targetPath === "string" && - path.basename(targetPath).startsWith("models.json.") && - targetPath.endsWith(".tmp"); - if (isModelsTempWrite) { - inFlightWrites += 1; - if (inFlightWrites > maxInFlightWrites) { - maxInFlightWrites = inFlightWrites; + writePrivateStoreTextWriteMock.mockImplementation( + async (params: { filePath: string; rootDir: string; content: string | Uint8Array }) => { + const isModelsWrite = path.basename(params.filePath) === "models.json"; + if (isModelsWrite) { + inFlightWrites += 1; + if (inFlightWrites > maxInFlightWrites) { + maxInFlightWrites = inFlightWrites; + } + await new Promise((resolve) => setTimeout(resolve, 10)); } - await new Promise((resolve) => setTimeout(resolve, 10)); - } - try { - return await originalWriteFile(...args); - } finally { - if (isModelsTempWrite) { - inFlightWrites -= 1; + try { + if (!actualPrivateFileStore) { + throw new Error("private file store mock not initialized"); + } + return await actualPrivateFileStore(params.rootDir).writeText( + path.basename(params.filePath), + params.content, + ); + } finally { + if (isModelsWrite) { + inFlightWrites -= 1; + } } - } - }); + }, + ); - try { - await Promise.all([ensureOpenClawModelsJson(first), ensureOpenClawModelsJson(second)]); - } finally { - writeSpy.mockRestore(); - } + await Promise.all([ensureOpenClawModelsJson(first), ensureOpenClawModelsJson(second)]); expect(maxInFlightWrites).toBe(1); const parsed = await readGeneratedModelsJson<{ diff --git a/src/agents/pi-auth-json.ts b/src/agents/pi-auth-json.ts index a82bfe607aa..16f9a1fb082 100644 --- a/src/agents/pi-auth-json.ts +++ b/src/agents/pi-auth-json.ts @@ -1,7 +1,7 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { z } from "zod"; -import { safeParseJsonWithSchema, safeParseWithSchema } from "../utils/zod-parse.js"; +import { privateFileStore } from "../infra/private-file-store.js"; +import { safeParseWithSchema } from "../utils/zod-parse.js"; import { ensureAuthProfileStore } from "./auth-profiles/store.js"; import { piCredentialsEqual, @@ -26,10 +26,12 @@ const PiCredentialSchema: z.ZodType = z.discriminatedUnion("type", const AuthJsonShapeSchema = z.record(z.string(), z.unknown()); -async function readAuthJson(filePath: string): Promise { +async function readAuthJson(rootDir: string, filePath: string): Promise { try { - const raw = await fs.readFile(filePath, "utf8"); - return safeParseJsonWithSchema(AuthJsonShapeSchema, raw) ?? {}; + const parsed = await privateFileStore(rootDir).readJsonIfExists( + path.relative(rootDir, filePath), + ); + return safeParseWithSchema(AuthJsonShapeSchema, parsed) ?? {}; } catch { return {}; } @@ -58,7 +60,7 @@ export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promis return { wrote: false, authPath }; } - const existing = await readAuthJson(authPath); + const existing = await readAuthJson(agentDir, authPath); let changed = false; for (const [provider, cred] of Object.entries(providerCredentials)) { @@ -73,8 +75,9 @@ export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promis return { wrote: false, authPath }; } - await fs.mkdir(agentDir, { recursive: true, mode: 0o700 }); - await fs.writeFile(authPath, `${JSON.stringify(existing, null, 2)}\n`, { mode: 0o600 }); + await privateFileStore(agentDir).writeJson(path.basename(authPath), existing, { + trailingNewline: true, + }); return { wrote: true, authPath }; } diff --git a/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts b/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts index 8d022981784..9ed04fd9118 100644 --- a/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts +++ b/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts @@ -18,11 +18,12 @@ * capabilities instead of the text-only fallback. */ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; import { resolveStateDir } from "../../config/paths.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { resolveProxyFetchFromEnv } from "../../infra/net/proxy-fetch.js"; +import { privateFileStoreSync } from "../../infra/private-file-store.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; const log = createSubsystemLogger("openrouter-model-capabilities"); @@ -89,14 +90,11 @@ function resolveDiskCachePath(): string { function writeDiskCache(map: Map): void { try { - const cacheDir = resolveDiskCacheDir(); - if (!existsSync(cacheDir)) { - mkdirSync(cacheDir, { recursive: true }); - } + const cachePath = resolveDiskCachePath(); const payload: DiskCachePayload = { models: Object.fromEntries(map), }; - writeFileSync(resolveDiskCachePath(), JSON.stringify(payload), "utf-8"); + privateFileStoreSync(dirname(cachePath)).writeJson(basename(cachePath), payload); } catch (err: unknown) { const message = formatErrorMessage(err); log.debug(`Failed to write OpenRouter disk cache: ${message}`); diff --git a/src/agents/pi-embedded-runner/transcript-file-state.ts b/src/agents/pi-embedded-runner/transcript-file-state.ts index db9c6c1a4b7..aa66f77d7fe 100644 --- a/src/agents/pi-embedded-runner/transcript-file-state.ts +++ b/src/agents/pi-embedded-runner/transcript-file-state.ts @@ -11,6 +11,8 @@ import { type SessionEntry, type SessionHeader, } from "@mariozechner/pi-coding-agent"; +import { appendRegularFile } from "../../infra/fs-safe.js"; +import { privateFileStore } from "../../infra/private-file-store.js"; type BranchSummaryEntry = Extract; type CompactionEntry = Extract; @@ -293,20 +295,10 @@ export async function writeTranscriptFileAtomic( filePath: string, entries: Array, ): Promise { - const dir = path.dirname(filePath); - await fs.mkdir(dir, { recursive: true }); - const tmpFile = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`); - try { - await fs.writeFile(tmpFile, serializeTranscriptFileEntries(entries), { - encoding: "utf-8", - mode: 0o600, - flag: "wx", - }); - await fs.rename(tmpFile, filePath); - } catch (err) { - await fs.unlink(tmpFile).catch(() => undefined); - throw err; - } + await privateFileStore(path.dirname(filePath)).writeText( + path.basename(filePath), + serializeTranscriptFileEntries(entries), + ); } export async function persistTranscriptStateMutation(params: { @@ -324,9 +316,9 @@ export async function persistTranscriptStateMutation(params: { ]); return; } - await fs.appendFile( - params.sessionFile, - params.appendedEntries.map((entry) => JSON.stringify(entry)).join("\n") + "\n", - "utf-8", - ); + await appendRegularFile({ + filePath: params.sessionFile, + content: `${params.appendedEntries.map((entry) => JSON.stringify(entry)).join("\n")}\n`, + rejectSymlinkParents: true, + }); } diff --git a/src/agents/pi-embedded-subscribe.raw-stream.ts b/src/agents/pi-embedded-subscribe.raw-stream.ts index 12efd4d0cac..144866f3e6a 100644 --- a/src/agents/pi-embedded-subscribe.raw-stream.ts +++ b/src/agents/pi-embedded-subscribe.raw-stream.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; import { isTruthyEnvValue } from "../infra/env.js"; +import { appendRegularFile } from "../infra/fs-safe.js"; let rawStreamReady = false; @@ -30,7 +31,11 @@ export function appendRawStream(payload: Record) { } } try { - void fs.promises.appendFile(rawStreamPath, `${JSON.stringify(payload)}\n`); + void appendRegularFile({ + filePath: rawStreamPath, + content: `${JSON.stringify(payload)}\n`, + rejectSymlinkParents: true, + }); } catch { // ignore raw stream write failures } diff --git a/src/agents/pi-hooks/compaction-safeguard.ts b/src/agents/pi-hooks/compaction-safeguard.ts index 7bf6222eaef..0f316563107 100644 --- a/src/agents/pi-hooks/compaction-safeguard.ts +++ b/src/agents/pi-hooks/compaction-safeguard.ts @@ -3,7 +3,7 @@ import path from "node:path"; import type { AgentMessage } from "@mariozechner/pi-agent-core"; import type { ExtensionAPI, ExtensionContext, FileOperations } from "@mariozechner/pi-coding-agent"; import { extractSections } from "../../auto-reply/reply/post-compaction-context.js"; -import { openBoundaryFile } from "../../infra/boundary-file-read.js"; +import { openRootFile } from "../../infra/boundary-file-read.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { isAbortError } from "../../infra/unhandled-rejections.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; @@ -735,7 +735,7 @@ async function readWorkspaceContextForSummary(): Promise { const agentsPath = path.join(workspaceDir, "AGENTS.md"); try { - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: agentsPath, rootPath: workspaceDir, boundaryLabel: "workspace root", diff --git a/src/agents/pi-project-settings-snapshot.ts b/src/agents/pi-project-settings-snapshot.ts index 39d3619848b..c6a98f40147 100644 --- a/src/agents/pi-project-settings-snapshot.ts +++ b/src/agents/pi-project-settings-snapshot.ts @@ -3,7 +3,7 @@ import path from "node:path"; import type { SettingsManager } from "@mariozechner/pi-coding-agent"; import { applyMergePatch } from "../config/merge-patch.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import type { BundleMcpServerConfig } from "../plugins/bundle-mcp.js"; import { @@ -43,7 +43,7 @@ function loadBundleSettingsFile(params: { relativePath: string; }): PiSettingsSnapshot | null { const absolutePath = path.join(params.rootDir, params.relativePath); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath, rootPath: params.rootDir, boundaryLabel: "plugin root", diff --git a/src/agents/pi-project-settings.bundle.test.ts b/src/agents/pi-project-settings.bundle.test.ts index 932ce1b5075..4c314746711 100644 --- a/src/agents/pi-project-settings.bundle.test.ts +++ b/src/agents/pi-project-settings.bundle.test.ts @@ -6,7 +6,7 @@ import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; vi.mock("../infra/boundary-file-read.js", async () => { const fs = await import("node:fs"); return { - openBoundaryFileSync: ({ absolutePath }: { absolutePath: string }) => ({ + openRootFileSync: ({ absolutePath }: { absolutePath: string }) => ({ ok: true, fd: fs.openSync(absolutePath, "r"), }), diff --git a/src/agents/pi-tools.read.ts b/src/agents/pi-tools.read.ts index 9278c9f3843..9dee15413a7 100644 --- a/src/agents/pi-tools.read.ts +++ b/src/agents/pi-tools.read.ts @@ -4,13 +4,7 @@ import { URL } from "node:url"; import type { AgentToolResult } from "@mariozechner/pi-agent-core"; import { createEditTool, createReadTool, createWriteTool } from "@mariozechner/pi-coding-agent"; import { isWindowsDrivePath } from "../infra/archive-path.js"; -import { - appendFileWithinRoot, - SafeOpenError, - openFileWithinRoot, - readFileWithinRoot, - writeFileWithinRoot, -} from "../infra/fs-safe.js"; +import { root as fsRoot, FsSafeError } from "../infra/fs-safe.js"; import { expandHomePrefix, resolveOsHomeDir } from "../infra/home-dir.js"; import { hasEncodedFileUrlSeparator, trySafeFileURLToPath } from "../infra/local-file-access.js"; import { detectMime } from "../media/mime.js"; @@ -491,10 +485,8 @@ async function appendMemoryFlushContent(params: { signal?: AbortSignal; }) { if (!params.sandbox) { - await appendFileWithinRoot({ - rootDir: params.root, - relativePath: params.relativePath, - data: params.content, + const root = await fsRoot(params.root); + await root.append(params.relativePath, params.content, { mkdir: true, prependNewlineIfNeeded: true, }); @@ -769,6 +761,7 @@ function createHostWriteOperations(root: string, options?: { workspaceOnly?: boo } // When workspaceOnly is true, enforce workspace boundary + const rootPromise = fsRoot(root); return { mkdir: async (dir: string) => { const relative = toRelativeWorkspacePath(root, dir, { allowRoot: true }); @@ -778,12 +771,7 @@ function createHostWriteOperations(root: string, options?: { workspaceOnly?: boo }, writeFile: async (absolutePath: string, content: string) => { const relative = toRelativeWorkspacePath(root, absolutePath); - await writeFileWithinRoot({ - rootDir: root, - relativePath: relative, - data: content, - mkdir: true, - }); + await (await rootPromise).write(relative, content, { mkdir: true }); }, } as const; } @@ -807,23 +795,16 @@ function createHostEditOperations(root: string, options?: { workspaceOnly?: bool } // When workspaceOnly is true, enforce workspace boundary + const rootPromise = fsRoot(root); return { readFile: async (absolutePath: string) => { const relative = toRelativeWorkspacePath(root, absolutePath); - const safeRead = await readFileWithinRoot({ - rootDir: root, - relativePath: relative, - }); + const safeRead = await (await rootPromise).read(relative); return safeRead.buffer; }, writeFile: async (absolutePath: string, content: string) => { const relative = toRelativeWorkspacePath(root, absolutePath); - await writeFileWithinRoot({ - rootDir: root, - relativePath: relative, - data: content, - mkdir: true, - }); + await (await rootPromise).write(relative, content, { mkdir: true }); }, access: async (absolutePath: string) => { let relative: string; @@ -838,16 +819,13 @@ function createHostEditOperations(root: string, options?: { workspaceOnly?: bool return; } try { - const opened = await openFileWithinRoot({ - rootDir: root, - relativePath: relative, - }); + const opened = await (await rootPromise).open(relative); await opened.handle.close().catch(() => {}); } catch (error) { - if (error instanceof SafeOpenError && error.code === "not-found") { + if (error instanceof FsSafeError && error.code === "not-found") { throw createFsAccessError("ENOENT", absolutePath); } - if (error instanceof SafeOpenError && error.code === "outside-workspace") { + if (error instanceof FsSafeError && error.code === "outside-workspace") { // Don't throw here – see the comment above about the upstream // library swallowing access errors as "File not found". return; diff --git a/src/agents/queued-file-writer.ts b/src/agents/queued-file-writer.ts index f02e5d9ef96..2b59a414049 100644 --- a/src/agents/queued-file-writer.ts +++ b/src/agents/queued-file-writer.ts @@ -1,6 +1,6 @@ -import nodeFs from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { appendRegularFile, resolveRegularFileAppendFlags } from "../infra/fs-safe.js"; export type QueuedFileWriteResult = "queued" | "dropped"; @@ -16,103 +16,19 @@ type QueuedFileWriterOptions = { yieldBeforeWrite?: boolean; }; -type QueuedFileAppendFlagConstants = Pick< - typeof nodeFs.constants, - "O_APPEND" | "O_CREAT" | "O_WRONLY" -> & - Partial>; - -export function resolveQueuedFileAppendFlags( - constants: QueuedFileAppendFlagConstants = nodeFs.constants, -): number { - const noFollow = constants.O_NOFOLLOW; - return ( - constants.O_CREAT | - constants.O_APPEND | - constants.O_WRONLY | - (typeof noFollow === "number" ? noFollow : 0) - ); -} - -async function assertNoSymlinkParents(filePath: string): Promise { - const resolvedDir = path.resolve(path.dirname(filePath)); - const parsed = path.parse(resolvedDir); - const relativeParts = path.relative(parsed.root, resolvedDir).split(path.sep).filter(Boolean); - let current = parsed.root; - for (const part of relativeParts) { - current = path.join(current, part); - const stat = await fs.lstat(current); - if (stat.isSymbolicLink()) { - if (path.dirname(current) === parsed.root) { - continue; - } - throw new Error(`Refusing to write queued log under symlinked directory: ${current}`); - } - if (!stat.isDirectory()) { - throw new Error(`Refusing to write queued log under non-directory: ${current}`); - } - } -} - -function verifyStableOpenedFile(params: { - preOpenStat?: nodeFs.Stats; - postOpenStat: nodeFs.Stats; - filePath: string; -}): void { - if (!params.postOpenStat.isFile()) { - throw new Error(`Refusing to write queued log to non-file: ${params.filePath}`); - } - if (params.postOpenStat.nlink > 1) { - throw new Error(`Refusing to write queued log to hardlinked file: ${params.filePath}`); - } - const pre = params.preOpenStat; - if (pre && (pre.dev !== params.postOpenStat.dev || pre.ino !== params.postOpenStat.ino)) { - throw new Error(`Refusing to write queued log after file changed: ${params.filePath}`); - } -} +export const resolveQueuedFileAppendFlags = resolveRegularFileAppendFlags; async function safeAppendFile( filePath: string, line: string, options: QueuedFileWriterOptions, ): Promise { - await assertNoSymlinkParents(filePath); - - let preOpenStat: nodeFs.Stats | undefined; - try { - const stat = await fs.lstat(filePath); - if (stat.isSymbolicLink()) { - throw new Error(`Refusing to write queued log through symlink: ${filePath}`); - } - if (!stat.isFile()) { - throw new Error(`Refusing to write queued log to non-file: ${filePath}`); - } - preOpenStat = stat; - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - } - const lineBytes = Buffer.byteLength(line, "utf8"); - if ( - options.maxFileBytes !== undefined && - (preOpenStat?.size ?? 0) + lineBytes > options.maxFileBytes - ) { - return; - } - - const handle = await fs.open(filePath, resolveQueuedFileAppendFlags(), 0o600); - try { - const stat = await handle.stat(); - verifyStableOpenedFile({ preOpenStat, postOpenStat: stat, filePath }); - if (options.maxFileBytes !== undefined && stat.size + lineBytes > options.maxFileBytes) { - return; - } - await handle.chmod(0o600); - await handle.appendFile(line, "utf8"); - } finally { - await handle.close(); - } + await appendRegularFile({ + filePath, + content: line, + maxFileBytes: options.maxFileBytes, + rejectSymlinkParents: true, + }); } function waitForImmediate(): Promise { diff --git a/src/agents/sandbox-paths.ts b/src/agents/sandbox-paths.ts index 441a06a7fa7..c4145a7473e 100644 --- a/src/agents/sandbox-paths.ts +++ b/src/agents/sandbox-paths.ts @@ -108,10 +108,15 @@ function isManagedMediaPathUnderRoot(candidate: string): boolean { return false; } const mediaRoot = path.join(resolveConfigDir(), "media"); - const relative = path.relative(path.resolve(mediaRoot), path.resolve(expanded)); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { + const resolvedMediaRoot = path.resolve(mediaRoot); + const resolvedExpanded = path.resolve(expanded); + if ( + resolvedExpanded === resolvedMediaRoot || + !isPathInside(resolvedMediaRoot, resolvedExpanded) + ) { return false; } + const relative = path.relative(resolvedMediaRoot, resolvedExpanded); const firstSegment = relative.split(path.sep)[0] ?? ""; return MANAGED_MEDIA_SUBDIRS.has(firstSegment) || firstSegment.startsWith("tool-"); } diff --git a/src/agents/sandbox/fs-bridge-path-safety.runtime.ts b/src/agents/sandbox/fs-bridge-path-safety.runtime.ts index 5c759b5cce9..8c87bd5fffb 100644 --- a/src/agents/sandbox/fs-bridge-path-safety.runtime.ts +++ b/src/agents/sandbox/fs-bridge-path-safety.runtime.ts @@ -1 +1 @@ -export { openBoundaryFile, type BoundaryFileOpenResult } from "../../infra/boundary-file-read.js"; +export { openRootFile, type RootFileOpenResult } from "../../infra/boundary-file-read.js"; diff --git a/src/agents/sandbox/fs-bridge-path-safety.ts b/src/agents/sandbox/fs-bridge-path-safety.ts index d0bfd00b78e..e75b02f3a6b 100644 --- a/src/agents/sandbox/fs-bridge-path-safety.ts +++ b/src/agents/sandbox/fs-bridge-path-safety.ts @@ -1,16 +1,17 @@ import fs from "node:fs"; import path from "node:path"; import type { PathAliasPolicy } from "../../infra/path-alias-guards.js"; -import type { SafeOpenSyncAllowedType } from "../../infra/safe-open-sync.js"; -import { openBoundaryFile, type BoundaryFileOpenResult } from "./fs-bridge-path-safety.runtime.js"; +import { openRootFile, type RootFileOpenResult } from "./fs-bridge-path-safety.runtime.js"; import type { SandboxResolvedFsPath, SandboxFsMount } from "./fs-paths.js"; import { isPathInsideContainerRoot, normalizeContainerPath } from "./path-utils.js"; +type BoundaryAllowedType = "file" | "directory"; + export type PathSafetyOptions = { action: string; aliasPolicy?: PathAliasPolicy; requireWritable?: boolean; - allowedType?: SafeOpenSyncAllowedType; + allowedType?: BoundaryAllowedType; }; export type PathSafetyCheck = { @@ -69,7 +70,7 @@ export class SandboxFsPathGuard { async openReadableFile( target: SandboxResolvedFsPath, - ): Promise { + ): Promise { const opened = await this.openBoundaryWithinRequiredMount(target, "read files"); if (!opened.ok) { throw opened.error instanceof Error @@ -110,7 +111,7 @@ export class SandboxFsPathGuard { private async assertGuardedPathSafety( target: SandboxResolvedFsPath, options: PathSafetyOptions, - guarded: BoundaryFileOpenResult, + guarded: RootFileOpenResult, ) { if (!guarded.ok) { if (guarded.reason !== "path") { @@ -145,11 +146,11 @@ export class SandboxFsPathGuard { action: string, options?: { aliasPolicy?: PathAliasPolicy; - allowedType?: SafeOpenSyncAllowedType; + allowedType?: BoundaryAllowedType; }, - ): Promise { + ): Promise { const lexicalMount = this.resolveRequiredMount(target.containerPath, action); - const guarded = await openBoundaryFile({ + const guarded = await openRootFile({ absolutePath: target.hostPath, rootPath: lexicalMount.hostRoot, boundaryLabel: "sandbox mount root", diff --git a/src/agents/sandbox/fs-bridge.shell.test.ts b/src/agents/sandbox/fs-bridge.shell.test.ts index 7d563beea6f..980330ed4b5 100644 --- a/src/agents/sandbox/fs-bridge.shell.test.ts +++ b/src/agents/sandbox/fs-bridge.shell.test.ts @@ -8,7 +8,7 @@ import { getScriptsFromCalls, installFsBridgeTestHarness, mockedExecDockerRaw, - mockedOpenBoundaryFile, + mockedOpenRootFile, withTempDir, } from "./fs-bridge.test-helpers.js"; @@ -159,7 +159,7 @@ describe("sandbox fs bridge shell compatibility", () => { }); it("re-validates target before the pinned write helper runs", async () => { - mockedOpenBoundaryFile + mockedOpenRootFile .mockImplementationOnce(async () => ({ ok: false, reason: "path" })) .mockImplementationOnce(async () => ({ ok: false, diff --git a/src/agents/sandbox/fs-bridge.test-helpers.ts b/src/agents/sandbox/fs-bridge.test-helpers.ts index dfef0f6b762..154e819fa11 100644 --- a/src/agents/sandbox/fs-bridge.test-helpers.ts +++ b/src/agents/sandbox/fs-bridge.test-helpers.ts @@ -4,21 +4,21 @@ import path from "node:path"; import { beforeEach, expect, vi, type Mock } from "vitest"; type ExecDockerRawFn = typeof import("./docker.js").execDockerRaw; -type OpenBoundaryFileFn = typeof import("./fs-bridge-path-safety.runtime.js").openBoundaryFile; +type OpenRootFileFn = typeof import("./fs-bridge-path-safety.runtime.js").openRootFile; type ExecDockerArgs = Parameters[0]; type ExecDockerRawMock = Mock; -type OpenBoundaryFileMock = Mock; +type OpenRootFileMock = Mock; type FsBridgeHoisted = { execDockerRaw: ExecDockerRawMock; - openBoundaryFile: OpenBoundaryFileMock; + openRootFile: OpenRootFileMock; }; -let actualOpenBoundaryFile: OpenBoundaryFileFn | undefined; +let actualOpenRootFile: OpenRootFileFn | undefined; const hoisted = vi.hoisted( (): FsBridgeHoisted => ({ execDockerRaw: vi.fn(), - openBoundaryFile: vi.fn(), + openRootFile: vi.fn(), }), ); @@ -31,11 +31,10 @@ vi.mock("./fs-bridge-path-safety.runtime.js", async () => { const actual = await vi.importActual( "./fs-bridge-path-safety.runtime.js", ); - actualOpenBoundaryFile = actual.openBoundaryFile; + actualOpenRootFile = actual.openRootFile; return { ...actual, - openBoundaryFile: (params: Parameters[0]) => - hoisted.openBoundaryFile(params), + openRootFile: (params: Parameters[0]) => hoisted.openRootFile(params), }; }); @@ -54,11 +53,10 @@ async function loadFreshFsBridgeModuleForTest() { const actual = await vi.importActual( "./fs-bridge-path-safety.runtime.js", ); - actualOpenBoundaryFile = actual.openBoundaryFile; + actualOpenRootFile = actual.openRootFile; return { ...actual, - openBoundaryFile: (params: Parameters[0]) => - hoisted.openBoundaryFile(params), + openRootFile: (params: Parameters[0]) => hoisted.openRootFile(params), }; }); ({ createSandboxFsBridge: createSandboxFsBridgeImpl } = await import("./fs-bridge.js")); @@ -74,7 +72,7 @@ export function createSandboxFsBridge( } export const mockedExecDockerRaw: ExecDockerRawMock = hoisted.execDockerRaw; -export const mockedOpenBoundaryFile: OpenBoundaryFileMock = hoisted.openBoundaryFile; +export const mockedOpenRootFile: OpenRootFileMock = hoisted.openRootFile; const DOCKER_SCRIPT_INDEX = 5; const DOCKER_FIRST_SCRIPT_ARG_INDEX = 7; @@ -206,7 +204,7 @@ export async function expectMkdirpAllowsExistingDirectory(params?: { await fs.mkdir(nestedDir, { recursive: true }); if (params?.forceBoundaryIoFallback) { - mockedOpenBoundaryFile.mockImplementationOnce(async () => ({ + mockedOpenRootFile.mockImplementationOnce(async () => ({ ok: false, reason: "io", error: Object.assign(new Error("EISDIR"), { code: "EISDIR" }), @@ -239,9 +237,9 @@ export function installFsBridgeTestHarness() { beforeEach(async () => { await loadFreshFsBridgeModuleForTest(); mockedExecDockerRaw.mockClear(); - mockedOpenBoundaryFile.mockClear(); - if (actualOpenBoundaryFile) { - mockedOpenBoundaryFile.mockImplementation(actualOpenBoundaryFile); + mockedOpenRootFile.mockClear(); + if (actualOpenRootFile) { + mockedOpenRootFile.mockImplementation(actualOpenRootFile); } installDockerReadMock(); }); diff --git a/src/agents/sandbox/fs-paths.ts b/src/agents/sandbox/fs-paths.ts index a319ed67116..1f134e511ee 100644 --- a/src/agents/sandbox/fs-paths.ts +++ b/src/agents/sandbox/fs-paths.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { isPathInside } from "../../infra/path-guards.js"; import { normalizeOptionalLowercaseString } from "../../shared/string-coerce.js"; import { resolveSandboxInputPath, resolveSandboxPath } from "../sandbox-paths.js"; import type { SandboxFsBridgeContext } from "./backend-handle.types.js"; @@ -228,11 +229,7 @@ function isPathInsideHost(root: string, target: string): boolean { path.dirname(resolvedTarget), ); const canonicalTarget = path.resolve(canonicalTargetParent, path.basename(resolvedTarget)); - const rel = path.relative(canonicalRoot, canonicalTarget); - if (!rel) { - return true; - } - return !(rel.startsWith("..") || path.isAbsolute(rel)); + return isPathInside(canonicalRoot, canonicalTarget); } function toHostSegments(relativePosix: string): string[] { diff --git a/src/agents/sandbox/registry.test.ts b/src/agents/sandbox/registry.test.ts index a30aff5bfc9..3314167ba32 100644 --- a/src/agents/sandbox/registry.test.ts +++ b/src/agents/sandbox/registry.test.ts @@ -46,10 +46,10 @@ vi.mock("../../infra/json-files.js", async () => { ); return { ...actual, - writeJsonAtomic: async ( + writeJson: async ( filePath: string, value: unknown, - options?: Parameters[2], + options?: Parameters[2], ) => { const payload = JSON.stringify(value); const gate = writeGateState.active; @@ -64,7 +64,7 @@ vi.mock("../../infra/json-files.js", async () => { } await gate.waitForRelease; } - await actual.writeJsonAtomic(filePath, value, options); + await actual.writeJson(filePath, value, options); }, }; }); diff --git a/src/agents/sandbox/registry.ts b/src/agents/sandbox/registry.ts index 99876823547..a43052ade02 100644 --- a/src/agents/sandbox/registry.ts +++ b/src/agents/sandbox/registry.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { z } from "zod"; -import { writeJsonAtomic } from "../../infra/json-files.js"; +import { writeJson } from "../../infra/json-files.js"; import { safeParseJsonWithSchema } from "../../utils/zod-parse.js"; import { acquireSessionWriteLock } from "../session-write-lock.js"; import { @@ -171,7 +171,7 @@ async function readShardedEntry( async function writeShardedEntry(dir: string, entry: RegistryEntryPayload): Promise { await fs.mkdir(dir, { recursive: true }); - await writeJsonAtomic(shardedEntryFilePath(dir, entry.containerName), entry, { + await writeJson(shardedEntryFilePath(dir, entry.containerName), entry, { trailingNewline: true, }); } diff --git a/src/agents/sandbox/ssh.ts b/src/agents/sandbox/ssh.ts index 758579f6793..b4e33868b38 100644 --- a/src/agents/sandbox/ssh.ts +++ b/src/agents/sandbox/ssh.ts @@ -2,7 +2,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { resolveBoundaryPath } from "../../infra/boundary-path.js"; +import { resolveRootPath } from "../../infra/boundary-path.js"; import { parseSshTarget } from "../../infra/ssh-tunnel.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import { resolveUserPath } from "../../utils.js"; @@ -349,7 +349,7 @@ async function assertSafeUploadSymlinks(localDir: string): Promise { const entryPath = path.join(currentDir, entry.name); if (entry.isSymbolicLink()) { try { - await resolveBoundaryPath({ + await resolveRootPath({ absolutePath: entryPath, rootPath: rootDir, boundaryLabel: "SSH sandbox upload tree", diff --git a/src/agents/sandbox/workspace.ts b/src/agents/sandbox/workspace.ts index 667847d5844..9d9321c76dc 100644 --- a/src/agents/sandbox/workspace.ts +++ b/src/agents/sandbox/workspace.ts @@ -2,7 +2,7 @@ import syncFs from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { OptionalBootstrapFileName } from "../../config/types.agent-defaults.js"; -import { openBoundaryFile } from "../../infra/boundary-file-read.js"; +import { openRootFile } from "../../infra/boundary-file-read.js"; import { resolveUserPath } from "../../utils.js"; import { DEFAULT_AGENTS_FILENAME, @@ -40,7 +40,7 @@ export async function ensureSandboxWorkspace( await fs.access(dest); } catch { try { - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: src, rootPath: seed, boundaryLabel: "sandbox seed workspace", diff --git a/src/agents/session-file-repair.ts b/src/agents/session-file-repair.ts index bcb2267964d..4db76f1e57b 100644 --- a/src/agents/session-file-repair.ts +++ b/src/agents/session-file-repair.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { replaceFileAtomic } from "../infra/replace-file.js"; import { STREAM_ERROR_FALLBACK_TEXT } from "./stream-message-shared.js"; /** Placeholder for blank user messages — preserves the user turn so strict @@ -278,28 +279,19 @@ export async function repairSessionFileIfNeeded(params: { const cleaned = `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`; const backupPath = `${sessionFile}.bak-${process.pid}-${Date.now()}`; - const tmpPath = `${sessionFile}.repair-${process.pid}-${Date.now()}.tmp`; try { const stat = await fs.stat(sessionFile).catch(() => null); await fs.writeFile(backupPath, content, "utf-8"); if (stat) { await fs.chmod(backupPath, stat.mode); } - await fs.writeFile(tmpPath, cleaned, "utf-8"); - if (stat) { - await fs.chmod(tmpPath, stat.mode); - } - await fs.rename(tmpPath, sessionFile); + await replaceFileAtomic({ + filePath: sessionFile, + content: cleaned, + preserveExistingMode: true, + tempPrefix: `${path.basename(sessionFile)}.repair`, + }); } catch (err) { - try { - await fs.unlink(tmpPath); - } catch (cleanupErr) { - params.warn?.( - `session file repair cleanup failed: ${cleanupErr instanceof Error ? cleanupErr.message : "unknown error"} (${path.basename( - tmpPath, - )})`, - ); - } return { repaired: false, droppedLines, diff --git a/src/agents/session-write-lock.ts b/src/agents/session-write-lock.ts index e15e812c4e0..84efd0bddba 100644 --- a/src/agents/session-write-lock.ts +++ b/src/agents/session-write-lock.ts @@ -1,8 +1,9 @@ -import fsSync from "node:fs"; +import "../infra/fs-safe-defaults.js"; +import type fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { createFileLockManager } from "../infra/file-lock-manager.js"; import { getProcessStartTime, isPidAlive } from "../shared/pid-alive.js"; -import { resolveProcessScopedMap } from "../shared/process-scoped-map.js"; import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js"; type LockFilePayload = { @@ -16,19 +17,6 @@ function isValidLockNumber(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } -type HeldLock = { - count: number; - handle: fs.FileHandle; - lockPath: string; - acquiredAt: number; - maxHoldMs: number; - releasePromise?: Promise; -}; - -type SyncClosableFileHandle = fs.FileHandle & { - [key: symbol]: unknown; -}; - export type SessionLockInspection = { lockPath: string; pid: number | null; @@ -43,7 +31,6 @@ export type SessionLockInspection = { const CLEANUP_SIGNALS = ["SIGINT", "SIGTERM", "SIGQUIT", "SIGABRT"] as const; type CleanupSignal = (typeof CLEANUP_SIGNALS)[number]; const CLEANUP_STATE_KEY = Symbol.for("openclaw.sessionWriteLockCleanupState"); -const HELD_LOCKS_KEY = Symbol.for("openclaw.sessionWriteLockHeldLocks"); const WATCHDOG_STATE_KEY = Symbol.for("openclaw.sessionWriteLockWatchdogState"); const DEFAULT_STALE_MS = 30 * 60 * 1000; @@ -73,7 +60,7 @@ type LockInspectionDetails = Pick< "pid" | "pidAlive" | "createdAt" | "ageMs" | "stale" | "staleReasons" >; -const HELD_LOCKS = resolveProcessScopedMap(HELD_LOCKS_KEY); +const SESSION_LOCKS = createFileLockManager("openclaw.session-write-lock"); export type SessionWriteLockAcquireTimeoutConfig = { session?: { @@ -151,105 +138,30 @@ export function resolveSessionLockMaxHoldFromTimeout(params: { return Math.min(MAX_LOCK_HOLD_MS, Math.max(minMs, timeoutMs + graceMs)); } -async function releaseHeldLock( - normalizedSessionFile: string, - held: HeldLock, - opts: { force?: boolean } = {}, -): Promise { - const current = HELD_LOCKS.get(normalizedSessionFile); - if (current !== held) { - return false; - } - - if (opts.force) { - held.count = 0; - } else { - held.count -= 1; - if (held.count > 0) { - return false; - } - } - - if (held.releasePromise) { - await held.releasePromise.catch(() => undefined); - return true; - } - - HELD_LOCKS.delete(normalizedSessionFile); - held.releasePromise = (async () => { - try { - await held.handle.close(); - } catch { - // Ignore errors during cleanup - best effort. - } - try { - await fs.rm(held.lockPath, { force: true }); - } catch { - // Ignore errors during cleanup - best effort. - } - })(); - - try { - await held.releasePromise; - return true; - } finally { - held.releasePromise = undefined; - if (HELD_LOCKS.size === 0) { - stopWatchdogTimer(); - } - } -} - /** * Synchronously release all held locks. * Used during process exit when async operations aren't reliable. */ function releaseAllLocksSync(): void { - for (const [sessionFile, held] of HELD_LOCKS) { - closeFileHandleSyncBestEffort(held.handle); - try { - fsSync.rmSync(held.lockPath, { force: true }); - } catch { - // Ignore errors during cleanup - best effort - } - HELD_LOCKS.delete(sessionFile); - } - if (HELD_LOCKS.size === 0) { - stopWatchdogTimer(); - } -} - -function closeFileHandleSyncBestEffort(handle: fs.FileHandle): void { - const syncCloseSymbol = Object.getOwnPropertySymbols(Object.getPrototypeOf(handle)).find( - (symbol) => symbol.description === "kCloseSync", - ); - if (syncCloseSymbol) { - const closeSync = (handle as SyncClosableFileHandle)[syncCloseSymbol]; - if (typeof closeSync === "function") { - try { - closeSync.call(handle); - return; - } catch { - // Fall back to async close below. - } - } - } - void handle.close().catch(() => undefined); + SESSION_LOCKS.reset(); + stopWatchdogTimer(); } async function runLockWatchdogCheck(nowMs = Date.now()): Promise { let released = 0; - for (const [sessionFile, held] of HELD_LOCKS.entries()) { + for (const held of SESSION_LOCKS.heldEntries()) { + const maxHoldMs = + typeof held.metadata.maxHoldMs === "number" ? held.metadata.maxHoldMs : DEFAULT_MAX_HOLD_MS; const heldForMs = nowMs - held.acquiredAt; - if (heldForMs <= held.maxHoldMs) { + if (heldForMs <= maxHoldMs) { continue; } process.stderr.write( - `[session-write-lock] releasing lock held for ${heldForMs}ms (max=${held.maxHoldMs}ms): ${held.lockPath}\n`, + `[session-write-lock] releasing lock held for ${heldForMs}ms (max=${maxHoldMs}ms): ${held.lockPath}\n`, ); - const didRelease = await releaseHeldLock(sessionFile, held, { force: true }); + const didRelease = await held.forceRelease(); if (didRelease) { released += 1; } @@ -458,14 +370,14 @@ async function shouldReclaimContendedLockFile( function shouldTreatAsOrphanSelfLock(params: { payload: LockFilePayload | null; - normalizedSessionFile: string; + heldByThisProcess: boolean; reclaimLockWithoutStarttime: boolean; }): boolean { const pid = isValidLockNumber(params.payload?.pid) ? params.payload.pid : null; if (pid !== process.pid) { return false; } - if (HELD_LOCKS.has(params.normalizedSessionFile)) { + if (params.heldByThisProcess) { return false; } @@ -484,14 +396,14 @@ function inspectLockPayloadForSession(params: { payload: LockFilePayload | null; staleMs: number; nowMs: number; - normalizedSessionFile: string; + heldByThisProcess: boolean; reclaimLockWithoutStarttime: boolean; }): LockInspectionDetails { const inspected = inspectLockPayload(params.payload, params.staleMs, params.nowMs); if ( !shouldTreatAsOrphanSelfLock({ payload: params.payload, - normalizedSessionFile: params.normalizedSessionFile, + heldByThisProcess: params.heldByThisProcess, reclaimLockWithoutStarttime: params.reclaimLockWithoutStarttime, }) ) { @@ -541,13 +453,11 @@ export async function cleanStaleLockFiles(params: { for (const entry of lockEntries) { const lockPath = path.join(sessionsDir, entry.name); const payload = await readLockPayload(lockPath); - const sessionFile = lockPath.slice(0, -".lock".length); - const normalizedSessionFile = await resolveNormalizedSessionFile(sessionFile); const inspected = inspectLockPayloadForSession({ payload, staleMs, nowMs, - normalizedSessionFile, + heldByThisProcess: false, reclaimLockWithoutStarttime: false, }); const lockInfo: SessionLockInspection = { @@ -589,97 +499,46 @@ export async function acquireSessionWriteLock(params: { const maxHoldMs = resolvePositiveMs(params.maxHoldMs, DEFAULT_MAX_HOLD_MS); const sessionFile = path.resolve(params.sessionFile); const sessionDir = path.dirname(sessionFile); - await fs.mkdir(sessionDir, { recursive: true }); const normalizedSessionFile = await resolveNormalizedSessionFile(sessionFile); const lockPath = `${normalizedSessionFile}.lock`; - - const held = HELD_LOCKS.get(normalizedSessionFile); - if (allowReentrant && held) { - held.count += 1; - return { - release: async () => { - await releaseHeldLock(normalizedSessionFile, held); + await fs.mkdir(sessionDir, { recursive: true }); + try { + const lock = await SESSION_LOCKS.acquire(sessionFile, { + staleMs, + timeoutMs, + retry: { minTimeout: 50, maxTimeout: 1000, factor: 1 }, + allowReentrant, + metadata: { maxHoldMs }, + payload: () => { + const createdAt = new Date().toISOString(); + const starttime = getProcessStartTime(process.pid); + const lockPayload: LockFilePayload = { pid: process.pid, createdAt }; + if (starttime !== null) { + lockPayload.starttime = starttime; + } + return lockPayload as Record; }, - }; - } - - const startedAt = Date.now(); - let attempt = 0; - while (Date.now() - startedAt < timeoutMs) { - attempt += 1; - let handle: fs.FileHandle | null = null; - try { - handle = await fs.open(lockPath, "wx"); - const createdHeld: HeldLock = { - count: 1, - handle, - lockPath, - acquiredAt: Date.now(), - maxHoldMs, - }; - HELD_LOCKS.set(normalizedSessionFile, createdHeld); - const createdAt = new Date().toISOString(); - const starttime = getProcessStartTime(process.pid); - const lockPayload: LockFilePayload = { pid: process.pid, createdAt }; - if (starttime !== null) { - lockPayload.starttime = starttime; - } - await handle.writeFile(JSON.stringify(lockPayload, null, 2), "utf8"); - return { - release: async () => { - await releaseHeldLock(normalizedSessionFile, createdHeld); - }, - }; - } catch (err) { - if (handle) { - const currentHeld = HELD_LOCKS.get(normalizedSessionFile); - if (currentHeld?.handle === handle) { - HELD_LOCKS.delete(normalizedSessionFile); - if (HELD_LOCKS.size === 0) { - stopWatchdogTimer(); - } - } - try { - await handle.close(); - } catch { - // Ignore cleanup errors on failed lock initialization. - } - try { - await fs.rm(lockPath, { force: true }); - } catch { - // Ignore cleanup errors on failed lock initialization. - } - } - const code = (err as { code?: unknown }).code; - if (code !== "EEXIST") { - throw err; - } - const payload = await readLockPayload(lockPath); - const nowMs = Date.now(); - const inspected = inspectLockPayloadForSession({ - payload, - staleMs, - nowMs, - normalizedSessionFile, - reclaimLockWithoutStarttime: true, - }); - if (await shouldReclaimContendedLockFile(lockPath, inspected, staleMs, nowMs)) { - await fs.rm(lockPath, { force: true }); - continue; - } - - const remainingMs = timeoutMs - (Date.now() - startedAt); - if (remainingMs <= 0) { - break; - } - const delay = Math.min(1000, 50 * attempt, remainingMs); - await new Promise((r) => setTimeout(r, delay)); + shouldReclaim: async ({ payload, nowMs, heldByThisProcess }) => { + const inspected = inspectLockPayloadForSession({ + payload: payload as LockFilePayload | null, + staleMs, + nowMs, + heldByThisProcess, + reclaimLockWithoutStarttime: true, + }); + return await shouldReclaimContendedLockFile(lockPath, inspected, staleMs, nowMs); + }, + }); + return { release: lock.release }; + } catch (err) { + if ((err as { code?: unknown }).code !== "file_lock_timeout") { + throw err; } + const timeoutLockPath = (err as { lockPath?: string }).lockPath ?? lockPath; + const payload = await readLockPayload(timeoutLockPath); + const owner = typeof payload?.pid === "number" ? `pid=${payload.pid}` : "unknown"; + throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath: timeoutLockPath }); } - - const payload = await readLockPayload(lockPath); - const owner = typeof payload?.pid === "number" ? `pid=${payload.pid}` : "unknown"; - throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath }); } export const __testing = { @@ -690,9 +549,7 @@ export const __testing = { }; export async function drainSessionWriteLockStateForTest(): Promise { - for (const [sessionFile, held] of Array.from(HELD_LOCKS.entries())) { - await releaseHeldLock(sessionFile, held, { force: true }).catch(() => undefined); - } + await SESSION_LOCKS.drain(); stopWatchdogTimer(); unregisterCleanupHandlers(); } diff --git a/src/agents/skills-clawhub.test.ts b/src/agents/skills-clawhub.test.ts index eaceb6e1c29..74836ea04de 100644 --- a/src/agents/skills-clawhub.test.ts +++ b/src/agents/skills-clawhub.test.ts @@ -11,7 +11,7 @@ const searchClawHubSkillsMock = vi.fn(); const archiveCleanupMock = vi.fn(); const withExtractedArchiveRootMock = vi.fn(); const installPackageDirMock = vi.fn(); -const fileExistsMock = vi.fn(); +const pathExistsMock = vi.fn(); vi.mock("../infra/clawhub.js", () => ({ fetchClawHubSkillDetail: fetchClawHubSkillDetailMock, @@ -29,8 +29,8 @@ vi.mock("../infra/install-package-dir.js", () => ({ installPackageDir: installPackageDirMock, })); -vi.mock("../infra/archive.js", () => ({ - fileExists: fileExistsMock, +vi.mock("../infra/fs-safe.js", () => ({ + pathExists: pathExistsMock, })); const { installSkillFromClawHub, searchSkillsFromClawHub, updateSkillsFromClawHub } = @@ -46,10 +46,10 @@ describe("skills-clawhub", () => { archiveCleanupMock.mockReset(); withExtractedArchiveRootMock.mockReset(); installPackageDirMock.mockReset(); - fileExistsMock.mockReset(); + pathExistsMock.mockReset(); resolveClawHubBaseUrlMock.mockReturnValue("https://clawhub.ai"); - fileExistsMock.mockImplementation(async (input: string) => input.endsWith("SKILL.md")); + pathExistsMock.mockImplementation(async (input: string) => input.endsWith("SKILL.md")); fetchClawHubSkillDetailMock.mockResolvedValue({ skill: { slug: "agentreceipt", diff --git a/src/agents/skills-clawhub.ts b/src/agents/skills-clawhub.ts index a88f9be113d..e53a76dc782 100644 --- a/src/agents/skills-clawhub.ts +++ b/src/agents/skills-clawhub.ts @@ -1,6 +1,5 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { fileExists } from "../infra/archive.js"; import { downloadClawHubSkillArchive, fetchClawHubSkillDetail, @@ -10,6 +9,7 @@ import { type ClawHubSkillSearchResult, } from "../infra/clawhub.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { pathExists } from "../infra/fs-safe.js"; import { withExtractedArchiveRoot } from "../infra/install-flow.js"; import { installPackageDir } from "../infra/install-package-dir.js"; import { resolveSafeInstallDir } from "../infra/install-safe-path.js"; @@ -133,7 +133,7 @@ function resolveSkillInstallDir(workspaceDir: string, slug: string): string { async function ensureSkillRoot(rootDir: string): Promise { for (const candidate of ["SKILL.md", "skill.md", "skills.md", "SKILL.MD"]) { - if (await fileExists(path.join(rootDir, candidate))) { + if (await pathExists(path.join(rootDir, candidate))) { return; } } @@ -274,7 +274,7 @@ async function performClawHubSkillInstall( baseUrl: params.baseUrl, }); const targetDir = resolveSkillInstallDir(params.workspaceDir, params.slug); - if (!params.force && (await fileExists(targetDir))) { + if (!params.force && (await pathExists(targetDir))) { return { ok: false, error: `Skill already exists at ${targetDir}. Re-run with force/update.`, diff --git a/src/agents/skills-install-download.ts b/src/agents/skills-install-download.ts index 39e5d865278..7d3aafc6632 100644 --- a/src/agents/skills-install-download.ts +++ b/src/agents/skills-install-download.ts @@ -6,7 +6,7 @@ import { pipeline } from "node:stream/promises"; import type { ReadableStream as NodeReadableStream } from "node:stream/web"; import { isWindowsDrivePath } from "../infra/archive-path.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { writeFileFromPathWithinRoot } from "../infra/fs-safe.js"; +import { root as fsRoot } from "../infra/fs-safe.js"; import { assertCanonicalPathWithinBase } from "../infra/install-safe-path.js"; import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js"; import { isWithinDir } from "../infra/path-safety.js"; @@ -29,21 +29,21 @@ function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { } function resolveDownloadTargetDir(entry: SkillEntry, spec: SkillInstallSpec): string { - const safeRoot = resolveSkillToolsRootDir(entry); + const root = resolveSkillToolsRootDir(entry); const raw = spec.targetDir?.trim(); if (!raw) { - return safeRoot; + return root; } // Treat non-absolute paths as relative to the per-skill tools root. const resolved = raw.startsWith("~") || path.isAbsolute(raw) || isWindowsDrivePath(raw) ? resolveUserPath(raw) - : path.resolve(safeRoot, raw); + : path.resolve(root, raw); - if (!isWithinDir(safeRoot, resolved)) { + if (!isWithinDir(root, resolved)) { throw new Error( - `Refusing to install outside the skill tools directory. targetDir="${raw}" resolves to "${resolved}". Allowed root: "${safeRoot}".`, + `Refusing to install outside the skill tools directory. targetDir="${raw}" resolves to "${resolved}". Allowed root: "${root}".`, ); } return resolved; @@ -99,11 +99,8 @@ async function downloadFile(params: { ? body : Readable.fromWeb(body as NodeReadableStream); await pipeline(readable, file); - await writeFileFromPathWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - sourcePath: tempPath, - }); + const root = await fsRoot(params.rootDir); + await root.copyIn(params.relativePath, tempPath); const stat = await fs.promises.stat(destPath); return { bytes: stat.size }; } finally { @@ -118,7 +115,7 @@ export async function installDownloadSpec(params: { timeoutMs: number; }): Promise { const { entry, spec, timeoutMs } = params; - const safeRoot = resolveSkillToolsRootDir(entry); + const root = resolveSkillToolsRootDir(entry); const url = spec.url?.trim(); if (!url) { return { @@ -141,33 +138,33 @@ export async function installDownloadSpec(params: { filename = "download"; } - let canonicalSafeRoot = ""; + let canonicalRoot = ""; let targetDir = ""; try { - await ensureDir(safeRoot); + await ensureDir(root); await assertCanonicalPathWithinBase({ - baseDir: safeRoot, - candidatePath: safeRoot, + baseDir: root, + candidatePath: root, boundaryLabel: "skill tools directory", }); - canonicalSafeRoot = await fs.promises.realpath(safeRoot); + canonicalRoot = await fs.promises.realpath(root); const requestedTargetDir = resolveDownloadTargetDir(entry, spec); await ensureDir(requestedTargetDir); await assertCanonicalPathWithinBase({ - baseDir: safeRoot, + baseDir: root, candidatePath: requestedTargetDir, boundaryLabel: "skill tools directory", }); - const targetRelativePath = path.relative(safeRoot, requestedTargetDir); - targetDir = path.join(canonicalSafeRoot, targetRelativePath); + const targetRelativePath = path.relative(root, requestedTargetDir); + targetDir = path.join(canonicalRoot, targetRelativePath); } catch (err) { const message = formatErrorMessage(err); return { ok: false, message, stdout: "", stderr: message, code: null }; } const archivePath = path.join(targetDir, filename); - const archiveRelativePath = path.relative(canonicalSafeRoot, archivePath); + const archiveRelativePath = path.relative(canonicalRoot, archivePath); if ( !archiveRelativePath || archiveRelativePath === ".." || @@ -186,7 +183,7 @@ export async function installDownloadSpec(params: { try { const result = await downloadFile({ url, - rootDir: canonicalSafeRoot, + rootDir: canonicalRoot, relativePath: archiveRelativePath, timeoutMs, }); @@ -220,7 +217,7 @@ export async function installDownloadSpec(params: { try { await assertCanonicalPathWithinBase({ - baseDir: canonicalSafeRoot, + baseDir: canonicalRoot, candidatePath: targetDir, boundaryLabel: "skill tools directory", }); diff --git a/src/agents/skills-install.download.test.ts b/src/agents/skills-install.download.test.ts index fe93bebedd1..8175883a410 100644 --- a/src/agents/skills-install.download.test.ts +++ b/src/agents/skills-install.download.test.ts @@ -22,60 +22,6 @@ vi.mock("../infra/net/fetch-guard.js", () => ({ fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args), })); -// Download tests cover installer path handling; fs-safe has dedicated pinned-helper coverage. -vi.mock("../infra/fs-pinned-write-helper.js", async () => { - const fs = await import("node:fs/promises"); - const path = await import("node:path"); - const { pipeline } = await import("node:stream/promises"); - - type PinnedWriteParams = { - rootPath: string; - relativeParentPath: string; - basename: string; - mkdir: boolean; - mode: number; - input: - | { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding } - | { kind: "stream"; stream: NodeJS.ReadableStream }; - }; - - async function resolveParentPath(params: PinnedWriteParams): Promise { - const parentPath = params.relativeParentPath - ? path.join(params.rootPath, ...params.relativeParentPath.split("/")) - : params.rootPath; - if (params.mkdir) { - await fs.mkdir(parentPath, { recursive: true }); - } - return parentPath; - } - - async function writePinnedTarget(params: PinnedWriteParams, targetPath: string) { - if (params.input.kind === "buffer") { - await fs.writeFile(targetPath, params.input.data, { - encoding: params.input.encoding, - mode: params.mode, - }); - return; - } - const handle = await fs.open(targetPath, "w", params.mode); - try { - await pipeline(params.input.stream, handle.createWriteStream()); - } finally { - await handle.close().catch(() => undefined); - } - } - - return { - runPinnedWriteHelper: async (params: PinnedWriteParams) => { - const parentPath = await resolveParentPath(params); - const targetPath = path.join(parentPath, params.basename); - await writePinnedTarget(params, targetPath); - const stat = await fs.stat(targetPath); - return { dev: stat.dev, ino: stat.ino }; - }, - }; -}); - vi.mock("./skills.js", () => ({ hasBinary: (bin: string) => hasBinaryMock(bin), })); @@ -262,7 +208,7 @@ describe("installDownloadSpec extraction safety", () => { "fails closed when the lexical tools root is rebound before the final copy", async () => { const entry = buildEntry("base-rebind"); - const safeRoot = resolveSkillToolsRootDir(entry); + const safeToolsRoot = resolveSkillToolsRootDir(entry); const outsideRoot = path.join(workspaceDir, "outside-root"); await fs.mkdir(outsideRoot, { recursive: true }); @@ -274,9 +220,9 @@ describe("installDownloadSpec extraction safety", () => { body: Readable.from( (async function* () { yield Buffer.from("payload"); - const reboundRoot = `${safeRoot}-rebound`; - await fs.rename(safeRoot, reboundRoot); - await fs.symlink(outsideRoot, safeRoot); + const reboundRoot = `${safeToolsRoot}-rebound`; + await fs.rename(safeToolsRoot, reboundRoot); + await fs.symlink(outsideRoot, safeToolsRoot); })(), ), }, diff --git a/src/agents/skills/local-loader.ts b/src/agents/skills/local-loader.ts index c7f367e62b8..7993f5071a8 100644 --- a/src/agents/skills/local-loader.ts +++ b/src/agents/skills/local-loader.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import { openVerifiedFileSync } from "../../infra/safe-open-sync.js"; +import { openRootFileSync } from "../../infra/boundary-file-read.js"; import { parseFrontmatter, resolveSkillInvocationPolicy } from "./frontmatter.js"; import { createSyntheticSourceInfo, type Skill } from "./skill-contract.js"; import type { ParsedSkillFrontmatter } from "./types.js"; @@ -10,31 +10,22 @@ type LoadedLocalSkill = { frontmatter: ParsedSkillFrontmatter; }; -function isPathWithinRoot(rootRealPath: string, candidatePath: string): boolean { - const relative = path.relative(rootRealPath, candidatePath); - return ( - relative === "" || - (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)) - ); -} - function readSkillFileSync(params: { rootRealPath: string; filePath: string; maxBytes?: number; }): string | null { - const opened = openVerifiedFileSync({ - filePath: params.filePath, - rejectPathSymlink: true, + const opened = openRootFileSync({ + absolutePath: params.filePath, + rootPath: params.rootRealPath, + rootRealPath: params.rootRealPath, + boundaryLabel: "skill root", maxBytes: params.maxBytes, }); if (!opened.ok) { return null; } try { - if (!isPathWithinRoot(params.rootRealPath, opened.path)) { - return null; - } return fs.readFileSync(opened.fd, "utf8"); } finally { fs.closeSync(opened.fd); diff --git a/src/agents/skills/plugin-skills.ts b/src/agents/skills/plugin-skills.ts index 7c2ed971db0..5c2fc3e8480 100644 --- a/src/agents/skills/plugin-skills.ts +++ b/src/agents/skills/plugin-skills.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { isAcpRuntimeSpawnAvailable } from "../../acp/runtime/availability.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { walkDirectorySync } from "../../infra/fs-safe.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { normalizePluginsConfigWithResolver, @@ -130,15 +131,13 @@ function collectSkillTargets(dir: string, targets: Map): void { return; } - let entries: fs.Dirent[]; - try { - entries = fs.readdirSync(dir, { withFileTypes: true }); - } catch { - return; - } + const entries = walkDirectorySync(dir, { + maxDepth: 1, + symlinks: "skip", + include: (entry) => entry.kind === "directory", + }).entries; for (const entry of entries) { - if (!entry.isDirectory()) continue; - const childPath = path.join(dir, entry.name); + const childPath = entry.path; if (!hasPublishableSkillFile({ skillDir: childPath, rootDir: dir })) continue; const basename = entry.name; const existing = targets.get(basename); diff --git a/src/agents/skills/workspace.ts b/src/agents/skills/workspace.ts index 0b4b569779a..5e85e7a70d2 100644 --- a/src/agents/skills/workspace.ts +++ b/src/agents/skills/workspace.ts @@ -1,7 +1,8 @@ -import fs, { type Dirent } from "node:fs"; +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { walkDirectorySync } from "../../infra/fs-safe.js"; import { resolveOsHomeDir } from "../../infra/home-dir.js"; import { isPathInside } from "../../infra/path-guards.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; @@ -181,44 +182,21 @@ function listChildDirectories( opts?.maxRawEntriesToScan === undefined ? resolveRawEntryScanLimit(opts?.maxCandidateDirs) : Math.max(0, opts.maxRawEntriesToScan); - try { - const dirs: string[] = []; - let scannedEntryCount = 0; - let truncated = false; - const handle = fs.opendirSync(dir); - try { - let entry: Dirent | null; - while ((entry = handle.readSync()) !== null) { - if (scannedEntryCount >= maxRawEntriesToScan) { - truncated = true; - break; - } - scannedEntryCount += 1; - - if (entry.name.startsWith(".")) continue; - if (entry.name === "node_modules") continue; - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - dirs.push(entry.name); - continue; - } - if (entry.isSymbolicLink()) { - try { - if (fs.statSync(fullPath).isDirectory()) { - dirs.push(entry.name); - } - } catch { - // ignore broken symlinks - } - } - } - } finally { - handle.closeSync(); - } - return { dirs, scannedEntryCount, truncated }; - } catch { + const scan = walkDirectorySync(dir, { + maxDepth: 1, + maxEntries: maxRawEntriesToScan, + symlinks: "follow", + include: (entry) => + entry.kind === "directory" && !entry.name.startsWith(".") && entry.name !== "node_modules", + }); + if (scan.scannedEntryCount === 0 && scan.entries.length === 0) { return { dirs: [], scannedEntryCount: 0, truncated: false }; } + return { + dirs: scan.entries.map((entry) => entry.name), + scannedEntryCount: scan.scannedEntryCount, + truncated: scan.truncated, + }; } function resolveRawEntryScanLimit(maxCandidateDirs: number | undefined): number { diff --git a/src/agents/subagent-attachments.ts b/src/agents/subagent-attachments.ts index 5eca5bcd543..7474fd29e82 100644 --- a/src/agents/subagent-attachments.ts +++ b/src/agents/subagent-attachments.ts @@ -2,6 +2,7 @@ import crypto from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { privateFileStore } from "../infra/private-file-store.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; import { resolveAgentWorkspaceDir } from "./agent-scope.js"; @@ -131,6 +132,7 @@ export async function materializeSubagentAttachments(params: { try { await fs.mkdir(absDir, { recursive: true, mode: 0o700 }); + const store = privateFileStore(absDir); const seen = new Set(); const files: SubagentAttachmentReceiptFile[] = []; @@ -192,14 +194,11 @@ export async function materializeSubagentAttachments(params: { } const sha256 = crypto.createHash("sha256").update(buf).digest("hex"); - const outPath = path.join(absDir, name); - writeJobs.push({ outPath, buf }); + writeJobs.push({ outPath: name, buf }); files.push({ name, bytes, sha256 }); } - await Promise.all( - writeJobs.map(({ outPath, buf }) => fs.writeFile(outPath, buf, { mode: 0o600, flag: "wx" })), - ); + await Promise.all(writeJobs.map(({ outPath, buf }) => store.writeText(outPath, buf))); const manifest = { relDir, @@ -207,14 +206,7 @@ export async function materializeSubagentAttachments(params: { totalBytes, files, }; - await fs.writeFile( - path.join(absDir, ".manifest.json"), - JSON.stringify(manifest, null, 2) + "\n", - { - mode: 0o600, - flag: "wx", - }, - ); + await store.writeJson(".manifest.json", manifest, { trailingNewline: true }); return { status: "ok", diff --git a/src/agents/tools/canvas-tool.ts b/src/agents/tools/canvas-tool.ts index 7ec8d521681..41c56e70678 100644 --- a/src/agents/tools/canvas-tool.ts +++ b/src/agents/tools/canvas-tool.ts @@ -1,12 +1,10 @@ import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; import { Type } from "typebox"; import { writeBase64ToFile } from "../../cli/nodes-camera.js"; import { canvasSnapshotTempPath, parseCanvasSnapshotPayload } from "../../cli/nodes-canvas.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { logVerbose, shouldLogVerbose } from "../../globals.js"; -import { isInboundPathAllowed } from "../../media/inbound-path-policy.js"; +import { readLocalFileFromRoots } from "../../infra/fs-safe.js"; import { getDefaultMediaLocalRoots } from "../../media/local-roots.js"; import { imageMimeFromFormat } from "../../media/mime.js"; import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js"; @@ -33,22 +31,19 @@ async function readJsonlFromPath(jsonlPath: string): Promise { if (!trimmed) { return ""; } - const resolved = path.resolve(trimmed); const roots = getDefaultMediaLocalRoots(); - if (!isInboundPathAllowed({ filePath: resolved, roots })) { + const result = await readLocalFileFromRoots({ + filePath: trimmed, + roots, + label: "canvas jsonlPath", + }); + if (!result) { if (shouldLogVerbose()) { - logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${resolved}`); + logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${trimmed}`); } throw new Error("jsonlPath outside allowed roots"); } - const canonical = await fs.realpath(resolved).catch(() => resolved); - if (!isInboundPathAllowed({ filePath: canonical, roots })) { - if (shouldLogVerbose()) { - logVerbose(`Blocked canvas jsonlPath outside allowed roots: ${canonical}`); - } - throw new Error("jsonlPath outside allowed roots"); - } - return await fs.readFile(canonical, "utf8"); + return result.buffer.toString("utf8"); } // Flattened schema: runtime validates per-action requirements. diff --git a/src/agents/tools/common.ts b/src/agents/tools/common.ts index 5276dfb477e..75043d3b51b 100644 --- a/src/agents/tools/common.ts +++ b/src/agents/tools/common.ts @@ -1,10 +1,10 @@ -import fs from "node:fs/promises"; import type { AgentTool, AgentToolResult, AgentToolUpdateCallback, } from "@mariozechner/pi-agent-core"; import type { TSchema } from "typebox"; +import { readLocalFileSafely } from "../../infra/fs-safe.js"; import { detectMime } from "../../media/mime.js"; import { readSnakeCaseParamRaw } from "../../param-key.js"; import type { ImageSanitizationLimits } from "../image-sanitization.js"; @@ -345,7 +345,7 @@ export async function imageResultFromFile(params: { details?: Record; imageSanitization?: ImageSanitizationLimits; }): Promise> { - const buf = await fs.readFile(params.path); + const buf = (await readLocalFileSafely({ filePath: params.path })).buffer; const mimeType = (await detectMime({ buffer: buf.slice(0, 256) })) ?? "image/png"; return await imageResult({ label: params.label, diff --git a/src/agents/workspace.ts b/src/agents/workspace.ts index 69bf64ca604..af528da8151 100644 --- a/src/agents/workspace.ts +++ b/src/agents/workspace.ts @@ -1,7 +1,9 @@ import syncFs from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; -import { openBoundaryFile } from "../infra/boundary-file-read.js"; +import { openRootFile } from "../infra/boundary-file-read.js"; +import { pathExists } from "../infra/fs-safe.js"; +import { replaceFileAtomic } from "../infra/replace-file.js"; import { CANONICAL_ROOT_MEMORY_FILENAME, exactWorkspaceEntryExists, @@ -55,7 +57,7 @@ async function readWorkspaceFileWithGuards(params: { filePath: string; workspaceDir: string; }): Promise { - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: params.filePath, rootPath: params.workspaceDir, boundaryLabel: "workspace root", @@ -197,15 +199,6 @@ async function writeFileIfMissing(filePath: string, content: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function fileContentDiffersFromTemplate( filePath: string, template: string, @@ -274,7 +267,7 @@ async function reconcileWorkspaceBootstrapCompletionState(params: { state: WorkspaceSetupState; bootstrapExists?: boolean; }): Promise { - const bootstrapExists = params.bootstrapExists ?? (await fileExists(params.bootstrapPath)); + const bootstrapExists = params.bootstrapExists ?? (await pathExists(params.bootstrapPath)); if ( typeof params.state.setupCompletedAt === "string" && params.state.setupCompletedAt.trim().length > 0 @@ -384,7 +377,7 @@ export async function resolveWorkspaceBootstrapStatus( return "complete"; } const bootstrapPath = path.join(resolvedDir, DEFAULT_BOOTSTRAP_FILENAME); - const bootstrapExists = await fileExists(bootstrapPath); + const bootstrapExists = await pathExists(bootstrapPath); if (!bootstrapExists) { return "complete"; } @@ -416,16 +409,11 @@ async function writeWorkspaceSetupState( statePath: string, state: WorkspaceSetupState, ): Promise { - await fs.mkdir(path.dirname(statePath), { recursive: true }); - const payload = `${JSON.stringify(state, null, 2)}\n`; - const tmpPath = `${statePath}.tmp-${process.pid}-${Date.now().toString(36)}`; - try { - await fs.writeFile(tmpPath, payload, { encoding: "utf-8" }); - await fs.rename(tmpPath, statePath); - } catch (err) { - await fs.unlink(tmpPath).catch(() => {}); - throw err; - } + await replaceFileAtomic({ + filePath: statePath, + content: `${JSON.stringify(state, null, 2)}\n`, + tempPrefix: ".workspace-state", + }); } async function hasGitRepo(dir: string): Promise { @@ -561,7 +549,7 @@ export async function ensureAgentWorkspace(params?: { }; const nowIso = () => new Date().toISOString(); - let bootstrapExists = await fileExists(bootstrapPath); + let bootstrapExists = await pathExists(bootstrapPath); if (!state.bootstrapSeededAt && bootstrapExists) { markState({ bootstrapSeededAt: nowIso() }); } @@ -596,7 +584,7 @@ export async function ensureAgentWorkspace(params?: { const bootstrapTemplate = await loadTemplate(DEFAULT_BOOTSTRAP_FILENAME); const wroteBootstrap = await writeFileIfMissing(bootstrapPath, bootstrapTemplate); if (!wroteBootstrap) { - bootstrapExists = await fileExists(bootstrapPath); + bootstrapExists = await pathExists(bootstrapPath); } else { bootstrapExists = true; } diff --git a/src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts b/src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts index 567cbb62779..c0c73934251 100644 --- a/src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts +++ b/src/auto-reply/reply.triggers.trigger-handling.stages-inbound-media-into-sandbox-workspace.test.ts @@ -17,19 +17,20 @@ const childProcessMocks = vi.hoisted(() => ({ spawn: vi.fn(), })); const fsSafeMocks = vi.hoisted(() => { - class MockSafeOpenError extends Error { + class MockFsSafeError extends Error { readonly code: string; constructor(code: string, message: string) { super(message); - this.name = "SafeOpenError"; + this.name = "FsSafeError"; this.code = code; } } return { - SafeOpenError: MockSafeOpenError, - copyFileWithinRoot: vi.fn(), + FsSafeError: MockFsSafeError, + rootCopyFrom: vi.fn(), + root: vi.fn(), readLocalFileSafely: vi.fn(), }; }); @@ -51,7 +52,7 @@ vi.mock("node:child_process", async () => { vi.mock("../infra/fs-safe.js", () => fsSafeMocks); vi.mock("../media/channel-inbound-roots.js", () => mediaRootMocks); -async function copyFileWithinRootForTest({ +async function rootCopyFromForTest({ sourcePath, rootDir, relativePath, @@ -64,7 +65,7 @@ async function copyFileWithinRootForTest({ }) { const sourceStat = await fs.stat(sourcePath); if (typeof maxBytes === "number" && sourceStat.size > maxBytes) { - throw new fsSafeMocks.SafeOpenError( + throw new fsSafeMocks.FsSafeError( "too-large", `file exceeds limit of ${maxBytes} bytes (got ${sourceStat.size})`, ); @@ -75,7 +76,7 @@ async function copyFileWithinRootForTest({ const destPath = path.resolve(rootReal, relativePath); const rootPrefix = `${rootReal}${path.sep}`; if (destPath !== rootReal && !destPath.startsWith(rootPrefix)) { - throw new fsSafeMocks.SafeOpenError("outside-workspace", "file is outside workspace root"); + throw new fsSafeMocks.FsSafeError("outside-workspace", "file is outside workspace root"); } const parentDir = dirname(destPath); @@ -87,7 +88,7 @@ async function copyFileWithinRootForTest({ try { const stat = await fs.lstat(cursor); if (stat.isSymbolicLink()) { - throw new fsSafeMocks.SafeOpenError("symlink", "symlink not allowed"); + throw new fsSafeMocks.FsSafeError("symlink", "symlink not allowed"); } } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") { @@ -102,7 +103,7 @@ async function copyFileWithinRootForTest({ try { const destStat = await fs.lstat(destPath); if (destStat.isSymbolicLink()) { - throw new fsSafeMocks.SafeOpenError("symlink", "symlink not allowed"); + throw new fsSafeMocks.FsSafeError("symlink", "symlink not allowed"); } } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") { @@ -117,7 +118,16 @@ beforeEach(() => { sandboxMocks.ensureSandboxWorkspaceForSession.mockReset(); sandboxMocks.assertSandboxPath.mockReset().mockResolvedValue({ resolved: "", relative: "" }); childProcessMocks.spawn.mockClear(); - fsSafeMocks.copyFileWithinRoot.mockReset().mockImplementation(copyFileWithinRootForTest); + fsSafeMocks.rootCopyFrom.mockReset().mockImplementation(rootCopyFromForTest); + fsSafeMocks.root.mockReset().mockImplementation(async (rootDir: string) => ({ + copyFrom: async (sourcePath: string, relativePath: string, options?: { maxBytes?: number }) => + await rootCopyFromForTest({ + sourcePath, + rootDir, + relativePath, + maxBytes: options?.maxBytes, + }), + })); mediaRootMocks.resolveChannelRemoteInboundAttachmentRoots .mockReset() .mockReturnValue(["/Users/demo/Library/Messages/Attachments"]); diff --git a/src/auto-reply/reply/commands-export-session.ts b/src/auto-reply/reply/commands-export-session.ts index d372aff20b0..3d585f7db69 100644 --- a/src/auto-reply/reply/commands-export-session.ts +++ b/src/auto-reply/reply/commands-export-session.ts @@ -7,6 +7,7 @@ import { type SessionEntry as PiSessionEntry, type SessionHeader, } from "@mariozechner/pi-coding-agent"; +import { pathExists } from "../../infra/fs-safe.js"; import type { ReplyPayload } from "../types.js"; import { isReplyPayload, @@ -122,15 +123,6 @@ async function generateHtml(sessionData: SessionData): Promise { ].reduce((html, [name, value]) => replaceHtmlPlaceholder(html, name, value), template); } -async function fileExists(pathName: string): Promise { - try { - await fsp.access(pathName); - return true; - } catch { - return false; - } -} - function addCollisionSuffix(filePath: string, suffix: number): string { const ext = path.extname(filePath); const baseName = path.basename(filePath, ext); @@ -152,7 +144,6 @@ async function writeNewDefaultExportFile(filePath: string, html: string): Promis } throw new Error(`Could not find an unused export filename near ${filePath}`); } - async function readSessionDataFromTranscript(sessionFile: string): Promise<{ header: SessionHeader | null; entries: PiSessionEntry[]; @@ -183,7 +174,7 @@ export async function buildExportSessionReply(params: HandleCommandsParams): Pro } const { entry, sessionFile } = sessionTarget; - if (!(await fileExists(sessionFile))) { + if (!(await pathExists(sessionFile))) { return { text: `❌ Session file not found: ${sessionFile}` }; } diff --git a/src/auto-reply/reply/commands-export-trajectory.test.ts b/src/auto-reply/reply/commands-export-trajectory.test.ts index 16376aef9f7..b27b626497d 100644 --- a/src/auto-reply/reply/commands-export-trajectory.test.ts +++ b/src/auto-reply/reply/commands-export-trajectory.test.ts @@ -27,6 +27,11 @@ const hoisted = await vi.hoisted(async () => { await actualAccess(file); }, ), + statMock: vi.fn( + async (file: fs.PathLike, actualStat: (path: fs.PathLike) => Promise) => { + return await actualStat(file); + }, + ), }; }); @@ -59,6 +64,7 @@ vi.mock("node:fs/promises", async () => { const mockedFs = { ...actual, access: (file: fs.PathLike) => hoisted.accessMock(file, actual.access), + stat: (file: fs.PathLike) => hoisted.statMock(file, actual.stat), }; return { ...mockedFs, @@ -67,6 +73,7 @@ vi.mock("node:fs/promises", async () => { }); const tempDirs: string[] = []; +const mockedSessionFile = "/tmp/target-store/session.jsonl"; function makeTempDir(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-export-command-")); @@ -173,9 +180,20 @@ describe("buildExportTrajectoryReply", () => { await actualAccess(file); }, ); + hoisted.statMock.mockImplementation( + async (file: fs.PathLike, actualStat: (path: fs.PathLike) => Promise) => { + if (file.toString() === "/tmp/target-store/session.jsonl") { + return {}; + } + return await actualStat(file); + }, + ); + fs.mkdirSync(path.dirname(mockedSessionFile), { recursive: true }); + fs.writeFileSync(mockedSessionFile, "{}\n"); }); afterEach(() => { + fs.rmSync(mockedSessionFile, { force: true }); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -238,6 +256,7 @@ describe("buildExportTrajectoryReply", () => { it("does not echo absolute session paths when the transcript is missing", async () => { const { buildExportTrajectoryReply } = await import("./commands-export-trajectory.js"); + fs.rmSync(mockedSessionFile, { force: true }); hoisted.accessMock.mockImplementation( async (file: fs.PathLike, actualAccess: (path: fs.PathLike) => Promise) => { if (file.toString() === "/tmp/target-store/session.jsonl") { @@ -246,6 +265,14 @@ describe("buildExportTrajectoryReply", () => { await actualAccess(file); }, ); + hoisted.statMock.mockImplementation( + async (file: fs.PathLike, actualStat: (path: fs.PathLike) => Promise) => { + if (file.toString() === "/tmp/target-store/session.jsonl") { + throw Object.assign(new Error("missing"), { code: "ENOENT" }); + } + return await actualStat(file); + }, + ); const reply = await buildExportTrajectoryReply(makeParams()); diff --git a/src/auto-reply/reply/commands-export-trajectory.ts b/src/auto-reply/reply/commands-export-trajectory.ts index 8ac1cd58e31..b0359d414a4 100644 --- a/src/auto-reply/reply/commands-export-trajectory.ts +++ b/src/auto-reply/reply/commands-export-trajectory.ts @@ -1,9 +1,9 @@ -import fsp from "node:fs/promises"; import { resolveSessionAgentId } from "../../agents/agent-scope.js"; import { createExecTool } from "../../agents/bash-tools.js"; import type { ExecToolDetails } from "../../agents/bash-tools.js"; import { formatErrorMessage } from "../../infra/errors.js"; import type { ExecApprovalRequest } from "../../infra/exec-approvals.js"; +import { pathExists } from "../../infra/fs-safe.js"; import { exportTrajectoryForCommand, formatTrajectoryCommandExportSummary, @@ -56,15 +56,6 @@ const defaultExportTrajectoryCommandDeps: ExportTrajectoryCommandDeps = { deliverPrivateTrajectoryReply: deliverPrivateTrajectoryReply, }; -async function fileExists(pathName: string): Promise { - try { - await fsp.access(pathName); - return true; - } catch { - return false; - } -} - export async function buildExportTrajectoryCommandReply( params: HandleCommandsParams, deps: Partial = {}, @@ -146,7 +137,7 @@ export async function buildExportTrajectoryReply( } const { entry, sessionFile } = sessionTarget; - if (!(await fileExists(sessionFile))) { + if (!(await pathExists(sessionFile))) { return { text: "❌ Session file not found." }; } diff --git a/src/auto-reply/reply/post-compaction-context.ts b/src/auto-reply/reply/post-compaction-context.ts index 241e20d2c04..f29594b981d 100644 --- a/src/auto-reply/reply/post-compaction-context.ts +++ b/src/auto-reply/reply/post-compaction-context.ts @@ -4,7 +4,7 @@ import { resolveAgentContextLimits } from "../../agents/agent-scope.js"; import { resolveCronStyleNow } from "../../agents/current-time.js"; import { resolveUserTimezone } from "../../agents/date-time.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { openBoundaryFile } from "../../infra/boundary-file-read.js"; +import { openRootFile } from "../../infra/boundary-file-read.js"; import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js"; const MAX_CONTEXT_CHARS = 1800; @@ -78,7 +78,7 @@ export async function readPostCompactionContext( const agentsPath = path.join(workspaceDir, "AGENTS.md"); try { - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: agentsPath, rootPath: workspaceDir, boundaryLabel: "workspace root", diff --git a/src/auto-reply/reply/session-fork.runtime.ts b/src/auto-reply/reply/session-fork.runtime.ts index 91a649dd1e8..2c6197bf500 100644 --- a/src/auto-reply/reply/session-fork.runtime.ts +++ b/src/auto-reply/reply/session-fork.runtime.ts @@ -19,6 +19,7 @@ import { type SessionEntry as StoreSessionEntry, } from "../../config/sessions/types.js"; import { readLatestRecentSessionUsageFromTranscriptAsync } from "../../gateway/session-utils.fs.js"; +import { readRegularFile } from "../../infra/fs-safe.js"; type ForkSourceTranscript = { cwd: string; @@ -169,7 +170,7 @@ function collectBranchLabels(params: { async function readForkSourceTranscript( parentSessionFile: string, ): Promise { - const raw = await fs.readFile(parentSessionFile, "utf-8"); + const raw = (await readRegularFile({ filePath: parentSessionFile })).buffer.toString("utf-8"); const fileEntries = parseSessionEntries(raw); migrateSessionEntries(fileEntries); const header = @@ -281,15 +282,6 @@ async function writeBranchedSession(params: { return { sessionId, sessionFile }; } -async function fileExists(filePath: string): Promise { - try { - const stat = await fs.stat(filePath); - return stat.isFile(); - } catch { - return false; - } -} - export async function forkSessionFromParentRuntime(params: { parentEntry: StoreSessionEntry; agentId: string; @@ -300,7 +292,7 @@ export async function forkSessionFromParentRuntime(params: { params.parentEntry, { agentId: params.agentId, sessionsDir: params.sessionsDir }, ); - if (!parentSessionFile || !(await fileExists(parentSessionFile))) { + if (!parentSessionFile) { return null; } try { diff --git a/src/auto-reply/reply/stage-sandbox-media.ts b/src/auto-reply/reply/stage-sandbox-media.ts index d10a92c80e2..adc254327a5 100644 --- a/src/auto-reply/reply/stage-sandbox-media.ts +++ b/src/auto-reply/reply/stage-sandbox-media.ts @@ -7,7 +7,7 @@ import { ensureSandboxWorkspaceForSession } from "../../agents/sandbox.js"; import { slugifySessionKey } from "../../agents/sandbox/shared.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { logVerbose } from "../../globals.js"; -import { copyFileWithinRoot, SafeOpenError } from "../../infra/fs-safe.js"; +import { root as fsRoot, FsSafeError } from "../../infra/fs-safe.js"; import { normalizeScpRemoteHost, normalizeScpRemotePath } from "../../infra/scp-host.js"; import { resolvePreferredOpenClawTmpDir } from "../../infra/tmp-openclaw-dir.js"; import { resolveChannelRemoteInboundAttachmentRoots } from "../../media/channel-inbound-roots.js"; @@ -107,7 +107,7 @@ export async function stageSandboxMedia(params: { }); } } catch (err) { - if (err instanceof SafeOpenError && err.code === "too-large") { + if (err instanceof FsSafeError && err.code === "too-large") { logVerbose( `Blocking inbound media staging above ${STAGED_MEDIA_MAX_BYTES} bytes: ${source}`, ); @@ -139,10 +139,8 @@ async function stageLocalFileIntoRoot(params: { relativeDestPath: string; maxBytes?: number; }): Promise { - await copyFileWithinRoot({ - sourcePath: params.sourcePath, - rootDir: params.rootDir, - relativePath: params.relativeDestPath, + const root = await fsRoot(params.rootDir); + await root.copyIn(params.relativeDestPath, params.sourcePath, { maxBytes: params.maxBytes, }); } diff --git a/src/auto-reply/reply/startup-context.ts b/src/auto-reply/reply/startup-context.ts index e876f2144e3..a98b740de6b 100644 --- a/src/auto-reply/reply/startup-context.ts +++ b/src/auto-reply/reply/startup-context.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveUserTimezone } from "../../agents/date-time.js"; import type { OpenClawConfig } from "../../config/config.js"; -import { openBoundaryFile } from "../../infra/boundary-file-read.js"; +import { openRootFile } from "../../infra/boundary-file-read.js"; const STARTUP_MEMORY_FILE_MAX_BYTES = 16_384; const STARTUP_MEMORY_FILE_MAX_CHARS = 1_200; @@ -205,7 +205,7 @@ async function readStartupMemoryFile(params: { maxFileBytes: number; }): Promise { const absolutePath = path.join(params.workspaceDir, params.relativePath); - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath, rootPath: params.workspaceDir, boundaryLabel: "workspace root", diff --git a/src/canvas-host/file-resolver.test.ts b/src/canvas-host/file-resolver.test.ts new file mode 100644 index 00000000000..f8877faf3d7 --- /dev/null +++ b/src/canvas-host/file-resolver.test.ts @@ -0,0 +1,49 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; +import { normalizeUrlPath, resolveFileWithinRoot } from "./file-resolver.js"; + +const tempDirs = createTrackedTempDirs(); + +afterEach(async () => { + await tempDirs.cleanup(); +}); + +describe("resolveFileWithinRoot", () => { + it("normalizes URL paths", () => { + expect(normalizeUrlPath("/nested/../file.txt")).toBe("/file.txt"); + expect(normalizeUrlPath("plain.txt")).toBe("/plain.txt"); + }); + + it("opens directory index files through the fs-safe root", async () => { + const root = await tempDirs.make("openclaw-canvas-resolver-"); + await fs.mkdir(path.join(root, "docs"), { recursive: true }); + await fs.writeFile(path.join(root, "docs", "index.html"), "

docs

"); + + const result = await resolveFileWithinRoot(root, "/docs"); + expect(result).not.toBeNull(); + try { + await expect(result?.handle.readFile({ encoding: "utf8" })).resolves.toBe("

docs

"); + } finally { + await result?.handle.close().catch(() => {}); + } + }); + + it("rejects traversal paths", async () => { + const root = await tempDirs.make("openclaw-canvas-resolver-"); + + await expect(resolveFileWithinRoot(root, "/../outside.txt")).resolves.toBeNull(); + }); + + it.runIf(process.platform !== "win32")("rejects symlink entries", async () => { + const root = await tempDirs.make("openclaw-canvas-resolver-"); + const outside = await tempDirs.make("openclaw-canvas-resolver-outside-"); + const target = path.join(outside, "outside.html"); + const link = path.join(root, "link.html"); + await fs.writeFile(target, "outside"); + await fs.symlink(target, link); + + await expect(resolveFileWithinRoot(root, "/link.html")).resolves.toBeNull(); + }); +}); diff --git a/src/canvas-host/file-resolver.ts b/src/canvas-host/file-resolver.ts index 7f7ffc772e4..6f5f1a2e758 100644 --- a/src/canvas-host/file-resolver.ts +++ b/src/canvas-host/file-resolver.ts @@ -1,6 +1,5 @@ -import fs from "node:fs/promises"; import path from "node:path"; -import { SafeOpenError, openFileWithinRoot, type SafeOpenResult } from "../infra/fs-safe.js"; +import { root as fsRoot, FsSafeError, type OpenResult } from "../infra/fs-safe.js"; export function normalizeUrlPath(rawPath: string): string { const decoded = decodeURIComponent(rawPath || "/"); @@ -11,18 +10,19 @@ export function normalizeUrlPath(rawPath: string): string { export async function resolveFileWithinRoot( rootReal: string, urlPath: string, -): Promise { +): Promise { const normalized = normalizeUrlPath(urlPath); const rel = normalized.replace(/^\/+/, ""); if (rel.split("/").some((p) => p === "..")) { return null; } + const root = await fsRoot(rootReal); const tryOpen = async (relative: string) => { try { - return await openFileWithinRoot({ rootDir: rootReal, relativePath: relative }); + return await root.open(relative); } catch (err) { - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { return null; } throw err; @@ -33,17 +33,19 @@ export async function resolveFileWithinRoot( return await tryOpen(path.posix.join(rel, "index.html")); } - const candidate = path.join(rootReal, rel); try { - const st = await fs.lstat(candidate); - if (st.isSymbolicLink()) { + const st = await root.stat(rel); + if (st.isSymbolicLink) { return null; } - if (st.isDirectory()) { + if (st.isDirectory) { return await tryOpen(path.posix.join(rel, "index.html")); } - } catch { - // ignore + } catch (err) { + if (err instanceof FsSafeError) { + return null; + } + throw err; } return await tryOpen(rel); diff --git a/src/channels/plugins/bundled.shape-guard.test.ts b/src/channels/plugins/bundled.shape-guard.test.ts index a24cd96f2d7..01b8cee6b0f 100644 --- a/src/channels/plugins/bundled.shape-guard.test.ts +++ b/src/channels/plugins/bundled.shape-guard.test.ts @@ -890,7 +890,7 @@ module.exports = { }; }); vi.doMock("../../infra/boundary-file-read.js", () => ({ - openBoundaryFileSync: ({ absolutePath }: { absolutePath: string }) => ({ + openRootFileSync: ({ absolutePath }: { absolutePath: string }) => ({ ok: true, path: absolutePath, fd: fs.openSync(absolutePath, "r"), diff --git a/src/channels/plugins/module-loader.ts b/src/channels/plugins/module-loader.ts index 9b0feac15fb..4498b6c9e34 100644 --- a/src/channels/plugins/module-loader.ts +++ b/src/channels/plugins/module-loader.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import { openBoundaryFileSync } from "../../infra/boundary-file-read.js"; +import { openRootFileSync } from "../../infra/boundary-file-read.js"; import { isJavaScriptModulePath } from "../../plugins/native-module-require.js"; import { getCachedPluginModuleLoader, @@ -88,7 +88,7 @@ export function loadChannelPluginModule(params: { boundaryRootDir?: string; boundaryLabel?: string; }): unknown { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: params.modulePath, rootPath: params.boundaryRootDir ?? params.rootDir, boundaryLabel: params.boundaryLabel ?? "plugin root", diff --git a/src/cli/update-cli/update-command.ts b/src/cli/update-cli/update-command.ts index 0e298a59b34..2a5012853ab 100644 --- a/src/cli/update-cli/update-command.ts +++ b/src/cli/update-cli/update-command.ts @@ -27,6 +27,7 @@ import { type GatewayService, } from "../../daemon/service.js"; import { createLowDiskSpaceWarning } from "../../infra/disk-space.js"; +import { pathExists } from "../../infra/fs-safe.js"; import { runGlobalPackageUpdateSteps } from "../../infra/package-update-steps.js"; import { getSelfAndAncestorPidsSync } from "../../infra/restart-stale-pids.js"; import { nodeVersionSatisfiesEngine } from "../../infra/runtime-guard.js"; @@ -179,15 +180,6 @@ function isTrackedPackageInstallRecord(record: PluginInstallRecord): boolean { ); } -async function pathExists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - export async function collectMissingPluginInstallPayloads(params: { records: Record; config?: OpenClawConfig; diff --git a/src/commands/agents.commands.add.ts b/src/commands/agents.commands.add.ts index 9a798032eef..ec96d12cdf9 100644 --- a/src/commands/agents.commands.add.ts +++ b/src/commands/agents.commands.add.ts @@ -13,6 +13,7 @@ import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js"; import { commitConfigWithPendingPluginInstalls } from "../cli/plugins-install-record-commit.js"; import { logConfigUpdated } from "../config/logging.js"; +import { pathExists } from "../infra/fs-safe.js"; import { saveJsonFile } from "../infra/json-file.js"; import { DEFAULT_AGENT_ID, normalizeAgentId } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; @@ -48,15 +49,6 @@ type AgentsAddOptions = { json?: boolean; }; -async function fileExists(pathname: string): Promise { - try { - await fs.stat(pathname); - return true; - } catch { - return false; - } -} - async function copyPortableAuthProfiles(params: { destAuthPath: string; sourceAgentDir: string; @@ -291,8 +283,8 @@ export async function agentsAddCommand( normalizeLowercaseStringOrEmpty(path.resolve(mainAuthPath)); if ( !sameAuthPath && - (await fileExists(sourceAuthPath)) && - !(await fileExists(destAuthPath)) + (await pathExists(sourceAuthPath)) && + !(await pathExists(destAuthPath)) ) { const sourceStore = loadPersistedAuthProfileStore(sourceAgentDir); const portable = sourceStore diff --git a/src/commands/cleanup-utils.ts b/src/commands/cleanup-utils.ts index 7dc08bd1003..41945d27617 100644 --- a/src/commands/cleanup-utils.ts +++ b/src/commands/cleanup-utils.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { resolveDefaultAgentWorkspaceDir } from "../agents/workspace.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isPathInside } from "../infra/path-guards.js"; import type { RuntimeEnv } from "../runtime.js"; import { resolveHomeDir, resolveUserPath, shortenHomeInString } from "../utils.js"; @@ -55,8 +56,7 @@ export function buildCleanupPlan(params: { } export function isPathWithin(child: string, parent: string): boolean { - const relative = path.relative(parent, child); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); + return isPathInside(parent, child); } function isUnsafeRemovalTarget(target: string): boolean { diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index 9588fc49113..5a8c4d647cc 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -164,13 +164,13 @@ vi.mock("../infra/json-files.js", async () => { writeTextAtomic: async ( filePath: string, content: string, - options?: { mode?: number; ensureDirMode?: number; appendTrailingNewline?: boolean }, + options?: { mode?: number; dirMode?: number; trailingNewline?: boolean }, ) => { const payload = - options?.appendTrailingNewline && !content.endsWith("\n") ? `${content}\n` : content; + options?.trailingNewline && !content.endsWith("\n") ? `${content}\n` : content; await fs.promises.mkdir(path.dirname(filePath), { recursive: true, - ...(typeof options?.ensureDirMode === "number" ? { mode: options.ensureDirMode } : {}), + ...(typeof options?.dirMode === "number" ? { mode: options.dirMode } : {}), }); await fs.promises.writeFile(filePath, payload, { encoding: "utf8", diff --git a/src/commands/export-trajectory.ts b/src/commands/export-trajectory.ts index e56aa1fc485..43ad2497c2a 100644 --- a/src/commands/export-trajectory.ts +++ b/src/commands/export-trajectory.ts @@ -1,4 +1,3 @@ -import fsp from "node:fs/promises"; import path from "node:path"; import { resolveDefaultSessionStorePath, @@ -8,6 +7,7 @@ import { import { loadSessionStore } from "../config/sessions/store.js"; import type { SessionEntry } from "../config/sessions/types.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { pathExists } from "../infra/fs-safe.js"; import { resolveAgentIdFromSessionKey } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { @@ -72,15 +72,6 @@ function resolveExportTrajectoryOptions( }; } -async function fileExists(pathName: string): Promise { - try { - await fsp.access(pathName); - return true; - } catch { - return false; - } -} - export async function exportTrajectoryCommand( opts: ExportTrajectoryCommandOptions, runtime: RuntimeEnv, @@ -123,7 +114,7 @@ export async function exportTrajectoryCommand( runtime.exit(1); return; } - if (!(await fileExists(sessionFile))) { + if (!(await pathExists(sessionFile))) { runtime.error("Session file not found."); runtime.exit(1); return; diff --git a/src/commands/status.agent-local.ts b/src/commands/status.agent-local.ts index 1a29ab45a46..9ef1bfa5592 100644 --- a/src/commands/status.agent-local.ts +++ b/src/commands/status.agent-local.ts @@ -1,10 +1,10 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { resolveStorePath } from "../config/sessions/paths.js"; import { readSessionStoreReadOnly } from "../config/sessions/store-read.js"; import type { OpenClawConfig } from "../config/types.js"; import { listGatewayAgentsBasic } from "../gateway/agent-list.js"; +import { pathExists } from "../infra/fs-safe.js"; export type AgentLocalStatus = { id: string; @@ -24,15 +24,6 @@ type AgentLocalStatusesResult = { bootstrapPendingCount: number; }; -async function fileExists(p: string): Promise { - try { - await fs.access(p); - return true; - } catch { - return false; - } -} - export async function getAgentLocalStatuses( cfg: OpenClawConfig, ): Promise { @@ -51,7 +42,7 @@ export async function getAgentLocalStatuses( })(); const bootstrapPath = workspaceDir != null ? path.join(workspaceDir, "BOOTSTRAP.md") : null; - const bootstrapPending = bootstrapPath != null ? await fileExists(bootstrapPath) : null; + const bootstrapPending = bootstrapPath != null ? await pathExists(bootstrapPath) : null; const sessionsPath = resolveStorePath(cfg.session?.store, { agentId }); const store = readSessionStoreReadOnly(sessionsPath); diff --git a/src/commitments/store.ts b/src/commitments/store.ts index 2fdd7a3db73..7afd8205b60 100644 --- a/src/commitments/store.ts +++ b/src/commitments/store.ts @@ -1,9 +1,9 @@ import { randomBytes } from "node:crypto"; -import fs from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/config.js"; import { resolveStateDir } from "../config/paths.js"; import { expandHomePrefix } from "../infra/home-dir.js"; +import { privateFileStore } from "../infra/private-file-store.js"; import { DEFAULT_COMMITMENT_EXPIRE_AFTER_HOURS, DEFAULT_COMMITMENT_MAX_PER_HEARTBEAT, @@ -111,8 +111,9 @@ function sanitizeStoreForWrite(store: CommitmentStoreFile): CommitmentStoreFile async function loadCommitmentStoreInternal(storePath?: string): Promise { const resolved = resolveCommitmentStorePath(storePath); try { - const raw = await fs.promises.readFile(resolved, "utf-8"); - const parsed = JSON.parse(raw) as unknown; + const parsed = await privateFileStore(path.dirname(resolved)).readJsonIfExists( + path.basename(resolved), + ); if ( !isRecord(parsed) || parsed.version !== STORE_VERSION || @@ -149,15 +150,10 @@ export async function saveCommitmentStore( store: CommitmentStoreFile, ): Promise { const resolved = resolveCommitmentStorePath(storePath); - const dir = path.dirname(resolved); - await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 }); - await fs.promises.chmod(dir, 0o700).catch(() => undefined); - const json = JSON.stringify(sanitizeStoreForWrite(store), null, 2); - const tmp = `${resolved}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; - await fs.promises.writeFile(tmp, json, { encoding: "utf-8", mode: 0o600 }); - await fs.promises.chmod(tmp, 0o600).catch(() => undefined); - await fs.promises.rename(tmp, resolved); - await fs.promises.chmod(resolved, 0o600).catch(() => undefined); + await privateFileStore(path.dirname(resolved)).writeJson( + path.basename(resolved), + sanitizeStoreForWrite(store), + ); } function generateCommitmentId(nowMs: number): string { diff --git a/src/config/doc-baseline.ts b/src/config/doc-baseline.ts index 229f0559fb0..e5f42b6c64f 100644 --- a/src/config/doc-baseline.ts +++ b/src/config/doc-baseline.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; +import { replaceFileAtomicSync } from "../infra/replace-file.js"; import type { ConfigSchemaResponse } from "./schema.js"; import { schemaHasChildren } from "./schema.shared.js"; @@ -597,8 +598,13 @@ function readFileIfExists(filePath: string): string | null { } function writeFileAtomic(filePath: string, content: string): void { - fsSync.mkdirSync(path.dirname(filePath), { recursive: true }); - fsSync.writeFileSync(filePath, content, "utf8"); + replaceFileAtomicSync({ + filePath, + content, + dirMode: 0o755, + mode: 0o644, + tempPrefix: path.basename(filePath), + }); } function sha256(content: string): string { diff --git a/src/config/includes.ts b/src/config/includes.ts index fac11de33f1..251b7c3966b 100644 --- a/src/config/includes.ts +++ b/src/config/includes.ts @@ -13,7 +13,7 @@ import fs from "node:fs"; import path from "node:path"; import JSON5 from "json5"; -import { canUseBoundaryFileOpen, openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { canUseRootFileOpen, openRootFileSync } from "../infra/boundary-file-read.js"; import { isPathInside } from "../security/scan-paths.js"; import { isPlainObject } from "../utils.js"; import { isBlockedObjectKey } from "./prototype-keys.js"; @@ -359,11 +359,11 @@ function isNotFoundError(error: unknown): boolean { export function readConfigIncludeFileWithGuards(params: IncludeFileReadParams): string { const ioFs = params.ioFs ?? fs; const maxBytes = params.maxBytes ?? MAX_INCLUDE_FILE_BYTES; - if (!canUseBoundaryFileOpen(ioFs)) { + if (!canUseRootFileOpen(ioFs)) { return ioFs.readFileSync(params.resolvedPath, "utf-8"); } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: params.resolvedPath, rootPath: params.rootRealDir, rootRealPath: params.rootRealDir, diff --git a/src/config/io.ts b/src/config/io.ts index 39aa174114c..91d045b4551 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -8,6 +8,7 @@ import { ensureOwnerDisplaySecret } from "../agents/owner-display.js"; import { loadDotEnv } from "../infra/dotenv.js"; import { formatErrorMessage } from "../infra/errors.js"; import { resolveRequiredHomeDir } from "../infra/home-dir.js"; +import { replaceFileAtomic, replaceFileAtomicSync } from "../infra/replace-file.js"; import { loadShellEnvFallback, resolveShellEnvFallbackTimeoutMs, @@ -1313,38 +1314,15 @@ export function createConfigIO( } function replaceConfigFileSync(raw: string): void { - const dir = path.dirname(configPath); - deps.fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); - const tmp = path.join( - dir, - `${path.basename(configPath)}.${process.pid}.${crypto.randomUUID()}.tmp`, - ); - try { - deps.fs.writeFileSync(tmp, raw, { - encoding: "utf-8", - mode: 0o600, - }); - try { - deps.fs.renameSync(tmp, configPath); - } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code; - if (code !== "EPERM" && code !== "EEXIST") { - throw err; - } - deps.fs.copyFileSync(tmp, configPath); - deps.fs.chmodSync(configPath, 0o600); - deps.fs.unlinkSync(tmp); - } - } catch (err) { - try { - deps.fs.unlinkSync(tmp); - } catch (cleanupErr) { - if ((cleanupErr as NodeJS.ErrnoException)?.code !== "ENOENT") { - deps.logger.warn(`Failed to clean temporary config file ${tmp}: ${String(cleanupErr)}`); - } - } - throw err; - } + replaceFileAtomicSync({ + filePath: configPath, + content: raw, + dirMode: 0o700, + mode: 0o600, + tempPrefix: path.basename(configPath), + copyFallbackOnPermissionError: true, + fileSystem: deps.fs, + }); } function migrateAndStripShippedPluginInstallConfigRecords( @@ -2208,57 +2186,29 @@ export function createConfigIO( throw err; } - const tmp = path.join( - dir, - `${path.basename(configPath)}.${process.pid}.${crypto.randomUUID()}.tmp`, - ); - const pluginInstallConfigMigration = ensureShippedPluginInstallConfigRecordsMigratedForWrite(snapshot); let configCommitted = false; try { - await deps.fs.promises.writeFile(tmp, json, { - encoding: "utf-8", + const result = await replaceFileAtomic({ + filePath: configPath, + content: json, + dirMode: 0o700, mode: 0o600, + tempPrefix: path.basename(configPath), + copyFallbackOnPermissionError: true, + fileSystem: deps.fs, + beforeRename: async () => { + if (deps.fs.existsSync(configPath)) { + await maintainConfigBackups(configPath, deps.fs.promises); + } + }, }); - - if (deps.fs.existsSync(configPath)) { - await maintainConfigBackups(configPath, deps.fs.promises); - } - - try { - await deps.fs.promises.rename(tmp, configPath); - } catch (err) { - const code = (err as { code?: string }).code; - // Windows doesn't reliably support atomic replace via rename when dest exists. - if (code === "EPERM" || code === "EEXIST") { - await deps.fs.promises.copyFile(tmp, configPath); - await deps.fs.promises.chmod(configPath, 0o600).catch(() => { - // best-effort - }); - await deps.fs.promises.unlink(tmp).catch(() => { - // best-effort - }); - configCommitted = true; - logConfigOverwrite(); - logConfigWriteAnomalies(); - await appendWriteAudit( - "copy-fallback", - undefined, - await deps.fs.promises.stat(configPath).catch(() => null), - ); - return { persistedHash: nextHash, persistedConfig: stampedOutputConfig }; - } - await deps.fs.promises.unlink(tmp).catch(() => { - // best-effort - }); - throw err; - } configCommitted = true; logConfigOverwrite(); logConfigWriteAnomalies(); await appendWriteAudit( - "rename", + result.method, undefined, await deps.fs.promises.stat(configPath).catch(() => null), ); diff --git a/src/config/mutate.ts b/src/config/mutate.ts index 95019d8f3da..851228d4b5e 100644 --- a/src/config/mutate.ts +++ b/src/config/mutate.ts @@ -1,8 +1,8 @@ -import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; import { formatErrorMessage } from "../infra/errors.js"; +import { replaceFileAtomic } from "../infra/replace-file.js"; import { isPathInside } from "../security/scan-paths.js"; import { isRecord } from "../utils.js"; import { maintainConfigBackups } from "./backup-rotation.js"; @@ -102,31 +102,19 @@ function getSingleTopLevelIncludeTarget(params: { } async function writeJsonFileAtomic(filePath: string, value: unknown): Promise { - const dir = path.dirname(filePath); - const tmp = path.join( - dir, - `${path.basename(filePath)}.${process.pid}.${crypto.randomUUID()}.tmp`, - ); - try { - await fs.mkdir(dir, { recursive: true }); - await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, { - encoding: "utf-8", - mode: 0o600, - }); - await fs.access(filePath).then( - async () => await maintainConfigBackups(filePath, fs), - () => undefined, - ); - await fs.rename(tmp, filePath); - await fs.chmod(filePath, 0o600).catch(() => { - // best-effort - }); - } catch (err) { - await fs.unlink(tmp).catch(() => { - // best-effort - }); - throw err; - } + await replaceFileAtomic({ + filePath, + content: `${JSON.stringify(value, null, 2)}\n`, + dirMode: 0o700, + mode: 0o600, + tempPrefix: path.basename(filePath), + beforeRename: async () => { + await fs.access(filePath).then( + async () => await maintainConfigBackups(filePath, fs), + () => undefined, + ); + }, + }); } async function tryWriteSingleTopLevelIncludeMutation(params: { diff --git a/src/config/validation.ts b/src/config/validation.ts index debc4f1526c..21243cc9954 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; import { CHANNEL_IDS, normalizeChatChannelId } from "../channels/ids.js"; +import { isPathInside } from "../infra/path-guards.js"; import { planManifestModelCatalogSuppressions } from "../model-catalog/index.js"; import { withBundledPluginAllowlistCompat } from "../plugins/bundled-compat.js"; import { @@ -1419,8 +1420,8 @@ function validateConfigObjectWithPluginsBase( } if ( sourcePath === resolvedLoadPath || - sourcePath.startsWith(`${resolvedLoadPath}${path.sep}`) || - resolvedLoadPath.startsWith(`${sourcePath}${path.sep}`) + isPathInside(resolvedLoadPath, sourcePath) || + isPathInside(sourcePath, resolvedLoadPath) ) { return true; } diff --git a/src/crestodian/audit.ts b/src/crestodian/audit.ts index e5b032e0655..0760292f7db 100644 --- a/src/crestodian/audit.ts +++ b/src/crestodian/audit.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; +import { appendRegularFile } from "../infra/fs-safe.js"; type CrestodianAuditEntry = { timestamp: string; @@ -29,9 +30,10 @@ export async function appendCrestodianAuditEntry( timestamp: new Date().toISOString(), ...entry, } satisfies CrestodianAuditEntry); - await fs.appendFile(auditPath, `${line}\n`, { encoding: "utf8", mode: 0o600 }); - await fs.chmod(auditPath, 0o600).catch(() => { - // Best-effort on platforms/filesystems without POSIX modes. + await appendRegularFile({ + filePath: auditPath, + content: `${line}\n`, + rejectSymlinkParents: true, }); return auditPath; } diff --git a/src/cron/run-log.ts b/src/cron/run-log.ts index d9b70f55345..5f7f937142e 100644 --- a/src/cron/run-log.ts +++ b/src/cron/run-log.ts @@ -1,9 +1,10 @@ -import { randomBytes } from "node:crypto"; import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { parseByteSize } from "../cli/parse-bytes.js"; import type { CronConfig } from "../config/types.cron.js"; +import { appendRegularFile, isPathInside, pathExists, root as fsRoot } from "../infra/fs-safe.js"; +import { privateFileStore } from "../infra/private-file-store.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, @@ -84,7 +85,7 @@ export function resolveCronRunLogPath(params: { storePath: string; jobId: string const runsDir = path.resolve(dir, "runs"); const safeJobId = assertSafeCronRunLogJobId(params.jobId); const resolvedPath = path.resolve(runsDir, `${safeJobId}.jsonl`); - if (!resolvedPath.startsWith(`${runsDir}${path.sep}`)) { + if (!isPathInside(runsDir, resolvedPath)) { throw new Error("invalid cron run log job id"); } return resolvedPath; @@ -147,11 +148,10 @@ async function pruneIfNeeded(filePath: string, opts: { maxBytes: number; keepLin .map((l) => l.trim()) .filter(Boolean); const kept = lines.slice(Math.max(0, lines.length - opts.keepLines)); - const tmp = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; - await fs.writeFile(tmp, `${kept.join("\n")}\n`, { encoding: "utf-8", mode: 0o600 }); - await setSecureFileMode(tmp); - await fs.rename(tmp, filePath); - await setSecureFileMode(filePath); + await privateFileStore(path.dirname(filePath)).writeText( + path.basename(filePath), + `${kept.join("\n")}\n`, + ); } export async function appendCronRunLog( @@ -167,9 +167,10 @@ export async function appendCronRunLog( const runDir = path.dirname(resolved); await fs.mkdir(runDir, { recursive: true, mode: 0o700 }); await fs.chmod(runDir, 0o700).catch(() => undefined); - await fs.appendFile(resolved, `${JSON.stringify(entry)}\n`, { - encoding: "utf-8", - mode: 0o600, + await appendRegularFile({ + filePath: resolved, + content: `${JSON.stringify(entry)}\n`, + rejectSymlinkParents: true, }); await setSecureFileMode(resolved); await pruneIfNeeded(resolved, { @@ -447,10 +448,31 @@ export async function readCronRunLogEntriesPageAll( const query = normalizeLowercaseStringOrEmpty(opts.query); const sortDir: CronRunLogSortDir = opts.sortDir === "asc" ? "asc" : "desc"; const runsDir = path.resolve(path.dirname(path.resolve(opts.storePath)), "runs"); - const files = await fs.readdir(runsDir, { withFileTypes: true }).catch(() => []); + if (!(await pathExists(runsDir))) { + return { + entries: [], + total: 0, + offset: 0, + limit, + hasMore: false, + nextOffset: null, + }; + } + const runsRoot = await fsRoot(runsDir).catch(() => null); + if (!runsRoot) { + return { + entries: [], + total: 0, + offset: 0, + limit, + hasMore: false, + nextOffset: null, + }; + } + const files = await runsRoot.list(".", { withFileTypes: true }).catch(() => []); const jsonlFiles = files - .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) - .map((entry) => path.join(runsDir, entry.name)); + .filter((entry) => entry.isFile && entry.name.endsWith(".jsonl")) + .map((entry) => entry.name); if (jsonlFiles.length === 0) { return { entries: [], @@ -461,10 +483,10 @@ export async function readCronRunLogEntriesPageAll( nextOffset: null, }; } - await Promise.all(jsonlFiles.map((f) => drainPendingWrite(f))); + await Promise.all(jsonlFiles.map((fileName) => drainPendingWrite(path.join(runsDir, fileName)))); const chunks = await Promise.all( - jsonlFiles.map(async (filePath) => { - const raw = await fs.readFile(filePath, "utf-8").catch(() => ""); + jsonlFiles.map(async (fileName) => { + const raw = await runsRoot.readText(fileName).catch(() => ""); return parseAllRunLogEntries(raw); }), ); diff --git a/src/cron/store.ts b/src/cron/store.ts index 534c3d4e9a1..8fbd1c6b1af 100644 --- a/src/cron/store.ts +++ b/src/cron/store.ts @@ -1,7 +1,7 @@ -import { randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { expandHomePrefix } from "../infra/home-dir.js"; +import { replaceFileAtomic } from "../infra/replace-file.js"; import { resolveConfigDir } from "../utils.js"; import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; import { tryCronScheduleIdentity } from "./schedule-identity.js"; @@ -329,13 +329,15 @@ async function setSecureFileMode(filePath: string): Promise { } async function atomicWrite(filePath: string, content: string, dirMode = 0o700): Promise { - const dir = path.dirname(filePath); - await fs.promises.mkdir(dir, { recursive: true, mode: dirMode }); - await fs.promises.chmod(dir, dirMode).catch(() => undefined); - const tmp = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; - await fs.promises.writeFile(tmp, content, { encoding: "utf-8", mode: 0o600 }); - await renameWithRetry(tmp, filePath); - await setSecureFileMode(filePath); + await replaceFileAtomic({ + filePath, + content, + dirMode, + mode: 0o600, + tempPrefix: ".openclaw-cron", + renameMaxRetries: 3, + copyFallbackOnPermissionError: true, + }); } async function serializedFileNeedsWrite( @@ -409,28 +411,3 @@ export async function saveCronStore( } updatedCache.needsSplitMigration = stateOnly && migrating; } - -const RENAME_MAX_RETRIES = 3; -const RENAME_BASE_DELAY_MS = 50; - -async function renameWithRetry(src: string, dest: string): Promise { - for (let attempt = 0; attempt <= RENAME_MAX_RETRIES; attempt++) { - try { - await fs.promises.rename(src, dest); - return; - } catch (err) { - const code = (err as { code?: string }).code; - if (code === "EBUSY" && attempt < RENAME_MAX_RETRIES) { - await new Promise((resolve) => setTimeout(resolve, RENAME_BASE_DELAY_MS * 2 ** attempt)); - continue; - } - // Windows doesn't reliably support atomic replace via rename when dest exists. - if (code === "EPERM" || code === "EEXIST") { - await fs.promises.copyFile(src, dest); - await fs.promises.unlink(src).catch(() => {}); - return; - } - throw err; - } - } -} diff --git a/src/daemon/service-layout.ts b/src/daemon/service-layout.ts index 14d054b074e..0eebd610462 100644 --- a/src/daemon/service-layout.ts +++ b/src/daemon/service-layout.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { pathExists } from "../infra/fs-safe.js"; import { readPackageName, readPackageVersion } from "../infra/package-json.js"; import type { GatewayServiceCommandConfig } from "./service-types.js"; @@ -64,15 +65,6 @@ async function tryRealpath(value: string | undefined): Promise { - try { - await fs.access(candidate); - return true; - } catch { - return false; - } -} - async function isSourceCheckoutRoot(candidate: string): Promise { const hasRepoMarker = (await pathExists(path.join(candidate, ".git"))) || diff --git a/src/gateway/canvas-documents.ts b/src/gateway/canvas-documents.ts index e220fa96980..16c084c06da 100644 --- a/src/gateway/canvas-documents.ts +++ b/src/gateway/canvas-documents.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { CANVAS_HOST_PATH } from "../canvas-host/a2ui.js"; import { resolveStateDir } from "../config/paths.js"; +import { root as fsRoot, sanitizeUntrustedFileName } from "../infra/fs-safe.js"; import { resolveUserPath } from "../utils.js"; type CanvasDocumentKind = "html_bundle" | "url_embed" | "document" | "image" | "video_asset"; @@ -74,12 +75,27 @@ function escapeHtml(value: string): string { function normalizeLogicalPath(value: string): string { const normalized = value.replaceAll("\\", "/").replace(/^\/+/, ""); const parts = normalized.split("/").filter(Boolean); - if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) { + if ( + parts.length === 0 || + parts.some( + (part) => part === "." || part === ".." || part.includes(":") || hasControlCharacter(part), + ) + ) { throw new Error("canvas document logicalPath invalid"); } return parts.join("/"); } +function hasControlCharacter(value: string): boolean { + for (const char of value) { + const code = char.charCodeAt(0); + if (code < 0x20 || code === 0x7f) { + return true; + } + } + return false; +} + function canvasDocumentId(): string { return `cv_${randomUUID().replaceAll("-", "")}`; } @@ -172,16 +188,17 @@ export function resolveCanvasHttpPathToLocalPath( } } -async function writeManifest(rootDir: string, manifest: CanvasDocumentManifest): Promise { - await fs.writeFile( - path.join(rootDir, "manifest.json"), - `${JSON.stringify(manifest, null, 2)}\n`, - "utf8", - ); +type CanvasDocumentRoot = Awaited>; + +async function writeManifest( + root: CanvasDocumentRoot, + manifest: CanvasDocumentManifest, +): Promise { + await root.writeJson("manifest.json", manifest, { space: 2 }); } async function copyAssets( - rootDir: string, + root: CanvasDocumentRoot, assets: CanvasDocumentAsset[] | undefined, workspaceDir: string, ): Promise { @@ -193,9 +210,7 @@ async function copyAssets( : path.isAbsolute(asset.sourcePath) ? path.resolve(asset.sourcePath) : path.resolve(workspaceDir, asset.sourcePath); - const destination = path.join(rootDir, logicalPath); - await fs.mkdir(path.dirname(destination), { recursive: true }); - await fs.copyFile(sourcePath, destination); + await root.copyIn(logicalPath, sourcePath); copied.push({ logicalPath, ...(asset.contentType ? { contentType: asset.contentType } : {}), @@ -206,6 +221,7 @@ async function copyAssets( async function materializeEntrypoint( rootDir: string, + root: CanvasDocumentRoot, input: CanvasDocumentCreateInput, workspaceDir: string, ): Promise> { @@ -215,7 +231,7 @@ async function materializeEntrypoint( } if (entrypoint.type === "html") { const fileName = "index.html"; - await fs.writeFile(path.join(rootDir, fileName), entrypoint.value, "utf8"); + await root.write(fileName, entrypoint.value); return { localEntrypoint: fileName, entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), fileName), @@ -224,7 +240,7 @@ async function materializeEntrypoint( if (entrypoint.type === "url") { if (input.kind === "document" && isPdfPathLike(entrypoint.value)) { const fileName = "index.html"; - await fs.writeFile(path.join(rootDir, fileName), buildPdfWrapper(entrypoint.value), "utf8"); + await root.write(fileName, buildPdfWrapper(entrypoint.value)); return { localEntrypoint: fileName, externalUrl: entrypoint.value, @@ -244,23 +260,23 @@ async function materializeEntrypoint( : path.resolve(workspaceDir, entrypoint.value); if (input.kind === "image" || input.kind === "video_asset") { - const copiedName = path.basename(resolvedPath); - await fs.copyFile(resolvedPath, path.join(rootDir, copiedName)); + const copiedName = sanitizeUntrustedFileName(path.basename(resolvedPath), "asset"); + await root.copyIn(copiedName, resolvedPath); const wrapper = input.kind === "image" ? `` : ``; - await fs.writeFile(path.join(rootDir, "index.html"), wrapper, "utf8"); + await root.write("index.html", wrapper); return { localEntrypoint: "index.html", entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), "index.html"), }; } - const fileName = path.basename(resolvedPath); - await fs.copyFile(resolvedPath, path.join(rootDir, fileName)); + const fileName = sanitizeUntrustedFileName(path.basename(resolvedPath), "document"); + await root.copyIn(fileName, resolvedPath); if (input.kind === "document" && isPdfPathLike(fileName)) { - await fs.writeFile(path.join(rootDir, "index.html"), buildPdfWrapper(fileName), "utf8"); + await root.write("index.html", buildPdfWrapper(fileName)); return { localEntrypoint: "index.html", entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), "index.html"), @@ -284,8 +300,9 @@ export async function createCanvasDocument( }); await fs.rm(rootDir, { recursive: true, force: true }).catch(() => undefined); await fs.mkdir(rootDir, { recursive: true }); - const assets = await copyAssets(rootDir, input.assets, workspaceDir); - const entry = await materializeEntrypoint(rootDir, input, workspaceDir); + const root = await fsRoot(rootDir); + const assets = await copyAssets(root, input.assets, workspaceDir); + const entry = await materializeEntrypoint(rootDir, root, input, workspaceDir); const manifest: CanvasDocumentManifest = { id, kind: input.kind, @@ -300,7 +317,7 @@ export async function createCanvasDocument( ...(entry.externalUrl ? { externalUrl: entry.externalUrl } : {}), assets, }; - await writeManifest(rootDir, manifest); + await writeManifest(root, manifest); return manifest; } diff --git a/src/gateway/control-ui.ts b/src/gateway/control-ui.ts index d74a830cce7..e160d4aac15 100644 --- a/src/gateway/control-ui.ts +++ b/src/gateway/control-ui.ts @@ -4,17 +4,16 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import path from "node:path"; import { resolveAgentAvatar, resolvePublicAgentAvatarSource } from "../agents/identity-avatar.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { matchBoundaryFileOpenFailure, openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { matchRootFileOpenFailure, openRootFileSync } from "../infra/boundary-file-read.js"; import { isPackageProvenControlUiRootSync, resolveControlUiRootSync, } from "../infra/control-ui-assets.js"; import { listDevicePairing, verifyDeviceToken } from "../infra/device-pairing.js"; -import { openLocalFileSafely, SafeOpenError } from "../infra/fs-safe.js"; +import { openLocalFileSafely, FsSafeError, readSecureFile } from "../infra/fs-safe.js"; import { safeFileURLToPath } from "../infra/local-file-access.js"; import { verifyPairingToken } from "../infra/pairing-token.js"; import { isWithinDir } from "../infra/path-safety.js"; -import { openVerifiedFileSync } from "../infra/safe-open-sync.js"; import { assertLocalMediaAllowed, getDefaultLocalRoots } from "../media/local-media-access.js"; import { getAgentScopedMediaLocalRoots } from "../media/local-roots.js"; import { resolveMediaReferenceLocalPath } from "../media/media-reference.js"; @@ -441,7 +440,7 @@ function verifyAssistantMediaTicket(ticket: string | null, source: string, nowMs } function classifyAssistantMediaError(err: unknown): AssistantMediaAvailability { - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { switch (err.code) { case "not-found": return { available: false, code: "file-not-found", reason: "File not found" }; @@ -687,21 +686,17 @@ export async function handleControlUiAvatarRequest( return true; } - const safeAvatar = resolveSafeAvatarFile(resolved.filePath); + const safeAvatar = await resolveSafeAvatarFile(resolved.filePath); if (!safeAvatar) { respondControlUiNotFound(res); return true; } - try { - if (respondHeadForFile(req, res, safeAvatar.path)) { - return true; - } - - serveResolvedFile(res, safeAvatar.path, fs.readFileSync(safeAvatar.fd)); + if (respondHeadForFile(req, res, safeAvatar.path)) { return true; - } finally { - fs.closeSync(safeAvatar.fd); } + + serveResolvedFile(res, safeAvatar.path, safeAvatar.buffer); + return true; } function setStaticFileHeaders(res: ServerResponse, filePath: string) { @@ -736,16 +731,20 @@ function isExpectedSafePathError(error: unknown): boolean { return code === "ENOENT" || code === "ENOTDIR" || code === "ELOOP"; } -function resolveSafeAvatarFile(filePath: string): { path: string; fd: number } | null { - const opened = openVerifiedFileSync({ - filePath, - rejectPathSymlink: true, - maxBytes: AVATAR_MAX_BYTES, - }); - if (!opened.ok) { +async function resolveSafeAvatarFile( + filePath: string, +): Promise<{ path: string; buffer: Buffer } | null> { + try { + const read = await readSecureFile({ + filePath, + label: "Control UI avatar", + permissions: { allowInsecure: true, allowReadableByOthers: true }, + io: { maxBytes: AVATAR_MAX_BYTES }, + }); + return { path: read.realPath, buffer: read.buffer }; + } catch { return null; } - return { path: opened.path, fd: opened.fd }; } function resolveSafeControlUiFile( @@ -753,7 +752,7 @@ function resolveSafeControlUiFile( filePath: string, rejectHardlinks: boolean, ): { path: string; fd: number } | null { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: filePath, rootPath: rootReal, rootRealPath: rootReal, @@ -762,7 +761,7 @@ function resolveSafeControlUiFile( rejectHardlinks, }); if (!opened.ok) { - return matchBoundaryFileOpenFailure(opened, { + return matchRootFileOpenFailure(opened, { io: (failure) => { throw failure.error; }, diff --git a/src/gateway/managed-image-attachments.ts b/src/gateway/managed-image-attachments.ts index 248f366ec7d..c73d38f5688 100644 --- a/src/gateway/managed-image-attachments.ts +++ b/src/gateway/managed-image-attachments.ts @@ -4,6 +4,7 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import path from "node:path"; import { getLatestSubagentRunByChildSessionKey } from "../agents/subagent-registry.js"; import { resolveStateDir } from "../config/paths.js"; +import { readLocalFileSafely } from "../infra/fs-safe.js"; import { safeFileURLToPath } from "../infra/local-file-access.js"; import { getImageMetadata, @@ -366,7 +367,7 @@ function parseImageDataUrl( } async function getVariantStats(filePath: string) { - const [stats, metadataBuffer] = await Promise.all([fs.stat(filePath), fs.readFile(filePath)]); + const { buffer: metadataBuffer, stat } = await readLocalFileSafely({ filePath }); const metadata = (await getImageMetadata(metadataBuffer).catch(() => null)) ?? { width: null, height: null, @@ -374,7 +375,7 @@ async function getVariantStats(filePath: string) { return { width: metadata.width ?? null, height: metadata.height ?? null, - sizeBytes: Number.isFinite(stats.size) ? stats.size : null, + sizeBytes: Number.isFinite(stat.size) ? stat.size : null, }; } @@ -866,7 +867,7 @@ export async function createManagedOutgoingImageBlocks(params: { let originalBuffer = parsedDataUrl.kind === "image-data-url" ? parsedDataUrl.buffer - : await fs.readFile(savedOriginal.path); + : (await readLocalFileSafely({ filePath: savedOriginal.path })).buffer; validateManagedImageBuffer(originalBuffer, alt, limits); let originalStats = await getVariantStats(savedOriginal.path); @@ -1081,7 +1082,7 @@ export async function handleManagedOutgoingImageHttpRequest( let body: Buffer; try { - body = await fs.readFile(record.original.path); + body = (await readLocalFileSafely({ filePath: record.original.path })).buffer; } catch { sendStatus(res, 404, "not found"); return true; diff --git a/src/gateway/server-methods/agents-mutate.test.ts b/src/gateway/server-methods/agents-mutate.test.ts index 35c83f73579..4edeab726a0 100644 --- a/src/gateway/server-methods/agents-mutate.test.ts +++ b/src/gateway/server-methods/agents-mutate.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; -import { SafeOpenError } from "../../infra/fs-safe.js"; +import { FsSafeError } from "../../infra/fs-safe.js"; /* ------------------------------------------------------------------ */ /* Mocks */ /* ------------------------------------------------------------------ */ @@ -39,7 +39,17 @@ const mocks = vi.hoisted(() => ({ fsRealpath: vi.fn(async (p: string) => p), fsReadlink: vi.fn(async () => ""), fsOpen: vi.fn(async () => ({}) as unknown), - writeFileWithinRoot: vi.fn(async () => {}), + rootRead: vi.fn(async (_params?: unknown) => ({ + buffer: Buffer.from(""), + realPath: "/workspace/test-agent/AGENTS.md", + stat: { size: 0, mtimeMs: 0 }, + })), + rootOpen: vi.fn(async (_params?: unknown) => ({ + handle: { close: vi.fn(async () => {}) }, + realPath: "/workspace/test-agent/AGENTS.md", + stat: { size: 0, mtimeMs: 0 }, + })), + rootWrite: vi.fn(async (_params?: unknown) => {}), })); vi.mock("../../config/config.js", async () => { @@ -106,7 +116,23 @@ vi.mock("../../infra/fs-safe.js", async () => { await vi.importActual("../../infra/fs-safe.js"); return { ...actual, - writeFileWithinRoot: mocks.writeFileWithinRoot, + root: vi.fn(async (rootDir: string) => ({ + open: async (relativePath: string, options?: Record) => + await mocks.rootOpen({ rootDir, relativePath, ...options }), + read: async (relativePath: string, options?: Record) => + await mocks.rootRead({ rootDir, relativePath, ...options }), + write: async ( + relativePath: string, + data: string | Buffer, + options?: Record, + ) => + await mocks.rootWrite({ + rootDir, + relativePath, + data, + ...options, + }), + })), }; }); @@ -154,9 +180,44 @@ beforeEach(() => { mocks.resolveAgentWorkspaceDir.mockImplementation((cfg: unknown, agentId?: string) => resolveMockWorkspaceDir(cfg, agentId), ); - mocks.writeFileWithinRoot.mockResolvedValue(undefined); + mocks.rootOpen.mockResolvedValue({ + handle: { close: vi.fn(async () => {}) }, + realPath: "/workspace/test-agent/AGENTS.md", + stat: { size: 0, mtimeMs: 0 }, + }); + mocks.rootRead.mockResolvedValue({ + buffer: Buffer.from(""), + realPath: "/workspace/test-agent/AGENTS.md", + stat: { size: 0, mtimeMs: 0 }, + }); + mocks.rootWrite.mockResolvedValue(undefined); }); +function makeRootForTest(overrides?: { + open?: (params: Record) => Promise; + read?: (params: Record) => Promise; + write?: (params: Record) => Promise; +}) { + return async (rootDir: string) => + ({ + open: async (relativePath: string, options?: Record) => + await (overrides?.open ?? mocks.rootOpen)({ rootDir, relativePath, ...options }), + read: async (relativePath: string, options?: Record) => + await (overrides?.read ?? mocks.rootRead)({ rootDir, relativePath, ...options }), + write: async ( + relativePath: string, + data: string | Buffer, + options?: Record, + ) => + await (overrides?.write ?? mocks.rootWrite)({ + rootDir, + relativePath, + data, + ...options, + }), + }) as never; +} + function makeCall(method: keyof typeof agentsHandlers, params: Record) { const respond = vi.fn(); const handler = agentsHandlers[method]; @@ -466,7 +527,7 @@ describe("agents.create", () => { identity: expect.objectContaining({ name: "Plain Agent" }), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/resolved/tmp/ws", relativePath: "IDENTITY.md", @@ -494,7 +555,7 @@ describe("agents.create", () => { }), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/resolved/tmp/ws", relativePath: "IDENTITY.md", @@ -503,9 +564,9 @@ describe("agents.create", () => { ); }); - it("does not persist config when IDENTITY.md write fails with SafeOpenError", async () => { - mocks.writeFileWithinRoot.mockRejectedValueOnce( - new SafeOpenError("path-mismatch", "path escapes workspace root"), + it("does not persist config when IDENTITY.md write fails with FsSafeError", async () => { + mocks.rootWrite.mockRejectedValueOnce( + new FsSafeError("path-mismatch", "path escapes workspace root"), ); const { respond, promise } = makeCall("agents.create", { @@ -524,9 +585,11 @@ describe("agents.create", () => { it("does not persist config when IDENTITY.md read fails", async () => { agentsTesting.setDepsForTests({ - readFileWithinRoot: async () => { - throw createErrnoError("EACCES"); - }, + root: makeRootForTest({ + read: async () => { + throw createErrnoError("EACCES"); + }, + }), }); mocks.ensureAgentWorkspace.mockResolvedValueOnce({ dir: "/resolved/tmp/ws", @@ -540,14 +603,16 @@ describe("agents.create", () => { await expect(promise).rejects.toMatchObject({ code: "EACCES" }); expect(mocks.writeConfigFile).not.toHaveBeenCalled(); - expect(mocks.writeFileWithinRoot).not.toHaveBeenCalled(); + expect(mocks.rootWrite).not.toHaveBeenCalled(); }); it("treats unsafe IDENTITY.md reads as invalid create requests", async () => { agentsTesting.setDepsForTests({ - readFileWithinRoot: async () => { - throw new SafeOpenError("invalid-path", "path is not a regular file under root"); - }, + root: makeRootForTest({ + read: async () => { + throw new FsSafeError("invalid-path", "path is not a regular file under root"); + }, + }), }); const { respond, promise } = makeCall("agents.create", { @@ -564,14 +629,14 @@ describe("agents.create", () => { }), ); expect(mocks.writeConfigFile).not.toHaveBeenCalled(); - expect(mocks.writeFileWithinRoot).not.toHaveBeenCalled(); + expect(mocks.rootWrite).not.toHaveBeenCalled(); }); it("uses non-blocking reads for IDENTITY.md during agents.create", async () => { - const readFileWithinRoot = vi.fn(async () => { - throw new SafeOpenError("not-found", "file not found"); + const rootRead = vi.fn(async () => { + throw new FsSafeError("not-found", "file not found"); }); - agentsTesting.setDepsForTests({ readFileWithinRoot }); + agentsTesting.setDepsForTests({ root: makeRootForTest({ read: rootRead }) }); const { promise } = makeCall("agents.create", { name: "NB Agent", @@ -579,7 +644,7 @@ describe("agents.create", () => { }); await promise; - expect(readFileWithinRoot).toHaveBeenCalledWith( + expect(rootRead).toHaveBeenCalledWith( expect.objectContaining({ relativePath: "IDENTITY.md", nonBlockingRead: true, @@ -685,7 +750,7 @@ describe("agents.update", () => { }), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/workspace/test-agent", relativePath: "IDENTITY.md", @@ -710,7 +775,7 @@ describe("agents.update", () => { identity: expect.objectContaining({ emoji: "🦀" }), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/workspace/test-agent", relativePath: "IDENTITY.md", @@ -742,7 +807,7 @@ describe("agents.update", () => { }), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/workspace/test-agent", relativePath: "IDENTITY.md", @@ -759,50 +824,52 @@ describe("agents.update", () => { identityPathCreated: true, }); agentsTesting.setDepsForTests({ - readFileWithinRoot: async ({ rootDir, relativePath }) => { - const filePath = `${rootDir}/${relativePath}`; - if (filePath === "/workspace/test-agent/IDENTITY.md") { - return { - buffer: Buffer.from( - [ - "# IDENTITY.md - Agent Identity", - "", - "- **Name:** Current Agent", - "- **Creature:** Steady Turtle", - "- **Vibe:** Calm and methodical", - "- **Emoji:** 🐢", - "", - "## Role", - "", - "Protect the queue.", - "", - ].join("\n"), - ), - realPath: filePath, - stat: makeFileStat(), - }; - } - if (filePath === "/resolved/new/workspace/IDENTITY.md") { - return { - buffer: Buffer.from( - [ - "# IDENTITY.md - Agent Identity", - "", - "- **Name:** C-3PO (Clawd's Third Protocol Observer)", - "- **Creature:** Flustered Protocol Droid", - "", - "## Role", - "", - "Debug agent for `--dev` mode.", - "", - ].join("\n"), - ), - realPath: filePath, - stat: makeFileStat(), - }; - } - throw createEnoentError(); - }, + root: makeRootForTest({ + read: async ({ rootDir, relativePath }) => { + const filePath = `${String(rootDir)}/${String(relativePath)}`; + if (filePath === "/workspace/test-agent/IDENTITY.md") { + return { + buffer: Buffer.from( + [ + "# IDENTITY.md - Agent Identity", + "", + "- **Name:** Current Agent", + "- **Creature:** Steady Turtle", + "- **Vibe:** Calm and methodical", + "- **Emoji:** 🐢", + "", + "## Role", + "", + "Protect the queue.", + "", + ].join("\n"), + ), + realPath: filePath, + stat: makeFileStat(), + }; + } + if (filePath === "/resolved/new/workspace/IDENTITY.md") { + return { + buffer: Buffer.from( + [ + "# IDENTITY.md - Agent Identity", + "", + "- **Name:** C-3PO (Clawd's Third Protocol Observer)", + "- **Creature:** Flustered Protocol Droid", + "", + "## Role", + "", + "Debug agent for `--dev` mode.", + "", + ].join("\n"), + ), + realPath: filePath, + stat: makeFileStat(), + }; + } + throw createEnoentError(); + }, + }), }); const { respond, promise } = makeCall("agents.update", { @@ -812,19 +879,19 @@ describe("agents.update", () => { await promise; expect(respond).toHaveBeenCalledWith(true, { ok: true, agentId: "test-agent" }, undefined); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/resolved/new/workspace", relativePath: "IDENTITY.md", data: expect.stringContaining("- **Creature:** Steady Turtle"), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ data: expect.stringContaining("## Role"), }), ); - expect(mocks.writeFileWithinRoot).not.toHaveBeenCalledWith( + expect(mocks.rootWrite).not.toHaveBeenCalledWith( expect.objectContaining({ data: expect.stringContaining("Flustered Protocol Droid"), }), @@ -837,48 +904,50 @@ describe("agents.update", () => { identityPathCreated: false, }); agentsTesting.setDepsForTests({ - readFileWithinRoot: async ({ rootDir, relativePath }) => { - const filePath = `${rootDir}/${relativePath}`; - if (filePath === "/workspace/test-agent/IDENTITY.md") { - return { - buffer: Buffer.from( - [ - "# IDENTITY.md - Agent Identity", - "", - "- **Name:** Current Agent", - "- **Creature:** Old Turtle", - "", - "## Role", - "", - "Old workspace role.", - "", - ].join("\n"), - ), - realPath: filePath, - stat: makeFileStat(), - }; - } - if (filePath === "/resolved/new/workspace/IDENTITY.md") { - return { - buffer: Buffer.from( - [ - "# IDENTITY.md - Agent Identity", - "", - "- **Name:** Destination Agent", - "- **Creature:** Destination Fox", - "", - "## Role", - "", - "Destination workspace role.", - "", - ].join("\n"), - ), - realPath: filePath, - stat: makeFileStat(), - }; - } - throw createEnoentError(); - }, + root: makeRootForTest({ + read: async ({ rootDir, relativePath }) => { + const filePath = `${String(rootDir)}/${String(relativePath)}`; + if (filePath === "/workspace/test-agent/IDENTITY.md") { + return { + buffer: Buffer.from( + [ + "# IDENTITY.md - Agent Identity", + "", + "- **Name:** Current Agent", + "- **Creature:** Old Turtle", + "", + "## Role", + "", + "Old workspace role.", + "", + ].join("\n"), + ), + realPath: filePath, + stat: makeFileStat(), + }; + } + if (filePath === "/resolved/new/workspace/IDENTITY.md") { + return { + buffer: Buffer.from( + [ + "# IDENTITY.md - Agent Identity", + "", + "- **Name:** Destination Agent", + "- **Creature:** Destination Fox", + "", + "## Role", + "", + "Destination workspace role.", + "", + ].join("\n"), + ), + realPath: filePath, + stat: makeFileStat(), + }; + } + throw createEnoentError(); + }, + }), }); const { respond, promise } = makeCall("agents.update", { @@ -888,19 +957,19 @@ describe("agents.update", () => { await promise; expect(respond).toHaveBeenCalledWith(true, { ok: true, agentId: "test-agent" }, undefined); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/resolved/new/workspace", relativePath: "IDENTITY.md", data: expect.stringContaining("- **Creature:** Destination Fox"), }), ); - expect(mocks.writeFileWithinRoot).toHaveBeenCalledWith( + expect(mocks.rootWrite).toHaveBeenCalledWith( expect.objectContaining({ data: expect.stringContaining("Destination workspace role."), }), ); - expect(mocks.writeFileWithinRoot).not.toHaveBeenCalledWith( + expect(mocks.rootWrite).not.toHaveBeenCalledWith( expect.objectContaining({ data: expect.stringContaining("Old workspace role."), }), @@ -908,8 +977,8 @@ describe("agents.update", () => { }); it("does not persist config when IDENTITY.md write fails on update", async () => { - mocks.writeFileWithinRoot.mockRejectedValueOnce( - new SafeOpenError("path-mismatch", "path escapes workspace root"), + mocks.rootWrite.mockRejectedValueOnce( + new FsSafeError("path-mismatch", "path escapes workspace root"), ); const { respond, promise } = makeCall("agents.update", { @@ -929,9 +998,11 @@ describe("agents.update", () => { it("treats unsafe IDENTITY.md reads as invalid update requests", async () => { agentsTesting.setDepsForTests({ - readFileWithinRoot: async () => { - throw new SafeOpenError("invalid-path", "path is not a regular file under root"); - }, + root: makeRootForTest({ + read: async () => { + throw new FsSafeError("invalid-path", "path is not a regular file under root"); + }, + }), }); const { respond, promise } = makeCall("agents.update", { @@ -948,14 +1019,14 @@ describe("agents.update", () => { }), ); expect(mocks.writeConfigFile).not.toHaveBeenCalled(); - expect(mocks.writeFileWithinRoot).not.toHaveBeenCalled(); + expect(mocks.rootWrite).not.toHaveBeenCalled(); }); it("uses non-blocking reads for IDENTITY.md during agents.update", async () => { - const readFileWithinRoot = vi.fn(async () => { - throw new SafeOpenError("not-found", "file not found"); + const rootRead = vi.fn(async () => { + throw new FsSafeError("not-found", "file not found"); }); - agentsTesting.setDepsForTests({ readFileWithinRoot }); + agentsTesting.setDepsForTests({ root: makeRootForTest({ read: rootRead }) }); const { promise } = makeCall("agents.update", { agentId: "test-agent", @@ -963,7 +1034,7 @@ describe("agents.update", () => { }); await promise; - expect(readFileWithinRoot).toHaveBeenCalledWith( + expect(rootRead).toHaveBeenCalledWith( expect.objectContaining({ relativePath: "IDENTITY.md", nonBlockingRead: true, @@ -1082,10 +1153,10 @@ describe("agents.files.list", () => { }); it("reports unreadable workspace files as present in list responses", async () => { - const openFileWithinRoot = vi.fn(async () => { + const rootOpen = vi.fn(async () => { throw createErrnoError("EACCES"); }); - agentsTesting.setDepsForTests({ openFileWithinRoot }); + agentsTesting.setDepsForTests({ root: makeRootForTest({ open: rootOpen }) }); mocks.fsLstat.mockImplementation(async (...args: unknown[]) => { if (args[0] === "/workspace/main/AGENTS.md") { return makeFileStat({ size: 17, mtimeMs: 4567 }); @@ -1112,7 +1183,7 @@ describe("agents.files.list", () => { size: 17, }), ); - expect(openFileWithinRoot).not.toHaveBeenCalled(); + expect(rootOpen).not.toHaveBeenCalled(); }); }); @@ -1128,32 +1199,33 @@ describe("agents.files.get/set symlink safety", () => { }); function mockWorkspaceEscapeSymlink() { - const safeOpenError = new SafeOpenError("invalid-path", "path escapes workspace root"); + const safeOpenError = new FsSafeError("invalid-path", "path escapes workspace root"); agentsTesting.setDepsForTests({ - openFileWithinRoot: async () => { - throw safeOpenError; - }, - readFileWithinRoot: async () => { - throw safeOpenError; - }, + root: makeRootForTest({ + open: async () => { + throw safeOpenError; + }, + read: async () => { + throw safeOpenError; + }, + }), }); - mocks.writeFileWithinRoot.mockRejectedValue(safeOpenError); + mocks.rootWrite.mockRejectedValue(safeOpenError); } function mockInWorkspaceSymlinkAlias() { - const safeOpenError = new SafeOpenError( - "invalid-path", - "path is not a regular file under root", - ); + const safeOpenError = new FsSafeError("invalid-path", "path is not a regular file under root"); agentsTesting.setDepsForTests({ - openFileWithinRoot: async () => { - throw safeOpenError; - }, - readFileWithinRoot: async () => { - throw safeOpenError; - }, + root: makeRootForTest({ + open: async () => { + throw safeOpenError; + }, + read: async () => { + throw safeOpenError; + }, + }), }); - mocks.writeFileWithinRoot.mockRejectedValue(safeOpenError); + mocks.rootWrite.mockRejectedValue(safeOpenError); } it.each([ @@ -1179,16 +1251,18 @@ describe("agents.files.get/set symlink safety", () => { ); function mockHardlinkedWorkspaceAlias() { - const safeOpenError = new SafeOpenError("invalid-path", "hardlinked path not allowed"); + const safeOpenError = new FsSafeError("invalid-path", "hardlinked path not allowed"); agentsTesting.setDepsForTests({ - openFileWithinRoot: async () => { - throw safeOpenError; - }, - readFileWithinRoot: async () => { - throw safeOpenError; - }, + root: makeRootForTest({ + open: async () => { + throw safeOpenError; + }, + read: async () => { + throw safeOpenError; + }, + }), }); - mocks.writeFileWithinRoot.mockRejectedValue(safeOpenError); + mocks.rootWrite.mockRejectedValue(safeOpenError); } it.each([ @@ -1206,12 +1280,12 @@ describe("agents.files.get/set symlink safety", () => { ); it("uses non-blocking safe reads for agents.files.get", async () => { - const readFileWithinRoot = vi.fn(async () => ({ + const rootRead = vi.fn(async () => ({ buffer: Buffer.from("hello"), realPath: "/workspace/test-agent/AGENTS.md", stat: makeFileStat({ size: 5 }), })); - agentsTesting.setDepsForTests({ readFileWithinRoot }); + agentsTesting.setDepsForTests({ root: makeRootForTest({ read: rootRead }) }); const { respond, promise } = makeCall("agents.files.get", { agentId: "main", @@ -1219,11 +1293,11 @@ describe("agents.files.get/set symlink safety", () => { }); await promise; - expect(readFileWithinRoot).toHaveBeenCalledWith( + expect(rootRead).toHaveBeenCalledWith( expect.objectContaining({ rootDir: "/workspace/test-agent", relativePath: "AGENTS.md", - rejectHardlinks: true, + hardlinks: "reject", nonBlockingRead: true, }), ); diff --git a/src/gateway/server-methods/agents.ts b/src/gateway/server-methods/agents.ts index cf86cb979e7..7a053627f7d 100644 --- a/src/gateway/server-methods/agents.ts +++ b/src/gateway/server-methods/agents.ts @@ -33,13 +33,8 @@ import { } from "../../config/sessions.js"; import type { IdentityConfig } from "../../config/types.base.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { sameFileIdentity } from "../../infra/file-identity.js"; -import { - openFileWithinRoot, - readFileWithinRoot, - SafeOpenError, - writeFileWithinRoot, -} from "../../infra/fs-safe.js"; +import { sameFileIdentity } from "../../infra/fs-safe-advanced.js"; +import { root, FsSafeError, type ReadResult } from "../../infra/fs-safe.js"; import { movePathToTrash } from "../../plugin-sdk/browser-maintenance.js"; import { DEFAULT_AGENT_ID, normalizeAgentId } from "../../routing/session-key.js"; import { resolveUserPath } from "../../utils.js"; @@ -72,28 +67,27 @@ const BOOTSTRAP_FILE_NAMES_POST_ONBOARDING = BOOTSTRAP_FILE_NAMES.filter( ); const agentsHandlerDeps = { + root, isWorkspaceSetupCompleted, - openFileWithinRoot, - readFileWithinRoot, - writeFileWithinRoot, }; export const __testing = { setDepsForTests( overrides: Partial<{ + root: typeof root; isWorkspaceSetupCompleted: typeof isWorkspaceSetupCompleted; - openFileWithinRoot: typeof openFileWithinRoot; - readFileWithinRoot: typeof readFileWithinRoot; - writeFileWithinRoot: typeof writeFileWithinRoot; }>, ) { - Object.assign(agentsHandlerDeps, overrides); + if (overrides.isWorkspaceSetupCompleted) { + agentsHandlerDeps.isWorkspaceSetupCompleted = overrides.isWorkspaceSetupCompleted; + } + if (overrides.root) { + agentsHandlerDeps.root = overrides.root; + } }, resetDepsForTests() { + agentsHandlerDeps.root = root; agentsHandlerDeps.isWorkspaceSetupCompleted = isWorkspaceSetupCompleted; - agentsHandlerDeps.openFileWithinRoot = openFileWithinRoot; - agentsHandlerDeps.readFileWithinRoot = readFileWithinRoot; - agentsHandlerDeps.writeFileWithinRoot = writeFileWithinRoot; }, }; @@ -315,14 +309,10 @@ async function writeWorkspaceFileOrRespond(params: { }): Promise { await fs.mkdir(params.workspaceDir, { recursive: true }); try { - await agentsHandlerDeps.writeFileWithinRoot({ - rootDir: params.workspaceDir, - relativePath: params.name, - data: params.content, - encoding: "utf8", - }); + const workspaceRoot = await agentsHandlerDeps.root(params.workspaceDir); + await workspaceRoot.write(params.name, params.content, { encoding: "utf8" }); } catch (err) { - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { respondWorkspaceFileUnsafe(params.respond, params.name); return false; } @@ -354,15 +344,14 @@ async function readWorkspaceFileContent( name: string, ): Promise { try { - const safeRead = await agentsHandlerDeps.readFileWithinRoot({ - rootDir: workspaceDir, - relativePath: name, - rejectHardlinks: true, + const workspaceRoot = await agentsHandlerDeps.root(workspaceDir); + const safeRead = await workspaceRoot.read(name, { + hardlinks: "reject", nonBlockingRead: true, }); return safeRead.buffer.toString("utf-8"); } catch (err) { - if (err instanceof SafeOpenError && err.code === "not-found") { + if (err instanceof FsSafeError && err.code === "not-found") { return undefined; } throw err; @@ -407,7 +396,7 @@ async function buildIdentityMarkdownOrRespondUnsafe(params: { try { return await buildIdentityMarkdownForWrite(params); } catch (err) { - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { respondWorkspaceFileUnsafe(params.respond, DEFAULT_IDENTITY_FILENAME); return null; } @@ -716,20 +705,19 @@ export const agentsHandlers: GatewayRequestHandlers = { } const { agentId, workspaceDir, name } = resolved; const filePath = path.join(workspaceDir, name); - let safeRead: Awaited>; + let safeRead: ReadResult; try { - safeRead = await agentsHandlerDeps.readFileWithinRoot({ - rootDir: workspaceDir, - relativePath: name, - rejectHardlinks: true, + const workspaceRoot = await agentsHandlerDeps.root(workspaceDir); + safeRead = await workspaceRoot.read(name, { + hardlinks: "reject", nonBlockingRead: true, }); } catch (err) { - if (err instanceof SafeOpenError && err.code === "not-found") { + if (err instanceof FsSafeError && err.code === "not-found") { respondWorkspaceFileMissing({ respond, agentId, workspaceDir, name, filePath }); return; } - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { respondWorkspaceFileUnsafe(respond, name); return; } @@ -770,14 +758,10 @@ export const agentsHandlers: GatewayRequestHandlers = { const filePath = path.join(workspaceDir, name); const content = params.content; try { - await agentsHandlerDeps.writeFileWithinRoot({ - rootDir: workspaceDir, - relativePath: name, - data: content, - encoding: "utf8", - }); + const workspaceRoot = await agentsHandlerDeps.root(workspaceDir); + await workspaceRoot.write(name, content, { encoding: "utf8" }); } catch (err) { - if (!(err instanceof SafeOpenError)) { + if (!(err instanceof FsSafeError)) { throw err; } respondWorkspaceFileUnsafe(respond, name); diff --git a/src/gateway/server-methods/chat-webchat-media.test.ts b/src/gateway/server-methods/chat-webchat-media.test.ts index ae41dd9bed2..cac8467208c 100644 --- a/src/gateway/server-methods/chat-webchat-media.test.ts +++ b/src/gateway/server-methods/chat-webchat-media.test.ts @@ -1,4 +1,5 @@ import fs from "node:fs"; +import fsPromises from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -116,8 +117,7 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { }); it("drops tool-result file:// URLs with remote hosts before touching the filesystem", async () => { - const statSpy = vi.spyOn(fs, "statSync"); - const readSpy = vi.spyOn(fs, "readFileSync"); + const openSpy = vi.spyOn(fsPromises, "open"); const blocks = await buildWebchatAudioContentBlocksFromReplyPayloads([ { @@ -128,11 +128,9 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { ]); expect(blocks).toHaveLength(0); - expect(statSpy).not.toHaveBeenCalled(); - expect(readSpy).not.toHaveBeenCalled(); + expect(openSpy).not.toHaveBeenCalled(); - statSpy.mockRestore(); - readSpy.mockRestore(); + openSpy.mockRestore(); }); it("rejects a local audio file outside configured localRoots", async () => { @@ -174,19 +172,11 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { expect((blocks[0] as { type?: string }).type).toBe("audio"); }); - it("does not read file contents when stat reports size over the cap", async () => { + it("skips local audio when the opened file stat is over the cap", async () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-webchat-audio-")); const audioPath = path.join(tmpDir, "huge.mp3"); fs.writeFileSync(audioPath, Buffer.from([0x02])); - - const origStat = fs.statSync.bind(fs); - const statSpy = vi.spyOn(fs, "statSync").mockImplementation((p: fs.PathLike) => { - if (String(p) === audioPath) { - return { isFile: () => true, size: 16 * 1024 * 1024 } as fs.Stats; - } - return origStat(p); - }); - const readSpy = vi.spyOn(fs, "readFileSync"); + fs.truncateSync(audioPath, 16 * 1024 * 1024); const blocks = await buildWebchatAudioContentBlocksFromReplyPayloads( [{ mediaUrl: audioPath, trustedLocalMedia: true }], @@ -194,10 +184,6 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { ); expect(blocks).toHaveLength(0); - expect(readSpy).not.toHaveBeenCalled(); - - statSpy.mockRestore(); - readSpy.mockRestore(); }); it("rejects untrusted local audio paths", async () => { diff --git a/src/gateway/server-methods/chat-webchat-media.ts b/src/gateway/server-methods/chat-webchat-media.ts index e1b76902c36..cb9e5c27f36 100644 --- a/src/gateway/server-methods/chat-webchat-media.ts +++ b/src/gateway/server-methods/chat-webchat-media.ts @@ -1,6 +1,6 @@ -import fs from "node:fs"; import path from "node:path"; import type { ReplyPayload } from "../../auto-reply/reply-payload.js"; +import { openLocalFileSafely } from "../../infra/fs-safe.js"; import { assertNoWindowsNetworkPath, safeFileURLToPath } from "../../infra/local-file-access.js"; import { assertLocalMediaAllowed, LocalMediaAccessError } from "../../media/local-media-access.js"; import { isAudioFileName } from "../../media/mime.js"; @@ -41,6 +41,11 @@ type WebchatAudioEmbeddingOptions = { type WebchatAssistantMediaOptions = WebchatAudioEmbeddingOptions; +type LocalAudioContentBlock = { + path: string; + block: Record; +}; + /** Map `mediaUrl` strings to an absolute filesystem path for local embedding (plain paths or `file:` URLs). */ function resolveLocalMediaPathForEmbedding(raw: string): string | null { const trimmed = raw.trim(); @@ -75,12 +80,11 @@ function resolveLocalMediaPathForEmbedding(raw: string): string | null { return trimmed; } -/** Returns a readable local file path when it is a regular file and within the size cap (single stat before read). */ -async function resolveLocalAudioFileForEmbedding( +async function readLocalAudioContentBlockForEmbedding( payload: ReplyPayload, raw: string, options: WebchatAudioEmbeddingOptions | undefined, -): Promise { +): Promise { if (payload.trustedLocalMedia !== true) { return null; } @@ -91,18 +95,36 @@ async function resolveLocalAudioFileForEmbedding( if (!isAudioFileName(resolved)) { return null; } + let opened: Awaited> | undefined; try { await assertLocalMediaAllowed(resolved, options?.localRoots); - const st = fs.statSync(resolved); - if (!st.isFile() || st.size > MAX_WEBCHAT_AUDIO_BYTES) { + opened = await openLocalFileSafely({ filePath: resolved }); + await assertLocalMediaAllowed(opened.realPath, options?.localRoots); + if (opened.stat.size > MAX_WEBCHAT_AUDIO_BYTES) { return null; } - return resolved; + const buf = await opened.handle.readFile(); + if (buf.length > MAX_WEBCHAT_AUDIO_BYTES) { + return null; + } + return { + path: opened.realPath, + block: { + type: "audio", + source: { + type: "base64", + media_type: mimeTypeForPath(opened.realPath), + data: buf.toString("base64"), + }, + }, + }; } catch (err) { if (err instanceof LocalMediaAccessError) { options?.onLocalAudioAccessDenied?.(err); } return null; + } finally { + await opened?.handle.close().catch(() => {}); } } @@ -171,15 +193,12 @@ export async function buildWebchatAudioContentBlocksFromReplyPayloads( if (!url) { continue; } - const resolved = await resolveLocalAudioFileForEmbedding(payload, url, options); - if (!resolved || seen.has(resolved)) { + const audio = await readLocalAudioContentBlockForEmbedding(payload, url, options); + if (!audio || seen.has(audio.path)) { continue; } - seen.add(resolved); - const block = tryReadLocalAudioContentBlock(resolved); - if (block) { - blocks.push(block); - } + seen.add(audio.path); + blocks.push(audio.block); } } return blocks; @@ -213,18 +232,15 @@ export async function buildWebchatAssistantMessageFromReplyPayloads( if (!url) { continue; } - const resolvedAudioPath = await resolveLocalAudioFileForEmbedding(payload, url, options); - if (resolvedAudioPath) { - if (seenAudio.has(resolvedAudioPath)) { + const audio = await readLocalAudioContentBlockForEmbedding(payload, url, options); + if (audio) { + if (seenAudio.has(audio.path)) { continue; } - seenAudio.add(resolvedAudioPath); - const block = tryReadLocalAudioContentBlock(resolvedAudioPath); - if (block) { - payloadMediaBlocks.push(block); - hasAudio = true; - payloadHasAudio = true; - } + seenAudio.add(audio.path); + payloadMediaBlocks.push(audio.block); + hasAudio = true; + payloadHasAudio = true; continue; } const imageUrl = resolveEmbeddableImageUrl(url); @@ -270,20 +286,3 @@ export async function buildWebchatAssistantMessageFromReplyPayloads( } return { content, transcriptText }; } - -function tryReadLocalAudioContentBlock(filePath: string): Record | null { - try { - const buf = fs.readFileSync(filePath); - if (buf.length > MAX_WEBCHAT_AUDIO_BYTES) { - return null; - } - const mediaType = mimeTypeForPath(filePath); - const base64Data = buf.toString("base64"); - return { - type: "audio", - source: { type: "base64", media_type: mediaType, data: base64Data }, - }; - } catch { - return null; - } -} diff --git a/src/gateway/server.auth.control-ui.suite.ts b/src/gateway/server.auth.control-ui.suite.ts index a8589113cd5..3f0aa6d0400 100644 --- a/src/gateway/server.auth.control-ui.suite.ts +++ b/src/gateway/server.auth.control-ui.suite.ts @@ -184,24 +184,24 @@ export function registerControlUiAndPairingSuite(): void { }; const stripPairedMetadataRolesAndScopes = async (deviceId: string) => { - const { resolvePairingPaths, readJsonFile } = await import("../infra/pairing-files.js"); - const { writeJsonAtomic } = await import("../infra/json-files.js"); + const { resolvePairingPaths, tryReadJson } = await import("../infra/pairing-files.js"); + const { writeJson } = await import("../infra/json-files.js"); const { pairedPath } = resolvePairingPaths(undefined, "devices"); - const paired = (await readJsonFile>>(pairedPath)) ?? {}; + const paired = (await tryReadJson>>(pairedPath)) ?? {}; const legacy = getRequiredPairedMetadata(paired, deviceId); delete legacy.roles; delete legacy.scopes; - await writeJsonAtomic(pairedPath, paired); + await writeJson(pairedPath, paired); }; const overwritePairedPublicKey = async (deviceId: string, publicKey: string) => { - const { resolvePairingPaths, readJsonFile } = await import("../infra/pairing-files.js"); - const { writeJsonAtomic } = await import("../infra/json-files.js"); + const { resolvePairingPaths, tryReadJson } = await import("../infra/pairing-files.js"); + const { writeJson } = await import("../infra/json-files.js"); const { pairedPath } = resolvePairingPaths(undefined, "devices"); - const paired = (await readJsonFile>>(pairedPath)) ?? {}; + const paired = (await tryReadJson>>(pairedPath)) ?? {}; const metadata = getRequiredPairedMetadata(paired, deviceId); metadata.publicKey = publicKey; - await writeJsonAtomic(pairedPath, paired); + await writeJson(pairedPath, paired); }; const seedApprovedOperatorReadPairing = async (params: { diff --git a/src/gateway/session-utils.ts b/src/gateway/session-utils.ts index 56ba9ddaa19..6c1422da410 100644 --- a/src/gateway/session-utils.ts +++ b/src/gateway/session-utils.ts @@ -56,7 +56,7 @@ import { type SessionScope, } from "../config/sessions.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { projectPluginSessionExtensionsSync } from "../plugins/host-hook-state.js"; import { DEFAULT_AGENT_ID, @@ -168,7 +168,7 @@ function resolveIdentityAvatarUrl( return undefined; } try { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: resolvedCandidate, rootPath: workspaceRoot, rootRealPath: workspaceRoot, diff --git a/src/hooks/bundled/command-logger/handler.ts b/src/hooks/bundled/command-logger/handler.ts index b923e04605b..055199cafd9 100644 --- a/src/hooks/bundled/command-logger/handler.ts +++ b/src/hooks/bundled/command-logger/handler.ts @@ -28,6 +28,7 @@ import os from "node:os"; import path from "node:path"; import { resolveStateDir } from "../../../config/paths.js"; import { formatErrorMessage } from "../../../infra/errors.js"; +import { appendRegularFile } from "../../../infra/fs-safe.js"; import { createSubsystemLogger } from "../../../logging/subsystem.js"; import type { HookHandler } from "../../hooks.js"; @@ -59,7 +60,11 @@ const logCommand: HookHandler = async (event) => { source: event.context.commandSource ?? "unknown", }) + "\n"; - await fs.appendFile(logFile, logLine, "utf-8"); + await appendRegularFile({ + filePath: logFile, + content: logLine, + rejectSymlinkParents: true, + }); } catch (err) { const message = formatErrorMessage(err); log.error(`Failed to log command: ${message}`); diff --git a/src/hooks/bundled/session-memory/handler.ts b/src/hooks/bundled/session-memory/handler.ts index 3d6847f4371..7777ea49523 100644 --- a/src/hooks/bundled/session-memory/handler.ts +++ b/src/hooks/bundled/session-memory/handler.ts @@ -14,7 +14,7 @@ import { } from "../../../agents/agent-scope.js"; import { resolveStateDir } from "../../../config/paths.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; -import { writeFileWithinRoot } from "../../../infra/fs-safe.js"; +import { root } from "../../../infra/fs-safe.js"; import { createSubsystemLogger } from "../../../logging/subsystem.js"; import { parseAgentSessionKey, @@ -277,12 +277,8 @@ async function saveSessionMemoryNow(event: Parameters[0]): Promise< const entry = entryParts.join("\n"); // Write under memory root with alias-safe file validation. - await writeFileWithinRoot({ - rootDir: memoryDir, - relativePath: filename, - data: entry, - encoding: "utf-8", - }); + const memoryRoot = await root(memoryDir); + await memoryRoot.write(filename, entry, { encoding: "utf-8" }); log.debug("Memory file written successfully"); // Log completion (but don't send user-visible confirmation - it's internal housekeeping) diff --git a/src/hooks/install.runtime.ts b/src/hooks/install.runtime.ts index 406f859fed5..38ab18a921c 100644 --- a/src/hooks/install.runtime.ts +++ b/src/hooks/install.runtime.ts @@ -1,4 +1,5 @@ -import { fileExists, readJsonFile, resolveArchiveKind } from "../infra/archive.js"; +import { resolveArchiveKind } from "../infra/archive.js"; +import { pathExists } from "../infra/fs-safe.js"; import { resolveExistingInstallPath, withExtractedArchiveRoot } from "../infra/install-flow.js"; import { installFromValidatedNpmSpecArchive } from "../infra/install-from-npm-spec.js"; import { @@ -18,19 +19,20 @@ import { ensureInstallTargetAvailable, resolveCanonicalInstallTarget, } from "../infra/install-target.js"; +import { readJson } from "../infra/json-files.js"; import { isPathInside, isPathInsideWithRealpath } from "../security/scan-paths.js"; export type { NpmIntegrityDrift, NpmSpecResolution }; export { ensureInstallTargetAvailable, - fileExists, + pathExists as fileExists, installFromValidatedNpmSpecArchive, installPackageDir, installPackageDirWithManifestDeps, isPathInside, isPathInsideWithRealpath, - readJsonFile, + readJson as readJsonFile, resolveArchiveKind, resolveArchiveSourcePath, resolveCanonicalInstallTarget, diff --git a/src/hooks/loader.ts b/src/hooks/loader.ts index 1d069dfbbaa..2b4af989a0f 100644 --- a/src/hooks/loader.ts +++ b/src/hooks/loader.ts @@ -8,7 +8,7 @@ import fs from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFile } from "../infra/boundary-file-read.js"; +import { openRootFile } from "../infra/boundary-file-read.js"; import { formatErrorMessage } from "../infra/errors.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveGlobalSingleton } from "../shared/global-singleton.js"; @@ -119,7 +119,7 @@ export async function loadInternalHooks( ); continue; } - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: entry.hook.handlerPath, rootPath: hookBaseDir, boundaryLabel: "hook directory", @@ -215,7 +215,7 @@ export async function loadInternalHooks( log.error(`Handler module path must stay within workspaceDir: ${safeLogValue(rawModule)}`); continue; } - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath: modulePathSafe, rootPath: baseDirReal, boundaryLabel: "workspace directory", diff --git a/src/hooks/workspace.ts b/src/hooks/workspace.ts index 7fb9463e8d9..d0c506e45cd 100644 --- a/src/hooks/workspace.ts +++ b/src/hooks/workspace.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { MANIFEST_KEY } from "../compat/legacy-names.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { isPathInsideWithRealpath } from "../security/scan-paths.js"; import { CONFIG_DIR, resolveUserPath } from "../utils.js"; @@ -28,7 +28,7 @@ type LoadedHook = { function readHookPackageManifest(dir: string): HookPackageManifest | null { const manifestPath = path.join(dir, "package.json"); - const raw = readBoundaryFileUtf8({ + const raw = readRootFileUtf8({ absolutePath: manifestPath, rootPath: dir, boundaryLabel: "hook package directory", @@ -71,7 +71,7 @@ function loadHookFromDir(params: { nameHint?: string; }): LoadedHook | null { const hookMdPath = path.join(params.hookDir, "HOOK.md"); - const content = readBoundaryFileUtf8({ + const content = readRootFileUtf8({ absolutePath: hookMdPath, rootPath: params.hookDir, boundaryLabel: "hook directory", @@ -89,7 +89,7 @@ function loadHookFromDir(params: { let handlerPath: string | undefined; for (const candidate of handlerCandidates) { const candidatePath = path.join(params.hookDir, candidate); - const safeCandidatePath = resolveBoundaryFilePath({ + const safeCandidatePath = resolveRootFilePath({ absolutePath: candidatePath, rootPath: params.hookDir, boundaryLabel: "hook directory", @@ -293,12 +293,12 @@ export function loadWorkspaceHookEntries( }); } -function readBoundaryFileUtf8(params: { +function readRootFileUtf8(params: { absolutePath: string; rootPath: string; boundaryLabel: string; }): string | null { - return withOpenedBoundaryFileSync(params, (opened) => { + return withOpenedRootFileSync(params, (opened) => { try { return fs.readFileSync(opened.fd, "utf-8"); } catch { @@ -307,7 +307,7 @@ function readBoundaryFileUtf8(params: { }); } -function withOpenedBoundaryFileSync( +function withOpenedRootFileSync( params: { absolutePath: string; rootPath: string; @@ -315,7 +315,7 @@ function withOpenedBoundaryFileSync( }, read: (opened: { fd: number; path: string }) => T, ): T | null { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: params.absolutePath, rootPath: params.rootPath, boundaryLabel: params.boundaryLabel, @@ -330,10 +330,10 @@ function withOpenedBoundaryFileSync( } } -function resolveBoundaryFilePath(params: { +function resolveRootFilePath(params: { absolutePath: string; rootPath: string; boundaryLabel: string; }): string | null { - return withOpenedBoundaryFileSync(params, (opened) => opened.path); + return withOpenedRootFileSync(params, (opened) => opened.path); } diff --git a/src/infra/archive-helpers.test.ts b/src/infra/archive-helpers.test.ts index a238d2bfcdb..440444f262a 100644 --- a/src/infra/archive-helpers.test.ts +++ b/src/infra/archive-helpers.test.ts @@ -4,12 +4,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; import { createTarEntryPreflightChecker, - fileExists, - readJsonFile, resolveArchiveKind, resolvePackedRootDir, - withTimeout, } from "./archive.js"; +import { pathExists, withTimeout } from "./fs-safe.js"; +import { readJsonFileStrict } from "./json-files.js"; const tempDirs = createTrackedTempDirs(); const createTempDir = () => tempDirs.make("openclaw-archive-helper-test-"); @@ -159,9 +158,9 @@ describe("archive helpers", () => { await fs.writeFile(jsonPath, '{"ok":true}', "utf8"); await fs.writeFile(badPath, "{not json", "utf8"); - await expect(readJsonFile<{ ok: boolean }>(jsonPath)).resolves.toEqual({ ok: true }); - await expect(readJsonFile(badPath)).rejects.toThrow(); - await expect(fileExists(jsonPath)).resolves.toBe(true); - await expect(fileExists(path.join(dir, "missing.json"))).resolves.toBe(false); + await expect(readJsonFileStrict<{ ok: boolean }>(jsonPath)).resolves.toEqual({ ok: true }); + await expect(readJsonFileStrict(badPath)).rejects.toThrow(); + await expect(pathExists(jsonPath)).resolves.toBe(true); + await expect(pathExists(path.join(dir, "missing.json"))).resolves.toBe(false); }); }); diff --git a/src/infra/archive-path.ts b/src/infra/archive-path.ts index 43da416f28b..5fb5de9aa63 100644 --- a/src/infra/archive-path.ts +++ b/src/infra/archive-path.ts @@ -1,63 +1,8 @@ -import path from "node:path"; -import { resolveSafeBaseDir } from "./path-safety.js"; - -export function isWindowsDrivePath(value: string): boolean { - return /^[a-zA-Z]:[\\/]/.test(value); -} - -export function normalizeArchiveEntryPath(raw: string): string { - return raw.replaceAll("\\", "/"); -} - -export function validateArchiveEntryPath( - entryPath: string, - params?: { escapeLabel?: string }, -): void { - if (!entryPath || entryPath === "." || entryPath === "./") { - return; - } - if (isWindowsDrivePath(entryPath)) { - throw new Error(`archive entry uses a drive path: ${entryPath}`); - } - const normalized = path.posix.normalize(normalizeArchiveEntryPath(entryPath)); - const escapeLabel = params?.escapeLabel ?? "destination"; - if (normalized === ".." || normalized.startsWith("../")) { - throw new Error(`archive entry escapes ${escapeLabel}: ${entryPath}`); - } - if (path.posix.isAbsolute(normalized) || normalized.startsWith("//")) { - throw new Error(`archive entry is absolute: ${entryPath}`); - } -} - -export function stripArchivePath(entryPath: string, stripComponents: number): string | null { - const raw = normalizeArchiveEntryPath(entryPath); - if (!raw || raw === "." || raw === "./") { - return null; - } - - // Mimic tar --strip-components semantics (raw segments before normalization) - // so strip-induced escapes like "a/../b" are visible to validators. - const parts = raw.split("/").filter((part) => part.length > 0 && part !== "."); - const strip = Math.max(0, Math.floor(stripComponents)); - const stripped = strip === 0 ? parts.join("/") : parts.slice(strip).join("/"); - const result = path.posix.normalize(stripped); - if (!result || result === "." || result === "./") { - return null; - } - return result; -} - -export function resolveArchiveOutputPath(params: { - rootDir: string; - relPath: string; - originalPath: string; - escapeLabel?: string; -}): string { - const safeBase = resolveSafeBaseDir(params.rootDir); - const outPath = path.resolve(params.rootDir, params.relPath); - const escapeLabel = params.escapeLabel ?? "destination"; - if (!outPath.startsWith(safeBase)) { - throw new Error(`archive entry escapes ${escapeLabel}: ${params.originalPath}`); - } - return outPath; -} +import "./fs-safe-defaults.js"; +export { + isWindowsDrivePath, + normalizeArchiveEntryPath, + resolveArchiveOutputPath, + stripArchivePath, + validateArchiveEntryPath, +} from "@openclaw/fs-safe/archive"; diff --git a/src/infra/archive-staging.ts b/src/infra/archive-staging.ts index 443e28e062e..f4896b95d99 100644 --- a/src/infra/archive-staging.ts +++ b/src/infra/archive-staging.ts @@ -1,218 +1,10 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { copyFileWithinRoot } from "./fs-safe.js"; -import { isNotFoundPathError, isPathInside } from "./path-guards.js"; - -const ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK = "archive entry traverses symlink in destination"; - -export type ArchiveSecurityErrorCode = - | "destination-not-directory" - | "destination-symlink" - | "destination-symlink-traversal"; - -export class ArchiveSecurityError extends Error { - code: ArchiveSecurityErrorCode; - - constructor(code: ArchiveSecurityErrorCode, message: string, options?: ErrorOptions) { - super(message, options); - this.code = code; - this.name = "ArchiveSecurityError"; - } -} - -function symlinkTraversalError(originalPath: string): ArchiveSecurityError { - return new ArchiveSecurityError( - "destination-symlink-traversal", - `${ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK}: ${originalPath}`, - ); -} - -export async function prepareArchiveDestinationDir(destDir: string): Promise { - const stat = await fs.lstat(destDir); - if (stat.isSymbolicLink()) { - throw new ArchiveSecurityError("destination-symlink", "archive destination is a symlink"); - } - if (!stat.isDirectory()) { - throw new ArchiveSecurityError( - "destination-not-directory", - "archive destination is not a directory", - ); - } - return await fs.realpath(destDir); -} - -async function assertNoSymlinkTraversal(params: { - rootDir: string; - relPath: string; - originalPath: string; -}): Promise { - const parts = params.relPath.split(/[\\/]+/).filter(Boolean); - let current = path.resolve(params.rootDir); - for (const part of parts) { - current = path.join(current, part); - let stat: Awaited>; - try { - stat = await fs.lstat(current); - } catch (err) { - if (isNotFoundPathError(err)) { - continue; - } - throw err; - } - if (stat.isSymbolicLink()) { - throw symlinkTraversalError(params.originalPath); - } - } -} - -async function assertResolvedInsideDestination(params: { - destinationRealDir: string; - targetPath: string; - originalPath: string; -}): Promise { - let resolved: string; - try { - resolved = await fs.realpath(params.targetPath); - } catch (err) { - if (isNotFoundPathError(err)) { - return; - } - throw err; - } - if (!isPathInside(params.destinationRealDir, resolved)) { - throw symlinkTraversalError(params.originalPath); - } -} - -export async function prepareArchiveOutputPath(params: { - destinationDir: string; - destinationRealDir: string; - relPath: string; - outPath: string; - originalPath: string; - isDirectory: boolean; -}): Promise { - await assertNoSymlinkTraversal({ - rootDir: params.destinationDir, - relPath: params.relPath, - originalPath: params.originalPath, - }); - - if (params.isDirectory) { - await fs.mkdir(params.outPath, { recursive: true }); - await assertResolvedInsideDestination({ - destinationRealDir: params.destinationRealDir, - targetPath: params.outPath, - originalPath: params.originalPath, - }); - return; - } - - const parentDir = path.dirname(params.outPath); - await fs.mkdir(parentDir, { recursive: true }); - await assertResolvedInsideDestination({ - destinationRealDir: params.destinationRealDir, - targetPath: parentDir, - originalPath: params.originalPath, - }); -} - -async function applyStagedEntryMode(params: { - destinationRealDir: string; - relPath: string; - mode: number; - originalPath: string; -}): Promise { - const destinationPath = path.join(params.destinationRealDir, params.relPath); - await assertResolvedInsideDestination({ - destinationRealDir: params.destinationRealDir, - targetPath: destinationPath, - originalPath: params.originalPath, - }); - if (params.mode !== 0) { - await fs.chmod(destinationPath, params.mode).catch(() => undefined); - } -} - -export async function withStagedArchiveDestination(params: { - destinationRealDir: string; - run: (stagingDir: string) => Promise; -}): Promise { - const stagingDir = await fs.mkdtemp(path.join(params.destinationRealDir, ".openclaw-archive-")); - try { - return await params.run(stagingDir); - } finally { - await fs.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); - } -} - -export async function mergeExtractedTreeIntoDestination(params: { - sourceDir: string; - destinationDir: string; - destinationRealDir: string; -}): Promise { - const walk = async (currentSourceDir: string): Promise => { - const entries = await fs.readdir(currentSourceDir, { withFileTypes: true }); - for (const entry of entries) { - const sourcePath = path.join(currentSourceDir, entry.name); - const relPath = path.relative(params.sourceDir, sourcePath); - const originalPath = relPath.split(path.sep).join("/"); - const destinationPath = path.join(params.destinationDir, relPath); - const sourceStat = await fs.lstat(sourcePath); - - if (sourceStat.isSymbolicLink()) { - throw symlinkTraversalError(originalPath); - } - - if (sourceStat.isDirectory()) { - await prepareArchiveOutputPath({ - destinationDir: params.destinationDir, - destinationRealDir: params.destinationRealDir, - relPath, - outPath: destinationPath, - originalPath, - isDirectory: true, - }); - await walk(sourcePath); - await applyStagedEntryMode({ - destinationRealDir: params.destinationRealDir, - relPath, - mode: sourceStat.mode & 0o777, - originalPath, - }); - continue; - } - - if (!sourceStat.isFile()) { - throw new Error(`archive staging contains unsupported entry: ${originalPath}`); - } - - await prepareArchiveOutputPath({ - destinationDir: params.destinationDir, - destinationRealDir: params.destinationRealDir, - relPath, - outPath: destinationPath, - originalPath, - isDirectory: false, - }); - await copyFileWithinRoot({ - sourcePath, - rootDir: params.destinationRealDir, - relativePath: relPath, - mkdir: true, - }); - await applyStagedEntryMode({ - destinationRealDir: params.destinationRealDir, - relPath, - mode: sourceStat.mode & 0o777, - originalPath, - }); - } - }; - - await walk(params.sourceDir); -} - -export function createArchiveSymlinkTraversalError(originalPath: string): ArchiveSecurityError { - return symlinkTraversalError(originalPath); -} +import "./fs-safe-defaults.js"; +export { + ArchiveSecurityError, + createArchiveSymlinkTraversalError, + mergeExtractedTreeIntoDestination, + prepareArchiveDestinationDir, + prepareArchiveOutputPath, + withStagedArchiveDestination, + type ArchiveSecurityErrorCode, +} from "@openclaw/fs-safe/archive"; diff --git a/src/infra/archive.ts b/src/infra/archive.ts index b4222e520a9..edd80d1bcda 100644 --- a/src/infra/archive.ts +++ b/src/infra/archive.ts @@ -1,891 +1,27 @@ -import { randomUUID } from "node:crypto"; -import { constants as fsConstants } from "node:fs"; -import type { Stats } from "node:fs"; -import type { FileHandle } from "node:fs/promises"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { Readable, Transform } from "node:stream"; -import { pipeline } from "node:stream/promises"; -import JSZip from "jszip"; -import * as tar from "tar"; -import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; -import { - resolveArchiveOutputPath, - stripArchivePath, - validateArchiveEntryPath, -} from "./archive-path.js"; -import { - createArchiveSymlinkTraversalError, - mergeExtractedTreeIntoDestination, - prepareArchiveDestinationDir, - prepareArchiveOutputPath, - withStagedArchiveDestination, -} from "./archive-staging.js"; -import { sameFileIdentity } from "./file-identity.js"; -import { openFileWithinRoot, openWritableFileWithinRoot, SafeOpenError } from "./fs-safe.js"; -import { isNotFoundPathError } from "./path-guards.js"; - -export type ArchiveKind = "tar" | "zip"; - -export type ArchiveLogger = { - info?: (message: string) => void; - warn?: (message: string) => void; -}; - -export type ArchiveExtractLimits = { - /** - * Max archive file bytes (compressed). - */ - maxArchiveBytes?: number; - /** Max number of extracted entries (files + dirs). */ - maxEntries?: number; - /** Max extracted bytes (sum of all files). */ - maxExtractedBytes?: number; - /** Max extracted bytes for a single file entry. */ - maxEntryBytes?: number; -}; - -export { ArchiveSecurityError, type ArchiveSecurityErrorCode } from "./archive-staging.js"; +import "./fs-safe-defaults.js"; export { + ARCHIVE_LIMIT_ERROR_CODE, + ArchiveLimitError, + ArchiveSecurityError, + DEFAULT_MAX_ARCHIVE_BYTES_ZIP, + DEFAULT_MAX_ENTRIES, + DEFAULT_MAX_EXTRACTED_BYTES, + DEFAULT_MAX_ENTRY_BYTES, + createArchiveSymlinkTraversalError, + createTarEntryPreflightChecker, + extractArchive, + loadZipArchiveWithPreflight, mergeExtractedTreeIntoDestination, prepareArchiveDestinationDir, prepareArchiveOutputPath, + readZipCentralDirectoryEntryCount, + resolveArchiveKind, + resolvePackedRootDir, withStagedArchiveDestination, -} from "./archive-staging.js"; - -/** @internal */ -export const DEFAULT_MAX_ARCHIVE_BYTES_ZIP = 256 * 1024 * 1024; -/** @internal */ -export const DEFAULT_MAX_ENTRIES = 50_000; -/** @internal */ -export const DEFAULT_MAX_EXTRACTED_BYTES = 512 * 1024 * 1024; -/** @internal */ -export const DEFAULT_MAX_ENTRY_BYTES = 256 * 1024 * 1024; - -export const ARCHIVE_LIMIT_ERROR_CODE = { - ARCHIVE_SIZE_EXCEEDS_LIMIT: "archive-size-exceeds-limit", - ENTRY_COUNT_EXCEEDS_LIMIT: "archive-entry-count-exceeds-limit", - ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT: "archive-entry-extracted-size-exceeds-limit", - EXTRACTED_SIZE_EXCEEDS_LIMIT: "archive-extracted-size-exceeds-limit", -} as const; - -export type ArchiveLimitErrorCode = - (typeof ARCHIVE_LIMIT_ERROR_CODE)[keyof typeof ARCHIVE_LIMIT_ERROR_CODE]; - -const ARCHIVE_LIMIT_ERROR_MESSAGE = { - [ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT]: "archive size exceeds limit", - [ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT]: "archive entry count exceeds limit", - [ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT]: - "archive entry extracted size exceeds limit", - [ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT]: "archive extracted size exceeds limit", -} as const satisfies Record; - -export class ArchiveLimitError extends Error { - readonly code: ArchiveLimitErrorCode; - - constructor(code: ArchiveLimitErrorCode) { - super(ARCHIVE_LIMIT_ERROR_MESSAGE[code]); - this.name = "ArchiveLimitError"; - this.code = code; - } -} - -const ZIP_EOCD_SIGNATURE = 0x06054b50; -const ZIP64_EOCD_SIGNATURE = 0x06064b50; -const ZIP64_EOCD_LOCATOR_SIGNATURE = 0x07064b50; -const ZIP_EOCD_MIN_BYTES = 22; -const ZIP_EOCD_MAX_COMMENT_BYTES = 0xffff; -const ZIP64_ENTRY_COUNT_SENTINEL = 0xffff; -const ZIP64_UINT32_SENTINEL = 0xffffffff; -const ZIP_CENTRAL_FILE_HEADER_SIGNATURE = 0x02014b50; -const ZIP_CENTRAL_FILE_HEADER_MIN_BYTES = 46; -const ZIP_CENTRAL_FILE_HEADER_NAME_LENGTH_OFFSET = 28; -const ZIP_CENTRAL_FILE_HEADER_EXTRA_LENGTH_OFFSET = 30; -const ZIP_CENTRAL_FILE_HEADER_COMMENT_LENGTH_OFFSET = 32; -const ZIP_EOCD_TOTAL_ENTRIES_OFFSET = 10; -const ZIP_EOCD_CENTRAL_DIRECTORY_SIZE_OFFSET = 12; -const ZIP_EOCD_CENTRAL_DIRECTORY_OFFSET_OFFSET = 16; -const ZIP_EOCD_COMMENT_LENGTH_OFFSET = 20; -const ZIP64_EOCD_LOCATOR_BYTES = 20; -const ZIP64_EOCD_OFFSET_OFFSET = 8; -const ZIP64_EOCD_TOTAL_ENTRIES_OFFSET = 32; -const ZIP64_EOCD_CENTRAL_DIRECTORY_SIZE_OFFSET = 40; -const ZIP64_EOCD_CENTRAL_DIRECTORY_OFFSET_OFFSET = 48; -const SUPPORTS_NOFOLLOW = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants; -const OPEN_WRITE_CREATE_FLAGS = - fsConstants.O_WRONLY | - fsConstants.O_CREAT | - fsConstants.O_EXCL | - (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); - -const TAR_SUFFIXES = [".tgz", ".tar.gz", ".tar"]; - -export function resolveArchiveKind(filePath: string): ArchiveKind | null { - const lower = normalizeLowercaseStringOrEmpty(filePath); - if (lower.endsWith(".zip")) { - return "zip"; - } - if (TAR_SUFFIXES.some((suffix) => lower.endsWith(suffix))) { - return "tar"; - } - return null; -} - -type ResolvePackedRootDirOptions = { - rootMarkers?: string[]; -}; - -async function hasPackedRootMarker(extractDir: string, rootMarkers: string[]): Promise { - for (const marker of rootMarkers) { - const trimmed = marker.trim(); - if (!trimmed) { - continue; - } - try { - await fs.stat(path.join(extractDir, trimmed)); - return true; - } catch { - // ignore - } - } - return false; -} - -export async function resolvePackedRootDir( - extractDir: string, - options?: ResolvePackedRootDirOptions, -): Promise { - const direct = path.join(extractDir, "package"); - try { - const stat = await fs.stat(direct); - if (stat.isDirectory()) { - return direct; - } - } catch { - // ignore - } - - if ((options?.rootMarkers?.length ?? 0) > 0) { - const hasMarker = await hasPackedRootMarker(extractDir, options?.rootMarkers ?? []); - if (hasMarker) { - return extractDir; - } - } - - const entries = await fs.readdir(extractDir, { withFileTypes: true }); - const dirs = entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name); - if (dirs.length !== 1) { - throw new Error(`unexpected archive layout (dirs: ${dirs.join(", ")})`); - } - const onlyDir = dirs[0]; - if (!onlyDir) { - throw new Error("unexpected archive layout (no package dir found)"); - } - return path.join(extractDir, onlyDir); -} - -export async function withTimeout( - promise: Promise, - timeoutMs: number, - label: string, -): Promise { - let timeoutId: ReturnType | undefined; - try { - return await Promise.race([ - promise, - new Promise((_, reject) => { - timeoutId = setTimeout( - () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), - timeoutMs, - ); - }), - ]); - } finally { - if (timeoutId) { - clearTimeout(timeoutId); - } - } -} - -type ResolvedArchiveExtractLimits = Required; - -function clampLimit(value: number | undefined): number | undefined { - if (typeof value !== "number" || !Number.isFinite(value)) { - return undefined; - } - const v = Math.floor(value); - return v > 0 ? v : undefined; -} - -function resolveExtractLimits(limits?: ArchiveExtractLimits): ResolvedArchiveExtractLimits { - // Defaults: defensive, but should not break normal installs. - return { - maxArchiveBytes: clampLimit(limits?.maxArchiveBytes) ?? DEFAULT_MAX_ARCHIVE_BYTES_ZIP, - maxEntries: clampLimit(limits?.maxEntries) ?? DEFAULT_MAX_ENTRIES, - maxExtractedBytes: clampLimit(limits?.maxExtractedBytes) ?? DEFAULT_MAX_EXTRACTED_BYTES, - maxEntryBytes: clampLimit(limits?.maxEntryBytes) ?? DEFAULT_MAX_ENTRY_BYTES, - }; -} - -function assertArchiveEntryCountWithinLimit( - entryCount: number, - limits: ResolvedArchiveExtractLimits, -) { - if (entryCount > limits.maxEntries) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT); - } -} - -function asBufferView(buffer: Buffer | Uint8Array): Buffer { - if (Buffer.isBuffer(buffer)) { - return buffer; - } - return Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength); -} - -function readSafeUInt64LE(buffer: Buffer, offset: number): number { - const value = buffer.readBigUInt64LE(offset); - if (value > BigInt(Number.MAX_SAFE_INTEGER)) { - return Number.MAX_SAFE_INTEGER; - } - return Number(value); -} - -function findZipEndOfCentralDirectory(buffer: Buffer): number { - if (buffer.byteLength < ZIP_EOCD_MIN_BYTES) { - return -1; - } - const minOffset = Math.max( - 0, - buffer.byteLength - ZIP_EOCD_MIN_BYTES - ZIP_EOCD_MAX_COMMENT_BYTES, - ); - for (let offset = buffer.byteLength - ZIP_EOCD_MIN_BYTES; offset >= minOffset; offset -= 1) { - if (buffer.readUInt32LE(offset) !== ZIP_EOCD_SIGNATURE) { - continue; - } - const commentLength = buffer.readUInt16LE(offset + ZIP_EOCD_COMMENT_LENGTH_OFFSET); - if (offset + ZIP_EOCD_MIN_BYTES + commentLength === buffer.byteLength) { - return offset; - } - } - return -1; -} - -type ZipCentralDirectoryInfo = { - declaredEntryCount: number; - centralDirectoryOffset: number; - centralDirectorySize: number; - endOfCentralDirectoryOffset: number; -}; - -function readZip64CentralDirectoryInfo( - buffer: Buffer, - eocdOffset: number, -): ZipCentralDirectoryInfo | null { - const locatorOffset = eocdOffset - ZIP64_EOCD_LOCATOR_BYTES; - if (locatorOffset < 0 || buffer.readUInt32LE(locatorOffset) !== ZIP64_EOCD_LOCATOR_SIGNATURE) { - return null; - } - const zip64EocdOffset = readSafeUInt64LE(buffer, locatorOffset + ZIP64_EOCD_OFFSET_OFFSET); - if ( - zip64EocdOffset < 0 || - zip64EocdOffset + ZIP64_EOCD_CENTRAL_DIRECTORY_OFFSET_OFFSET + 8 > buffer.byteLength || - buffer.readUInt32LE(zip64EocdOffset) !== ZIP64_EOCD_SIGNATURE - ) { - return null; - } - return { - declaredEntryCount: readSafeUInt64LE(buffer, zip64EocdOffset + ZIP64_EOCD_TOTAL_ENTRIES_OFFSET), - centralDirectorySize: readSafeUInt64LE( - buffer, - zip64EocdOffset + ZIP64_EOCD_CENTRAL_DIRECTORY_SIZE_OFFSET, - ), - centralDirectoryOffset: readSafeUInt64LE( - buffer, - zip64EocdOffset + ZIP64_EOCD_CENTRAL_DIRECTORY_OFFSET_OFFSET, - ), - endOfCentralDirectoryOffset: eocdOffset, - }; -} - -function readZipCentralDirectoryInfo(buffer: Buffer): ZipCentralDirectoryInfo | null { - const eocdOffset = findZipEndOfCentralDirectory(buffer); - if (eocdOffset < 0) { - return null; - } - const declaredEntryCount = buffer.readUInt16LE(eocdOffset + ZIP_EOCD_TOTAL_ENTRIES_OFFSET); - const centralDirectorySize = buffer.readUInt32LE( - eocdOffset + ZIP_EOCD_CENTRAL_DIRECTORY_SIZE_OFFSET, - ); - const centralDirectoryOffset = buffer.readUInt32LE( - eocdOffset + ZIP_EOCD_CENTRAL_DIRECTORY_OFFSET_OFFSET, - ); - const usesZip64 = - declaredEntryCount === ZIP64_ENTRY_COUNT_SENTINEL || - centralDirectorySize === ZIP64_UINT32_SENTINEL || - centralDirectoryOffset === ZIP64_UINT32_SENTINEL; - if (usesZip64) { - return ( - readZip64CentralDirectoryInfo(buffer, eocdOffset) ?? { - declaredEntryCount, - centralDirectoryOffset, - centralDirectorySize, - endOfCentralDirectoryOffset: eocdOffset, - } - ); - } - return { - declaredEntryCount, - centralDirectoryOffset, - centralDirectorySize, - endOfCentralDirectoryOffset: eocdOffset, - }; -} - -function countZipCentralDirectoryHeaders( - buffer: Buffer, - info: ZipCentralDirectoryInfo, -): number | null { - const start = info.centralDirectoryOffset; - const declaredEnd = start + info.centralDirectorySize; - const scanEnd = info.endOfCentralDirectoryOffset; - if ( - !Number.isSafeInteger(start) || - !Number.isSafeInteger(declaredEnd) || - !Number.isSafeInteger(scanEnd) || - start < 0 || - declaredEnd < start || - scanEnd < start || - scanEnd > buffer.byteLength - ) { - return null; - } - let offset = start; - let count = 0; - while (offset < scanEnd) { - if (scanEnd - offset < ZIP_CENTRAL_FILE_HEADER_MIN_BYTES) { - break; - } - if (buffer.readUInt32LE(offset) !== ZIP_CENTRAL_FILE_HEADER_SIGNATURE) { - break; - } - const nameLength = buffer.readUInt16LE(offset + ZIP_CENTRAL_FILE_HEADER_NAME_LENGTH_OFFSET); - const extraLength = buffer.readUInt16LE(offset + ZIP_CENTRAL_FILE_HEADER_EXTRA_LENGTH_OFFSET); - const commentLength = buffer.readUInt16LE( - offset + ZIP_CENTRAL_FILE_HEADER_COMMENT_LENGTH_OFFSET, - ); - const nextOffset = - offset + ZIP_CENTRAL_FILE_HEADER_MIN_BYTES + nameLength + extraLength + commentLength; - if (nextOffset <= offset || nextOffset > scanEnd) { - return null; - } - count += 1; - offset = nextOffset; - } - return count > 0 || info.declaredEntryCount === 0 ? count : null; -} - -/** @internal */ -export function readZipCentralDirectoryEntryCount(buffer: Buffer | Uint8Array): number | null { - const view = asBufferView(buffer); - const info = readZipCentralDirectoryInfo(view); - if (!info) { - return null; - } - const countedEntryCount = countZipCentralDirectoryHeaders(view, info); - return countedEntryCount === null - ? info.declaredEntryCount - : Math.max(info.declaredEntryCount, countedEntryCount); -} - -export async function loadZipArchiveWithPreflight( - buffer: Buffer | Uint8Array, - limits?: ArchiveExtractLimits, -): Promise { - const resolvedLimits = resolveExtractLimits(limits); - if (buffer.byteLength > resolvedLimits.maxArchiveBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); - } - const entryCount = readZipCentralDirectoryEntryCount(buffer); - if (entryCount !== null) { - assertArchiveEntryCountWithinLimit(entryCount, resolvedLimits); - } - return await JSZip.loadAsync(buffer); -} - -function createByteBudgetTracker(limits: ResolvedArchiveExtractLimits): { - startEntry: () => void; - addBytes: (bytes: number) => void; - addEntrySize: (size: number) => void; -} { - let entryBytes = 0; - let extractedBytes = 0; - - const addBytes = (bytes: number) => { - const b = Math.max(0, Math.floor(bytes)); - if (b === 0) { - return; - } - entryBytes += b; - if (entryBytes > limits.maxEntryBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT); - } - extractedBytes += b; - if (extractedBytes > limits.maxExtractedBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT); - } - }; - - return { - startEntry() { - entryBytes = 0; - }, - addBytes, - addEntrySize(size: number) { - const s = Math.max(0, Math.floor(size)); - if (s > limits.maxEntryBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT); - } - // Note: tar budgets are based on the header-declared size. - addBytes(s); - }, - }; -} - -function createExtractBudgetTransform(params: { - onChunkBytes: (bytes: number) => void; -}): Transform { - return new Transform({ - transform(chunk, _encoding, callback) { - try { - const buf = chunk instanceof Buffer ? chunk : Buffer.from(chunk as Uint8Array); - params.onChunkBytes(buf.byteLength); - callback(null, buf); - } catch (err) { - callback(err instanceof Error ? err : new Error(String(err))); - } - }, - }); -} - -function symlinkTraversalError(originalPath: string) { - return createArchiveSymlinkTraversalError(originalPath); -} - -type OpenZipOutputFileResult = { - handle: FileHandle; - createdForWrite: boolean; - openedRealPath: string; - openedStat: Stats; -}; - -async function openZipOutputFile(params: { - relPath: string; - originalPath: string; - destinationRealDir: string; -}): Promise { - try { - return await openWritableFileWithinRoot({ - rootDir: params.destinationRealDir, - relativePath: params.relPath, - mkdir: false, - mode: 0o666, - }); - } catch (err) { - if ( - err instanceof SafeOpenError && - (err.code === "invalid-path" || - err.code === "outside-workspace" || - err.code === "path-mismatch") - ) { - throw symlinkTraversalError(params.originalPath); - } - throw err; - } -} - -async function cleanupPartialRegularFile(filePath: string): Promise { - let stat: Awaited>; - try { - stat = await fs.lstat(filePath); - } catch (err) { - if (isNotFoundPathError(err)) { - return; - } - throw err; - } - if (stat.isFile()) { - await fs.unlink(filePath).catch(() => undefined); - } -} - -function buildArchiveAtomicTempPath(targetPath: string): string { - return path.join( - path.dirname(targetPath), - `.${path.basename(targetPath)}.${process.pid}.${randomUUID()}.tmp`, - ); -} - -async function verifyZipWriteResult(params: { - destinationRealDir: string; - relPath: string; - expectedStat: Stats; -}): Promise { - const opened = await openFileWithinRoot({ - rootDir: params.destinationRealDir, - relativePath: params.relPath, - rejectHardlinks: true, - }); - try { - if (!sameFileIdentity(opened.stat, params.expectedStat)) { - throw new SafeOpenError("path-mismatch", "path changed during zip extract"); - } - return opened.realPath; - } finally { - await opened.handle.close().catch(() => undefined); - } -} - -type ZipEntry = { - name: string; - dir: boolean; - unixPermissions?: number; - nodeStream?: () => NodeJS.ReadableStream; - async: (type: "nodebuffer") => Promise; -}; - -type ZipExtractBudget = ReturnType; - -async function readZipEntryStream(entry: ZipEntry): Promise { - if (typeof entry.nodeStream === "function") { - return entry.nodeStream(); - } - // Old JSZip: fall back to buffering, but still extract via a stream. - const buf = await entry.async("nodebuffer"); - return Readable.from(buf); -} - -function resolveZipOutputPath(params: { - entryPath: string; - strip: number; - destinationDir: string; -}): { relPath: string; outPath: string } | null { - validateArchiveEntryPath(params.entryPath); - const relPath = stripArchivePath(params.entryPath, params.strip); - if (!relPath) { - return null; - } - validateArchiveEntryPath(relPath); - return { - relPath, - outPath: resolveArchiveOutputPath({ - rootDir: params.destinationDir, - relPath, - originalPath: params.entryPath, - }), - }; -} - -async function prepareZipOutputPath(params: { - destinationDir: string; - destinationRealDir: string; - relPath: string; - outPath: string; - originalPath: string; - isDirectory: boolean; -}): Promise { - await prepareArchiveOutputPath(params); -} - -async function writeZipFileEntry(params: { - entry: ZipEntry; - relPath: string; - destinationRealDir: string; - budget: ZipExtractBudget; -}): Promise { - const opened = await openZipOutputFile({ - relPath: params.relPath, - originalPath: params.entry.name, - destinationRealDir: params.destinationRealDir, - }); - params.budget.startEntry(); - const readable = await readZipEntryStream(params.entry); - const destinationPath = opened.openedRealPath; - const targetMode = opened.openedStat.mode & 0o777; - await opened.handle.close().catch(() => undefined); - - let tempHandle: FileHandle | null = null; - let tempPath: string | null = null; - let tempStat: Stats | null = null; - let handleClosedByStream = false; - - try { - tempPath = buildArchiveAtomicTempPath(destinationPath); - tempHandle = await fs.open(tempPath, OPEN_WRITE_CREATE_FLAGS, targetMode || 0o666); - const writable = tempHandle.createWriteStream(); - writable.once("close", () => { - handleClosedByStream = true; - }); - - await pipeline( - readable, - createExtractBudgetTransform({ onChunkBytes: params.budget.addBytes }), - writable, - ); - tempStat = await fs.stat(tempPath); - if (!tempStat) { - throw new Error("zip temp write did not produce file metadata"); - } - if (!handleClosedByStream) { - await tempHandle.close().catch(() => undefined); - handleClosedByStream = true; - } - tempHandle = null; - await fs.rename(tempPath, destinationPath); - tempPath = null; - const verifiedPath = await verifyZipWriteResult({ - destinationRealDir: params.destinationRealDir, - relPath: params.relPath, - expectedStat: tempStat, - }); - - // Best-effort permission restore for zip entries created on unix. - if (typeof params.entry.unixPermissions === "number") { - const mode = params.entry.unixPermissions & 0o777; - if (mode !== 0) { - await fs.chmod(verifiedPath, mode).catch(() => undefined); - } - } - } catch (err) { - if (tempPath) { - await fs.rm(tempPath, { force: true }).catch(() => undefined); - } else { - await cleanupPartialRegularFile(destinationPath).catch(() => undefined); - } - if (err instanceof SafeOpenError) { - throw symlinkTraversalError(params.entry.name); - } - throw err; - } finally { - if (tempHandle && !handleClosedByStream) { - await tempHandle.close().catch(() => undefined); - } - } -} - -async function extractZip(params: { - archivePath: string; - destDir: string; - stripComponents?: number; - limits?: ArchiveExtractLimits; -}): Promise { - const limits = resolveExtractLimits(params.limits); - const destinationRealDir = await prepareArchiveDestinationDir(params.destDir); - const stat = await fs.stat(params.archivePath); - if (stat.size > limits.maxArchiveBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); - } - - const buffer = await fs.readFile(params.archivePath); - const zip = await loadZipArchiveWithPreflight(buffer, limits); - const entries = Object.values(zip.files) as ZipEntry[]; - const strip = Math.max(0, Math.floor(params.stripComponents ?? 0)); - - assertArchiveEntryCountWithinLimit(entries.length, limits); - - const budget = createByteBudgetTracker(limits); - - for (const entry of entries) { - const output = resolveZipOutputPath({ - entryPath: entry.name, - strip, - destinationDir: params.destDir, - }); - if (!output) { - continue; - } - - await prepareZipOutputPath({ - destinationDir: params.destDir, - destinationRealDir, - relPath: output.relPath, - outPath: output.outPath, - originalPath: entry.name, - isDirectory: entry.dir, - }); - if (entry.dir) { - continue; - } - - await writeZipFileEntry({ - entry, - relPath: output.relPath, - destinationRealDir, - budget, - }); - } -} - -export type TarEntryInfo = { path: string; type: string; size: number }; - -const BLOCKED_TAR_ENTRY_TYPES = new Set([ - "SymbolicLink", - "Link", - "BlockDevice", - "CharacterDevice", - "FIFO", - "Socket", -]); - -function readTarEntryInfo(entry: unknown): TarEntryInfo { - const p = - typeof entry === "object" && entry !== null && "path" in entry - ? String((entry as { path: unknown }).path) - : ""; - const t = - typeof entry === "object" && entry !== null && "type" in entry - ? String((entry as { type: unknown }).type) - : ""; - const s = - typeof entry === "object" && - entry !== null && - "size" in entry && - typeof (entry as { size?: unknown }).size === "number" && - Number.isFinite((entry as { size: number }).size) - ? Math.max(0, Math.floor((entry as { size: number }).size)) - : 0; - return { path: p, type: t, size: s }; -} - -export function createTarEntryPreflightChecker(params: { - rootDir: string; - stripComponents?: number; - limits?: ArchiveExtractLimits; - escapeLabel?: string; -}): (entry: TarEntryInfo) => void { - const strip = Math.max(0, Math.floor(params.stripComponents ?? 0)); - const limits = resolveExtractLimits(params.limits); - let entryCount = 0; - const budget = createByteBudgetTracker(limits); - - return (entry: TarEntryInfo) => { - validateArchiveEntryPath(entry.path, { escapeLabel: params.escapeLabel }); - - const relPath = stripArchivePath(entry.path, strip); - if (!relPath) { - return; - } - validateArchiveEntryPath(relPath, { escapeLabel: params.escapeLabel }); - resolveArchiveOutputPath({ - rootDir: params.rootDir, - relPath, - originalPath: entry.path, - escapeLabel: params.escapeLabel, - }); - - if (BLOCKED_TAR_ENTRY_TYPES.has(entry.type)) { - throw new Error(`tar entry is a link: ${entry.path}`); - } - - entryCount += 1; - assertArchiveEntryCountWithinLimit(entryCount, limits); - budget.addEntrySize(entry.size); - }; -} - -export async function extractArchive(params: { - archivePath: string; - destDir: string; - timeoutMs: number; - kind?: ArchiveKind; - stripComponents?: number; - tarGzip?: boolean; - limits?: ArchiveExtractLimits; - logger?: ArchiveLogger; -}): Promise { - const kind = params.kind ?? resolveArchiveKind(params.archivePath); - if (!kind) { - throw new Error(`unsupported archive: ${params.archivePath}`); - } - - const label = kind === "zip" ? "extract zip" : "extract tar"; - if (kind === "tar") { - await withTimeout( - (async () => { - const limits = resolveExtractLimits(params.limits); - const stat = await fs.stat(params.archivePath); - if (stat.size > limits.maxArchiveBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); - } - - const destinationRealDir = await prepareArchiveDestinationDir(params.destDir); - await withStagedArchiveDestination({ - destinationRealDir, - run: async (stagingDir) => { - const checkTarEntrySafety = createTarEntryPreflightChecker({ - rootDir: destinationRealDir, - stripComponents: params.stripComponents, - limits, - }); - // A canonical cwd is not enough here: tar can still follow - // pre-existing child symlinks in the live destination tree. - // Extract into a private staging dir first, then merge through - // the same safe-open boundary checks used by direct file writes. - await tar.x({ - file: params.archivePath, - cwd: stagingDir, - strip: Math.max(0, Math.floor(params.stripComponents ?? 0)), - gzip: params.tarGzip, - preservePaths: false, - strict: true, - onReadEntry(entry) { - try { - checkTarEntrySafety(readTarEntryInfo(entry)); - } catch (err) { - const error = err instanceof Error ? err : new Error(String(err)); - // Node's EventEmitter calls listeners with `this` bound to the - // emitter (tar.Unpack), which exposes Parser.abort(). - const emitter = this as unknown as { abort?: (error: Error) => void }; - emitter.abort?.(error); - } - }, - }); - await mergeExtractedTreeIntoDestination({ - sourceDir: stagingDir, - destinationDir: destinationRealDir, - destinationRealDir, - }); - }, - }); - })(), - params.timeoutMs, - label, - ); - return; - } - - await withTimeout( - extractZip({ - archivePath: params.archivePath, - destDir: params.destDir, - stripComponents: params.stripComponents, - limits: params.limits, - }), - params.timeoutMs, - label, - ); -} - -export async function fileExists(filePath: string): Promise { - try { - await fs.stat(filePath); - return true; - } catch { - return false; - } -} - -export async function readJsonFile(filePath: string): Promise { - const raw = await fs.readFile(filePath, "utf-8"); - return JSON.parse(raw) as T; -} + type ArchiveExtractLimits, + type ArchiveKind, + type ArchiveLimitErrorCode, + type ArchiveLogger, + type ArchiveSecurityErrorCode, + type TarEntryInfo, +} from "@openclaw/fs-safe/archive"; diff --git a/src/infra/boundary-file-read.test.ts b/src/infra/boundary-file-read.test.ts index fb0e563d726..4fd13de120a 100644 --- a/src/infra/boundary-file-read.test.ts +++ b/src/infra/boundary-file-read.test.ts @@ -1,240 +1,12 @@ -import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import * as upstream from "@openclaw/fs-safe/advanced"; +import { describe, expect, it } from "vitest"; +import * as shim from "./boundary-file-read.js"; -const resolveBoundaryPathSyncMock = vi.hoisted(() => vi.fn()); -const resolveBoundaryPathMock = vi.hoisted(() => vi.fn()); -const openVerifiedFileSyncMock = vi.hoisted(() => vi.fn()); - -vi.mock("./boundary-path.js", () => ({ - resolveBoundaryPathSync: (...args: unknown[]) => resolveBoundaryPathSyncMock(...args), - resolveBoundaryPath: (...args: unknown[]) => resolveBoundaryPathMock(...args), -})); - -vi.mock("./safe-open-sync.js", () => ({ - openVerifiedFileSync: (...args: unknown[]) => openVerifiedFileSyncMock(...args), -})); - -let canUseBoundaryFileOpen: typeof import("./boundary-file-read.js").canUseBoundaryFileOpen; -let matchBoundaryFileOpenFailure: typeof import("./boundary-file-read.js").matchBoundaryFileOpenFailure; -let openBoundaryFile: typeof import("./boundary-file-read.js").openBoundaryFile; -let openBoundaryFileSync: typeof import("./boundary-file-read.js").openBoundaryFileSync; - -describe("boundary-file-read", () => { - beforeEach(async () => { - vi.resetModules(); - ({ - canUseBoundaryFileOpen, - matchBoundaryFileOpenFailure, - openBoundaryFile, - openBoundaryFileSync, - } = await import("./boundary-file-read.js")); - resolveBoundaryPathSyncMock.mockReset(); - resolveBoundaryPathMock.mockReset(); - openVerifiedFileSyncMock.mockReset(); - }); - - it("recognizes the required sync fs surface", () => { - const validFs = { - openSync() {}, - closeSync() {}, - fstatSync() {}, - lstatSync() {}, - realpathSync() {}, - readFileSync() {}, - constants: {}, - }; - - expect(canUseBoundaryFileOpen(validFs as never)).toBe(true); - expect( - canUseBoundaryFileOpen({ - ...validFs, - openSync: undefined, - } as never), - ).toBe(false); - expect( - canUseBoundaryFileOpen({ - ...validFs, - constants: null, - } as never), - ).toBe(false); - }); - - it("maps sync boundary resolution into verified file opens", () => { - const stat = { size: 3 } as never; - const ioFs = { marker: "io" } as never; - const absolutePath = path.resolve("plugin.json"); - - resolveBoundaryPathSyncMock.mockReturnValue({ - canonicalPath: "/real/plugin.json", - rootCanonicalPath: "/real/root", - }); - openVerifiedFileSyncMock.mockReturnValue({ - ok: true, - path: "/real/plugin.json", - fd: 7, - stat, - }); - - const opened = openBoundaryFileSync({ - absolutePath: "plugin.json", - rootPath: "/workspace", - boundaryLabel: "plugin root", - ioFs, - }); - - expect(resolveBoundaryPathSyncMock).toHaveBeenCalledWith({ - absolutePath, - rootPath: "/workspace", - rootCanonicalPath: undefined, - boundaryLabel: "plugin root", - skipLexicalRootCheck: undefined, - }); - expect(openVerifiedFileSyncMock).toHaveBeenCalledWith({ - filePath: absolutePath, - resolvedPath: "/real/plugin.json", - rejectHardlinks: true, - maxBytes: undefined, - allowedType: undefined, - ioFs, - }); - expect(opened).toEqual({ - ok: true, - path: "/real/plugin.json", - fd: 7, - stat, - rootRealPath: "/real/root", - }); - }); - - it("returns validation errors when sync boundary resolution throws", () => { - const error = new Error("outside root"); - resolveBoundaryPathSyncMock.mockImplementation(() => { - throw error; - }); - - const opened = openBoundaryFileSync({ - absolutePath: "plugin.json", - rootPath: "/workspace", - boundaryLabel: "plugin root", - }); - - expect(opened).toEqual({ - ok: false, - reason: "validation", - error, - }); - expect(openVerifiedFileSyncMock).not.toHaveBeenCalled(); - }); - - it("guards against unexpected async sync-resolution results", () => { - resolveBoundaryPathSyncMock.mockReturnValue( - Promise.resolve({ - canonicalPath: "/real/plugin.json", - rootCanonicalPath: "/real/root", - }), - ); - - const opened = openBoundaryFileSync({ - absolutePath: "plugin.json", - rootPath: "/workspace", - boundaryLabel: "plugin root", - }); - - expect(opened.ok).toBe(false); - if (opened.ok) { - return; - } - expect(opened.reason).toBe("validation"); - expect(String(opened.error)).toContain("Unexpected async boundary resolution"); - }); - - it("awaits async boundary resolution before verifying the file", async () => { - const ioFs = { marker: "io" } as never; - const absolutePath = path.resolve("notes.txt"); - - resolveBoundaryPathMock.mockResolvedValue({ - canonicalPath: "/real/notes.txt", - rootCanonicalPath: "/real/root", - }); - openVerifiedFileSyncMock.mockReturnValue({ - ok: false, - reason: "validation", - error: new Error("blocked"), - }); - - const opened = await openBoundaryFile({ - absolutePath: "notes.txt", - rootPath: "/workspace", - boundaryLabel: "workspace", - aliasPolicy: { allowFinalSymlinkForUnlink: true }, - ioFs, - }); - - expect(resolveBoundaryPathMock).toHaveBeenCalledWith({ - absolutePath, - rootPath: "/workspace", - rootCanonicalPath: undefined, - boundaryLabel: "workspace", - policy: { allowFinalSymlinkForUnlink: true }, - skipLexicalRootCheck: undefined, - }); - expect(openVerifiedFileSyncMock).toHaveBeenCalledWith({ - filePath: absolutePath, - resolvedPath: "/real/notes.txt", - rejectHardlinks: true, - maxBytes: undefined, - allowedType: undefined, - ioFs, - }); - expect(opened).toEqual({ - ok: false, - reason: "validation", - error: expect.any(Error), - }); - }); - - it("maps async boundary resolution failures to validation errors", async () => { - const error = new Error("escaped"); - resolveBoundaryPathMock.mockRejectedValue(error); - - const opened = await openBoundaryFile({ - absolutePath: "notes.txt", - rootPath: "/workspace", - boundaryLabel: "workspace", - }); - - expect(opened).toEqual({ - ok: false, - reason: "validation", - error, - }); - expect(openVerifiedFileSyncMock).not.toHaveBeenCalled(); - }); - - it("matches boundary file failures by reason with fallback support", () => { - const missing = matchBoundaryFileOpenFailure( - { ok: false, reason: "path", error: new Error("missing") }, - { - path: () => "missing", - fallback: () => "fallback", - }, - ); - const io = matchBoundaryFileOpenFailure( - { ok: false, reason: "io", error: new Error("io") }, - { - io: () => "io", - fallback: () => "fallback", - }, - ); - const validation = matchBoundaryFileOpenFailure( - { ok: false, reason: "validation", error: new Error("blocked") }, - { - fallback: (failure) => failure.reason, - }, - ); - - expect(missing).toBe("missing"); - expect(io).toBe("io"); - expect(validation).toBe("validation"); +describe("root file open shim", () => { + it("re-exports the fs-safe root file helpers", () => { + expect(shim.canUseRootFileOpen).toBe(upstream.canUseRootFileOpen); + expect(shim.matchRootFileOpenFailure).toBe(upstream.matchRootFileOpenFailure); + expect(shim.openRootFile).toBe(upstream.openRootFile); + expect(shim.openRootFileSync).toBe(upstream.openRootFileSync); }); }); diff --git a/src/infra/boundary-file-read.ts b/src/infra/boundary-file-read.ts index 31a3a10bed4..036ab82b168 100644 --- a/src/infra/boundary-file-read.ts +++ b/src/infra/boundary-file-read.ts @@ -1,224 +1,12 @@ -import fs from "node:fs"; -import path from "node:path"; -import { - resolveBoundaryPath, - resolveBoundaryPathSync, - type ResolvedBoundaryPath, -} from "./boundary-path.js"; -import type { PathAliasPolicy } from "./path-alias-guards.js"; -import { - openVerifiedFileSync, - type SafeOpenSyncAllowedType, - type SafeOpenSyncFailureReason, -} from "./safe-open-sync.js"; - -type BoundaryReadFs = Pick< - typeof fs, - | "closeSync" - | "constants" - | "fstatSync" - | "lstatSync" - | "openSync" - | "readFileSync" - | "realpathSync" ->; - -export type BoundaryFileOpenFailureReason = SafeOpenSyncFailureReason | "validation"; - -export type BoundaryFileOpenResult = - | { ok: true; path: string; fd: number; stat: fs.Stats; rootRealPath: string } - | { ok: false; reason: BoundaryFileOpenFailureReason; error?: unknown }; - -export type BoundaryFileOpenFailure = Extract; - -export type OpenBoundaryFileSyncParams = { - absolutePath: string; - rootPath: string; - boundaryLabel: string; - rootRealPath?: string; - maxBytes?: number; - rejectHardlinks?: boolean; - allowedType?: SafeOpenSyncAllowedType; - skipLexicalRootCheck?: boolean; - ioFs?: BoundaryReadFs; -}; - -export type OpenBoundaryFileParams = OpenBoundaryFileSyncParams & { - aliasPolicy?: PathAliasPolicy; -}; - -type ResolvedBoundaryFilePath = { - absolutePath: string; - resolvedPath: string; - rootRealPath: string; -}; - -export function canUseBoundaryFileOpen(ioFs: typeof fs): boolean { - return ( - typeof ioFs.openSync === "function" && - typeof ioFs.closeSync === "function" && - typeof ioFs.fstatSync === "function" && - typeof ioFs.lstatSync === "function" && - typeof ioFs.realpathSync === "function" && - typeof ioFs.readFileSync === "function" && - typeof ioFs.constants === "object" && - ioFs.constants !== null - ); -} - -export function openBoundaryFileSync(params: OpenBoundaryFileSyncParams): BoundaryFileOpenResult { - const ioFs = params.ioFs ?? fs; - const resolved = resolveBoundaryFilePathGeneric({ - absolutePath: params.absolutePath, - resolve: (absolutePath) => - resolveBoundaryPathSync({ - absolutePath, - rootPath: params.rootPath, - rootCanonicalPath: params.rootRealPath, - boundaryLabel: params.boundaryLabel, - skipLexicalRootCheck: params.skipLexicalRootCheck, - }), - }); - if (resolved instanceof Promise) { - return toBoundaryValidationError(new Error("Unexpected async boundary resolution")); - } - return finalizeBoundaryFileOpen({ - resolved, - maxBytes: params.maxBytes, - rejectHardlinks: params.rejectHardlinks, - allowedType: params.allowedType, - ioFs, - }); -} - -export function matchBoundaryFileOpenFailure( - failure: BoundaryFileOpenFailure, - handlers: { - path?: (failure: BoundaryFileOpenFailure) => T; - validation?: (failure: BoundaryFileOpenFailure) => T; - io?: (failure: BoundaryFileOpenFailure) => T; - fallback: (failure: BoundaryFileOpenFailure) => T; - }, -): T { - switch (failure.reason) { - case "path": - return handlers.path ? handlers.path(failure) : handlers.fallback(failure); - case "validation": - return handlers.validation ? handlers.validation(failure) : handlers.fallback(failure); - case "io": - return handlers.io ? handlers.io(failure) : handlers.fallback(failure); - } - return handlers.fallback(failure); -} - -function openBoundaryFileResolved(params: { - absolutePath: string; - resolvedPath: string; - rootRealPath: string; - maxBytes?: number; - rejectHardlinks?: boolean; - allowedType?: SafeOpenSyncAllowedType; - ioFs: BoundaryReadFs; -}): BoundaryFileOpenResult { - const opened = openVerifiedFileSync({ - filePath: params.absolutePath, - resolvedPath: params.resolvedPath, - rejectHardlinks: params.rejectHardlinks ?? true, - maxBytes: params.maxBytes, - allowedType: params.allowedType, - ioFs: params.ioFs, - }); - if (!opened.ok) { - return opened; - } - return { - ok: true, - path: opened.path, - fd: opened.fd, - stat: opened.stat, - rootRealPath: params.rootRealPath, - }; -} - -function finalizeBoundaryFileOpen(params: { - resolved: ResolvedBoundaryFilePath | BoundaryFileOpenResult; - maxBytes?: number; - rejectHardlinks?: boolean; - allowedType?: SafeOpenSyncAllowedType; - ioFs: BoundaryReadFs; -}): BoundaryFileOpenResult { - if ("ok" in params.resolved) { - return params.resolved; - } - return openBoundaryFileResolved({ - absolutePath: params.resolved.absolutePath, - resolvedPath: params.resolved.resolvedPath, - rootRealPath: params.resolved.rootRealPath, - maxBytes: params.maxBytes, - rejectHardlinks: params.rejectHardlinks, - allowedType: params.allowedType, - ioFs: params.ioFs, - }); -} - -export async function openBoundaryFile( - params: OpenBoundaryFileParams, -): Promise { - const ioFs = params.ioFs ?? fs; - const maybeResolved = resolveBoundaryFilePathGeneric({ - absolutePath: params.absolutePath, - resolve: (absolutePath) => - resolveBoundaryPath({ - absolutePath, - rootPath: params.rootPath, - rootCanonicalPath: params.rootRealPath, - boundaryLabel: params.boundaryLabel, - policy: params.aliasPolicy, - skipLexicalRootCheck: params.skipLexicalRootCheck, - }), - }); - const resolved = maybeResolved instanceof Promise ? await maybeResolved : maybeResolved; - return finalizeBoundaryFileOpen({ - resolved, - maxBytes: params.maxBytes, - rejectHardlinks: params.rejectHardlinks, - allowedType: params.allowedType, - ioFs, - }); -} - -function toBoundaryValidationError(error: unknown): BoundaryFileOpenResult { - return { ok: false, reason: "validation", error }; -} - -function mapResolvedBoundaryPath( - absolutePath: string, - resolved: ResolvedBoundaryPath, -): ResolvedBoundaryFilePath { - return { - absolutePath, - resolvedPath: resolved.canonicalPath, - rootRealPath: resolved.rootCanonicalPath, - }; -} - -function resolveBoundaryFilePathGeneric(params: { - absolutePath: string; - resolve: (absolutePath: string) => ResolvedBoundaryPath | Promise; -}): - | ResolvedBoundaryFilePath - | BoundaryFileOpenResult - | Promise { - const absolutePath = path.resolve(params.absolutePath); - try { - const resolved = params.resolve(absolutePath); - if (resolved instanceof Promise) { - return resolved - .then((value) => mapResolvedBoundaryPath(absolutePath, value)) - .catch((error) => toBoundaryValidationError(error)); - } - return mapResolvedBoundaryPath(absolutePath, resolved); - } catch (error) { - return toBoundaryValidationError(error); - } -} +import "./fs-safe-defaults.js"; +export { + canUseRootFileOpen, + matchRootFileOpenFailure, + openRootFile, + openRootFileSync, + type OpenRootFileParams, + type OpenRootFileSyncParams, + type RootFileOpenFailure, + type RootFileOpenFailureReason, + type RootFileOpenResult, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/boundary-path.test.ts b/src/infra/boundary-path.test.ts index bf7b20ffcc0..4d45a2c7cf3 100644 --- a/src/infra/boundary-path.test.ts +++ b/src/infra/boundary-path.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { withTempDir } from "../test-helpers/temp-dir.js"; -import { resolveBoundaryPath, resolveBoundaryPathSync } from "./boundary-path.js"; +import { resolveRootPath, resolveRootPathSync } from "./boundary-path.js"; import { isPathInside } from "./path-guards.js"; function createSeededRandom(seed: number): () => number { @@ -13,7 +13,7 @@ function createSeededRandom(seed: number): () => number { }; } -describe("resolveBoundaryPath", () => { +describe("resolveRootPath", () => { it("resolves symlink parents with non-existent leafs inside root", async () => { if (process.platform === "win32") { return; @@ -27,7 +27,7 @@ describe("resolveBoundaryPath", () => { await fs.symlink(targetDir, linkPath); const unresolved = path.join(linkPath, "missing.txt"); - const result = await resolveBoundaryPath({ + const result = await resolveRootPath({ absolutePath: unresolved, rootPath: root, boundaryLabel: "sandbox root", @@ -56,14 +56,14 @@ describe("resolveBoundaryPath", () => { const dangling = path.join(linkPath, "missing.txt"); await expect( - resolveBoundaryPath({ + resolveRootPath({ absolutePath: dangling, rootPath: root, boundaryLabel: "sandbox root", }), ).rejects.toThrow(/Symlink escapes sandbox root/i); expect(() => - resolveBoundaryPathSync({ + resolveRootPathSync({ absolutePath: dangling, rootPath: root, boundaryLabel: "sandbox root", @@ -88,14 +88,14 @@ describe("resolveBoundaryPath", () => { await fs.symlink(outsideFile, linkPath); await expect( - resolveBoundaryPath({ + resolveRootPath({ absolutePath: linkPath, rootPath: root, boundaryLabel: "sandbox root", }), ).rejects.toThrow(/Symlink escapes sandbox root/i); - const allowed = await resolveBoundaryPath({ + const allowed = await resolveRootPath({ absolutePath: linkPath, rootPath: root, boundaryLabel: "sandbox root", @@ -121,7 +121,7 @@ describe("resolveBoundaryPath", () => { await fs.writeFile(path.join(root, fileName), "export default {}", "utf8"); await fs.symlink(root, aliasRoot); - const resolved = await resolveBoundaryPath({ + const resolved = await resolveRootPath({ absolutePath: path.join(aliasRoot, fileName), rootPath: await fs.realpath(root), boundaryLabel: "plugin root", @@ -129,7 +129,7 @@ describe("resolveBoundaryPath", () => { expect(resolved.exists).toBe(true); expect(isPathInside(resolved.rootCanonicalPath, resolved.canonicalPath)).toBe(true); - const resolvedSync = resolveBoundaryPathSync({ + const resolvedSync = resolveRootPathSync({ absolutePath: path.join(aliasRoot, fileName), rootPath: await fs.realpath(root), boundaryLabel: "plugin root", @@ -167,7 +167,7 @@ describe("resolveBoundaryPath", () => { const useLink = rand() > 0.5; const safeBase = useLink ? safeLinkBase : safeRealBase; const safeCandidate = path.join(safeBase, `new-${token}.txt`); - const safeResolved = await resolveBoundaryPath({ + const safeResolved = await resolveRootPath({ absolutePath: safeCandidate, rootPath: root, boundaryLabel: "sandbox root", @@ -176,7 +176,7 @@ describe("resolveBoundaryPath", () => { const unsafeCandidate = path.join(escapeLink, `new-${token}.txt`); await expect( - resolveBoundaryPath({ + resolveRootPath({ absolutePath: unsafeCandidate, rootPath: root, boundaryLabel: "sandbox root", diff --git a/src/infra/boundary-path.ts b/src/infra/boundary-path.ts index 812642fa10c..3bf8520704a 100644 --- a/src/infra/boundary-path.ts +++ b/src/infra/boundary-path.ts @@ -1,861 +1,9 @@ -import fs from "node:fs"; -import fsp from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { isNotFoundPathError, isPathInside } from "./path-guards.js"; - -type BoundaryPathIntent = "read" | "write" | "create" | "delete" | "stat"; - -export type BoundaryPathAliasPolicy = { - allowFinalSymlinkForUnlink?: boolean; - allowFinalHardlinkForUnlink?: boolean; -}; - -export const BOUNDARY_PATH_ALIAS_POLICIES = { - strict: Object.freeze({ - allowFinalSymlinkForUnlink: false, - allowFinalHardlinkForUnlink: false, - }), - unlinkTarget: Object.freeze({ - allowFinalSymlinkForUnlink: true, - allowFinalHardlinkForUnlink: true, - }), -} as const; - -type ResolveBoundaryPathParams = { - absolutePath: string; - rootPath: string; - boundaryLabel: string; - intent?: BoundaryPathIntent; - policy?: BoundaryPathAliasPolicy; - skipLexicalRootCheck?: boolean; - rootCanonicalPath?: string; -}; - -type ResolvedBoundaryPathKind = "missing" | "file" | "directory" | "symlink" | "other"; - -export type ResolvedBoundaryPath = { - absolutePath: string; - canonicalPath: string; - rootPath: string; - rootCanonicalPath: string; - relativePath: string; - exists: boolean; - kind: ResolvedBoundaryPathKind; -}; - -export async function resolveBoundaryPath( - params: ResolveBoundaryPathParams, -): Promise { - const rootPath = path.resolve(params.rootPath); - const absolutePath = path.resolve(params.absolutePath); - const rootCanonicalPath = params.rootCanonicalPath - ? path.resolve(params.rootCanonicalPath) - : await resolvePathViaExistingAncestor(rootPath); - const context = createBoundaryResolutionContext({ - resolveParams: params, - rootPath, - absolutePath, - rootCanonicalPath, - outsideLexicalCanonicalPath: await resolveOutsideLexicalCanonicalPathAsync({ - rootPath, - absolutePath, - }), - }); - - const outsideResult = await resolveOutsideBoundaryPathAsync({ - boundaryLabel: params.boundaryLabel, - context, - }); - if (outsideResult) { - return outsideResult; - } - - return resolveBoundaryPathLexicalAsync({ - params, - absolutePath: context.absolutePath, - rootPath: context.rootPath, - rootCanonicalPath: context.rootCanonicalPath, - }); -} - -export function resolveBoundaryPathSync(params: ResolveBoundaryPathParams): ResolvedBoundaryPath { - const rootPath = path.resolve(params.rootPath); - const absolutePath = path.resolve(params.absolutePath); - const rootCanonicalPath = params.rootCanonicalPath - ? path.resolve(params.rootCanonicalPath) - : resolvePathViaExistingAncestorSync(rootPath); - const context = createBoundaryResolutionContext({ - resolveParams: params, - rootPath, - absolutePath, - rootCanonicalPath, - outsideLexicalCanonicalPath: resolveOutsideLexicalCanonicalPathSync({ - rootPath, - absolutePath, - }), - }); - - const outsideResult = resolveOutsideBoundaryPathSync({ - boundaryLabel: params.boundaryLabel, - context, - }); - if (outsideResult) { - return outsideResult; - } - - return resolveBoundaryPathLexicalSync({ - params, - absolutePath: context.absolutePath, - rootPath: context.rootPath, - rootCanonicalPath: context.rootCanonicalPath, - }); -} - -type LexicalTraversalState = { - segments: string[]; - allowFinalSymlink: boolean; - canonicalCursor: string; - lexicalCursor: string; - preserveFinalSymlink: boolean; -}; - -type BoundaryResolutionContext = { - rootPath: string; - absolutePath: string; - rootCanonicalPath: string; - lexicalInside: boolean; - canonicalOutsideLexicalPath: string; -}; - -function isPromiseLike(value: unknown): value is PromiseLike { - return Boolean( - value && - (typeof value === "object" || typeof value === "function") && - "then" in value && - typeof (value as { then?: unknown }).then === "function", - ); -} - -function createLexicalTraversalState(params: { - params: ResolveBoundaryPathParams; - rootPath: string; - rootCanonicalPath: string; - absolutePath: string; -}): LexicalTraversalState { - const relative = path.relative(params.rootPath, params.absolutePath); - return { - segments: relative.split(path.sep).filter(Boolean), - allowFinalSymlink: params.params.policy?.allowFinalSymlinkForUnlink === true, - canonicalCursor: params.rootCanonicalPath, - lexicalCursor: params.rootPath, - preserveFinalSymlink: false, - }; -} - -function assertLexicalCursorInsideBoundary(params: { - params: ResolveBoundaryPathParams; - rootCanonicalPath: string; - absolutePath: string; - candidatePath: string; -}): void { - assertInsideBoundary({ - boundaryLabel: params.params.boundaryLabel, - rootCanonicalPath: params.rootCanonicalPath, - candidatePath: params.candidatePath, - absolutePath: params.absolutePath, - }); -} - -function applyMissingSuffixToCanonicalCursor(params: { - state: LexicalTraversalState; - missingFromIndex: number; - rootCanonicalPath: string; - params: ResolveBoundaryPathParams; - absolutePath: string; -}): void { - const missingSuffix = params.state.segments.slice(params.missingFromIndex); - params.state.canonicalCursor = path.resolve(params.state.canonicalCursor, ...missingSuffix); - assertLexicalCursorInsideBoundary({ - params: params.params, - rootCanonicalPath: params.rootCanonicalPath, - candidatePath: params.state.canonicalCursor, - absolutePath: params.absolutePath, - }); -} - -function advanceCanonicalCursorForSegment(params: { - state: LexicalTraversalState; - segment: string; - rootCanonicalPath: string; - params: ResolveBoundaryPathParams; - absolutePath: string; -}): void { - params.state.canonicalCursor = path.resolve(params.state.canonicalCursor, params.segment); - assertLexicalCursorInsideBoundary({ - params: params.params, - rootCanonicalPath: params.rootCanonicalPath, - candidatePath: params.state.canonicalCursor, - absolutePath: params.absolutePath, - }); -} - -function finalizeLexicalResolution(params: { - params: ResolveBoundaryPathParams; - rootPath: string; - rootCanonicalPath: string; - absolutePath: string; - state: LexicalTraversalState; - kind: { exists: boolean; kind: ResolvedBoundaryPathKind }; -}): ResolvedBoundaryPath { - assertLexicalCursorInsideBoundary({ - params: params.params, - rootCanonicalPath: params.rootCanonicalPath, - candidatePath: params.state.canonicalCursor, - absolutePath: params.absolutePath, - }); - return buildResolvedBoundaryPath({ - absolutePath: params.absolutePath, - canonicalPath: params.state.canonicalCursor, - rootPath: params.rootPath, - rootCanonicalPath: params.rootCanonicalPath, - kind: params.kind, - }); -} - -function handleLexicalLstatFailure(params: { - error: unknown; - state: LexicalTraversalState; - missingFromIndex: number; - rootCanonicalPath: string; - resolveParams: ResolveBoundaryPathParams; - absolutePath: string; -}): boolean { - if (!isNotFoundPathError(params.error)) { - return false; - } - applyMissingSuffixToCanonicalCursor({ - state: params.state, - missingFromIndex: params.missingFromIndex, - rootCanonicalPath: params.rootCanonicalPath, - params: params.resolveParams, - absolutePath: params.absolutePath, - }); - return true; -} - -function handleLexicalStatReadFailure(params: { - error: unknown; - state: LexicalTraversalState; - missingFromIndex: number; - rootCanonicalPath: string; - resolveParams: ResolveBoundaryPathParams; - absolutePath: string; -}): null { - if ( - handleLexicalLstatFailure({ - error: params.error, - state: params.state, - missingFromIndex: params.missingFromIndex, - rootCanonicalPath: params.rootCanonicalPath, - resolveParams: params.resolveParams, - absolutePath: params.absolutePath, - }) - ) { - return null; - } - throw params.error; -} - -function handleLexicalStatDisposition(params: { - state: LexicalTraversalState; - isSymbolicLink: boolean; - segment: string; - isLast: boolean; - rootCanonicalPath: string; - resolveParams: ResolveBoundaryPathParams; - absolutePath: string; -}): "continue" | "break" | "resolve-link" { - if (!params.isSymbolicLink) { - advanceCanonicalCursorForSegment({ - state: params.state, - segment: params.segment, - rootCanonicalPath: params.rootCanonicalPath, - params: params.resolveParams, - absolutePath: params.absolutePath, - }); - return "continue"; - } - - if (params.state.allowFinalSymlink && params.isLast) { - params.state.preserveFinalSymlink = true; - advanceCanonicalCursorForSegment({ - state: params.state, - segment: params.segment, - rootCanonicalPath: params.rootCanonicalPath, - params: params.resolveParams, - absolutePath: params.absolutePath, - }); - return "break"; - } - - return "resolve-link"; -} - -function applyResolvedSymlinkHop(params: { - state: LexicalTraversalState; - linkCanonical: string; - rootCanonicalPath: string; - boundaryLabel: string; -}): void { - if (!isPathInside(params.rootCanonicalPath, params.linkCanonical)) { - throw symlinkEscapeError({ - boundaryLabel: params.boundaryLabel, - rootCanonicalPath: params.rootCanonicalPath, - symlinkPath: params.state.lexicalCursor, - }); - } - params.state.canonicalCursor = params.linkCanonical; - params.state.lexicalCursor = params.linkCanonical; -} - -function readLexicalStat(params: { - state: LexicalTraversalState; - missingFromIndex: number; - rootCanonicalPath: string; - resolveParams: ResolveBoundaryPathParams; - absolutePath: string; - read: (cursor: string) => fs.Stats | Promise; -}): fs.Stats | null | Promise { - try { - const stat = params.read(params.state.lexicalCursor); - if (isPromiseLike(stat)) { - return Promise.resolve(stat).catch((error) => - handleLexicalStatReadFailure({ ...params, error }), - ); - } - return stat; - } catch (error) { - return handleLexicalStatReadFailure({ ...params, error }); - } -} - -function resolveAndApplySymlinkHop(params: { - state: LexicalTraversalState; - rootCanonicalPath: string; - boundaryLabel: string; - resolveLinkCanonical: (cursor: string) => string | Promise; -}): void | Promise { - const linkCanonical = params.resolveLinkCanonical(params.state.lexicalCursor); - if (isPromiseLike(linkCanonical)) { - return Promise.resolve(linkCanonical).then((value) => - applyResolvedSymlinkHop({ - state: params.state, - linkCanonical: value, - rootCanonicalPath: params.rootCanonicalPath, - boundaryLabel: params.boundaryLabel, - }), - ); - } - applyResolvedSymlinkHop({ - state: params.state, - linkCanonical, - rootCanonicalPath: params.rootCanonicalPath, - boundaryLabel: params.boundaryLabel, - }); -} - -type LexicalTraversalStep = { - idx: number; - segment: string; - isLast: boolean; -}; - -function* iterateLexicalTraversal(state: LexicalTraversalState): Iterable { - for (let idx = 0; idx < state.segments.length; idx += 1) { - const segment = state.segments[idx] ?? ""; - const isLast = idx === state.segments.length - 1; - state.lexicalCursor = path.join(state.lexicalCursor, segment); - yield { idx, segment, isLast }; - } -} - -async function resolveBoundaryPathLexicalAsync(params: { - params: ResolveBoundaryPathParams; - absolutePath: string; - rootPath: string; - rootCanonicalPath: string; -}): Promise { - const state = createLexicalTraversalState(params); - const sharedStepParams = { - state, - rootCanonicalPath: params.rootCanonicalPath, - resolveParams: params.params, - absolutePath: params.absolutePath, - }; - - for (const { idx, segment, isLast } of iterateLexicalTraversal(state)) { - const stat = await readLexicalStat({ - ...sharedStepParams, - missingFromIndex: idx, - read: (cursor) => fsp.lstat(cursor), - }); - if (!stat) { - break; - } - - const disposition = handleLexicalStatDisposition({ - ...sharedStepParams, - isSymbolicLink: stat.isSymbolicLink(), - segment, - isLast, - }); - if (disposition === "continue") { - continue; - } - if (disposition === "break") { - break; - } - - await resolveAndApplySymlinkHop({ - state, - rootCanonicalPath: params.rootCanonicalPath, - boundaryLabel: params.params.boundaryLabel, - resolveLinkCanonical: (cursor) => resolveSymlinkHopPath(cursor), - }); - } - - const kind = await getPathKind(params.absolutePath, state.preserveFinalSymlink); - return finalizeLexicalResolution({ - ...params, - state, - kind, - }); -} - -function resolveBoundaryPathLexicalSync(params: { - params: ResolveBoundaryPathParams; - absolutePath: string; - rootPath: string; - rootCanonicalPath: string; -}): ResolvedBoundaryPath { - const state = createLexicalTraversalState(params); - for (let idx = 0; idx < state.segments.length; idx += 1) { - const segment = state.segments[idx] ?? ""; - const isLast = idx === state.segments.length - 1; - state.lexicalCursor = path.join(state.lexicalCursor, segment); - const maybeStat = readLexicalStat({ - state, - missingFromIndex: idx, - rootCanonicalPath: params.rootCanonicalPath, - resolveParams: params.params, - absolutePath: params.absolutePath, - read: (cursor) => fs.lstatSync(cursor), - }); - if (isPromiseLike(maybeStat)) { - throw new Error("Unexpected async lexical stat"); - } - const stat = maybeStat; - if (!stat) { - break; - } - - const disposition = handleLexicalStatDisposition({ - state, - isSymbolicLink: stat.isSymbolicLink(), - segment, - isLast, - rootCanonicalPath: params.rootCanonicalPath, - resolveParams: params.params, - absolutePath: params.absolutePath, - }); - if (disposition === "continue") { - continue; - } - if (disposition === "break") { - break; - } - - const maybeApplied = resolveAndApplySymlinkHop({ - state, - rootCanonicalPath: params.rootCanonicalPath, - boundaryLabel: params.params.boundaryLabel, - resolveLinkCanonical: (cursor) => resolveSymlinkHopPathSync(cursor), - }); - if (isPromiseLike(maybeApplied)) { - throw new Error("Unexpected async symlink resolution"); - } - } - - const kind = getPathKindSync(params.absolutePath, state.preserveFinalSymlink); - return finalizeLexicalResolution({ - ...params, - state, - kind, - }); -} - -function resolveCanonicalOutsideLexicalPath(params: { - absolutePath: string; - outsideLexicalCanonicalPath?: string; -}): string { - return params.outsideLexicalCanonicalPath ?? params.absolutePath; -} - -function createBoundaryResolutionContext(params: { - resolveParams: ResolveBoundaryPathParams; - rootPath: string; - absolutePath: string; - rootCanonicalPath: string; - outsideLexicalCanonicalPath?: string; -}): BoundaryResolutionContext { - const lexicalInside = isPathInside(params.rootPath, params.absolutePath); - const canonicalOutsideLexicalPath = resolveCanonicalOutsideLexicalPath({ - absolutePath: params.absolutePath, - outsideLexicalCanonicalPath: params.outsideLexicalCanonicalPath, - }); - assertLexicalBoundaryOrCanonicalAlias({ - skipLexicalRootCheck: params.resolveParams.skipLexicalRootCheck, - lexicalInside, - canonicalOutsideLexicalPath, - rootCanonicalPath: params.rootCanonicalPath, - boundaryLabel: params.resolveParams.boundaryLabel, - rootPath: params.rootPath, - absolutePath: params.absolutePath, - }); - return { - rootPath: params.rootPath, - absolutePath: params.absolutePath, - rootCanonicalPath: params.rootCanonicalPath, - lexicalInside, - canonicalOutsideLexicalPath, - }; -} - -async function resolveOutsideBoundaryPathAsync(params: { - boundaryLabel: string; - context: BoundaryResolutionContext; -}): Promise { - if (params.context.lexicalInside) { - return null; - } - const kind = await getPathKind(params.context.absolutePath, false); - return buildOutsideBoundaryPathFromContext({ - boundaryLabel: params.boundaryLabel, - context: params.context, - kind, - }); -} - -function resolveOutsideBoundaryPathSync(params: { - boundaryLabel: string; - context: BoundaryResolutionContext; -}): ResolvedBoundaryPath | null { - if (params.context.lexicalInside) { - return null; - } - const kind = getPathKindSync(params.context.absolutePath, false); - return buildOutsideBoundaryPathFromContext({ - boundaryLabel: params.boundaryLabel, - context: params.context, - kind, - }); -} - -function buildOutsideBoundaryPathFromContext(params: { - boundaryLabel: string; - context: BoundaryResolutionContext; - kind: { exists: boolean; kind: ResolvedBoundaryPathKind }; -}): ResolvedBoundaryPath { - return buildOutsideLexicalBoundaryPath({ - boundaryLabel: params.boundaryLabel, - rootCanonicalPath: params.context.rootCanonicalPath, - absolutePath: params.context.absolutePath, - canonicalOutsideLexicalPath: params.context.canonicalOutsideLexicalPath, - rootPath: params.context.rootPath, - kind: params.kind, - }); -} - -async function resolveOutsideLexicalCanonicalPathAsync(params: { - rootPath: string; - absolutePath: string; -}): Promise { - if (isPathInside(params.rootPath, params.absolutePath)) { - return undefined; - } - return await resolvePathViaExistingAncestor(params.absolutePath); -} - -function resolveOutsideLexicalCanonicalPathSync(params: { - rootPath: string; - absolutePath: string; -}): string | undefined { - if (isPathInside(params.rootPath, params.absolutePath)) { - return undefined; - } - return resolvePathViaExistingAncestorSync(params.absolutePath); -} - -function buildOutsideLexicalBoundaryPath(params: { - boundaryLabel: string; - rootCanonicalPath: string; - absolutePath: string; - canonicalOutsideLexicalPath: string; - rootPath: string; - kind: { exists: boolean; kind: ResolvedBoundaryPathKind }; -}): ResolvedBoundaryPath { - assertInsideBoundary({ - boundaryLabel: params.boundaryLabel, - rootCanonicalPath: params.rootCanonicalPath, - candidatePath: params.canonicalOutsideLexicalPath, - absolutePath: params.absolutePath, - }); - return buildResolvedBoundaryPath({ - absolutePath: params.absolutePath, - canonicalPath: params.canonicalOutsideLexicalPath, - rootPath: params.rootPath, - rootCanonicalPath: params.rootCanonicalPath, - kind: params.kind, - }); -} - -function assertLexicalBoundaryOrCanonicalAlias(params: { - skipLexicalRootCheck?: boolean; - lexicalInside: boolean; - canonicalOutsideLexicalPath: string; - rootCanonicalPath: string; - boundaryLabel: string; - rootPath: string; - absolutePath: string; -}): void { - if (params.skipLexicalRootCheck || params.lexicalInside) { - return; - } - if (isPathInside(params.rootCanonicalPath, params.canonicalOutsideLexicalPath)) { - return; - } - throw pathEscapeError({ - boundaryLabel: params.boundaryLabel, - rootPath: params.rootPath, - absolutePath: params.absolutePath, - }); -} - -function buildResolvedBoundaryPath(params: { - absolutePath: string; - canonicalPath: string; - rootPath: string; - rootCanonicalPath: string; - kind: { exists: boolean; kind: ResolvedBoundaryPathKind }; -}): ResolvedBoundaryPath { - return { - absolutePath: params.absolutePath, - canonicalPath: params.canonicalPath, - rootPath: params.rootPath, - rootCanonicalPath: params.rootCanonicalPath, - relativePath: relativeInsideRoot(params.rootCanonicalPath, params.canonicalPath), - exists: params.kind.exists, - kind: params.kind.kind, - }; -} - -async function resolvePathViaExistingAncestor(targetPath: string): Promise { - const normalized = path.resolve(targetPath); - let cursor = normalized; - const missingSuffix: string[] = []; - - while (!isFilesystemRoot(cursor) && !(await pathExists(cursor))) { - missingSuffix.unshift(path.basename(cursor)); - const parent = path.dirname(cursor); - if (parent === cursor) { - break; - } - cursor = parent; - } - - if (!(await pathExists(cursor))) { - return normalized; - } - - try { - const resolvedAncestor = path.resolve(await fsp.realpath(cursor)); - if (missingSuffix.length === 0) { - return resolvedAncestor; - } - return path.resolve(resolvedAncestor, ...missingSuffix); - } catch { - return normalized; - } -} - -export function resolvePathViaExistingAncestorSync(targetPath: string): string { - const normalized = path.resolve(targetPath); - let cursor = normalized; - const missingSuffix: string[] = []; - - while (!isFilesystemRoot(cursor) && !fs.existsSync(cursor)) { - missingSuffix.unshift(path.basename(cursor)); - const parent = path.dirname(cursor); - if (parent === cursor) { - break; - } - cursor = parent; - } - - if (!fs.existsSync(cursor)) { - return normalized; - } - - try { - // Keep sync behavior aligned with async (`fsp.realpath`) to avoid - // platform-specific canonical alias drift (notably on Windows). - const resolvedAncestor = path.resolve(fs.realpathSync(cursor)); - if (missingSuffix.length === 0) { - return resolvedAncestor; - } - return path.resolve(resolvedAncestor, ...missingSuffix); - } catch { - return normalized; - } -} - -async function getPathKind( - absolutePath: string, - preserveFinalSymlink: boolean, -): Promise<{ exists: boolean; kind: ResolvedBoundaryPathKind }> { - try { - const stat = preserveFinalSymlink - ? await fsp.lstat(absolutePath) - : await fsp.stat(absolutePath); - return { exists: true, kind: toResolvedKind(stat) }; - } catch (error) { - if (isNotFoundPathError(error)) { - return { exists: false, kind: "missing" }; - } - throw error; - } -} - -function getPathKindSync( - absolutePath: string, - preserveFinalSymlink: boolean, -): { exists: boolean; kind: ResolvedBoundaryPathKind } { - try { - const stat = preserveFinalSymlink ? fs.lstatSync(absolutePath) : fs.statSync(absolutePath); - return { exists: true, kind: toResolvedKind(stat) }; - } catch (error) { - if (isNotFoundPathError(error)) { - return { exists: false, kind: "missing" }; - } - throw error; - } -} - -function toResolvedKind(stat: fs.Stats): ResolvedBoundaryPathKind { - if (stat.isFile()) { - return "file"; - } - if (stat.isDirectory()) { - return "directory"; - } - if (stat.isSymbolicLink()) { - return "symlink"; - } - return "other"; -} - -function relativeInsideRoot(rootPath: string, targetPath: string): string { - const relative = path.relative(path.resolve(rootPath), path.resolve(targetPath)); - if (!relative || relative === ".") { - return ""; - } - if (relative.startsWith("..") || path.isAbsolute(relative)) { - return ""; - } - return relative; -} - -function assertInsideBoundary(params: { - boundaryLabel: string; - rootCanonicalPath: string; - candidatePath: string; - absolutePath: string; -}): void { - if (isPathInside(params.rootCanonicalPath, params.candidatePath)) { - return; - } - throw new Error( - `Path resolves outside ${params.boundaryLabel} (${shortPath(params.rootCanonicalPath)}): ${shortPath(params.absolutePath)}`, - ); -} - -function pathEscapeError(params: { - boundaryLabel: string; - rootPath: string; - absolutePath: string; -}): Error { - return new Error( - `Path escapes ${params.boundaryLabel} (${shortPath(params.rootPath)}): ${shortPath(params.absolutePath)}`, - ); -} - -function symlinkEscapeError(params: { - boundaryLabel: string; - rootCanonicalPath: string; - symlinkPath: string; -}): Error { - return new Error( - `Symlink escapes ${params.boundaryLabel} (${shortPath(params.rootCanonicalPath)}): ${shortPath(params.symlinkPath)}`, - ); -} - -function shortPath(value: string): string { - const home = os.homedir(); - if (value.startsWith(home)) { - return `~${value.slice(home.length)}`; - } - return value; -} - -function isFilesystemRoot(candidate: string): boolean { - return path.parse(candidate).root === candidate; -} - -async function pathExists(targetPath: string): Promise { - try { - await fsp.lstat(targetPath); - return true; - } catch (error) { - if (isNotFoundPathError(error)) { - return false; - } - throw error; - } -} - -async function resolveSymlinkHopPath(symlinkPath: string): Promise { - try { - return path.resolve(await fsp.realpath(symlinkPath)); - } catch (error) { - if (!isNotFoundPathError(error)) { - throw error; - } - const linkTarget = await fsp.readlink(symlinkPath); - const linkAbsolute = path.resolve(path.dirname(symlinkPath), linkTarget); - return resolvePathViaExistingAncestor(linkAbsolute); - } -} - -function resolveSymlinkHopPathSync(symlinkPath: string): string { - try { - return path.resolve(fs.realpathSync(symlinkPath)); - } catch (error) { - if (!isNotFoundPathError(error)) { - throw error; - } - const linkTarget = fs.readlinkSync(symlinkPath); - const linkAbsolute = path.resolve(path.dirname(symlinkPath), linkTarget); - return resolvePathViaExistingAncestorSync(linkAbsolute); - } -} +import "./fs-safe-defaults.js"; +export { + ROOT_PATH_ALIAS_POLICIES, + resolvePathViaExistingAncestorSync, + resolveRootPath, + resolveRootPathSync, + type ResolvedRootPath, + type RootPathAliasPolicy, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/device-auth-store.ts b/src/infra/device-auth-store.ts index e9b6dc6e1e6..3aca1e776c6 100644 --- a/src/infra/device-auth-store.ts +++ b/src/infra/device-auth-store.ts @@ -10,6 +10,7 @@ import { } from "../shared/device-auth-store.js"; import type { DeviceAuthStore } from "../shared/device-auth.js"; import { safeParseJsonWithSchema } from "../utils/zod-parse.js"; +import { privateFileStoreSync } from "./private-file-store.js"; const DEVICE_AUTH_FILE = "device-auth.json"; const DeviceAuthStoreSchema = z.object({ @@ -35,13 +36,9 @@ function readStore(filePath: string): DeviceAuthStore | null { } function writeStore(filePath: string, store: DeviceAuthStore): void { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, `${JSON.stringify(store, null, 2)}\n`, { mode: 0o600 }); - try { - fs.chmodSync(filePath, 0o600); - } catch { - // best-effort - } + privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), store, { + trailingNewline: true, + }); } export function loadDeviceAuthToken(params: { diff --git a/src/infra/device-bootstrap.ts b/src/infra/device-bootstrap.ts index df589280cb7..54cf81ed7b1 100644 --- a/src/infra/device-bootstrap.ts +++ b/src/infra/device-bootstrap.ts @@ -11,12 +11,7 @@ import { import { roleScopesAllow } from "../shared/operator-scope-compat.js"; import { normalizeDevicePublicKeyBase64Url } from "./device-identity.js"; import { resolvePairingPaths } from "./pairing-files.js"; -import { - createAsyncLock, - pruneExpiredPending, - readJsonFile, - writeJsonAtomic, -} from "./pairing-files.js"; +import { createAsyncLock, pruneExpiredPending, tryReadJson, writeJson } from "./pairing-files.js"; import { generatePairingToken, verifyPairingToken } from "./pairing-token.js"; export const DEVICE_BOOTSTRAP_TOKEN_TTL_MS = 10 * 60 * 1000; @@ -164,7 +159,7 @@ function normalizeBootstrapPublicKey(publicKey: string): string { async function loadState(baseDir?: string): Promise { const bootstrapPath = resolveBootstrapPath(baseDir); - const rawState = (await readJsonFile(bootstrapPath)) ?? {}; + const rawState = (await tryReadJson(bootstrapPath)) ?? {}; const state: DeviceBootstrapStateFile = {}; if (!rawState || typeof rawState !== "object" || Array.isArray(rawState)) { return state; @@ -195,7 +190,7 @@ async function loadState(baseDir?: string): Promise { async function persistState(state: DeviceBootstrapStateFile, baseDir?: string): Promise { const bootstrapPath = resolveBootstrapPath(baseDir); - await writeJsonAtomic(bootstrapPath, state); + await writeJson(bootstrapPath, state); } export async function issueDeviceBootstrapToken( diff --git a/src/infra/device-identity.ts b/src/infra/device-identity.ts index 53463f5dad5..62aeb4c45c4 100644 --- a/src/infra/device-identity.ts +++ b/src/infra/device-identity.ts @@ -2,6 +2,7 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; +import { privateFileStoreSync } from "./private-file-store.js"; export type DeviceIdentity = { deviceId: string; @@ -21,10 +22,6 @@ function resolveDefaultIdentityPath(): string { return path.join(resolveStateDir(), "identity", "device.json"); } -function ensureDir(filePath: string) { - fs.mkdirSync(path.dirname(filePath), { recursive: true }); -} - const ED25519_SPKI_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); function base64UrlEncode(buf: Buffer): string { @@ -81,12 +78,9 @@ export function loadOrCreateDeviceIdentity( ...parsed, deviceId: derivedId, }; - fs.writeFileSync(filePath, `${JSON.stringify(updated, null, 2)}\n`, { mode: 0o600 }); - try { - fs.chmodSync(filePath, 0o600); - } catch { - // best-effort - } + privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), updated, { + trailingNewline: true, + }); return { deviceId: derivedId, publicKeyPem: parsed.publicKeyPem, @@ -105,7 +99,6 @@ export function loadOrCreateDeviceIdentity( } const identity = generateIdentity(); - ensureDir(filePath); const stored: StoredIdentity = { version: 1, deviceId: identity.deviceId, @@ -113,12 +106,9 @@ export function loadOrCreateDeviceIdentity( privateKeyPem: identity.privateKeyPem, createdAtMs: Date.now(), }; - fs.writeFileSync(filePath, `${JSON.stringify(stored, null, 2)}\n`, { mode: 0o600 }); - try { - fs.chmodSync(filePath, 0o600); - } catch { - // best-effort - } + privateFileStoreSync(path.dirname(filePath)).writeJson(path.basename(filePath), stored, { + trailingNewline: true, + }); return identity; } diff --git a/src/infra/device-pairing.ts b/src/infra/device-pairing.ts index 26d53909f48..d56896b51ad 100644 --- a/src/infra/device-pairing.ts +++ b/src/infra/device-pairing.ts @@ -13,11 +13,11 @@ import { import { createAsyncLock, pruneExpiredPending, - readDurableJsonFile, + readJsonIfExists, reconcilePendingPairingRequests, coercePairingStateRecord, resolvePairingPaths, - writeJsonAtomic, + writeJson, } from "./pairing-files.js"; import { rejectPendingPairingRequest } from "./pairing-pending.js"; import { generatePairingToken, verifyPairingToken } from "./pairing-token.js"; @@ -154,8 +154,8 @@ export function formatDevicePairingForbiddenMessage(result: DevicePairingForbidd async function loadState(baseDir?: string): Promise { const { pendingPath, pairedPath } = resolvePairingPaths(baseDir, "devices"); const [pending, paired] = await Promise.all([ - readDurableJsonFile(pendingPath), - readDurableJsonFile(pairedPath), + readJsonIfExists(pendingPath), + readJsonIfExists(pairedPath), ]); const state: DevicePairingStateFile = { pendingById: coercePairingStateRecord(pending), @@ -174,16 +174,16 @@ async function persistState( ) { const { pendingPath, pairedPath } = resolvePairingPaths(baseDir, "devices"); if (target === "pending") { - await writeJsonAtomic(pendingPath, state.pendingById); + await writeJson(pendingPath, state.pendingById); return; } if (target === "paired") { - await writeJsonAtomic(pairedPath, state.pairedByDeviceId); + await writeJson(pairedPath, state.pairedByDeviceId); return; } await Promise.all([ - writeJsonAtomic(pendingPath, state.pendingById), - writeJsonAtomic(pairedPath, state.pairedByDeviceId), + writeJson(pendingPath, state.pendingById), + writeJson(pairedPath, state.pairedByDeviceId), ]); } diff --git a/src/infra/diagnostics-timeline.ts b/src/infra/diagnostics-timeline.ts index 5b2a179e1a0..b181ff23604 100644 --- a/src/infra/diagnostics-timeline.ts +++ b/src/infra/diagnostics-timeline.ts @@ -1,10 +1,11 @@ import { randomUUID } from "node:crypto"; -import { appendFileSync, mkdirSync } from "node:fs"; +import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { performance } from "node:perf_hooks"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isDiagnosticFlagEnabled } from "./diagnostic-flags.js"; import { isTruthyEnvValue } from "./env.js"; +import { appendRegularFileSync } from "./regular-file.js"; const OPENCLAW_DIAGNOSTICS_TIMELINE_SCHEMA_VERSION = "openclaw.diagnostics.v1"; @@ -167,7 +168,7 @@ export function emitDiagnosticsTimelineEvent( mkdirSync(dir, { recursive: true }); createdTimelineDirs.add(dir); } - appendFileSync(path, line, "utf8"); + appendRegularFileSync({ filePath: path, content: line }); } catch (error) { if (!warnedAboutTimelineWrite) { warnedAboutTimelineWrite = true; diff --git a/src/infra/exec-approvals.ts b/src/infra/exec-approvals.ts index 04c0f207738..6e732ab9ead 100644 --- a/src/infra/exec-approvals.ts +++ b/src/infra/exec-approvals.ts @@ -12,8 +12,10 @@ import type { CommandExplanationSummary } from "./command-analysis/explain.js"; import { resolveAllowAlwaysPatternEntries } from "./exec-approvals-allowlist.js"; import type { ExecCommandSegment } from "./exec-approvals-analysis.js"; import type { ExecAllowlistEntry } from "./exec-approvals.types.js"; +import { assertNoSymlinkParentsSync } from "./fs-safe-advanced.js"; import { expandHomePrefix, resolveRequiredHomeDir } from "./home-dir.js"; import { requestJsonlSocket } from "./jsonl-socket.js"; +import { privateFileStoreSync } from "./private-file-store.js"; export * from "./exec-approvals-analysis.js"; export * from "./exec-approvals-allowlist.js"; export type { ExecAllowlistEntry } from "./exec-approvals.types.js"; @@ -258,7 +260,7 @@ function mergeLegacyAgent( function ensureDir(filePath: string) { const dir = path.dirname(filePath); - assertNoSymlinkPathComponents(dir, resolveRequiredHomeDir()); + assertNoExecApprovalsSymlinkParents(dir, resolveRequiredHomeDir()); fs.mkdirSync(dir, { recursive: true }); const dirStat = fs.lstatSync(dir); if (!dirStat.isDirectory() || dirStat.isSymbolicLink()) { @@ -267,29 +269,13 @@ function ensureDir(filePath: string) { return dir; } -function assertNoSymlinkPathComponents(targetPath: string, trustedRoot: string): void { - const resolvedTarget = path.resolve(targetPath); - const resolvedRoot = path.resolve(trustedRoot); - if (resolvedTarget !== resolvedRoot && !resolvedTarget.startsWith(`${resolvedRoot}${path.sep}`)) { - return; - } - - const relative = path.relative(resolvedRoot, resolvedTarget); - const segments = relative && relative !== "." ? relative.split(path.sep) : []; - let current = resolvedRoot; - for (const segment of segments) { - current = path.join(current, segment); - try { - const stat = fs.lstatSync(current); - if (stat.isSymbolicLink()) { - throw new Error(`Refusing to traverse symlink in exec approvals path: ${current}`); - } - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - } - } +function assertNoExecApprovalsSymlinkParents(targetPath: string, trustedRoot: string): void { + assertNoSymlinkParentsSync({ + rootDir: trustedRoot, + targetPath, + allowOutsideRoot: true, + messagePrefix: "Refusing to traverse symlink in exec approvals path", + }); } function assertSafeExecApprovalsDestination(filePath: string): void { @@ -514,22 +500,7 @@ export function saveExecApprovals(file: ExecApprovalsFile) { function writeExecApprovalsRaw(filePath: string, raw: string) { const dir = ensureDir(filePath); assertSafeExecApprovalsDestination(filePath); - const tempPath = path.join(dir, `.exec-approvals.${process.pid}.${crypto.randomUUID()}.tmp`); - let tempWritten = false; - try { - fs.writeFileSync(tempPath, raw, { mode: 0o600, flag: "wx" }); - tempWritten = true; - fs.renameSync(tempPath, filePath); - } finally { - if (tempWritten && fs.existsSync(tempPath)) { - fs.rmSync(tempPath, { force: true }); - } - } - try { - fs.chmodSync(filePath, 0o600); - } catch { - // best-effort on platforms without chmod - } + privateFileStoreSync(dir).writeText(path.basename(filePath), raw); } export function restoreExecApprovalsSnapshot(snapshot: ExecApprovalsSnapshot): void { diff --git a/src/infra/file-identity.test.ts b/src/infra/file-identity.test.ts index 2a28255a1ac..1f013933d3a 100644 --- a/src/infra/file-identity.test.ts +++ b/src/infra/file-identity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; +import { sameFileIdentity, type FileIdentityStat } from "./fs-safe-advanced.js"; function stat(dev: number | bigint, ino: number | bigint): FileIdentityStat { return { dev, ino }; diff --git a/src/infra/file-identity.ts b/src/infra/file-identity.ts deleted file mode 100644 index 686d6dd086e..00000000000 --- a/src/infra/file-identity.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type FileIdentityStat = { - dev: number | bigint; - ino: number | bigint; -}; - -function isZero(value: number | bigint): boolean { - return value === 0 || value === 0n; -} - -export function sameFileIdentity( - left: FileIdentityStat, - right: FileIdentityStat, - platform: NodeJS.Platform = process.platform, -): boolean { - if (left.ino !== right.ino) { - return false; - } - - // On Windows, path-based stat calls can report dev=0 while fd-based stat - // reports a real volume serial; treat either-side dev=0 as "unknown device". - if (left.dev === right.dev) { - return true; - } - return platform === "win32" && (isZero(left.dev) || isZero(right.dev)); -} diff --git a/src/infra/file-lock-manager.ts b/src/infra/file-lock-manager.ts new file mode 100644 index 00000000000..ff5e50bb77e --- /dev/null +++ b/src/infra/file-lock-manager.ts @@ -0,0 +1,7 @@ +import "./fs-safe-defaults.js"; + +export { + createFileLockManager, + type FileLockHeldEntry, + type FileLockManager, +} from "@openclaw/fs-safe/file-lock"; diff --git a/src/infra/file-store.ts b/src/infra/file-store.ts new file mode 100644 index 00000000000..315ab14d407 --- /dev/null +++ b/src/infra/file-store.ts @@ -0,0 +1,8 @@ +import "./fs-safe-defaults.js"; +export { + fileStore, + type FileStore, + type FileStoreOptions, + type FileStorePruneOptions, + type FileStoreWriteOptions, +} from "@openclaw/fs-safe/store"; diff --git a/src/infra/fs-pinned-path-helper.ts b/src/infra/fs-pinned-path-helper.ts deleted file mode 100644 index 43e9131fba1..00000000000 --- a/src/infra/fs-pinned-path-helper.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { spawn } from "node:child_process"; -import fsSync from "node:fs"; - -const LOCAL_PINNED_PATH_PYTHON = [ - "import errno", - "import os", - "import stat", - "import sys", - "", - "operation = sys.argv[1]", - "root_path = sys.argv[2]", - "relative_path = sys.argv[3]", - "", - "DIR_FLAGS = os.O_RDONLY", - "if hasattr(os, 'O_DIRECTORY'):", - " DIR_FLAGS |= os.O_DIRECTORY", - "if hasattr(os, 'O_NOFOLLOW'):", - " DIR_FLAGS |= os.O_NOFOLLOW", - "", - "def open_dir(path_value, dir_fd=None):", - " return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd)", - "", - "def split_segments(relative_path):", - " return [part for part in relative_path.split('/') if part and part != '.']", - "", - "def validate_segment(segment):", - " if segment == '..':", - " raise OSError(errno.EPERM, 'path traversal is not allowed', segment)", - "", - "def walk_existing_path(root_fd, segments):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in segments:", - " validate_segment(segment)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " return current_fd", - " except Exception:", - " os.close(current_fd)", - " raise", - "", - "def mkdirp_within_root(root_fd, segments):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in segments:", - " validate_segment(segment)", - " try:", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " except FileNotFoundError:", - " os.mkdir(segment, 0o777, dir_fd=current_fd)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " finally:", - " os.close(current_fd)", - "", - "def remove_within_root(root_fd, segments):", - " if not segments:", - " raise OSError(errno.EPERM, 'refusing to remove root path')", - " parent_segments = segments[:-1]", - " basename = segments[-1]", - " validate_segment(basename)", - " parent_fd = walk_existing_path(root_fd, parent_segments)", - " try:", - " target_stat = os.lstat(basename, dir_fd=parent_fd)", - " if stat.S_ISDIR(target_stat.st_mode) and not stat.S_ISLNK(target_stat.st_mode):", - " os.rmdir(basename, dir_fd=parent_fd)", - " else:", - " os.unlink(basename, dir_fd=parent_fd)", - " finally:", - " os.close(parent_fd)", - "", - "root_fd = open_dir(root_path)", - "try:", - " segments = split_segments(relative_path)", - " if operation == 'mkdirp':", - " mkdirp_within_root(root_fd, segments)", - " elif operation == 'remove':", - " remove_within_root(root_fd, segments)", - " else:", - " raise RuntimeError(f'unknown pinned path operation: {operation}')", - "finally:", - " os.close(root_fd)", -].join("\n"); - -const PINNED_PATH_PYTHON_CANDIDATES = [ - process.env.OPENCLAW_PINNED_PYTHON, - // Keep the write-specific alias for backwards compatibility. - process.env.OPENCLAW_PINNED_WRITE_PYTHON, - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -].filter((value): value is string => Boolean(value)); - -let cachedPinnedPathPython = ""; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; - } -} - -function resolvePinnedPathPython(): string { - if (cachedPinnedPathPython) { - return cachedPinnedPathPython; - } - for (const candidate of PINNED_PATH_PYTHON_CANDIDATES) { - if (canExecute(candidate)) { - cachedPinnedPathPython = candidate; - return cachedPinnedPathPython; - } - } - cachedPinnedPathPython = "python3"; - return cachedPinnedPathPython; -} - -function buildPinnedPathError(stderr: string, code: number | null, signal: NodeJS.Signals | null) { - return new Error( - stderr.trim() || `Pinned path helper failed with code ${code ?? "null"} (${signal ?? "?"})`, - ); -} - -export function isPinnedPathHelperSpawnError(error: unknown): boolean { - if (!(error instanceof Error)) { - return false; - } - - const maybeErrno = error as NodeJS.ErrnoException; - if (typeof maybeErrno.syscall !== "string" || !maybeErrno.syscall.startsWith("spawn")) { - return false; - } - - return ["EACCES", "ENOENT", "ENOEXEC"].includes(maybeErrno.code ?? ""); -} - -export async function runPinnedPathHelper(params: { - operation: "mkdirp" | "remove"; - rootPath: string; - relativePath: string; -}): Promise { - const child = spawn( - resolvePinnedPathPython(), - ["-c", LOCAL_PINNED_PATH_PYTHON, params.operation, params.rootPath, params.relativePath], - { - stdio: ["ignore", "ignore", "pipe"], - }, - ); - - let stderr = ""; - child.stderr.setEncoding?.("utf8"); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - const [code, signal] = await new Promise<[number | null, NodeJS.Signals | null]>( - (resolve, reject) => { - child.once("error", reject); - child.once("close", (exitCode, exitSignal) => resolve([exitCode, exitSignal])); - }, - ); - if (code !== 0) { - throw buildPinnedPathError(stderr, code, signal); - } -} diff --git a/src/infra/fs-pinned-write-helper.test.ts b/src/infra/fs-pinned-write-helper.test.ts deleted file mode 100644 index c7f0fcd6069..00000000000 --- a/src/infra/fs-pinned-write-helper.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; -import { runPinnedWriteHelper } from "./fs-pinned-write-helper.js"; - -const tempDirs = createTrackedTempDirs(); - -afterEach(async () => { - await tempDirs.cleanup(); -}); - -describe("fs pinned write helper", () => { - it.runIf(process.platform !== "win32")("writes through a pinned parent directory", async () => { - const root = await tempDirs.make("openclaw-fs-pinned-root-"); - - const identity = await runPinnedWriteHelper({ - rootPath: root, - relativeParentPath: "nested/deeper", - basename: "note.txt", - mkdir: true, - mode: 0o600, - input: { - kind: "buffer", - data: "hello", - }, - }); - - await expect( - fs.readFile(path.join(root, "nested", "deeper", "note.txt"), "utf8"), - ).resolves.toBe("hello"); - expect(identity.dev).toBeGreaterThanOrEqual(0); - expect(identity.ino).toBeGreaterThan(0); - }); - - it.runIf(process.platform !== "win32")( - "rejects symlink-parent writes instead of creating a temp file outside root", - async () => { - const root = await tempDirs.make("openclaw-fs-pinned-root-"); - const outside = await tempDirs.make("openclaw-fs-pinned-outside-"); - await fs.symlink(outside, path.join(root, "alias")); - - await expect( - runPinnedWriteHelper({ - rootPath: root, - relativeParentPath: "alias", - basename: "escape.txt", - mkdir: false, - mode: 0o600, - input: { - kind: "buffer", - data: "owned", - }, - }), - ).rejects.toThrow(); - - await expect(fs.stat(path.join(outside, "escape.txt"))).rejects.toThrow(); - const outsideFiles = await fs.readdir(outside); - expect(outsideFiles).toEqual([]); - }, - ); - - it.runIf(process.platform !== "win32")("accepts streamed input", async () => { - const root = await tempDirs.make("openclaw-fs-pinned-root-"); - const sourcePath = path.join(await tempDirs.make("openclaw-fs-pinned-src-"), "source.txt"); - await fs.writeFile(sourcePath, "streamed", "utf8"); - const sourceHandle = await fs.open(sourcePath, "r"); - try { - await runPinnedWriteHelper({ - rootPath: root, - relativeParentPath: "", - basename: "stream.txt", - mkdir: true, - mode: 0o600, - input: { - kind: "stream", - stream: sourceHandle.createReadStream(), - }, - }); - } finally { - await sourceHandle.close(); - } - - await expect(fs.readFile(path.join(root, "stream.txt"), "utf8")).resolves.toBe("streamed"); - }); -}); diff --git a/src/infra/fs-pinned-write-helper.ts b/src/infra/fs-pinned-write-helper.ts deleted file mode 100644 index 6659a73e461..00000000000 --- a/src/infra/fs-pinned-write-helper.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { spawn } from "node:child_process"; -import { once } from "node:events"; -import fsSync from "node:fs"; -import fs from "node:fs/promises"; -import path from "node:path"; -import type { Readable } from "node:stream"; -import { pipeline } from "node:stream/promises"; -import type { FileIdentityStat } from "./file-identity.js"; - -type PinnedWriteInput = - | { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding } - | { kind: "stream"; stream: Readable }; - -const LOCAL_PINNED_WRITE_PYTHON = [ - "import errno", - "import os", - "import secrets", - "import stat", - "import sys", - "", - "root_path = sys.argv[1]", - "relative_parent = sys.argv[2]", - "basename = sys.argv[3]", - 'mkdir_enabled = sys.argv[4] == "1"', - "file_mode = int(sys.argv[5], 8)", - "", - "DIR_FLAGS = os.O_RDONLY", - "if hasattr(os, 'O_DIRECTORY'):", - " DIR_FLAGS |= os.O_DIRECTORY", - "if hasattr(os, 'O_NOFOLLOW'):", - " DIR_FLAGS |= os.O_NOFOLLOW", - "", - "WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL", - "if hasattr(os, 'O_NOFOLLOW'):", - " WRITE_FLAGS |= os.O_NOFOLLOW", - "", - "def open_dir(path_value, dir_fd=None):", - " return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd)", - "", - "def walk_parent(root_fd, rel_parent, mkdir_enabled):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in [part for part in rel_parent.split('/') if part and part != '.']:", - " if segment == '..':", - " raise OSError(errno.EPERM, 'path traversal is not allowed', segment)", - " try:", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " except FileNotFoundError:", - " if not mkdir_enabled:", - " raise", - " os.mkdir(segment, 0o777, dir_fd=current_fd)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " return current_fd", - " except Exception:", - " os.close(current_fd)", - " raise", - "", - "def create_temp_file(parent_fd, basename, mode):", - " prefix = '.' + basename + '.'", - " for _ in range(128):", - " candidate = prefix + secrets.token_hex(6) + '.tmp'", - " try:", - " fd = os.open(candidate, WRITE_FLAGS, mode, dir_fd=parent_fd)", - " return candidate, fd", - " except FileExistsError:", - " continue", - " raise RuntimeError('failed to allocate pinned temp file')", - "", - "root_fd = open_dir(root_path)", - "parent_fd = None", - "temp_fd = None", - "temp_name = None", - "try:", - " parent_fd = walk_parent(root_fd, relative_parent, mkdir_enabled)", - " temp_name, temp_fd = create_temp_file(parent_fd, basename, file_mode)", - " while True:", - " chunk = sys.stdin.buffer.read(65536)", - " if not chunk:", - " break", - " os.write(temp_fd, chunk)", - " os.fsync(temp_fd)", - " os.close(temp_fd)", - " temp_fd = None", - " os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)", - " temp_name = None", - " os.fsync(parent_fd)", - " result_stat = os.stat(basename, dir_fd=parent_fd, follow_symlinks=False)", - " print(f'{result_stat.st_dev}|{result_stat.st_ino}')", - "finally:", - " if temp_fd is not None:", - " os.close(temp_fd)", - " if temp_name is not None and parent_fd is not None:", - " try:", - " os.unlink(temp_name, dir_fd=parent_fd)", - " except FileNotFoundError:", - " pass", - " if parent_fd is not None:", - " os.close(parent_fd)", - " os.close(root_fd)", -].join("\n"); - -const PINNED_WRITE_PYTHON_CANDIDATES = [ - process.env.OPENCLAW_PINNED_WRITE_PYTHON, - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -].filter((value): value is string => Boolean(value)); - -let cachedPinnedWritePython = ""; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; - } -} - -function resolvePinnedWritePython(): string { - if (cachedPinnedWritePython) { - return cachedPinnedWritePython; - } - for (const candidate of PINNED_WRITE_PYTHON_CANDIDATES) { - if (canExecute(candidate)) { - cachedPinnedWritePython = candidate; - return cachedPinnedWritePython; - } - } - cachedPinnedWritePython = "python3"; - return cachedPinnedWritePython; -} - -function parsePinnedIdentity(stdout: string): FileIdentityStat { - const line = stdout - .trim() - .split(/\r?\n/) - .map((value) => value.trim()) - .findLast(Boolean); - if (!line) { - throw new Error("Pinned write helper returned no identity"); - } - const [devRaw, inoRaw] = line.split("|"); - const dev = Number.parseInt(devRaw ?? "", 10); - const ino = Number.parseInt(inoRaw ?? "", 10); - if (!Number.isFinite(dev) || !Number.isFinite(ino)) { - throw new Error(`Pinned write helper returned invalid identity: ${line}`); - } - return { dev, ino }; -} - -export async function runPinnedWriteHelper(params: { - rootPath: string; - relativeParentPath: string; - basename: string; - mkdir: boolean; - mode: number; - input: PinnedWriteInput; -}): Promise { - const child = spawn( - resolvePinnedWritePython(), - [ - "-c", - LOCAL_PINNED_WRITE_PYTHON, - params.rootPath, - params.relativeParentPath, - params.basename, - params.mkdir ? "1" : "0", - (params.mode || 0o600).toString(8), - ], - { - stdio: ["pipe", "pipe", "pipe"], - }, - ); - - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding?.("utf8"); - child.stderr.setEncoding?.("utf8"); - child.stdout.on("data", (chunk: string) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - const exitPromise = once(child, "close") as Promise<[number | null, NodeJS.Signals | null]>; - try { - if (!child.stdin) { - const identity = await runPinnedWriteFallback(params); - await exitPromise.catch(() => {}); - return identity; - } - - if (params.input.kind === "buffer") { - const input = params.input; - await new Promise((resolve, reject) => { - child.stdin.once("error", reject); - if (typeof input.data === "string") { - child.stdin.end(input.data, input.encoding ?? "utf8", () => resolve()); - return; - } - child.stdin.end(input.data, () => resolve()); - }); - } else { - await pipeline(params.input.stream, child.stdin); - } - - const [code, signal] = await exitPromise; - if (code !== 0) { - throw new Error( - stderr.trim() || - `Pinned write helper failed with code ${code ?? "null"} (${signal ?? "?"})`, - ); - } - return parsePinnedIdentity(stdout); - } catch (error) { - child.kill("SIGKILL"); - await exitPromise.catch(() => {}); - throw error; - } -} - -async function runPinnedWriteFallback(params: { - rootPath: string; - relativeParentPath: string; - basename: string; - mkdir: boolean; - mode: number; - input: PinnedWriteInput; -}): Promise { - const parentPath = params.relativeParentPath - ? path.join(params.rootPath, ...params.relativeParentPath.split("/")) - : params.rootPath; - if (params.mkdir) { - await fs.mkdir(parentPath, { recursive: true }); - } - const targetPath = path.join(parentPath, params.basename); - const tempPath = path.join(parentPath, `.${params.basename}.fallback.tmp`); - if (params.input.kind === "buffer") { - if (typeof params.input.data === "string") { - await fs.writeFile(tempPath, params.input.data, { - encoding: params.input.encoding ?? "utf8", - mode: params.mode, - }); - } else { - await fs.writeFile(tempPath, params.input.data, { mode: params.mode }); - } - } else { - const handle = await fs.open(tempPath, "w", params.mode); - try { - await pipeline(params.input.stream, handle.createWriteStream()); - } finally { - await handle.close().catch(() => {}); - } - } - await fs.rename(tempPath, targetPath); - const stat = await fs.stat(targetPath); - return { dev: stat.dev, ino: stat.ino }; -} diff --git a/src/infra/fs-safe-advanced.ts b/src/infra/fs-safe-advanced.ts new file mode 100644 index 00000000000..93debee5eba --- /dev/null +++ b/src/infra/fs-safe-advanced.ts @@ -0,0 +1,11 @@ +import "./fs-safe-defaults.js"; +export { + assertNoHardlinkedFinalPath, + assertNoSymlinkParents, + assertNoSymlinkParentsSync, + sameFileIdentity, + sanitizeUntrustedFileName, + writeViaSiblingTempPath, + type AssertNoSymlinkParentsOptions, + type FileIdentityStat, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/fs-safe-defaults.test.ts b/src/infra/fs-safe-defaults.test.ts new file mode 100644 index 00000000000..eb326d94b41 --- /dev/null +++ b/src/infra/fs-safe-defaults.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const { configureFsSafePython } = vi.hoisted(() => ({ + configureFsSafePython: vi.fn(), +})); + +vi.mock("@openclaw/fs-safe/config", () => ({ + configureFsSafePython, +})); + +async function importDefaults() { + vi.resetModules(); + await import("./fs-safe-defaults.js"); +} + +describe("fs-safe defaults", () => { + afterEach(() => { + configureFsSafePython.mockReset(); + delete process.env.FS_SAFE_PYTHON_MODE; + delete process.env.OPENCLAW_FS_SAFE_PYTHON_MODE; + }); + + it("disables the Python helper by default in OpenClaw", async () => { + await importDefaults(); + + expect(configureFsSafePython).toHaveBeenCalledWith({ mode: "off" }); + }); + + it("lets fs-safe env mode overrides opt back into the helper", async () => { + process.env.FS_SAFE_PYTHON_MODE = "require"; + + await importDefaults(); + + expect(configureFsSafePython).not.toHaveBeenCalled(); + }); + + it("honors the OpenClaw-specific env mode override", async () => { + process.env.OPENCLAW_FS_SAFE_PYTHON_MODE = "auto"; + + await importDefaults(); + + expect(configureFsSafePython).not.toHaveBeenCalled(); + }); +}); diff --git a/src/infra/fs-safe-defaults.ts b/src/infra/fs-safe-defaults.ts new file mode 100644 index 00000000000..5cbe997e9f3 --- /dev/null +++ b/src/infra/fs-safe-defaults.ts @@ -0,0 +1,8 @@ +import { configureFsSafePython } from "@openclaw/fs-safe/config"; + +const hasPythonModeOverride = + process.env.FS_SAFE_PYTHON_MODE != null || process.env.OPENCLAW_FS_SAFE_PYTHON_MODE != null; + +if (!hasPythonModeOverride) { + configureFsSafePython({ mode: "off" }); +} diff --git a/src/infra/fs-safe-import-boundary.test.ts b/src/infra/fs-safe-import-boundary.test.ts new file mode 100644 index 00000000000..6775709a779 --- /dev/null +++ b/src/infra/fs-safe-import-boundary.test.ts @@ -0,0 +1,51 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../.."); +const SCAN_ROOTS = ["src", "packages", "extensions"] as const; + +const ALLOWED_PREFIXES = ["src/infra/", "src/plugin-sdk/", "packages/memory-host-sdk/"] as const; + +function isSourceFile(filePath: string): boolean { + return filePath.endsWith(".ts") && !filePath.endsWith(".test.ts") && !filePath.endsWith(".d.ts"); +} + +function walk(dir: string): string[] { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === "node_modules" || entry.name === "dist") { + continue; + } + files.push(...walk(fullPath)); + continue; + } + if (entry.isFile() && isSourceFile(fullPath)) { + files.push(fullPath); + } + } + return files; +} + +function toRepoPath(filePath: string): string { + return path.relative(REPO_ROOT, filePath).replaceAll(path.sep, "/"); +} + +describe("fs-safe import boundary", () => { + it("keeps direct fs-safe imports behind OpenClaw policy wrappers", () => { + const violations = SCAN_ROOTS.flatMap((root) => walk(path.join(REPO_ROOT, root))) + .map(toRepoPath) + .filter((filePath) => { + if (ALLOWED_PREFIXES.some((prefix) => filePath.startsWith(prefix))) { + return false; + } + const source = fs.readFileSync(path.join(REPO_ROOT, filePath), "utf8"); + return source.includes('"@openclaw/fs-safe') || source.includes("'@openclaw/fs-safe"); + }); + + expect(violations).toEqual([]); + }); +}); diff --git a/src/infra/fs-safe-test-hooks.ts b/src/infra/fs-safe-test-hooks.ts new file mode 100644 index 00000000000..8bb89861a43 --- /dev/null +++ b/src/infra/fs-safe-test-hooks.ts @@ -0,0 +1,2 @@ +import "./fs-safe-defaults.js"; +export { __setFsSafeTestHooksForTest, type FsSafeTestHooks } from "@openclaw/fs-safe/test-hooks"; diff --git a/src/infra/fs-safe.test.ts b/src/infra/fs-safe.test.ts index a248729a68a..fa7ed242570 100644 --- a/src/infra/fs-safe.test.ts +++ b/src/infra/fs-safe.test.ts @@ -7,22 +7,12 @@ import { withRealpathSymlinkRebindRace, } from "../test-utils/symlink-rebind-race.js"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; -import * as pinnedPathHelperModule from "./fs-pinned-path-helper.js"; +import { __setFsSafeTestHooksForTest } from "./fs-safe-test-hooks.js"; import { - __setFsSafeTestHooksForTest, - appendFileWithinRoot, - copyFileWithinRoot, - createRootScopedReadFile, - mkdirPathWithinRoot, resolveOpenedFileRealPathForHandle, - SafeOpenError, - openFileWithinRoot, - readFileWithinRoot, - readPathWithinRoot, + FsSafeError, readLocalFileSafely, - removePathWithinRoot, - writeFileWithinRoot, - writeFileFromPathWithinRoot, + root as openRoot, } from "./fs-safe.js"; const tempDirs = createTrackedTempDirs(); @@ -45,7 +35,7 @@ async function expectWriteOpenRaceIsBlocked(params: { timing: "before-realpath", run: async () => { await expect(params.runWrite()).rejects.toMatchObject({ - code: expect.stringMatching(/outside-workspace|invalid-path/), + code: expect.stringMatching(/outside-workspace|path-mismatch|path-alias|invalid-path/), }); }, }); @@ -128,8 +118,8 @@ describe("fs-safe", () => { code: "not-file", }); const err = await readLocalFileSafely({ filePath: dir }).catch((e: unknown) => e); - expect(err).toBeInstanceOf(SafeOpenError); - expect((err as SafeOpenError).message).not.toMatch(/EISDIR/i); + expect(err).toBeInstanceOf(FsSafeError); + expect((err as FsSafeError).message).not.toMatch(/EISDIR/i); }); it("enforces maxBytes", async () => { @@ -187,10 +177,7 @@ describe("fs-safe", () => { await fs.writeFile(file, "outside"); await expect( - openFileWithinRoot({ - rootDir: root, - relativePath: path.join("..", path.basename(outside), "outside.txt"), - }), + (await openRoot(root)).open(path.join("..", path.basename(outside), "outside.txt")), ).rejects.toMatchObject({ code: "outside-workspace" }); }); @@ -198,41 +185,34 @@ describe("fs-safe", () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); await fs.mkdir(path.join(root, "memory"), { recursive: true }); - await expect( - openFileWithinRoot({ rootDir: root, relativePath: "memory" }), - ).rejects.toMatchObject({ code: expect.stringMatching(/invalid-path|not-file/) }); + const rootFs = await openRoot(root); + await expect(rootFs.open("memory")).rejects.toMatchObject({ + code: expect.stringMatching(/invalid-path|not-file/), + }); - const err = await openFileWithinRoot({ - rootDir: root, - relativePath: "memory", - }).catch((e: unknown) => e); - expect(err).toBeInstanceOf(SafeOpenError); - expect((err as SafeOpenError).message).not.toMatch(/EISDIR/i); + const err = await rootFs.open("memory").catch((e: unknown) => e); + expect(err).toBeInstanceOf(FsSafeError); + expect((err as FsSafeError).message).not.toMatch(/EISDIR/i); }); it("reads files within root through all read helpers", async () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); await fs.writeFile(path.join(root, "inside.txt"), "inside"); - const byRelativePath = await readFileWithinRoot({ - rootDir: root, - relativePath: "inside.txt", - }); + const rootFs = await openRoot(root); + const byRelativePath = await rootFs.read("inside.txt"); expect(byRelativePath.buffer.toString("utf8")).toBe("inside"); expect(byRelativePath.realPath).toContain("inside.txt"); expect(byRelativePath.stat.size).toBe(6); const absolutePath = path.join(root, "absolute.txt"); await fs.writeFile(absolutePath, "absolute"); - const byAbsolutePath = await readPathWithinRoot({ - rootDir: root, - filePath: absolutePath, - }); + const byAbsolutePath = await rootFs.readAbsolute(absolutePath); expect(byAbsolutePath.buffer.toString("utf8")).toBe("absolute"); const scopedPath = path.join(root, "scoped.txt"); await fs.writeFile(scopedPath, "scoped"); - const readScoped = createRootScopedReadFile({ rootDir: root }); + const readScoped = rootFs.reader(); await expect(readScoped(scopedPath)).resolves.toEqual(Buffer.from("scoped")); }); @@ -244,12 +224,9 @@ describe("fs-safe", () => { await fs.writeFile(target, "outside"); await fs.symlink(target, link); - await expect( - openFileWithinRoot({ - rootDir: root, - relativePath: "link.txt", - }), - ).rejects.toMatchObject({ code: "invalid-path" }); + await expect((await openRoot(root)).open("link.txt")).rejects.toMatchObject({ + code: "symlink", + }); }); it.runIf(process.platform !== "win32")( @@ -271,12 +248,10 @@ describe("fs-safe", () => { }); await expect( - readFileWithinRoot({ - rootDir: root, - relativePath: "link.txt", - allowSymlinkTargetWithinRoot: true, + (await openRoot(root)).read("link.txt", { + symlinks: "follow-within-root", }), - ).rejects.toMatchObject({ code: "invalid-path" }); + ).rejects.toMatchObject({ code: "path-mismatch" }); }, ); @@ -293,12 +268,7 @@ describe("fs-safe", () => { }, }); - await expect( - openFileWithinRoot({ - rootDir: root, - relativePath: "inside.txt", - }), - ).rejects.toThrow("after-open boom"); + await expect((await openRoot(root)).open("inside.txt")).rejects.toThrow("after-open boom"); expect(openedHandle).toBeDefined(); await expect(openedHandle?.readFile({ encoding: "utf8" })).rejects.toMatchObject({ code: "EBADF", @@ -322,23 +292,16 @@ describe("fs-safe", () => { await withOutsideHardlinkAlias({ aliasPath: hardlinkPath, run: async () => { - await expect( - openFileWithinRoot({ - rootDir: root, - relativePath: "link.txt", - }), - ).rejects.toMatchObject({ code: "invalid-path" }); + await expect((await openRoot(root)).open("link.txt")).rejects.toMatchObject({ + code: "hardlink", + }); }, }); }); it("writes a file within root safely", async () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); - await writeFileWithinRoot({ - rootDir: root, - relativePath: "nested/out.txt", - data: "hello", - }); + await (await openRoot(root)).write("nested/out.txt", "hello"); await expect(fs.readFile(path.join(root, "nested", "out.txt"), "utf8")).resolves.toBe("hello"); }); @@ -348,10 +311,9 @@ describe("fs-safe", () => { await fs.mkdir(path.dirname(targetPath), { recursive: true }); await fs.writeFile(targetPath, "seed"); - await appendFileWithinRoot({ - rootDir: root, - relativePath: "nested/out.txt", - data: "next", + await ( + await openRoot(root) + ).append("nested/out.txt", "next", { prependNewlineIfNeeded: true, }); @@ -364,11 +326,7 @@ describe("fs-safe", () => { const sourcePath = path.join(sourceDir, "in.txt"); await fs.writeFile(sourcePath, "copy-ok"); - await copyFileWithinRoot({ - sourcePath, - rootDir: root, - relativePath: "nested/copied.txt", - }); + await (await openRoot(root)).copyIn("nested/copied.txt", sourcePath); await expect(fs.readFile(path.join(root, "nested", "copied.txt"), "utf8")).resolves.toBe( "copy-ok", @@ -381,10 +339,7 @@ describe("fs-safe", () => { await fs.mkdir(path.dirname(targetPath), { recursive: true }); await fs.writeFile(targetPath, "hello"); - await removePathWithinRoot({ - rootDir: root, - relativePath: "nested/out.txt", - }); + await (await openRoot(root)).remove("nested/out.txt"); await expect(fs.stat(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); }); @@ -392,10 +347,7 @@ describe("fs-safe", () => { it("creates directories within root safely", async () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); - await mkdirPathWithinRoot({ - rootDir: root, - relativePath: "nested/deeper", - }); + await (await openRoot(root)).mkdir("nested/deeper"); const stat = await fs.stat(path.join(root, "nested", "deeper")); expect(stat.isDirectory()).toBe(true); @@ -410,10 +362,7 @@ describe("fs-safe", () => { await fs.mkdir(realDir, { recursive: true }); await fs.symlink(realDir, aliasDir); - await mkdirPathWithinRoot({ - rootDir: root, - relativePath: path.join("alias", "nested", "deeper"), - }); + await (await openRoot(root)).mkdir(path.join("alias", "nested", "deeper")); await expect(fs.stat(path.join(realDir, "nested", "deeper"))).resolves.toMatchObject({ isDirectory: expect.any(Function), @@ -431,10 +380,7 @@ describe("fs-safe", () => { await fs.symlink(realDir, aliasDir); await fs.writeFile(path.join(realDir, "target.txt"), "hello"); - await removePathWithinRoot({ - rootDir: root, - relativePath: path.join("alias", "target.txt"), - }); + await (await openRoot(root)).remove(path.join("alias", "target.txt")); await expect(fs.stat(path.join(realDir, "target.txt"))).rejects.toMatchObject({ code: "ENOENT", @@ -442,49 +388,6 @@ describe("fs-safe", () => { }, ); - it.runIf(process.platform !== "win32")( - "falls back to legacy remove when the pinned helper cannot spawn", - async () => { - const error = new Error("spawn missing python ENOENT") as NodeJS.ErrnoException; - error.code = "ENOENT"; - error.syscall = "spawn python3"; - vi.spyOn(pinnedPathHelperModule, "runPinnedPathHelper").mockRejectedValue(error); - - const root = await tempDirs.make("openclaw-fs-safe-root-"); - const targetPath = path.join(root, "nested", "out.txt"); - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.writeFile(targetPath, "hello"); - - await removePathWithinRoot({ - rootDir: root, - relativePath: "nested/out.txt", - }); - - await expect(fs.stat(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); - }, - ); - - it.runIf(process.platform !== "win32")( - "falls back to legacy mkdir when the pinned helper cannot spawn", - async () => { - const error = new Error("spawn missing python ENOENT") as NodeJS.ErrnoException; - error.code = "ENOENT"; - error.syscall = "spawn python3"; - vi.spyOn(pinnedPathHelperModule, "runPinnedPathHelper").mockRejectedValue(error); - - const root = await tempDirs.make("openclaw-fs-safe-root-"); - - await mkdirPathWithinRoot({ - rootDir: root, - relativePath: "nested/deeper", - }); - - await expect(fs.stat(path.join(root, "nested", "deeper"))).resolves.toMatchObject({ - isDirectory: expect.any(Function), - }); - }, - ); - it("enforces maxBytes when copying into root", async () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); const sourceDir = await tempDirs.make("openclaw-fs-safe-source-"); @@ -492,10 +395,7 @@ describe("fs-safe", () => { await fs.writeFile(sourcePath, Buffer.alloc(8)); await expect( - copyFileWithinRoot({ - sourcePath, - rootDir: root, - relativePath: "nested/big.bin", + (await openRoot(root)).copyIn("nested/big.bin", sourcePath, { maxBytes: 4, }), ).rejects.toMatchObject({ code: "too-large" }); @@ -509,24 +409,16 @@ describe("fs-safe", () => { const outside = await tempDirs.make("openclaw-fs-safe-src-"); const sourcePath = path.join(outside, "source.bin"); await fs.writeFile(sourcePath, "hello-from-source"); - await writeFileFromPathWithinRoot({ - rootDir: root, - relativePath: "nested/from-source.txt", - sourcePath, - }); + await (await openRoot(root)).copyIn("nested/from-source.txt", sourcePath); await expect(fs.readFile(path.join(root, "nested", "from-source.txt"), "utf8")).resolves.toBe( "hello-from-source", ); }); it("rejects write traversal outside root", async () => { const root = await tempDirs.make("openclaw-fs-safe-root-"); - await expect( - writeFileWithinRoot({ - rootDir: root, - relativePath: "../escape.txt", - data: "x", - }), - ).rejects.toMatchObject({ code: "outside-workspace" }); + await expect((await openRoot(root)).write("../escape.txt", "x")).rejects.toMatchObject({ + code: "outside-workspace", + }); }); it.runIf(process.platform !== "win32")("rejects writing through hardlink aliases", async () => { @@ -535,13 +427,9 @@ describe("fs-safe", () => { await withOutsideHardlinkAlias({ aliasPath: hardlinkPath, run: async (outsideFile) => { - await expect( - writeFileWithinRoot({ - rootDir: root, - relativePath: "alias.txt", - data: "pwned", - }), - ).rejects.toMatchObject({ code: "invalid-path" }); + await expect((await openRoot(root)).write("alias.txt", "pwned")).rejects.toMatchObject({ + code: "path-alias", + }); await expect(fs.readFile(outsideFile, "utf8")).resolves.toBe("outside"); }, }); @@ -554,13 +442,10 @@ describe("fs-safe", () => { aliasPath: hardlinkPath, run: async (outsideFile) => { await expect( - appendFileWithinRoot({ - rootDir: root, - relativePath: "alias.txt", - data: "pwned", + (await openRoot(root)).append("alias.txt", "pwned", { prependNewlineIfNeeded: true, }), - ).rejects.toMatchObject({ code: "invalid-path" }); + ).rejects.toMatchObject({ code: "path-alias" }); await expect(fs.readFile(outsideFile, "utf8")).resolves.toBe("outside"); }, }); @@ -575,10 +460,9 @@ describe("fs-safe", () => { slotPath: slot, outsideDir: outside, runWrite: async (relativePath) => - await writeFileWithinRoot({ - rootDir: root, - relativePath, - data: "new-content", + await ( + await openRoot(root) + ).write(relativePath, "new-content", { mkdir: false, }), }); @@ -595,10 +479,9 @@ describe("fs-safe", () => { slotPath: slot, outsideDir: outside, runWrite: async (relativePath) => - await appendFileWithinRoot({ - rootDir: root, - relativePath, - data: "new-content", + await ( + await openRoot(root) + ).append(relativePath, "new-content", { mkdir: false, prependNewlineIfNeeded: true, }), @@ -621,12 +504,9 @@ describe("fs-safe", () => { timing: "before-realpath", run: async () => { await expect( - removePathWithinRoot({ - rootDir: root, - relativePath: path.join("slot", "target.txt"), - }), + (await openRoot(root)).remove(path.join("slot", "target.txt")), ).rejects.toMatchObject({ - code: expect.stringMatching(/invalid-path|not-found/), + code: expect.stringMatching(/path-alias|not-found/), }); }, }); @@ -655,12 +535,9 @@ describe("fs-safe", () => { timing: "before-realpath", run: async () => { await expect( - mkdirPathWithinRoot({ - rootDir: root, - relativePath: path.join("slot", "nested", "deep"), - }), + (await openRoot(root)).mkdir(path.join("slot", "nested", "deep")), ).rejects.toMatchObject({ - code: "invalid-path", + code: "path-alias", }); }, }); @@ -679,10 +556,9 @@ describe("fs-safe", () => { slotPath: slot, outsideDir: outside, runWrite: async (relativePath) => - await writeFileFromPathWithinRoot({ - rootDir: root, - relativePath, - sourcePath, + await ( + await openRoot(root) + ).copyIn(relativePath, sourcePath, { mkdir: false, }), }); @@ -694,7 +570,7 @@ describe("fs-safe", () => { const dir = await tempDirs.make("openclaw-fs-safe-"); const missing = path.join(dir, "missing.txt"); - await expect(readLocalFileSafely({ filePath: missing })).rejects.toBeInstanceOf(SafeOpenError); + await expect(readLocalFileSafely({ filePath: missing })).rejects.toBeInstanceOf(FsSafeError); await expect(readLocalFileSafely({ filePath: missing })).rejects.toMatchObject({ code: "not-found", }); @@ -722,20 +598,14 @@ describe("tilde expansion in file tools", () => { process.env.OPENCLAW_HOME = root; try { await fs.writeFile(path.join(root, "hello.txt"), "tilde-works"); - const result = await openFileWithinRoot({ - rootDir: root, - relativePath: "~/hello.txt", - }); + const rootFs = await openRoot(root); + const result = await rootFs.open("~/hello.txt"); const buf = Buffer.alloc(result.stat.size); await result.handle.read(buf, 0, buf.length, 0); await result.handle.close(); expect(buf.toString("utf8")).toBe("tilde-works"); - await writeFileWithinRoot({ - rootDir: root, - relativePath: "~/output.txt", - data: "tilde-write-works", - }); + await rootFs.write("~/output.txt", "tilde-write-works"); const content = await fs.readFile(path.join(root, "output.txt"), "utf8"); expect(content).toBe("tilde-write-works"); } finally { @@ -744,12 +614,7 @@ describe("tilde expansion in file tools", () => { } const outsideRoot = await tempDirs.make("openclaw-tilde-outside-"); - await expect( - openFileWithinRoot({ - rootDir: outsideRoot, - relativePath: "~/escape.txt", - }), - ).rejects.toMatchObject({ + await expect((await openRoot(outsideRoot)).open("~/escape.txt")).rejects.toMatchObject({ code: expect.stringMatching(/outside-workspace|not-found|invalid-path/), }); }); diff --git a/src/infra/fs-safe.ts b/src/infra/fs-safe.ts index 46254451d87..298def7cdbc 100644 --- a/src/infra/fs-safe.ts +++ b/src/infra/fs-safe.ts @@ -1,271 +1,52 @@ -import { randomUUID } from "node:crypto"; -import type { Stats } from "node:fs"; -import { constants as fsConstants } from "node:fs"; -import type { FileHandle } from "node:fs/promises"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { pipeline } from "node:stream/promises"; -import { logWarn } from "../logger.js"; -import { resolveBoundaryPath } from "./boundary-path.js"; -import { sameFileIdentity } from "./file-identity.js"; -import { isPinnedPathHelperSpawnError, runPinnedPathHelper } from "./fs-pinned-path-helper.js"; -import { runPinnedWriteHelper } from "./fs-pinned-write-helper.js"; -import { expandHomePrefix } from "./home-dir.js"; -import { assertNoPathAliasEscape, PATH_ALIAS_POLICIES } from "./path-alias-guards.js"; -import { - hasNodeErrorCode, - isNotFoundPathError, - isPathInside, - isSymlinkOpenError, -} from "./path-guards.js"; +import "./fs-safe-defaults.js"; +import { root as fsSafeRoot, type ReadResult } from "@openclaw/fs-safe/root"; -export type SafeOpenErrorCode = - | "invalid-path" - | "not-found" - | "outside-workspace" - | "symlink" - | "not-file" - | "path-mismatch" - | "too-large"; - -export class SafeOpenError extends Error { - code: SafeOpenErrorCode; - - constructor(code: SafeOpenErrorCode, message: string, options?: ErrorOptions) { - super(message, options); - this.code = code; - this.name = "SafeOpenError"; - } -} - -export type SafeOpenResult = { - handle: FileHandle; - realPath: string; - stat: Stats; -}; - -export type SafeLocalReadResult = { - buffer: Buffer; - realPath: string; - stat: Stats; -}; - -export type FsSafeTestHooks = { - afterPreOpenLstat?: (filePath: string) => Promise | void; - beforeOpen?: (filePath: string, flags: number) => Promise | void; - afterOpen?: (filePath: string, handle: FileHandle) => Promise | void; -}; - -let fsSafeTestHooks: FsSafeTestHooks | undefined; - -function allowFsSafeTestHooks(): boolean { - return process.env.NODE_ENV === "test" || process.env.VITEST === "true"; -} - -export function __setFsSafeTestHooksForTest(hooks?: FsSafeTestHooks): void { - if (hooks && !allowFsSafeTestHooks()) { - throw new Error("__setFsSafeTestHooksForTest is only available in tests"); - } - fsSafeTestHooks = hooks; -} - -const SUPPORTS_NOFOLLOW = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants; -const NONBLOCK_OPEN_FLAG = "O_NONBLOCK" in fsConstants ? fsConstants.O_NONBLOCK : 0; -const OPEN_READ_FLAGS = fsConstants.O_RDONLY | (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); -const OPEN_READ_NONBLOCK_FLAGS = OPEN_READ_FLAGS | NONBLOCK_OPEN_FLAG; -const OPEN_READ_FOLLOW_FLAGS = fsConstants.O_RDONLY; -const OPEN_READ_FOLLOW_NONBLOCK_FLAGS = OPEN_READ_FOLLOW_FLAGS | NONBLOCK_OPEN_FLAG; -const OPEN_WRITE_EXISTING_FLAGS = - fsConstants.O_WRONLY | (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); -const OPEN_WRITE_CREATE_FLAGS = - fsConstants.O_WRONLY | - fsConstants.O_CREAT | - fsConstants.O_EXCL | - (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); -const OPEN_APPEND_EXISTING_FLAGS = - fsConstants.O_RDWR | fsConstants.O_APPEND | (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); -const OPEN_APPEND_CREATE_FLAGS = - fsConstants.O_RDWR | - fsConstants.O_APPEND | - fsConstants.O_CREAT | - fsConstants.O_EXCL | - (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); - -const ensureTrailingSep = (value: string) => (value.endsWith(path.sep) ? value : value + path.sep); - -async function expandRelativePathWithHome(relativePath: string): Promise { - let home = process.env.HOME || process.env.USERPROFILE || os.homedir(); - try { - home = await fs.realpath(home); - } catch { - // If the home dir cannot be canonicalized, keep lexical expansion behavior. - } - return expandHomePrefix(relativePath, { home }); -} - -async function openVerifiedLocalFile( - filePath: string, - options?: { - rejectHardlinks?: boolean; - nonBlockingRead?: boolean; - allowSymlinkTargetWithinRoot?: boolean; - }, -): Promise { - // Reject directories before opening so we never surface EISDIR to callers (e.g. tool - // results that get sent to messaging channels). See openclaw/openclaw#31186. - try { - const preStat = await fs.lstat(filePath); - if (preStat.isDirectory()) { - throw new SafeOpenError("not-file", "not a file"); - } - await fsSafeTestHooks?.afterPreOpenLstat?.(filePath); - } catch (err) { - if (err instanceof SafeOpenError) { - throw err; - } - // ENOENT and other lstat errors: fall through and let fs.open handle. - } - - let handle: FileHandle; - try { - const openFlags = options?.allowSymlinkTargetWithinRoot - ? options?.nonBlockingRead - ? OPEN_READ_FOLLOW_NONBLOCK_FLAGS - : OPEN_READ_FOLLOW_FLAGS - : options?.nonBlockingRead - ? OPEN_READ_NONBLOCK_FLAGS - : OPEN_READ_FLAGS; - await fsSafeTestHooks?.beforeOpen?.(filePath, openFlags); - handle = await fs.open(filePath, openFlags); - try { - await fsSafeTestHooks?.afterOpen?.(filePath, handle); - } catch (err) { - await handle.close().catch(() => {}); - throw err; - } - } catch (err) { - if (isNotFoundPathError(err)) { - throw new SafeOpenError("not-found", "file not found"); - } - if (isSymlinkOpenError(err)) { - throw new SafeOpenError("symlink", "symlink open blocked", { cause: err }); - } - // Defensive: if open still throws EISDIR (e.g. race), sanitize so it never leaks. - if (hasNodeErrorCode(err, "EISDIR")) { - throw new SafeOpenError("not-file", "not a file"); - } - throw err; - } - - try { - const stat = await handle.stat(); - if (!stat.isFile()) { - throw new SafeOpenError("not-file", "not a file"); - } - if (options?.rejectHardlinks && stat.nlink > 1) { - throw new SafeOpenError("invalid-path", "hardlinked path not allowed"); - } - - if (options?.allowSymlinkTargetWithinRoot) { - const pathStat = await fs.stat(filePath); - if (!sameFileIdentity(stat, pathStat)) { - throw new SafeOpenError("path-mismatch", "path changed during read"); - } - } else { - const pathStat = await fs.lstat(filePath); - if (pathStat.isSymbolicLink()) { - throw new SafeOpenError("symlink", "symlink not allowed"); - } - if (!sameFileIdentity(stat, pathStat)) { - throw new SafeOpenError("path-mismatch", "path changed during read"); - } - } - - const realPath = await resolveOpenedFileRealPathForHandle(handle, filePath); - const realStat = await fs.stat(realPath); - if (options?.rejectHardlinks && realStat.nlink > 1) { - throw new SafeOpenError("invalid-path", "hardlinked path not allowed"); - } - if (!sameFileIdentity(stat, realStat)) { - throw new SafeOpenError("path-mismatch", "path mismatch"); - } - - return { handle, realPath, stat }; - } catch (err) { - await handle.close().catch(() => {}); - if (err instanceof SafeOpenError) { - throw err; - } - if (isNotFoundPathError(err)) { - throw new SafeOpenError("not-found", "file not found"); - } - throw err; - } -} - -async function resolvePathWithinRoot(params: { - rootDir: string; - relativePath: string; -}): Promise<{ rootReal: string; rootWithSep: string; resolved: string }> { - let rootReal: string; - try { - rootReal = await fs.realpath(params.rootDir); - } catch (err) { - if (isNotFoundPathError(err)) { - throw new SafeOpenError("not-found", "root dir not found"); - } - throw err; - } - const rootWithSep = ensureTrailingSep(rootReal); - const expanded = await expandRelativePathWithHome(params.relativePath); - const resolved = path.resolve(rootWithSep, expanded); - if (!isPathInside(rootWithSep, resolved)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - return { rootReal, rootWithSep, resolved }; -} - -export async function openFileWithinRoot(params: { - rootDir: string; - relativePath: string; - rejectHardlinks?: boolean; - nonBlockingRead?: boolean; - allowSymlinkTargetWithinRoot?: boolean; -}): Promise { - const { rootWithSep, resolved } = await resolvePathWithinRoot(params); - - let opened: SafeOpenResult; - try { - opened = await openVerifiedLocalFile(resolved, { - nonBlockingRead: params.nonBlockingRead, - allowSymlinkTargetWithinRoot: params.allowSymlinkTargetWithinRoot, - }); - } catch (err) { - if (err instanceof SafeOpenError) { - if (err.code === "not-found") { - throw err; - } - throw new SafeOpenError("invalid-path", "path is not a regular file under root", { - cause: err, - }); - } - throw err; - } - - if (params.rejectHardlinks !== false && opened.stat.nlink > 1) { - await opened.handle.close().catch(() => {}); - throw new SafeOpenError("invalid-path", "hardlinked path not allowed"); - } - - if (!isPathInside(rootWithSep, opened.realPath)) { - await opened.handle.close().catch(() => {}); - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - - return opened; -} +export { FsSafeError, type FsSafeErrorCode } from "@openclaw/fs-safe/errors"; +export { + assertAbsolutePathInput, + canonicalPathFromExistingAncestor, + findExistingAncestor, + resolveAbsolutePathForRead, + resolveAbsolutePathForWrite, + type AbsolutePathSymlinkPolicy, + type ResolvedAbsolutePath, + type ResolvedWritableAbsolutePath, +} from "@openclaw/fs-safe/advanced"; +export { isPathInside } from "@openclaw/fs-safe/path"; +export { pathExists, pathExistsSync } from "@openclaw/fs-safe/advanced"; +export { readLocalFileFromRoots, resolveLocalPathFromRootsSync } from "@openclaw/fs-safe/advanced"; +export { + appendRegularFile, + appendRegularFileSync, + readRegularFile, + readRegularFileSync, + resolveRegularFileAppendFlags, + statRegularFileSync, +} from "@openclaw/fs-safe/advanced"; +export { + openLocalFileSafely, + readLocalFileSafely, + resolveOpenedFileRealPathForHandle, + root, + type OpenResult, + type ReadResult, +} from "@openclaw/fs-safe/root"; +export { sanitizeUntrustedFileName } from "@openclaw/fs-safe/advanced"; +export { + readSecureFile, + type SecureFileReadOptions, + type SecureFileReadResult, +} from "@openclaw/fs-safe/secure-file"; +export { + walkDirectory, + walkDirectorySync, + type WalkDirectoryEntry, + type WalkDirectoryOptions, + type WalkDirectoryResult, +} from "@openclaw/fs-safe/walk"; +export { withTimeout } from "@openclaw/fs-safe/advanced"; +/** @deprecated Use root(rootDir).read(relativePath, options). */ export async function readFileWithinRoot(params: { rootDir: string; relativePath: string; @@ -273,444 +54,17 @@ export async function readFileWithinRoot(params: { nonBlockingRead?: boolean; allowSymlinkTargetWithinRoot?: boolean; maxBytes?: number; -}): Promise { - const opened = await openFileWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - rejectHardlinks: params.rejectHardlinks, - nonBlockingRead: params.nonBlockingRead, - allowSymlinkTargetWithinRoot: params.allowSymlinkTargetWithinRoot, - }); - try { - return await readOpenedFileSafely({ opened, maxBytes: params.maxBytes }); - } finally { - await opened.handle.close().catch(() => {}); - } -} - -export async function readPathWithinRoot(params: { - rootDir: string; - filePath: string; - rejectHardlinks?: boolean; - maxBytes?: number; -}): Promise { - const rootDir = path.resolve(params.rootDir); - const candidatePath = path.isAbsolute(params.filePath) - ? path.resolve(params.filePath) - : path.resolve(rootDir, params.filePath); - const relativePath = path.relative(rootDir, candidatePath); - return await readFileWithinRoot({ - rootDir, - relativePath, - rejectHardlinks: params.rejectHardlinks, +}): Promise { + const root = await fsSafeRoot(params.rootDir); + return await root.read(params.relativePath, { + hardlinks: params.rejectHardlinks === false ? "allow" : "reject", maxBytes: params.maxBytes, + nonBlockingRead: params.nonBlockingRead, + symlinks: params.allowSymlinkTargetWithinRoot === true ? "follow-within-root" : "reject", }); } -export function createRootScopedReadFile(params: { - rootDir: string; - rejectHardlinks?: boolean; - maxBytes?: number; -}): (filePath: string) => Promise { - const rootDir = path.resolve(params.rootDir); - return async (filePath: string) => { - const safeRead = await readPathWithinRoot({ - rootDir, - filePath, - rejectHardlinks: params.rejectHardlinks, - maxBytes: params.maxBytes, - }); - return safeRead.buffer; - }; -} - -export async function readLocalFileSafely(params: { - filePath: string; - maxBytes?: number; -}): Promise { - const opened = await openLocalFileSafely({ filePath: params.filePath }); - try { - return await readOpenedFileSafely({ opened, maxBytes: params.maxBytes }); - } finally { - await opened.handle.close().catch(() => {}); - } -} - -export async function openLocalFileSafely(params: { filePath: string }): Promise { - return await openVerifiedLocalFile(params.filePath); -} - -async function readOpenedFileSafely(params: { - opened: SafeOpenResult; - maxBytes?: number; -}): Promise { - if (params.maxBytes !== undefined && params.opened.stat.size > params.maxBytes) { - throw new SafeOpenError( - "too-large", - `file exceeds limit of ${params.maxBytes} bytes (got ${params.opened.stat.size})`, - ); - } - const buffer = await params.opened.handle.readFile(); - return { - buffer, - realPath: params.opened.realPath, - stat: params.opened.stat, - }; -} - -export type SafeWritableOpenResult = { - handle: FileHandle; - createdForWrite: boolean; - openedRealPath: string; - openedStat: Stats; -}; - -function emitWriteBoundaryWarning(reason: string) { - logWarn(`security: fs-safe write boundary warning (${reason})`); -} - -function buildAtomicWriteTempPath(targetPath: string): string { - const dir = path.dirname(targetPath); - const base = path.basename(targetPath); - return path.join(dir, `.${base}.${process.pid}.${randomUUID()}.tmp`); -} - -async function writeTempFileForAtomicReplace(params: { - tempPath: string; - data: string | Buffer; - encoding?: BufferEncoding; - mode: number; -}): Promise { - const tempHandle = await fs.open(params.tempPath, OPEN_WRITE_CREATE_FLAGS, params.mode); - try { - if (typeof params.data === "string") { - await tempHandle.writeFile(params.data, params.encoding ?? "utf8"); - } else { - await tempHandle.writeFile(params.data); - } - return await tempHandle.stat(); - } finally { - await tempHandle.close().catch(() => {}); - } -} - -async function verifyAtomicWriteResult(params: { - rootDir: string; - targetPath: string; - expectedIdentity: { dev: number | bigint; ino: number | bigint }; -}): Promise { - const rootReal = await fs.realpath(params.rootDir); - const rootWithSep = ensureTrailingSep(rootReal); - const opened = await openVerifiedLocalFile(params.targetPath, { rejectHardlinks: true }); - try { - if (!sameFileIdentity(opened.stat, params.expectedIdentity)) { - throw new SafeOpenError("path-mismatch", "path changed during write"); - } - if (!isPathInside(rootWithSep, opened.realPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - } finally { - await opened.handle.close().catch(() => {}); - } -} - -export async function resolveOpenedFileRealPathForHandle( - handle: FileHandle, - ioPath: string, -): Promise { - const handleStat = await handle.stat(); - const fdCandidates = - process.platform === "linux" - ? [`/proc/self/fd/${handle.fd}`, `/dev/fd/${handle.fd}`] - : process.platform === "win32" - ? [] - : [`/dev/fd/${handle.fd}`]; - for (const fdPath of fdCandidates) { - try { - const fdRealPath = await fs.realpath(fdPath); - const fdRealStat = await fs.stat(fdRealPath); - if (sameFileIdentity(handleStat, fdRealStat)) { - return fdRealPath; - } - } catch { - // try next fd path - } - } - - try { - const ioRealPath = await fs.realpath(ioPath); - const ioRealStat = await fs.stat(ioRealPath); - if (sameFileIdentity(handleStat, ioRealStat)) { - return ioRealPath; - } - } catch (err) { - if (!isNotFoundPathError(err)) { - throw err; - } - } - const parentResolved = await resolveOpenedFileRealPathFromParent(handleStat, ioPath); - if (parentResolved) { - return parentResolved; - } - throw new SafeOpenError("path-mismatch", "unable to resolve opened file path"); -} - -async function resolveOpenedFileRealPathFromParent( - handleStat: Stats, - ioPath: string, -): Promise { - let parentReal: string; - try { - parentReal = await fs.realpath(path.dirname(ioPath)); - } catch (err) { - if (isNotFoundPathError(err)) { - return null; - } - throw err; - } - - let entries: string[]; - try { - entries = await fs.readdir(parentReal); - } catch (err) { - if (isNotFoundPathError(err)) { - return null; - } - throw err; - } - - for (const entry of entries.toSorted()) { - const candidatePath = path.join(parentReal, entry); - try { - const candidateStat = await fs.lstat(candidatePath); - if (candidateStat.isFile() && sameFileIdentity(handleStat, candidateStat)) { - return await fs.realpath(candidatePath); - } - } catch (err) { - if (!isNotFoundPathError(err)) { - throw err; - } - } - } - return null; -} - -export async function openWritableFileWithinRoot(params: { - rootDir: string; - relativePath: string; - mkdir?: boolean; - mode?: number; - truncateExisting?: boolean; - append?: boolean; -}): Promise { - const { rootReal, rootWithSep, resolved } = await resolvePathWithinRoot(params); - try { - await assertNoPathAliasEscape({ - absolutePath: resolved, - rootPath: rootReal, - boundaryLabel: "root", - }); - } catch (err) { - throw new SafeOpenError("invalid-path", "path alias escape blocked", { cause: err }); - } - if (params.mkdir !== false) { - await fs.mkdir(path.dirname(resolved), { recursive: true }); - } - - let ioPath = resolved; - try { - const resolvedRealPath = await fs.realpath(resolved); - if (!isPathInside(rootWithSep, resolvedRealPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - ioPath = resolvedRealPath; - } catch (err) { - if (err instanceof SafeOpenError) { - throw err; - } - if (!isNotFoundPathError(err)) { - throw err; - } - } - - const fileMode = params.mode ?? 0o600; - - let handle: FileHandle; - let createdForWrite = false; - const existingFlags = params.append ? OPEN_APPEND_EXISTING_FLAGS : OPEN_WRITE_EXISTING_FLAGS; - const createFlags = params.append ? OPEN_APPEND_CREATE_FLAGS : OPEN_WRITE_CREATE_FLAGS; - try { - try { - handle = await fs.open(ioPath, existingFlags, fileMode); - } catch (err) { - if (!isNotFoundPathError(err)) { - throw err; - } - handle = await fs.open(ioPath, createFlags, fileMode); - createdForWrite = true; - } - } catch (err) { - if (isNotFoundPathError(err)) { - throw new SafeOpenError("not-found", "file not found"); - } - if (isSymlinkOpenError(err)) { - throw new SafeOpenError("invalid-path", "symlink open blocked", { cause: err }); - } - throw err; - } - - let openedRealPath: string | null = null; - try { - const stat = await handle.stat(); - if (!stat.isFile()) { - throw new SafeOpenError("invalid-path", "path is not a regular file under root"); - } - if (stat.nlink > 1) { - throw new SafeOpenError("invalid-path", "hardlinked path not allowed"); - } - - try { - const lstat = await fs.lstat(ioPath); - if (lstat.isSymbolicLink() || !lstat.isFile()) { - throw new SafeOpenError("invalid-path", "path is not a regular file under root"); - } - if (!sameFileIdentity(stat, lstat)) { - throw new SafeOpenError("path-mismatch", "path changed during write"); - } - } catch (err) { - if (!isNotFoundPathError(err)) { - throw err; - } - } - - const realPath = await resolveOpenedFileRealPathForHandle(handle, ioPath); - openedRealPath = realPath; - const realStat = await fs.stat(realPath); - if (!sameFileIdentity(stat, realStat)) { - throw new SafeOpenError("path-mismatch", "path mismatch"); - } - if (realStat.nlink > 1) { - throw new SafeOpenError("invalid-path", "hardlinked path not allowed"); - } - if (!isPathInside(rootWithSep, realPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - - // Truncate only after boundary and identity checks complete. This avoids - // irreversible side effects if a symlink target changes before validation. - if (params.append !== true && params.truncateExisting !== false && !createdForWrite) { - await handle.truncate(0); - } - return { - handle, - createdForWrite, - openedRealPath: realPath, - openedStat: stat, - }; - } catch (err) { - const cleanupCreatedPath = createdForWrite && err instanceof SafeOpenError; - const cleanupPath = openedRealPath ?? ioPath; - await handle.close().catch(() => {}); - if (cleanupCreatedPath) { - await fs.rm(cleanupPath, { force: true }).catch(() => {}); - } - throw err; - } -} - -export async function appendFileWithinRoot(params: { - rootDir: string; - relativePath: string; - data: string | Buffer; - encoding?: BufferEncoding; - mkdir?: boolean; - prependNewlineIfNeeded?: boolean; -}): Promise { - const target = await openWritableFileWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - mkdir: params.mkdir, - truncateExisting: false, - append: true, - }); - try { - let prefix = ""; - if ( - params.prependNewlineIfNeeded === true && - !target.createdForWrite && - target.openedStat.size > 0 && - ((typeof params.data === "string" && !params.data.startsWith("\n")) || - (Buffer.isBuffer(params.data) && params.data.length > 0 && params.data[0] !== 0x0a)) - ) { - const lastByte = Buffer.alloc(1); - const { bytesRead } = await target.handle.read(lastByte, 0, 1, target.openedStat.size - 1); - if (bytesRead === 1 && lastByte[0] !== 0x0a) { - prefix = "\n"; - } - } - - if (typeof params.data === "string") { - await target.handle.appendFile(`${prefix}${params.data}`, params.encoding ?? "utf8"); - return; - } - - const payload = - prefix.length > 0 ? Buffer.concat([Buffer.from(prefix, "utf8"), params.data]) : params.data; - await target.handle.appendFile(payload); - } finally { - await target.handle.close().catch(() => {}); - } -} - -export async function removePathWithinRoot(params: { - rootDir: string; - relativePath: string; -}): Promise { - const resolved = await resolvePinnedRemovePathWithinRoot(params); - if (process.platform === "win32") { - await removePathWithinRootLegacy(resolved); - return; - } - try { - await runPinnedPathHelper({ - operation: "remove", - rootPath: resolved.rootReal, - relativePath: resolved.relativePosix, - }); - } catch (error) { - if (isPinnedPathHelperSpawnError(error)) { - await removePathWithinRootLegacy(resolved); - return; - } - throw normalizePinnedPathError(error); - } -} - -export async function mkdirPathWithinRoot(params: { - rootDir: string; - relativePath: string; - allowRoot?: boolean; -}): Promise { - const resolved = await resolvePinnedPathWithinRoot(params); - if (process.platform === "win32") { - await mkdirPathWithinRootLegacy(resolved); - return; - } - try { - await runPinnedPathHelper({ - operation: "mkdirp", - rootPath: resolved.rootReal, - relativePath: resolved.relativePosix, - }); - } catch (error) { - if (isPinnedPathHelperSpawnError(error)) { - await mkdirPathWithinRootLegacy(resolved); - return; - } - throw normalizePinnedPathError(error); - } -} - +/** @deprecated Use root(rootDir).write(relativePath, data, options). */ export async function writeFileWithinRoot(params: { rootDir: string; relativePath: string; @@ -718,432 +72,9 @@ export async function writeFileWithinRoot(params: { encoding?: BufferEncoding; mkdir?: boolean; }): Promise { - if (process.platform === "win32") { - await writeFileWithinRootLegacy(params); - return; - } - - const pinned = await resolvePinnedWriteTargetWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - }); - - const identity = await runPinnedWriteHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: pinned.mode, - input: { - kind: "buffer", - data: params.data, - encoding: params.encoding, - }, - }).catch((error) => { - throw normalizePinnedWriteError(error); - }); - - try { - await verifyAtomicWriteResult({ - rootDir: params.rootDir, - targetPath: pinned.targetPath, - expectedIdentity: identity, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-write verification failed: ${String(err)}`); - throw err; - } -} - -export async function copyFileWithinRoot(params: { - sourcePath: string; - rootDir: string; - relativePath: string; - maxBytes?: number; - mkdir?: boolean; - rejectSourceHardlinks?: boolean; -}): Promise { - const source = await openVerifiedLocalFile(params.sourcePath, { - rejectHardlinks: params.rejectSourceHardlinks, - }); - if (params.maxBytes !== undefined && source.stat.size > params.maxBytes) { - await source.handle.close().catch(() => {}); - throw new SafeOpenError( - "too-large", - `file exceeds limit of ${params.maxBytes} bytes (got ${source.stat.size})`, - ); - } - - try { - if (process.platform === "win32") { - await copyFileWithinRootLegacy(params, source); - return; - } - - const pinned = await resolvePinnedWriteTargetWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - }); - const sourceStream = source.handle.createReadStream(); - const identity = await runPinnedWriteHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: pinned.mode, - input: { - kind: "stream", - stream: sourceStream, - }, - }).catch((error) => { - throw normalizePinnedWriteError(error); - }); - try { - await verifyAtomicWriteResult({ - rootDir: params.rootDir, - targetPath: pinned.targetPath, - expectedIdentity: identity, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); - throw err; - } - } finally { - await source.handle.close().catch(() => {}); - } -} - -export async function writeFileFromPathWithinRoot(params: { - rootDir: string; - relativePath: string; - sourcePath: string; - mkdir?: boolean; -}): Promise { - await copyFileWithinRoot({ - sourcePath: params.sourcePath, - rootDir: params.rootDir, - relativePath: params.relativePath, + const root = await fsSafeRoot(params.rootDir); + await root.write(params.relativePath, params.data, { + encoding: params.encoding, mkdir: params.mkdir, - rejectSourceHardlinks: true, }); } - -async function resolvePinnedWriteTargetWithinRoot(params: { - rootDir: string; - relativePath: string; -}): Promise<{ - rootReal: string; - targetPath: string; - relativeParentPath: string; - basename: string; - mode: number; -}> { - const { rootReal, rootWithSep, resolved } = await resolvePathWithinRoot(params); - try { - await assertNoPathAliasEscape({ - absolutePath: resolved, - rootPath: rootReal, - boundaryLabel: "root", - }); - } catch (err) { - throw new SafeOpenError("invalid-path", "path alias escape blocked", { cause: err }); - } - - const relativeResolved = path.relative(rootReal, resolved); - if (relativeResolved.startsWith("..") || path.isAbsolute(relativeResolved)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - const relativePosix = relativeResolved - ? relativeResolved.split(path.sep).join(path.posix.sep) - : ""; - const basename = path.posix.basename(relativePosix); - if (!basename || basename === "." || basename === "/") { - throw new SafeOpenError("invalid-path", "invalid target path"); - } - let mode = 0o600; - try { - const opened = await openFileWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - rejectHardlinks: true, - nonBlockingRead: true, - }); - try { - mode = opened.stat.mode & 0o777; - if (!isPathInside(rootWithSep, opened.realPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - } finally { - await opened.handle.close().catch(() => {}); - } - } catch (err) { - if (!(err instanceof SafeOpenError) || err.code !== "not-found") { - throw err; - } - } - - return { - rootReal, - targetPath: resolved, - relativeParentPath: - path.posix.dirname(relativePosix) === "." ? "" : path.posix.dirname(relativePosix), - basename, - mode: mode || 0o600, - }; -} - -async function resolvePinnedPathWithinRoot(params: { - rootDir: string; - relativePath: string; - allowRoot?: boolean; -}): Promise<{ rootReal: string; resolved: string; relativePosix: string }> { - const resolved = await resolvePinnedBoundaryPathWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - policy: PATH_ALIAS_POLICIES.strict, - }); - const relativeResolved = path.relative(resolved.rootReal, resolved.canonicalPath); - if ((relativeResolved === "" || relativeResolved === ".") && params.allowRoot === true) { - return { rootReal: resolved.rootReal, resolved: resolved.canonicalPath, relativePosix: "" }; - } - if ( - relativeResolved === "" || - relativeResolved === "." || - relativeResolved.startsWith("..") || - path.isAbsolute(relativeResolved) - ) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - - const relativePosix = relativeResolved.split(path.sep).join(path.posix.sep); - if (!isPathInside(resolved.rootWithSep, resolved.canonicalPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - - return { rootReal: resolved.rootReal, resolved: resolved.canonicalPath, relativePosix }; -} - -async function resolvePinnedRemovePathWithinRoot(params: { - rootDir: string; - relativePath: string; -}): Promise<{ rootReal: string; resolved: string; relativePosix: string }> { - const resolved = await resolvePinnedBoundaryPathWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - policy: PATH_ALIAS_POLICIES.unlinkTarget, - }); - const relativeResolved = path.relative(resolved.rootReal, resolved.canonicalPath); - if ( - relativeResolved === "" || - relativeResolved === "." || - relativeResolved.startsWith("..") || - path.isAbsolute(relativeResolved) - ) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - const relativePosix = relativeResolved.split(path.sep).join(path.posix.sep); - if (!isPathInside(resolved.rootWithSep, resolved.canonicalPath)) { - throw new SafeOpenError("outside-workspace", "file is outside workspace root"); - } - - const parentRelative = path.posix.dirname(relativePosix); - if (parentRelative === "." || parentRelative === "") { - return { rootReal: resolved.rootReal, resolved: resolved.canonicalPath, relativePosix }; - } - return { rootReal: resolved.rootReal, resolved: resolved.canonicalPath, relativePosix }; -} - -async function resolvePinnedBoundaryPathWithinRoot(params: { - rootDir: string; - relativePath: string; - policy: (typeof PATH_ALIAS_POLICIES)[keyof typeof PATH_ALIAS_POLICIES]; -}): Promise<{ rootReal: string; rootWithSep: string; canonicalPath: string }> { - const { rootReal } = await resolvePathWithinRoot({ - rootDir: params.rootDir, - relativePath: ".", - }); - let resolved; - try { - resolved = await resolveBoundaryPath({ - absolutePath: path.resolve(rootReal, await expandRelativePathWithHome(params.relativePath)), - rootPath: rootReal, - rootCanonicalPath: rootReal, - boundaryLabel: "root", - policy: params.policy, - }); - } catch (err) { - throw new SafeOpenError("invalid-path", "path alias escape blocked", { cause: err }); - } - const rootWithSep = ensureTrailingSep(resolved.rootCanonicalPath); - return { - rootReal: resolved.rootCanonicalPath, - rootWithSep, - canonicalPath: resolved.canonicalPath, - }; -} - -function normalizePinnedWriteError(error: unknown): Error { - if (error instanceof SafeOpenError) { - return error; - } - return new SafeOpenError("invalid-path", "path is not a regular file under root", { - cause: error instanceof Error ? error : undefined, - }); -} - -function normalizePinnedPathError(error: unknown): Error { - if (error instanceof SafeOpenError) { - return error; - } - if (error instanceof Error) { - const message = error.message; - if (/No such file or directory/i.test(message)) { - return new SafeOpenError("not-found", "file not found", { cause: error }); - } - if (/Not a directory|symbolic link|Too many levels of symbolic links/i.test(message)) { - return new SafeOpenError("invalid-path", "path is not under root", { cause: error }); - } - if (/Directory not empty/i.test(message)) { - return new SafeOpenError("invalid-path", "directory is not empty", { cause: error }); - } - if (/Is a directory|Operation not permitted|Permission denied/i.test(message)) { - return new SafeOpenError("invalid-path", "path is not removable under root", { - cause: error, - }); - } - } - return new SafeOpenError("invalid-path", "path is not under root", { - cause: error instanceof Error ? error : undefined, - }); -} - -async function removePathWithinRootLegacy(resolved: { resolved: string }): Promise { - await fs.rm(resolved.resolved); -} - -async function mkdirPathWithinRootLegacy(resolved: { resolved: string }): Promise { - await fs.mkdir(resolved.resolved, { recursive: true }); -} - -async function writeFileWithinRootLegacy(params: { - rootDir: string; - relativePath: string; - data: string | Buffer; - encoding?: BufferEncoding; - mkdir?: boolean; -}): Promise { - const target = await openWritableFileWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - mkdir: params.mkdir, - truncateExisting: false, - }); - const destinationPath = target.openedRealPath; - const targetMode = target.openedStat.mode & 0o777; - await target.handle.close().catch(() => {}); - let tempPath: string | null = null; - try { - tempPath = buildAtomicWriteTempPath(destinationPath); - const writtenStat = await writeTempFileForAtomicReplace({ - tempPath, - data: params.data, - encoding: params.encoding, - mode: targetMode || 0o600, - }); - await fs.rename(tempPath, destinationPath); - tempPath = null; - try { - await verifyAtomicWriteResult({ - rootDir: params.rootDir, - targetPath: destinationPath, - expectedIdentity: writtenStat, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-write verification failed: ${String(err)}`); - throw err; - } - } finally { - if (tempPath) { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } - } -} - -async function copyFileWithinRootLegacy( - params: { - sourcePath: string; - rootDir: string; - relativePath: string; - maxBytes?: number; - mkdir?: boolean; - rejectSourceHardlinks?: boolean; - }, - source: SafeOpenResult, -): Promise { - let target: SafeWritableOpenResult | null = null; - let sourceClosedByStream = false; - let targetClosedByUs = false; - let tempHandle: FileHandle | null = null; - let tempPath: string | null = null; - let tempClosedByStream = false; - try { - target = await openWritableFileWithinRoot({ - rootDir: params.rootDir, - relativePath: params.relativePath, - mkdir: params.mkdir, - truncateExisting: false, - }); - const destinationPath = target.openedRealPath; - const targetMode = target.openedStat.mode & 0o777; - await target.handle.close().catch(() => {}); - targetClosedByUs = true; - - tempPath = buildAtomicWriteTempPath(destinationPath); - tempHandle = await fs.open(tempPath, OPEN_WRITE_CREATE_FLAGS, targetMode || 0o600); - const sourceStream = source.handle.createReadStream(); - const targetStream = tempHandle.createWriteStream(); - sourceStream.once("close", () => { - sourceClosedByStream = true; - }); - targetStream.once("close", () => { - tempClosedByStream = true; - }); - await pipeline(sourceStream, targetStream); - const writtenStat = await fs.stat(tempPath); - if (!tempClosedByStream) { - await tempHandle.close().catch(() => {}); - tempClosedByStream = true; - } - tempHandle = null; - await fs.rename(tempPath, destinationPath); - tempPath = null; - try { - await verifyAtomicWriteResult({ - rootDir: params.rootDir, - targetPath: destinationPath, - expectedIdentity: writtenStat, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); - throw err; - } - } catch (err) { - if (target?.createdForWrite) { - await fs.rm(target.openedRealPath, { force: true }).catch(() => {}); - } - throw err; - } finally { - if (tempPath) { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } - if (!sourceClosedByStream) { - await source.handle.close().catch(() => {}); - } - if (tempHandle && !tempClosedByStream) { - await tempHandle.close().catch(() => {}); - } - if (target && !targetClosedByUs) { - await target.handle.close().catch(() => {}); - } - } -} diff --git a/src/infra/hardlink-guards.test.ts b/src/infra/hardlink-guards.test.ts index b1981b8bd2d..e1eb30b5ac6 100644 --- a/src/infra/hardlink-guards.test.ts +++ b/src/infra/hardlink-guards.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { withTempDir } from "../test-helpers/temp-dir.js"; -import { assertNoHardlinkedFinalPath } from "./hardlink-guards.js"; +import { assertNoHardlinkedFinalPath } from "./fs-safe-advanced.js"; async function withHardlinkFixture( cb: (context: { root: string; source: string; linked: string; dirPath: string }) => Promise, diff --git a/src/infra/hardlink-guards.ts b/src/infra/hardlink-guards.ts deleted file mode 100644 index ad99729b463..00000000000 --- a/src/infra/hardlink-guards.ts +++ /dev/null @@ -1,38 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import { isNotFoundPathError } from "./path-guards.js"; - -export async function assertNoHardlinkedFinalPath(params: { - filePath: string; - root: string; - boundaryLabel: string; - allowFinalHardlinkForUnlink?: boolean; -}): Promise { - if (params.allowFinalHardlinkForUnlink) { - return; - } - let stat: Awaited>; - try { - stat = await fs.stat(params.filePath); - } catch (err) { - if (isNotFoundPathError(err)) { - return; - } - throw err; - } - if (!stat.isFile()) { - return; - } - if (stat.nlink > 1) { - throw new Error( - `Hardlinked path is not allowed under ${params.boundaryLabel} (${shortPath(params.root)}): ${shortPath(params.filePath)}`, - ); - } -} - -function shortPath(value: string) { - if (value.startsWith(os.homedir())) { - return `~${value.slice(os.homedir().length)}`; - } - return value; -} diff --git a/src/infra/home-dir.ts b/src/infra/home-dir.ts index 840ef1c07fe..83ba0fce5c0 100644 --- a/src/infra/home-dir.ts +++ b/src/infra/home-dir.ts @@ -1,18 +1,38 @@ import os from "node:os"; import path from "node:path"; -import { normalizeOptionalString } from "../shared/string-coerce.js"; function normalize(value: string | undefined): string | undefined { - const trimmed = normalizeOptionalString(value); - if (!trimmed) { - return undefined; - } - if (trimmed === "undefined" || trimmed === "null") { + const trimmed = value?.trim(); + if (!trimmed || trimmed === "undefined" || trimmed === "null") { return undefined; } return trimmed; } +function normalizeSafe(homedir: () => string): string | undefined { + try { + return normalize(homedir()); + } catch { + return undefined; + } +} + +function resolveRawOsHomeDir(env: NodeJS.ProcessEnv, homedir: () => string): string | undefined { + return normalize(env.HOME) ?? normalize(env.USERPROFILE) ?? normalizeSafe(homedir); +} + +function resolveRawHomeDir(env: NodeJS.ProcessEnv, homedir: () => string): string | undefined { + const explicitHome = normalize(env.OPENCLAW_HOME); + if (!explicitHome) { + return resolveRawOsHomeDir(env, homedir); + } + if (explicitHome === "~" || explicitHome.startsWith("~/") || explicitHome.startsWith("~\\")) { + const fallbackHome = resolveRawOsHomeDir(env, homedir); + return fallbackHome ? explicitHome.replace(/^~(?=$|[\\/])/, fallbackHome) : undefined; + } + return explicitHome; +} + export function resolveEffectiveHomeDir( env: NodeJS.ProcessEnv = process.env, homedir: () => string = os.homedir, @@ -29,42 +49,6 @@ export function resolveOsHomeDir( return raw ? path.resolve(raw) : undefined; } -function resolveRawHomeDir(env: NodeJS.ProcessEnv, homedir: () => string): string | undefined { - const explicitHome = normalize(env.OPENCLAW_HOME); - if (explicitHome) { - if (explicitHome === "~" || explicitHome.startsWith("~/") || explicitHome.startsWith("~\\")) { - const fallbackHome = resolveRawOsHomeDir(env, homedir); - if (fallbackHome) { - return explicitHome.replace(/^~(?=$|[\\/])/, fallbackHome); - } - return undefined; - } - return explicitHome; - } - - return resolveRawOsHomeDir(env, homedir); -} - -function resolveRawOsHomeDir(env: NodeJS.ProcessEnv, homedir: () => string): string | undefined { - const envHome = normalize(env.HOME); - if (envHome) { - return envHome; - } - const userProfile = normalize(env.USERPROFILE); - if (userProfile) { - return userProfile; - } - return normalizeSafe(homedir); -} - -function normalizeSafe(homedir: () => string): string | undefined { - try { - return normalize(homedir()); - } catch { - return undefined; - } -} - export function resolveRequiredHomeDir( env: NodeJS.ProcessEnv = process.env, homedir: () => string = os.homedir, @@ -121,6 +105,14 @@ export function resolveHomeRelativePath( return path.resolve(trimmed); } +export function resolveUserPath( + input: string, + env: NodeJS.ProcessEnv = process.env, + homedir: () => string = os.homedir, +): string { + return resolveHomeRelativePath(input, { env, homedir }); +} + export function resolveOsHomeRelativePath( input: string, opts?: { diff --git a/src/infra/install-flow.ts b/src/infra/install-flow.ts index 07ede657f42..cda8a5f9a02 100644 --- a/src/infra/install-flow.ts +++ b/src/infra/install-flow.ts @@ -2,7 +2,8 @@ import type { Stats } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { resolveUserPath } from "../utils.js"; -import { type ArchiveLogger, extractArchive, fileExists, resolvePackedRootDir } from "./archive.js"; +import { type ArchiveLogger, extractArchive, resolvePackedRootDir } from "./archive.js"; +import { pathExists } from "./fs-safe.js"; import { withTempDir } from "./install-source-utils.js"; type ExistingInstallPathResult = @@ -20,7 +21,7 @@ export async function resolveExistingInstallPath( inputPath: string, ): Promise { const resolvedPath = resolveUserPath(inputPath); - if (!(await fileExists(resolvedPath))) { + if (!(await pathExists(resolvedPath))) { return { ok: false, error: `path not found: ${resolvedPath}` }; } const stat = await fs.stat(resolvedPath); diff --git a/src/infra/install-package-dir.ts b/src/infra/install-package-dir.ts index 5548855c970..e08b68f44e4 100644 --- a/src/infra/install-package-dir.ts +++ b/src/infra/install-package-dir.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { runCommandWithTimeout } from "../process/exec.js"; -import { fileExists } from "./archive.js"; +import { pathExists } from "./fs-safe.js"; import { assertCanonicalPathWithinBase } from "./install-safe-path.js"; import { createSafeNpmInstallArgs, createSafeNpmInstallEnv } from "./safe-package-install.js"; @@ -291,7 +291,7 @@ export async function installPackageDir(params: { } } - if (params.mode === "update" && (await fileExists(canonicalTargetDir))) { + if (params.mode === "update" && (await pathExists(canonicalTargetDir))) { const backupRoot = path.join(installBaseRealPath, ".openclaw-install-backups"); backupDir = path.join(backupRoot, `${path.basename(canonicalTargetDir)}-${Date.now()}`); try { diff --git a/src/infra/install-safe-path.ts b/src/infra/install-safe-path.ts index e9847ef5701..fc0c6a100f7 100644 --- a/src/infra/install-safe-path.ts +++ b/src/infra/install-safe-path.ts @@ -1,7 +1,10 @@ -import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { isPathInside } from "./path-guards.js"; +import "./fs-safe-defaults.js"; +export { + assertCanonicalPathWithinBase, + resolveSafeInstallDir, + safeDirName, + safePathSegmentHashed, +} from "@openclaw/fs-safe/advanced"; export function unscopedPackageName(name: string): string { const trimmed = name.trim(); @@ -24,113 +27,3 @@ export function packageNameMatchesId(packageName: string, id: string): boolean { return trimmedId === trimmedPackageName || trimmedId === unscopedPackageName(trimmedPackageName); } - -export function safeDirName(input: string): string { - const trimmed = input.trim(); - if (!trimmed) { - return trimmed; - } - return trimmed.replaceAll("/", "__").replaceAll("\\", "__"); -} - -export function safePathSegmentHashed(input: string): string { - const trimmed = input.trim(); - const base = trimmed - .replaceAll(/[\\/]/g, "-") - .replaceAll(/[^a-zA-Z0-9._-]/g, "-") - .replaceAll(/-+/g, "-") - .replaceAll(/^-+/g, "") - .replaceAll(/-+$/g, ""); - - const normalized = base.length > 0 ? base : "skill"; - const safe = normalized === "." || normalized === ".." ? "skill" : normalized; - - const hash = createHash("sha256").update(trimmed).digest("hex").slice(0, 10); - - if (safe !== trimmed) { - const prefix = safe.length > 50 ? safe.slice(0, 50) : safe; - return `${prefix}-${hash}`; - } - if (safe.length > 60) { - return `${safe.slice(0, 50)}-${hash}`; - } - return safe; -} - -export function resolveSafeInstallDir(params: { - baseDir: string; - id: string; - invalidNameMessage: string; - nameEncoder?: (id: string) => string; -}): { ok: true; path: string } | { ok: false; error: string } { - const encodedName = (params.nameEncoder ?? safeDirName)(params.id); - const targetDir = path.join(params.baseDir, encodedName); - const resolvedBase = path.resolve(params.baseDir); - const resolvedTarget = path.resolve(targetDir); - const relative = path.relative(resolvedBase, resolvedTarget); - if ( - !relative || - relative === ".." || - relative.startsWith(`..${path.sep}`) || - path.isAbsolute(relative) - ) { - return { ok: false, error: params.invalidNameMessage }; - } - return { ok: true, path: targetDir }; -} - -export async function assertCanonicalPathWithinBase(params: { - baseDir: string; - candidatePath: string; - boundaryLabel: string; -}): Promise { - const baseDir = path.resolve(params.baseDir); - const candidatePath = path.resolve(params.candidatePath); - if (!isPathInside(baseDir, candidatePath)) { - throw new Error(`Invalid path: must stay within ${params.boundaryLabel}`); - } - - const baseLstat = await fs.lstat(baseDir); - if (baseLstat.isSymbolicLink()) { - const baseStat = await fs.stat(baseDir); - if (!baseStat.isDirectory()) { - throw new Error( - `Invalid ${params.boundaryLabel}: base directory must resolve to a directory`, - ); - } - } else if (!baseLstat.isDirectory()) { - throw new Error(`Invalid ${params.boundaryLabel}: base directory must be a directory`); - } - const baseRealPath = await fs.realpath(baseDir); - - const validateDirectory = async (dirPath: string): Promise => { - const resolvedDirPath = path.resolve(dirPath); - const dirLstat = await fs.lstat(dirPath); - if (dirLstat.isSymbolicLink()) { - if (resolvedDirPath !== baseDir) { - throw new Error(`Invalid path: must stay within ${params.boundaryLabel}`); - } - const dirStat = await fs.stat(dirPath); - if (!dirStat.isDirectory()) { - throw new Error(`Invalid path: must stay within ${params.boundaryLabel}`); - } - } else if (!dirLstat.isDirectory()) { - throw new Error(`Invalid path: must stay within ${params.boundaryLabel}`); - } - const dirRealPath = await fs.realpath(dirPath); - if (!isPathInside(baseRealPath, dirRealPath)) { - throw new Error(`Invalid path: must stay within ${params.boundaryLabel}`); - } - }; - - try { - await validateDirectory(candidatePath); - return; - } catch (err) { - const code = (err as { code?: string }).code; - if (code !== "ENOENT") { - throw err; - } - } - await validateDirectory(path.dirname(candidatePath)); -} diff --git a/src/infra/install-source-utils.ts b/src/infra/install-source-utils.ts index 39ff1d27e83..c23f4a9a874 100644 --- a/src/infra/install-source-utils.ts +++ b/src/infra/install-source-utils.ts @@ -3,7 +3,9 @@ import os from "node:os"; import path from "node:path"; import { runCommandWithTimeout } from "../process/exec.js"; import { resolveUserPath } from "../utils.js"; -import { fileExists, resolveArchiveKind } from "./archive.js"; +import { resolveArchiveKind } from "./archive.js"; +import { pathExists } from "./fs-safe.js"; +import { withTempWorkspace } from "./private-temp-workspace.js"; export type NpmSpecResolution = { name?: string; @@ -105,12 +107,7 @@ export async function withTempDir( prefix: string, fn: (tmpDir: string) => Promise, ): Promise { - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); - try { - return await fn(tmpDir); - } finally { - await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); - } + return await withTempWorkspace({ rootDir: os.tmpdir(), prefix }, async (tmp) => fn(tmp.dir)); } export async function resolveArchiveSourcePath(archivePath: string): Promise< @@ -124,7 +121,7 @@ export async function resolveArchiveSourcePath(archivePath: string): Promise< } > { const resolved = resolveUserPath(archivePath); - if (!(await fileExists(resolved))) { + if (!(await pathExists(resolved))) { return { ok: false, error: `archive not found: ${resolved}` }; } @@ -310,7 +307,7 @@ export async function packNpmSpecToArchive(params: { } let archivePath = path.isAbsolute(packed) ? packed : path.join(params.cwd, packed); - if (!(await fileExists(archivePath))) { + if (!(await pathExists(archivePath))) { const fallbackPacked = await findPackedArchiveInDir(params.cwd); if (!fallbackPacked) { return { ok: false, error: "npm pack produced no archive" }; diff --git a/src/infra/install-target.test.ts b/src/infra/install-target.test.ts index 211d5c1a99d..ce6013789be 100644 --- a/src/infra/install-target.test.ts +++ b/src/infra/install-target.test.ts @@ -3,12 +3,12 @@ import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { withTempDir } from "../test-helpers/temp-dir.js"; -const fileExistsMock = vi.hoisted(() => vi.fn()); +const pathExistsMock = vi.hoisted(() => vi.fn()); const resolveSafeInstallDirMock = vi.hoisted(() => vi.fn()); const assertCanonicalPathWithinBaseMock = vi.hoisted(() => vi.fn()); -vi.mock("./archive.js", () => ({ - fileExists: (...args: unknown[]) => fileExistsMock(...args), +vi.mock("./fs-safe.js", () => ({ + pathExists: (...args: unknown[]) => pathExistsMock(...args), })); vi.mock("./install-safe-path.js", () => ({ @@ -19,7 +19,7 @@ vi.mock("./install-safe-path.js", () => ({ import { ensureInstallTargetAvailable, resolveCanonicalInstallTarget } from "./install-target.js"; beforeEach(() => { - fileExistsMock.mockReset(); + pathExistsMock.mockReset(); resolveSafeInstallDirMock.mockReset(); assertCanonicalPathWithinBaseMock.mockReset(); }); @@ -99,8 +99,8 @@ describe("resolveCanonicalInstallTarget", () => { describe("ensureInstallTargetAvailable", () => { it("blocks only install mode when the target already exists", async () => { - fileExistsMock.mockResolvedValueOnce(true); - fileExistsMock.mockResolvedValueOnce(false); + pathExistsMock.mockResolvedValueOnce(true); + pathExistsMock.mockResolvedValueOnce(false); await expect( ensureInstallTargetAvailable({ diff --git a/src/infra/install-target.ts b/src/infra/install-target.ts index c9debe8da31..54987146793 100644 --- a/src/infra/install-target.ts +++ b/src/infra/install-target.ts @@ -1,6 +1,6 @@ import fs from "node:fs/promises"; -import { fileExists } from "./archive.js"; import { formatErrorMessage } from "./errors.js"; +import { pathExists } from "./fs-safe.js"; import { assertCanonicalPathWithinBase, resolveSafeInstallDir } from "./install-safe-path.js"; export async function resolveCanonicalInstallTarget(params: { @@ -37,7 +37,7 @@ export async function ensureInstallTargetAvailable(params: { targetDir: string; alreadyExistsError: string; }): Promise<{ ok: true } | { ok: false; error: string }> { - if (params.mode === "install" && (await fileExists(params.targetDir))) { + if (params.mode === "install" && (await pathExists(params.targetDir))) { return { ok: false, error: params.alreadyExistsError }; } return { ok: true }; diff --git a/src/infra/json-file.ts b/src/infra/json-file.ts index 247d6b404db..f1cecb19e03 100644 --- a/src/infra/json-file.ts +++ b/src/infra/json-file.ts @@ -1,130 +1,47 @@ -import { randomUUID } from "node:crypto"; +import "./fs-safe-defaults.js"; import fs from "node:fs"; import path from "node:path"; +import { tryReadJsonSync, tryReadJson, writeJsonSync } from "@openclaw/fs-safe/json"; -const JSON_FILE_MODE = 0o600; -const JSON_DIR_MODE = 0o700; +export { tryReadJson, tryReadJsonSync, writeJsonSync }; +export const readJsonFile = tryReadJson; -function trySetSecureMode(pathname: string) { +function resolveJsonSymlinkTarget(pathname: string): string | undefined { + let stat: fs.Stats; try { - fs.chmodSync(pathname, JSON_FILE_MODE); - } catch { - // best-effort on platforms without chmod support - } -} - -function trySyncDirectory(pathname: string) { - let fd: number | undefined; - try { - fd = fs.openSync(path.dirname(pathname), "r"); - fs.fsyncSync(fd); - } catch { - // best-effort; some platforms/filesystems do not support syncing directories. - } finally { - if (fd !== undefined) { - try { - fs.closeSync(fd); - } catch { - // best-effort cleanup - } - } - } -} - -function readSymlinkTargetPath(linkPath: string): string { - const target = fs.readlinkSync(linkPath); - return path.resolve(path.dirname(linkPath), target); -} - -function resolveJsonWriteTarget(pathname: string): { targetPath: string; followsSymlink: boolean } { - let currentPath = pathname; - const visited = new Set(); - let followsSymlink = false; - - for (;;) { - let stat: fs.Stats; - try { - stat = fs.lstatSync(currentPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - return { targetPath: currentPath, followsSymlink }; - } - - if (!stat.isSymbolicLink()) { - return { targetPath: currentPath, followsSymlink }; - } - - if (visited.has(currentPath)) { - const err = new Error( - `Too many symlink levels while resolving ${pathname}`, - ) as NodeJS.ErrnoException; - err.code = "ELOOP"; - throw err; - } - - visited.add(currentPath); - followsSymlink = true; - currentPath = readSymlinkTargetPath(currentPath); - } -} - -function renameJsonFileWithFallback(tmpPath: string, pathname: string) { - try { - fs.renameSync(tmpPath, pathname); - return; + stat = fs.lstatSync(pathname); } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - // Windows does not reliably support rename-based overwrite for existing files. - if (code === "EPERM" || code === "EEXIST") { - fs.copyFileSync(tmpPath, pathname); - fs.rmSync(tmpPath, { force: true }); - return; + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; } throw error; } -} - -function writeTempJsonFile(pathname: string, payload: string) { - const fd = fs.openSync(pathname, "w", JSON_FILE_MODE); - try { - fs.writeFileSync(fd, payload, "utf8"); - fs.fsyncSync(fd); - } finally { - fs.closeSync(fd); - } -} - -// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- JSON loading helper lets callers ascribe the expected payload type. -export function loadJsonFile(pathname: string): T | undefined { - try { - const raw = fs.readFileSync(pathname, "utf8"); - return JSON.parse(raw) as T; - } catch { + if (!stat.isSymbolicLink()) { return undefined; } + + return path.resolve(path.dirname(pathname), fs.readlinkSync(pathname)); } -export function saveJsonFile(pathname: string, data: unknown) { - const { targetPath, followsSymlink } = resolveJsonWriteTarget(pathname); - const tmpPath = `${targetPath}.${randomUUID()}.tmp`; - const payload = `${JSON.stringify(data, null, 2)}\n`; - - if (!followsSymlink) { - fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: JSON_DIR_MODE }); - } - try { - writeTempJsonFile(tmpPath, payload); - trySetSecureMode(tmpPath); - renameJsonFileWithFallback(tmpPath, targetPath); - trySetSecureMode(targetPath); - trySyncDirectory(targetPath); - } finally { - try { - fs.rmSync(tmpPath, { force: true }); - } catch { - // best-effort cleanup when rename does not happen - } +function resolveJsonSaveTarget(pathname: string): string { + const target = resolveJsonSymlinkTarget(pathname); + if (!target) { + return pathname; } + fs.statSync(path.dirname(target)); + return target; +} + +export function saveJsonFile(pathname: string, data: unknown): void { + writeJsonSync(resolveJsonSaveTarget(pathname), data); +} + +// oxlint-disable-next-line typescript-eslint/no-unnecessary-type-parameters -- legacy typed JSON loader alias. +export function loadJsonFile(pathname: string): T | undefined { + const direct = tryReadJsonSync(pathname); + if (direct !== null) { + return direct; + } + const target = resolveJsonSymlinkTarget(pathname); + return target ? (tryReadJsonSync(target) ?? undefined) : undefined; } diff --git a/src/infra/json-files.test.ts b/src/infra/json-files.test.ts index 50ea62541f4..d0c3a29e09b 100644 --- a/src/infra/json-files.test.ts +++ b/src/infra/json-files.test.ts @@ -76,7 +76,7 @@ describe("json file helpers", () => { await writeJsonAtomic( filePath, { ok: true, nested: { value: 1 } }, - { trailingNewline: true, ensureDirMode: 0o755 }, + { trailingNewline: true, dirMode: 0o755 }, ); await expect(fs.readFile(filePath, "utf8")).resolves.toBe( @@ -91,7 +91,7 @@ describe("json file helpers", () => { ])("writes text atomically for %j", async ({ input, expected }) => { await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => { const filePath = path.join(base, "nested", "note.txt"); - await writeTextAtomic(filePath, input, { appendTrailingNewline: true }); + await writeTextAtomic(filePath, input, { trailingNewline: true }); await expect(fs.readFile(filePath, "utf8")).resolves.toBe(expected); }); }); @@ -114,7 +114,7 @@ describe("json file helpers", () => { }); }); - it("replaces symlink targets instead of writing through them on Windows rename fallback", async () => { + it("refuses Windows copy fallback through symlink destinations", async () => { await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => { const filePath = path.join(base, "state.json"); const outsidePath = path.join(base, "outside.json"); @@ -125,10 +125,11 @@ describe("json file helpers", () => { const renameError = Object.assign(new Error("EPERM"), { code: "EPERM" }); vi.spyOn(fs, "rename").mockRejectedValueOnce(renameError); - await writeTextAtomic(filePath, "new"); + await expect(writeTextAtomic(filePath, "new")).rejects.toThrow( + "Refusing copy fallback through symlink destination", + ); - await expect(fs.lstat(filePath)).resolves.toSatisfy((stat) => !stat.isSymbolicLink()); - await expect(fs.readFile(filePath, "utf8")).resolves.toBe("new"); + await expect(fs.lstat(filePath)).resolves.toSatisfy((stat) => stat.isSymbolicLink()); await expect(fs.readFile(outsidePath, "utf8")).resolves.toBe("outside"); }); }); diff --git a/src/infra/json-files.ts b/src/infra/json-files.ts index 8ca5802eee0..fda9494add1 100644 --- a/src/infra/json-files.ts +++ b/src/infra/json-files.ts @@ -1,161 +1,18 @@ -import { randomUUID } from "node:crypto"; -import { readFileSync } from "node:fs"; -import fs from "node:fs/promises"; -import path from "node:path"; - -function getErrorCode(err: unknown): string | undefined { - return err instanceof Error ? (err as NodeJS.ErrnoException).code : undefined; -} - -export class JsonFileReadError extends Error { - readonly filePath: string; - readonly reason: "read" | "parse"; - - constructor(filePath: string, reason: "read" | "parse", cause: unknown) { - super(`Failed to ${reason} JSON file: ${filePath}`, { cause }); - this.name = "JsonFileReadError"; - this.filePath = filePath; - this.reason = reason; - } -} - -async function replaceFileWithWindowsFallback(tempPath: string, filePath: string, mode: number) { - try { - await fs.rename(tempPath, filePath); - return; - } catch (err) { - const code = getErrorCode(err); - if (process.platform !== "win32" || (code !== "EPERM" && code !== "EEXIST")) { - throw err; - } - } - - const existing = await fs.lstat(filePath).catch(() => null); - if (existing?.isSymbolicLink()) { - await fs.rm(filePath, { force: true }); - await fs.rename(tempPath, filePath); - return; - } - - await fs.copyFile(tempPath, filePath); - try { - await fs.chmod(filePath, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - await fs.rm(tempPath, { force: true }).catch(() => undefined); -} - -export async function readJsonFile(filePath: string): Promise { - try { - const raw = await fs.readFile(filePath, "utf8"); - return JSON.parse(raw) as T; - } catch { - return null; - } -} - -export async function readDurableJsonFile(filePath: string): Promise { - let raw: string; - try { - raw = await fs.readFile(filePath, "utf8"); - } catch (err) { - if (getErrorCode(err) === "ENOENT") { - return null; - } - throw new JsonFileReadError(filePath, "read", err); - } - try { - return JSON.parse(raw) as T; - } catch (err) { - throw new JsonFileReadError(filePath, "parse", err); - } -} - -export function readJsonFileSync(filePath: string): unknown { - try { - const raw = readFileSync(filePath, "utf8"); - return JSON.parse(raw) as unknown; - } catch { - return null; - } -} - -export async function writeJsonAtomic( - filePath: string, - value: unknown, - options?: { mode?: number; trailingNewline?: boolean; ensureDirMode?: number }, -) { - const text = JSON.stringify(value, null, 2); - await writeTextAtomic(filePath, text, { - mode: options?.mode, - ensureDirMode: options?.ensureDirMode, - appendTrailingNewline: options?.trailingNewline, - }); -} - -export async function writeTextAtomic( - filePath: string, - content: string, - options?: { mode?: number; ensureDirMode?: number; appendTrailingNewline?: boolean }, -) { - const mode = options?.mode ?? 0o600; - const payload = - options?.appendTrailingNewline && !content.endsWith("\n") ? `${content}\n` : content; - const mkdirOptions: { recursive: true; mode?: number } = { recursive: true }; - if (typeof options?.ensureDirMode === "number") { - mkdirOptions.mode = options.ensureDirMode; - } - await fs.mkdir(path.dirname(filePath), mkdirOptions); - const parentDir = path.dirname(filePath); - const tmp = `${filePath}.${randomUUID()}.tmp`; - try { - const tmpHandle = await fs.open(tmp, "w", mode); - try { - await tmpHandle.writeFile(payload, { encoding: "utf8" }); - await tmpHandle.sync(); - } finally { - await tmpHandle.close().catch(() => undefined); - } - try { - await fs.chmod(tmp, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - await replaceFileWithWindowsFallback(tmp, filePath, mode); - try { - const dirHandle = await fs.open(parentDir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close().catch(() => undefined); - } - } catch { - // best-effort; some platforms/filesystems do not support syncing directories. - } - try { - await fs.chmod(filePath, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - } finally { - await fs.rm(tmp, { force: true }).catch(() => undefined); - } -} - -export function createAsyncLock() { - let lock: Promise = Promise.resolve(); - return async function withLock(fn: () => Promise): Promise { - const prev = lock; - let release: (() => void) | undefined; - lock = new Promise((resolve) => { - release = resolve; - }); - await prev; - try { - return await fn(); - } finally { - release?.(); - } - }; -} +import "./fs-safe-defaults.js"; +export { + JsonFileReadError, + readJson, + readJson as readJsonFileStrict, + readJsonIfExists, + readJsonIfExists as readDurableJsonFile, + readJsonSync, + tryReadJson, + tryReadJson as readJsonFile, + tryReadJsonSync, + tryReadJsonSync as readJsonFileSync, + writeJson, + writeJson as writeJsonAtomic, + writeJsonSync, +} from "@openclaw/fs-safe/json"; +export { writeTextAtomic } from "@openclaw/fs-safe/atomic"; +export { createAsyncLock } from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/local-file-access.ts b/src/infra/local-file-access.ts index 66354b00de2..cda4a2d83e6 100644 --- a/src/infra/local-file-access.ts +++ b/src/infra/local-file-access.ts @@ -1,75 +1,9 @@ -import path from "node:path"; -import { fileURLToPath, URL } from "node:url"; -import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; - -const ENCODED_FILE_URL_SEPARATOR_RE = /%(?:2f|5c)/i; - -function isLocalFileUrlHost(hostname: string): boolean { - const normalized = normalizeLowercaseStringOrEmpty(hostname); - return normalized === "" || normalized === "localhost"; -} - -export function hasEncodedFileUrlSeparator(pathname: string): boolean { - return ENCODED_FILE_URL_SEPARATOR_RE.test(pathname); -} - -export function isWindowsNetworkPath(filePath: string): boolean { - if (process.platform !== "win32") { - return false; - } - const normalized = filePath.replace(/\//g, "\\"); - return normalized.startsWith("\\\\?\\UNC\\") || normalized.startsWith("\\\\"); -} - -export function assertNoWindowsNetworkPath(filePath: string, label = "Path"): void { - if (isWindowsNetworkPath(filePath)) { - throw new Error(`${label} cannot use Windows network paths: ${filePath}`); - } -} - -export function safeFileURLToPath(fileUrl: string): string { - let parsed: URL; - try { - parsed = new URL(fileUrl); - } catch { - throw new Error(`Invalid file:// URL: ${fileUrl}`); - } - if (parsed.protocol !== "file:") { - throw new Error(`Invalid file:// URL: ${fileUrl}`); - } - if (!isLocalFileUrlHost(parsed.hostname)) { - throw new Error(`file:// URLs with remote hosts are not allowed: ${fileUrl}`); - } - if (hasEncodedFileUrlSeparator(parsed.pathname)) { - throw new Error(`file:// URLs cannot encode path separators: ${fileUrl}`); - } - const filePath = fileURLToPath(parsed); - assertNoWindowsNetworkPath(filePath, "Local file URL"); - return filePath; -} - -export function trySafeFileURLToPath(fileUrl: string): string | undefined { - try { - return safeFileURLToPath(fileUrl); - } catch { - return undefined; - } -} - -export function basenameFromMediaSource(source?: string): string | undefined { - if (!source) { - return undefined; - } - if (source.startsWith("file://")) { - const filePath = trySafeFileURLToPath(source); - return filePath ? path.basename(filePath) || undefined : undefined; - } - if (/^https?:\/\//i.test(source)) { - try { - return path.basename(new URL(source).pathname) || undefined; - } catch { - return undefined; - } - } - return path.basename(source) || undefined; -} +import "./fs-safe-defaults.js"; +export { + assertNoWindowsNetworkPath, + basenameFromMediaSource, + hasEncodedFileUrlSeparator, + isWindowsNetworkPath, + safeFileURLToPath, + trySafeFileURLToPath, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/node-pairing.ts b/src/infra/node-pairing.ts index 74920c8485d..53f25dcc6c3 100644 --- a/src/infra/node-pairing.ts +++ b/src/infra/node-pairing.ts @@ -5,11 +5,11 @@ import { type NodeApprovalScope, resolveNodePairApprovalScopes } from "./node-pa import { createAsyncLock, pruneExpiredPending, - readDurableJsonFile, + readJsonIfExists, reconcilePendingPairingRequests, coercePairingStateRecord, resolvePairingPaths, - writeJsonAtomic, + writeJson, } from "./pairing-files.js"; import { rejectPendingPairingRequest } from "./pairing-pending.js"; import { generatePairingToken, verifyPairingToken } from "./pairing-token.js"; @@ -137,8 +137,8 @@ type ApproveNodePairingResult = ApprovedNodePairingResult | ForbiddenNodePairing async function loadState(baseDir?: string): Promise { const { pendingPath, pairedPath } = resolvePairingPaths(baseDir, "nodes"); const [pending, paired] = await Promise.all([ - readDurableJsonFile(pendingPath), - readDurableJsonFile(pairedPath), + readJsonIfExists(pendingPath), + readJsonIfExists(pairedPath), ]); const state: NodePairingStateFile = { pendingById: coercePairingStateRecord(pending), @@ -151,8 +151,8 @@ async function loadState(baseDir?: string): Promise { async function persistState(state: NodePairingStateFile, baseDir?: string) { const { pendingPath, pairedPath } = resolvePairingPaths(baseDir, "nodes"); await Promise.all([ - writeJsonAtomic(pendingPath, state.pendingById), - writeJsonAtomic(pairedPath, state.pairedByNodeId), + writeJson(pendingPath, state.pendingById), + writeJson(pairedPath, state.pairedByNodeId), ]); } diff --git a/src/infra/outbound/delivery-queue-recovery.ts b/src/infra/outbound/delivery-queue-recovery.ts index 85421cbad58..ddd1e39a4fd 100644 --- a/src/infra/outbound/delivery-queue-recovery.ts +++ b/src/infra/outbound/delivery-queue-recovery.ts @@ -31,6 +31,8 @@ export type DeliverFn = ( params: { cfg: OpenClawConfig; } & QueuedDeliveryPayload & { + deliveryQueueId?: string; + deliveryQueueStateDir?: string; skipQueue?: boolean; deferCommitHooks?: boolean; }, @@ -120,7 +122,7 @@ export async function withActiveDeliveryClaim( } } -function buildRecoveryDeliverParams(entry: QueuedDelivery, cfg: OpenClawConfig) { +function buildRecoveryDeliverParams(entry: QueuedDelivery, cfg: OpenClawConfig, stateDir?: string) { return { cfg, channel: entry.channel, @@ -140,6 +142,8 @@ function buildRecoveryDeliverParams(entry: QueuedDelivery, cfg: OpenClawConfig) mirror: entry.mirror, session: entry.session, gatewayClientScopes: entry.gatewayClientScopes, + deliveryQueueId: entry.id, + deliveryQueueStateDir: stateDir, skipQueue: true, // Prevent re-enqueueing during recovery. deferCommitHooks: true, } satisfies Parameters[0]; @@ -413,7 +417,7 @@ async function drainQueuedEntry(opts: { : `delivery state is ${entry.recoveryState}; refusing blind replay without adapter reconciliation`; opts.log.warn(`Delivery entry ${entry.id} ${errMsg}`); opts.onFailed?.(entry, errMsg); - if (reconciliation === null || reconciliation.retryable === true) { + if (reconciliation?.status === "unresolved" && reconciliation.retryable === true) { try { await failDelivery(entry.id, errMsg, opts.stateDir); return "failed"; @@ -436,7 +440,7 @@ async function drainQueuedEntry(opts: { } } try { - const result = await opts.deliver(buildRecoveryDeliverParams(entry, opts.cfg)); + const result = await opts.deliver(buildRecoveryDeliverParams(entry, opts.cfg, opts.stateDir)); await ackDelivery(entry.id, opts.stateDir); if (isOutboundDeliveryResultArray(result)) { await runOutboundDeliveryCommitHooks(result); diff --git a/src/infra/outbound/delivery-queue-storage.ts b/src/infra/outbound/delivery-queue-storage.ts index 806f4fc3c34..419e7a79445 100644 --- a/src/infra/outbound/delivery-queue-storage.ts +++ b/src/infra/outbound/delivery-queue-storage.ts @@ -4,6 +4,7 @@ import type { ReplyPayload } from "../../auto-reply/types.js"; import type { RenderedMessageBatchPlanItem } from "../../channels/message/types.js"; import { resolveStateDir } from "../../config/paths.js"; import type { ReplyToMode } from "../../config/types.js"; +import { replaceFileAtomic } from "../replace-file.js"; import { generateSecureUuid } from "../secure-random.js"; import type { OutboundDeliveryFormattingOptions } from "./formatting.js"; import type { OutboundIdentity } from "./identity.js"; @@ -101,12 +102,12 @@ async function unlinkBestEffort(filePath: string): Promise { } async function writeQueueEntry(filePath: string, entry: QueuedDelivery): Promise { - const tmp = `${filePath}.${process.pid}.tmp`; - await fs.promises.writeFile(tmp, JSON.stringify(entry, null, 2), { - encoding: "utf-8", + await replaceFileAtomic({ + filePath, + content: JSON.stringify(entry, null, 2), mode: 0o600, + tempPrefix: ".delivery-queue", }); - await fs.promises.rename(tmp, filePath); } async function readQueueEntry(filePath: string): Promise { diff --git a/src/infra/outbound/delivery-queue.reconnect-drain.test.ts b/src/infra/outbound/delivery-queue.reconnect-drain.test.ts index 76512fc8b85..105dc9b7e83 100644 --- a/src/infra/outbound/delivery-queue.reconnect-drain.test.ts +++ b/src/infra/outbound/delivery-queue.reconnect-drain.test.ts @@ -8,6 +8,7 @@ import { enqueueDelivery, failDelivery, MAX_RETRIES, + markDeliveryPlatformOutcomeUnknown, type RecoveryLogger, recoverPendingDeliveries, withActiveDeliveryClaim, @@ -159,6 +160,22 @@ describe("drainPendingDeliveries for reconnect", () => { ).resolves.toBeUndefined(); }); + it("moves unknown-after-send entries to failed without replaying during reconnect drain", async () => { + const log = createRecoveryLog(); + const deliver = vi.fn(async () => {}); + const id = await enqueueFailedDirectChatDelivery({ accountId: "acct1", stateDir: tmpDir }); + await markDeliveryPlatformOutcomeUnknown(id, tmpDir); + + await drainAcct1DirectChatReconnect({ deliver, log, stateDir: tmpDir }); + + expect(deliver).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(tmpDir, "delivery-queue", `${id}.json`))).toBe(false); + expect(fs.existsSync(path.join(tmpDir, "delivery-queue", "failed", `${id}.json`))).toBe(true); + expect(log.warn).toHaveBeenCalledWith( + expect.stringContaining("refusing blind replay without adapter reconciliation"), + ); + }); + it("skips entries where retryCount >= MAX_RETRIES", async () => { const log = createRecoveryLog(); const deliver = vi.fn(async () => {}); diff --git a/src/infra/outbound/delivery-queue.recovery.test.ts b/src/infra/outbound/delivery-queue.recovery.test.ts index 10970aee3f5..24955b7ea99 100644 --- a/src/infra/outbound/delivery-queue.recovery.test.ts +++ b/src/infra/outbound/delivery-queue.recovery.test.ts @@ -5,6 +5,7 @@ import { attachOutboundDeliveryCommitHook } from "./delivery-commit-hooks.js"; import { enqueueDelivery, loadPendingDeliveries, + markDeliveryPlatformOutcomeUnknown, MAX_RETRIES, recoverPendingDeliveries, } from "./delivery-queue.js"; @@ -109,7 +110,7 @@ describe("delivery-queue recovery", () => { expect(entries[0]?.lastError).toBe("network down"); }); - it("retains entries abandoned after platform send may have started without reconciliation", async () => { + it("moves entries abandoned after platform send may have started to failed without reconciliation", async () => { const id = await enqueueDelivery( { channel: "demo-channel-a", to: "+1", payloads: [{ text: "maybe sent" }] }, tmpDir(), @@ -131,15 +132,12 @@ describe("delivery-queue recovery", () => { skippedMaxRetries: 0, deferredBackoff: 0, }); - const entries = await loadPendingDeliveries(tmpDir()); - expect(entries).toHaveLength(1); - expect(entries[0]?.id).toBe(id); - expect(entries[0]?.retryCount).toBe(1); - expect(entries[0]?.lastError).toContain("unknown_after_send"); + expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0); + expect(fs.existsSync(path.join(tmpDir(), "delivery-queue", "failed", `${id}.json`))).toBe(true); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("unknown_after_send")); }); - it("retains started entries without reconciliation instead of blindly replaying", async () => { + it("moves started entries without reconciliation to failed instead of blindly replaying", async () => { const id = await enqueueDelivery( { channel: "demo-channel-a", to: "+1", payloads: [{ text: "not yet sent" }] }, tmpDir(), @@ -161,11 +159,8 @@ describe("delivery-queue recovery", () => { skippedMaxRetries: 0, deferredBackoff: 0, }); - const entries = await loadPendingDeliveries(tmpDir()); - expect(entries).toHaveLength(1); - expect(entries[0]?.id).toBe(id); - expect(entries[0]?.retryCount).toBe(1); - expect(entries[0]?.lastError).toContain("send_attempt_started"); + expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0); + expect(fs.existsSync(path.join(tmpDir(), "delivery-queue", "failed", `${id}.json`))).toBe(true); expect(log.warn).toHaveBeenCalledWith( expect.stringContaining("refusing blind replay without adapter reconciliation"), ); @@ -447,10 +442,8 @@ describe("delivery-queue recovery", () => { expect(reconcileUnknownSend).not.toHaveBeenCalled(); expect(deliver).not.toHaveBeenCalled(); expect(result.failed).toBe(1); - const entries = await loadPendingDeliveries(tmpDir()); - expect(entries).toHaveLength(1); - expect(entries[0]?.id).toBe(id); - expect(entries[0]?.retryCount).toBe(1); + expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0); + expect(fs.existsSync(path.join(tmpDir(), "delivery-queue", "failed", `${id}.json`))).toBe(true); expect(log.warn).toHaveBeenCalledWith( expect.stringContaining("refusing blind replay without adapter reconciliation"), ); @@ -497,7 +490,7 @@ describe("delivery-queue recovery", () => { }); it("passes skipQueue: true to prevent re-enqueueing during recovery", async () => { - await enqueueDelivery( + const id = await enqueueDelivery( { channel: "demo-channel-a", to: "+1", payloads: [{ text: "a" }] }, tmpDir(), ); @@ -505,7 +498,37 @@ describe("delivery-queue recovery", () => { const deliver = vi.fn().mockResolvedValue([]); await runRecovery({ deliver }); - expect(deliver).toHaveBeenCalledWith(expect.objectContaining({ skipQueue: true })); + expect(deliver).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryQueueId: id, + deliveryQueueStateDir: tmpDir(), + skipQueue: true, + }), + ); + }); + + it("moves unknown-after-send entries to failed without replaying", async () => { + const id = await enqueueDelivery( + { channel: "demo-channel-a", to: "+1", payloads: [{ text: "a" }] }, + tmpDir(), + ); + await markDeliveryPlatformOutcomeUnknown(id, tmpDir()); + + const deliver = vi.fn().mockResolvedValue([]); + const { result, log } = await runRecovery({ deliver }); + + expect(deliver).not.toHaveBeenCalled(); + expect(result).toEqual({ + recovered: 0, + failed: 1, + skippedMaxRetries: 0, + deferredBackoff: 0, + }); + expect(await loadPendingDeliveries(tmpDir())).toHaveLength(0); + expect(fs.existsSync(path.join(tmpDir(), "delivery-queue", "failed", `${id}.json`))).toBe(true); + expect(log.warn).toHaveBeenCalledWith( + expect.stringContaining("refusing blind replay without adapter reconciliation"), + ); }); it("runs recovered send commit hooks only after the queue entry is acked", async () => { diff --git a/src/infra/outbound/message-action-params.ts b/src/infra/outbound/message-action-params.ts index 78f079025a6..5e22b9a4e48 100644 --- a/src/infra/outbound/message-action-params.ts +++ b/src/infra/outbound/message-action-params.ts @@ -3,7 +3,7 @@ import { readStringParam } from "../../agents/tools/common.js"; import { resolveChannelMessageToolMediaSourceParamKeys } from "../../channels/plugins/message-action-discovery.js"; import type { ChannelId, ChannelMessageActionName } from "../../channels/plugins/types.public.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { createRootScopedReadFile } from "../../infra/fs-safe.js"; +import { root } from "../../infra/fs-safe.js"; import { basenameFromMediaSource } from "../../infra/local-file-access.js"; import { resolveChannelAccountMediaMaxMb } from "../../media/configured-max-bytes.js"; import { @@ -215,9 +215,12 @@ function buildAttachmentMediaLoadOptions(params: { hostReadCapability?: boolean; } { if (params.policy.mode === "sandbox") { - const readSandboxFile = createRootScopedReadFile({ - rootDir: params.policy.sandboxRoot.trim(), - }); + const sandboxRoot = params.policy.sandboxRoot.trim(); + let sandboxFsPromise: ReturnType | undefined; + const readSandboxFile = async (filePath: string): Promise => { + sandboxFsPromise ??= root(sandboxRoot); + return await (await sandboxFsPromise).readBytes(filePath); + }; return { maxBytes: params.maxBytes, sandboxValidated: true, diff --git a/src/infra/package-update-steps.ts b/src/infra/package-update-steps.ts index 3a3f15e7cd9..0ce5d02fd74 100644 --- a/src/infra/package-update-steps.ts +++ b/src/infra/package-update-steps.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { pathExists } from "./fs-safe.js"; import { readPackageVersion } from "./package-json.js"; import { collectInstalledGlobalPackageErrors, @@ -52,15 +53,6 @@ function formatError(err: unknown): string { return err instanceof Error ? err.message : String(err); } -async function pathExists(targetPath: string): Promise { - try { - await fs.access(targetPath); - return true; - } catch { - return false; - } -} - async function removePathBestEffort(targetPath: string): Promise { await fs .rm(targetPath, { diff --git a/src/infra/package-update-utils.ts b/src/infra/package-update-utils.ts index 9582d9f6688..4cc5c564d18 100644 --- a/src/infra/package-update-utils.ts +++ b/src/infra/package-update-utils.ts @@ -1,6 +1,6 @@ import fsSync from "node:fs"; import path from "node:path"; -import { openBoundaryFileSync } from "./boundary-file-read.js"; +import { openRootFileSync } from "./boundary-file-read.js"; export function expectedIntegrityForUpdate( spec: string | undefined, @@ -30,7 +30,7 @@ function isRecord(value: unknown): value is Record { function readInstalledPackageManifest(dir: string): Record | undefined { const manifestPath = path.join(dir, "package.json"); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: manifestPath, rootPath: dir, boundaryLabel: "installed package directory", diff --git a/src/infra/pairing-files.ts b/src/infra/pairing-files.ts index bed3f735701..8050a4334fe 100644 --- a/src/infra/pairing-files.ts +++ b/src/infra/pairing-files.ts @@ -1,12 +1,7 @@ import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; -export { - createAsyncLock, - readDurableJsonFile, - readJsonFile, - writeJsonAtomic, -} from "./json-files.js"; +export { createAsyncLock, readJsonIfExists, tryReadJson, writeJson } from "./json-files.js"; export function resolvePairingPaths(baseDir: string | undefined, subdir: string) { const root = baseDir ?? resolveStateDir(); diff --git a/src/infra/path-alias-guards.ts b/src/infra/path-alias-guards.ts index e7b0aa42a0e..13264304b58 100644 --- a/src/infra/path-alias-guards.ts +++ b/src/infra/path-alias-guards.ts @@ -1,34 +1,6 @@ -import { - BOUNDARY_PATH_ALIAS_POLICIES, - resolveBoundaryPath, - type BoundaryPathAliasPolicy, -} from "./boundary-path.js"; -import { assertNoHardlinkedFinalPath } from "./hardlink-guards.js"; - -export type PathAliasPolicy = BoundaryPathAliasPolicy; - -export const PATH_ALIAS_POLICIES = BOUNDARY_PATH_ALIAS_POLICIES; - -export async function assertNoPathAliasEscape(params: { - absolutePath: string; - rootPath: string; - boundaryLabel: string; - policy?: PathAliasPolicy; -}): Promise { - const resolved = await resolveBoundaryPath({ - absolutePath: params.absolutePath, - rootPath: params.rootPath, - boundaryLabel: params.boundaryLabel, - policy: params.policy, - }); - const allowFinalSymlink = params.policy?.allowFinalSymlinkForUnlink === true; - if (allowFinalSymlink && resolved.kind === "symlink") { - return; - } - await assertNoHardlinkedFinalPath({ - filePath: resolved.absolutePath, - root: resolved.rootPath, - boundaryLabel: params.boundaryLabel, - allowFinalHardlinkForUnlink: params.policy?.allowFinalHardlinkForUnlink, - }); -} +import "./fs-safe-defaults.js"; +export { + PATH_ALIAS_POLICIES, + assertNoPathAliasEscape, + type PathAliasPolicy, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/path-guards.ts b/src/infra/path-guards.ts index 52139375e4b..c2d6005bb2b 100644 --- a/src/infra/path-guards.ts +++ b/src/infra/path-guards.ts @@ -1,64 +1,16 @@ -import path from "node:path"; -import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; - -const NOT_FOUND_CODES = new Set(["ENOENT", "ENOTDIR"]); -const SYMLINK_OPEN_CODES = new Set(["ELOOP", "EINVAL", "ENOTSUP"]); -const PARENT_SEGMENT_PREFIX = /^\.\.(?:[\\/]|$)/u; -const POSIX_SEPARATOR_CHAR_CODE = 0x2f; - -export function normalizeWindowsPathForComparison(input: string): string { - let normalized = path.win32.normalize(input); - if (normalized.startsWith("\\\\?\\")) { - normalized = normalized.slice(4); - if (normalized.toUpperCase().startsWith("UNC\\")) { - normalized = `\\\\${normalized.slice(4)}`; - } - } - return normalizeLowercaseStringOrEmpty(normalized.replaceAll("/", "\\")); -} - -export function isNodeError(value: unknown): value is NodeJS.ErrnoException { - return Boolean( - value && typeof value === "object" && "code" in (value as Record), - ); -} - -export function hasNodeErrorCode(value: unknown, code: string): boolean { - return isNodeError(value) && value.code === code; -} - -export function isNotFoundPathError(value: unknown): boolean { - return isNodeError(value) && typeof value.code === "string" && NOT_FOUND_CODES.has(value.code); -} - -export function isSymlinkOpenError(value: unknown): boolean { - return isNodeError(value) && typeof value.code === "string" && SYMLINK_OPEN_CODES.has(value.code); -} - -export function isPathInside(root: string, target: string): boolean { - if (process.platform === "win32") { - const rootForCompare = normalizeWindowsPathForComparison(path.win32.resolve(root)); - const targetForCompare = normalizeWindowsPathForComparison(path.win32.resolve(target)); - const relative = path.win32.relative(rootForCompare, targetForCompare); - return ( - relative === "" || (!PARENT_SEGMENT_PREFIX.test(relative) && !path.win32.isAbsolute(relative)) - ); - } - - if ( - root.length > 0 && - root.charCodeAt(0) === POSIX_SEPARATOR_CHAR_CODE && - target.length >= root.length && - target.charCodeAt(0) === POSIX_SEPARATOR_CHAR_CODE && - !target.includes("/..") && - (target === root || - (target.startsWith(root) && target.charCodeAt(root.length) === POSIX_SEPARATOR_CHAR_CODE)) - ) { - return true; - } - - const resolvedRoot = path.resolve(root); - const resolvedTarget = path.resolve(target); - const relative = path.relative(resolvedRoot, resolvedTarget); - return relative === "" || (!PARENT_SEGMENT_PREFIX.test(relative) && !path.isAbsolute(relative)); -} +import "./fs-safe-defaults.js"; +export { + isNotFoundPathError, + hasNodeErrorCode, + isNodeError, + isPathInside, + isPathInsideWithRealpath, + isSymlinkOpenError, + isWithinDir, + normalizeWindowsPathForComparison, + resolveSafeBaseDir, + resolveSafeRelativePath, + safeRealpathSync, + safeStatSync, + splitSafeRelativePath, +} from "@openclaw/fs-safe/path"; diff --git a/src/infra/path-safety.ts b/src/infra/path-safety.ts index 40a72d81b13..018a082ec6d 100644 --- a/src/infra/path-safety.ts +++ b/src/infra/path-safety.ts @@ -1,11 +1,17 @@ -import path from "node:path"; -import { isPathInside } from "./path-guards.js"; - -export function resolveSafeBaseDir(rootDir: string): string { - const resolved = path.resolve(rootDir); - return resolved.endsWith(path.sep) ? resolved : `${resolved}${path.sep}`; -} - -export function isWithinDir(rootDir: string, targetPath: string): boolean { - return isPathInside(rootDir, targetPath); -} +import "./fs-safe-defaults.js"; +export { + isNotFoundPathError, + hasNodeErrorCode, + isNodeError, + isPathInside, + isPathInsideWithRealpath, + isSymlinkOpenError, + isWithinDir, + normalizeWindowsPathForComparison, + resolveSafeBaseDir, + resolveSafeRelativePath, + safeRealpathSync, + safeStatSync, + splitSafeRelativePath, +} from "@openclaw/fs-safe/path"; +export { formatPosixMode } from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/permissions.ts b/src/infra/permissions.ts new file mode 100644 index 00000000000..e19f6358e92 --- /dev/null +++ b/src/infra/permissions.ts @@ -0,0 +1,21 @@ +import "./fs-safe-defaults.js"; +export { + formatPermissionDetail, + formatPermissionRemediation, + inspectPathPermissions, + safeStat, + type PermissionCheck, + type PermissionCheckOptions, +} from "@openclaw/fs-safe/permissions"; +export { + createIcaclsResetCommand, + formatIcaclsResetCommand, + formatWindowsAclSummary, + inspectWindowsAcl, + parseIcaclsOutput, + resolveWindowsUserPrincipal, + summarizeWindowsAcl, + type PermissionExec as ExecFn, + type WindowsAclEntry, + type WindowsAclSummary, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/private-file-store.ts b/src/infra/private-file-store.ts new file mode 100644 index 00000000000..3c171ca7624 --- /dev/null +++ b/src/infra/private-file-store.ts @@ -0,0 +1,19 @@ +import "./fs-safe-defaults.js"; +import { + fileStore, + fileStoreSync, + type FileStore, + type FileStoreSync, +} from "@openclaw/fs-safe/store"; + +export type PrivateFileStore = FileStore; + +export function privateFileStore(rootDir: string): FileStore { + return fileStore({ rootDir, private: true }); +} + +export type PrivateFileStoreSync = FileStoreSync; + +export function privateFileStoreSync(rootDir: string): PrivateFileStoreSync { + return fileStoreSync({ rootDir, private: true }); +} diff --git a/src/infra/private-temp-workspace.ts b/src/infra/private-temp-workspace.ts new file mode 100644 index 00000000000..41d13028aa6 --- /dev/null +++ b/src/infra/private-temp-workspace.ts @@ -0,0 +1,10 @@ +import "./fs-safe-defaults.js"; +export { + tempWorkspace, + tempWorkspaceSync, + type TempWorkspace, + type TempWorkspaceOptions, + type TempWorkspaceSync, + withTempWorkspace, + withTempWorkspaceSync, +} from "@openclaw/fs-safe/temp"; diff --git a/src/infra/push-apns.ts b/src/infra/push-apns.ts index 63adadf0656..721148e51b8 100644 --- a/src/infra/push-apns.ts +++ b/src/infra/push-apns.ts @@ -8,7 +8,7 @@ import { } from "../shared/string-coerce.js"; import type { DeviceIdentity } from "./device-identity.js"; import { formatErrorMessage } from "./errors.js"; -import { createAsyncLock, readJsonFile, writeJsonAtomic } from "./json-files.js"; +import { createAsyncLock, tryReadJson, writeJson } from "./json-files.js"; import { APNS_HTTP2_CANCEL_CODE, connectApnsHttp2Session } from "./push-apns-http2.js"; import { type ApnsRelayConfig, @@ -355,7 +355,7 @@ function normalizeStoredRegistration(record: unknown): ApnsRegistration | null { async function loadRegistrationsState(baseDir?: string): Promise { const filePath = resolveApnsRegistrationPath(baseDir); - const existing = await readJsonFile(filePath); + const existing = await tryReadJson(filePath); if (!existing || typeof existing !== "object") { return { registrationsByNodeId: {} }; } @@ -382,9 +382,9 @@ async function persistRegistrationsState( baseDir?: string, ): Promise { const filePath = resolveApnsRegistrationPath(baseDir); - await writeJsonAtomic(filePath, state, { + await writeJson(filePath, state, { mode: 0o600, - ensureDirMode: 0o700, + dirMode: 0o700, trailingNewline: true, }); } diff --git a/src/infra/push-web.ts b/src/infra/push-web.ts index 236c8cfdfea..e1c35011361 100644 --- a/src/infra/push-web.ts +++ b/src/infra/push-web.ts @@ -1,7 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; -import { createAsyncLock, readJsonFile, writeJsonAtomic } from "./json-files.js"; +import { createAsyncLock, tryReadJson, writeJson } from "./json-files.js"; // --- Types --- @@ -88,13 +88,13 @@ function isValidKey(key: string): boolean { async function loadState(baseDir?: string): Promise { const filePath = resolveWebPushStatePath(baseDir); - const state = await readJsonFile(filePath); + const state = await tryReadJson(filePath); return state ?? { subscriptionsByEndpointHash: {} }; } async function persistState(state: WebPushRegistrationState, baseDir?: string): Promise { const filePath = resolveWebPushStatePath(baseDir); - await writeJsonAtomic(filePath, state, { trailingNewline: true }); + await writeJson(filePath, state, { trailingNewline: true }); } // --- VAPID keys --- @@ -116,7 +116,7 @@ export async function resolveVapidKeys(baseDir?: string): Promise // prevent concurrent bootstraps from writing different keypairs. return await withLock(async () => { const filePath = resolveVapidKeysPath(baseDir); - const existing = await readJsonFile(filePath); + const existing = await tryReadJson(filePath); if (existing?.publicKey && existing?.privateKey) { return { publicKey: existing.publicKey, @@ -133,7 +133,7 @@ export async function resolveVapidKeys(baseDir?: string): Promise privateKey: keys.privateKey, subject: resolveVapidSubjectFromEnv(), }; - await writeJsonAtomic(filePath, pair, { trailingNewline: true }); + await writeJson(filePath, pair, { trailingNewline: true }); return pair; }); } diff --git a/src/infra/regular-file.ts b/src/infra/regular-file.ts new file mode 100644 index 00000000000..64cdba8df0e --- /dev/null +++ b/src/infra/regular-file.ts @@ -0,0 +1,12 @@ +import "./fs-safe-defaults.js"; +export { + appendRegularFile, + appendRegularFileSync, + readRegularFile, + readRegularFileSync, + resolveRegularFileAppendFlags, + statRegularFile, + statRegularFileSync, + type AppendRegularFileOptions, + type RegularFileStatResult, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/replace-file.ts b/src/infra/replace-file.ts new file mode 100644 index 00000000000..86c08c78b09 --- /dev/null +++ b/src/infra/replace-file.ts @@ -0,0 +1,14 @@ +import "./fs-safe-defaults.js"; +export { + movePathWithCopyFallback, + replaceDirectoryAtomic, + replaceFileAtomic, + replaceFileAtomicSync, + type MovePathWithCopyFallbackOptions, + type ReplaceDirectoryAtomicOptions, + type ReplaceFileAtomicFileSystem, + type ReplaceFileAtomicOptions, + type ReplaceFileAtomicResult, + type ReplaceFileAtomicSyncFileSystem, + type ReplaceFileAtomicSyncOptions, +} from "@openclaw/fs-safe/atomic"; diff --git a/src/infra/restart-sentinel.ts b/src/infra/restart-sentinel.ts index 376a850e649..4baa4ce8a9c 100644 --- a/src/infra/restart-sentinel.ts +++ b/src/infra/restart-sentinel.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { formatCliCommand } from "../cli/command-format.js"; import { resolveStateDir } from "../config/paths.js"; import { resolveRuntimeServiceVersion } from "../version.js"; -import { writeJsonAtomic } from "./json-files.js"; +import { writeJson } from "./json-files.js"; export type RestartSentinelLog = { stdoutTail?: string | null; @@ -84,7 +84,7 @@ export async function writeRestartSentinel( ) { const filePath = resolveRestartSentinelPath(env); const data: RestartSentinel = { version: 1, payload }; - await writeJsonAtomic(filePath, data, { trailingNewline: true, ensureDirMode: 0o700 }); + await writeJson(filePath, data, { trailingNewline: true, dirMode: 0o700 }); return filePath; } diff --git a/src/infra/restart.ts b/src/infra/restart.ts index bf8b7fbd7c3..f6144080da5 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -1,5 +1,4 @@ import { spawnSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +9,7 @@ import { resolveGatewaySystemdServiceName, } from "../daemon/constants.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { replaceFileAtomicSync } from "./replace-file.js"; import { cleanStaleGatewayProcessesSync, findGatewayPidsOnPortSync } from "./restart-stale-pids.js"; import type { RestartAttempt } from "./restart.types.js"; import { relaunchGatewayScheduledTask } from "./windows-task-restart.js"; @@ -131,10 +131,8 @@ export function writeGatewayRestartIntentSync(opts: { return false; } const env = opts.env ?? process.env; - let tmpPath: string | undefined; try { const intentPath = resolveGatewayRestartIntentPath(env); - fs.mkdirSync(path.dirname(intentPath), { recursive: true }); const payload: GatewayRestartIntentPayload = { kind: "gateway-restart", pid: targetPid, @@ -146,25 +144,14 @@ export function writeGatewayRestartIntentSync(opts: { ? { waitMs: Math.floor(opts.intent.waitMs) } : {}), }; - tmpPath = path.join( - path.dirname(intentPath), - `.${path.basename(intentPath)}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`, - ); - let fd: number | undefined; - try { - fd = fs.openSync(tmpPath, "wx", 0o600); - fs.writeFileSync(fd, `${JSON.stringify(payload)}\n`, "utf8"); - } finally { - if (fd !== undefined) { - fs.closeSync(fd); - } - } - fs.renameSync(tmpPath, intentPath); + replaceFileAtomicSync({ + filePath: intentPath, + content: `${JSON.stringify(payload)}\n`, + mode: 0o600, + tempPrefix: ".gateway-restart-intent", + }); return true; } catch (err) { - if (tmpPath) { - unlinkGatewayRestartIntentFileSync(tmpPath); - } restartLog.warn(`failed to write gateway restart intent: ${String(err)}`); return false; } diff --git a/src/infra/root-paths.ts b/src/infra/root-paths.ts new file mode 100644 index 00000000000..95cec3bccb7 --- /dev/null +++ b/src/infra/root-paths.ts @@ -0,0 +1,10 @@ +import "./fs-safe-defaults.js"; +export { + ensureDirectoryWithinRoot, + resolveExistingPathsWithinRoot, + resolvePathsWithinRoot, + resolvePathWithinRoot, + resolveStrictExistingPathsWithinRoot, + resolveWritablePathWithinRoot, +} from "@openclaw/fs-safe/advanced"; +export { pathScope } from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/safe-open-sync.test.ts b/src/infra/safe-open-sync.test.ts deleted file mode 100644 index 7e966fe1b2c..00000000000 --- a/src/infra/safe-open-sync.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import fs from "node:fs"; -import fsp from "node:fs/promises"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { withTempDir } from "../test-helpers/temp-dir.js"; -import { openVerifiedFileSync } from "./safe-open-sync.js"; - -type SafeOpenSyncFs = NonNullable[0]["ioFs"]>; -type SafeOpenSyncLstatSync = SafeOpenSyncFs["lstatSync"]; -type SafeOpenSyncRealpathSync = SafeOpenSyncFs["realpathSync"]; -type SafeOpenSyncFstatSync = SafeOpenSyncFs["fstatSync"]; - -function mockStat(params: { - isFile?: boolean; - isDirectory?: boolean; - nlink?: number; - size?: number; - dev?: number; - ino?: number; -}): fs.Stats { - return { - isFile: () => params.isFile ?? false, - isDirectory: () => params.isDirectory ?? false, - isSymbolicLink: () => false, - nlink: params.nlink ?? 1, - size: params.size ?? 0, - dev: params.dev ?? 1, - ino: params.ino ?? 1, - } as unknown as fs.Stats; -} - -function mockRealpathSync(result: string): SafeOpenSyncRealpathSync { - const resolvePath = ((_: fs.PathLike) => result) as SafeOpenSyncRealpathSync; - resolvePath.native = ((_: fs.PathLike) => result) as typeof resolvePath.native; - return resolvePath; -} - -function mockLstatSync(read: (filePath: fs.PathLike) => fs.Stats): SafeOpenSyncLstatSync { - return ((filePath: fs.PathLike) => read(filePath)) as unknown as SafeOpenSyncLstatSync; -} - -function mockFstatSync(stat: fs.Stats): SafeOpenSyncFstatSync { - return ((_: number) => stat) as unknown as SafeOpenSyncFstatSync; -} - -async function expectOpenFailure(params: { - setup: (root: string) => Promise[0]>; - expectedReason: "path" | "validation" | "io"; -}): Promise { - await withTempDir({ prefix: "openclaw-safe-open-" }, async (root) => { - const opened = openVerifiedFileSync(await params.setup(root)); - expect(opened.ok).toBe(false); - if (!opened.ok) { - expect(opened.reason).toBe(params.expectedReason); - } - }); -} - -function expectOpenReason( - opened: ReturnType, - expectedReason: "path" | "validation" | "io", -): void { - expect(opened.ok).toBe(false); - if (opened.ok) { - return; - } - expect(opened.reason).toBe(expectedReason); -} - -describe("openVerifiedFileSync", () => { - it.each([ - { - name: "missing files", - expectedReason: "path" as const, - setup: async (root: string) => ({ filePath: path.join(root, "missing.txt") }), - }, - { - name: "directories by default", - expectedReason: "validation" as const, - setup: async (root: string) => { - const targetDir = path.join(root, "nested"); - await fsp.mkdir(targetDir, { recursive: true }); - return { filePath: targetDir }; - }, - }, - { - name: "symlink paths when rejectPathSymlink is enabled", - expectedReason: "validation" as const, - setup: async (root: string) => { - const targetFile = path.join(root, "target.txt"); - const linkFile = path.join(root, "link.txt"); - await fsp.writeFile(targetFile, "hello"); - await fsp.symlink(targetFile, linkFile); - return { - filePath: linkFile, - rejectPathSymlink: true, - }; - }, - }, - { - name: "files larger than maxBytes", - expectedReason: "validation" as const, - setup: async (root: string) => { - const filePath = path.join(root, "payload.txt"); - await fsp.writeFile(filePath, "hello"); - return { - filePath, - maxBytes: 4, - }; - }, - }, - ])("fails for $name", async ({ setup, expectedReason }) => { - await expectOpenFailure({ setup, expectedReason }); - }); - - it("accepts directories when allowedType is directory", async () => { - await withTempDir({ prefix: "openclaw-safe-open-" }, async (root) => { - const targetDir = path.join(root, "nested"); - await fsp.mkdir(targetDir, { recursive: true }); - - const opened = openVerifiedFileSync({ - filePath: targetDir, - allowedType: "directory", - rejectHardlinks: true, - }); - expect(opened.ok).toBe(true); - if (!opened.ok) { - return; - } - expect(opened.stat.isDirectory()).toBe(true); - fs.closeSync(opened.fd); - }); - }); - - it("rejects post-open validation mismatches and closes the fd", () => { - const closeSync = (fd: number) => { - closed.push(fd); - }; - const closed: number[] = []; - const ioFs: SafeOpenSyncFs = { - constants: fs.constants, - lstatSync: mockLstatSync((filePath) => - String(filePath) === "/real/file.txt" - ? mockStat({ isFile: true, size: 1, dev: 1, ino: 1 }) - : mockStat({ isFile: false }), - ), - realpathSync: mockRealpathSync("/real/file.txt"), - openSync: () => 42, - fstatSync: mockFstatSync(mockStat({ isFile: true, size: 1, dev: 2, ino: 1 })), - closeSync, - }; - - const opened = openVerifiedFileSync({ - filePath: "/input/file.txt", - ioFs, - }); - expectOpenReason(opened, "validation"); - expect(closed).toEqual([42]); - }); - - it("reports non-path filesystem failures as io errors", () => { - const ioFs: SafeOpenSyncFs = { - constants: fs.constants, - lstatSync: () => { - const err = new Error("permission denied") as NodeJS.ErrnoException; - err.code = "EACCES"; - throw err; - }, - realpathSync: mockRealpathSync("/real/file.txt"), - openSync: () => 42, - fstatSync: mockFstatSync(mockStat({ isFile: true })), - closeSync: () => {}, - }; - - const opened = openVerifiedFileSync({ - filePath: "/input/file.txt", - rejectPathSymlink: true, - ioFs, - }); - expectOpenReason(opened, "io"); - }); -}); diff --git a/src/infra/safe-open-sync.ts b/src/infra/safe-open-sync.ts deleted file mode 100644 index d220ddb4d94..00000000000 --- a/src/infra/safe-open-sync.ts +++ /dev/null @@ -1,101 +0,0 @@ -import fs from "node:fs"; -import { sameFileIdentity as hasSameFileIdentity } from "./file-identity.js"; - -export type SafeOpenSyncFailureReason = "path" | "validation" | "io"; - -type SafeOpenSyncResult = - | { ok: true; path: string; fd: number; stat: fs.Stats } - | { ok: false; reason: SafeOpenSyncFailureReason; error?: unknown }; - -export type SafeOpenSyncAllowedType = "file" | "directory"; - -type SafeOpenSyncFs = Pick< - typeof fs, - "constants" | "lstatSync" | "realpathSync" | "openSync" | "fstatSync" | "closeSync" ->; - -function isExpectedPathError(error: unknown): boolean { - const code = - typeof error === "object" && error !== null && "code" in error ? String(error.code) : ""; - return code === "ENOENT" || code === "ENOTDIR" || code === "ELOOP"; -} - -function sameFileIdentity(left: fs.Stats, right: fs.Stats): boolean { - return hasSameFileIdentity(left, right); -} - -export function openVerifiedFileSync(params: { - filePath: string; - resolvedPath?: string; - rejectPathSymlink?: boolean; - rejectHardlinks?: boolean; - maxBytes?: number; - allowedType?: SafeOpenSyncAllowedType; - ioFs?: SafeOpenSyncFs; -}): SafeOpenSyncResult { - const ioFs = params.ioFs ?? fs; - const allowedType = params.allowedType ?? "file"; - const openReadFlags = - ioFs.constants.O_RDONLY | - (typeof ioFs.constants.O_NOFOLLOW === "number" ? ioFs.constants.O_NOFOLLOW : 0); - let fd: number | null = null; - try { - if (params.rejectPathSymlink) { - const candidateStat = ioFs.lstatSync(params.filePath); - if (candidateStat.isSymbolicLink()) { - return { ok: false, reason: "validation" }; - } - } - - const realPath = params.resolvedPath ?? ioFs.realpathSync(params.filePath); - const preOpenStat = ioFs.lstatSync(realPath); - if (!isAllowedType(preOpenStat, allowedType)) { - return { ok: false, reason: "validation" }; - } - if (params.rejectHardlinks && preOpenStat.isFile() && preOpenStat.nlink > 1) { - return { ok: false, reason: "validation" }; - } - if ( - params.maxBytes !== undefined && - preOpenStat.isFile() && - preOpenStat.size > params.maxBytes - ) { - return { ok: false, reason: "validation" }; - } - - fd = ioFs.openSync(realPath, openReadFlags); - const openedStat = ioFs.fstatSync(fd); - if (!isAllowedType(openedStat, allowedType)) { - return { ok: false, reason: "validation" }; - } - if (params.rejectHardlinks && openedStat.isFile() && openedStat.nlink > 1) { - return { ok: false, reason: "validation" }; - } - if (params.maxBytes !== undefined && openedStat.isFile() && openedStat.size > params.maxBytes) { - return { ok: false, reason: "validation" }; - } - if (!sameFileIdentity(preOpenStat, openedStat)) { - return { ok: false, reason: "validation" }; - } - - const opened = { ok: true as const, path: realPath, fd, stat: openedStat }; - fd = null; - return opened; - } catch (error) { - if (isExpectedPathError(error)) { - return { ok: false, reason: "path", error }; - } - return { ok: false, reason: "io", error }; - } finally { - if (fd !== null) { - ioFs.closeSync(fd); - } - } -} - -function isAllowedType(stat: fs.Stats, allowedType: SafeOpenSyncAllowedType): boolean { - if (allowedType === "directory") { - return stat.isDirectory(); - } - return stat.isFile(); -} diff --git a/src/infra/secret-file.test.ts b/src/infra/secret-file.test.ts index 3f585bd41fc..2645132ffac 100644 --- a/src/infra/secret-file.test.ts +++ b/src/infra/secret-file.test.ts @@ -4,7 +4,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; import { DEFAULT_SECRET_FILE_MAX_BYTES, - loadSecretFileSync, PRIVATE_SECRET_DIR_MODE, PRIVATE_SECRET_FILE_MODE, readSecretFileSync, @@ -53,36 +52,18 @@ describe("readSecretFileSync", () => { expect(tryReadSecretFileSync(file, "Gateway password")).toBe("top-secret"); }); - it.each([ - { - name: "surfaces resolvedPath and error details for missing files", - assert: (file: string) => { - expect(loadSecretFileSync(file, "Gateway password")).toMatchObject({ - ok: false, - resolvedPath: file, - message: expect.stringContaining(`Failed to inspect Gateway password file at ${file}:`), - error: expect.any(Error), - }); - }, - }, - { - name: "preserves the underlying cause when throwing for missing files", - assert: (file: string) => { - let thrown: Error | undefined; - try { - readSecretFileSync(file, "Gateway password"); - } catch (error) { - thrown = error as Error; - } - - expect(thrown).toBeInstanceOf(Error); - expect(thrown?.message).toContain(`Failed to inspect Gateway password file at ${file}:`); - expect((thrown as Error & { cause?: unknown }).cause).toBeInstanceOf(Error); - }, - }, - ])("$name", async ({ assert }) => { + it("preserves the underlying cause when throwing for missing files", async () => { const file = await createSecretPath(async (dir) => path.join(dir, "missing-secret.txt")); - assert(file); + let thrown: Error | undefined; + try { + readSecretFileSync(file, "Gateway password"); + } catch (error) { + thrown = error as Error; + } + + expect(thrown).toBeInstanceOf(Error); + expect(thrown?.message).toContain(`Failed to inspect Gateway password file at ${file}:`); + expect((thrown as Error & { cause?: unknown }).cause).toBeInstanceOf(Error); }); it.each([ @@ -131,23 +112,6 @@ describe("readSecretFileSync", () => { }); it.each([ - { - name: "exposes resolvedPath on non-throwing read failures", - pathValue: async () => - createSecretPath(async (dir) => { - const file = path.join(dir, "secret.txt"); - await fsPromises.writeFile(file, " \n\t ", "utf8"); - return file; - }), - label: "Gateway password", - options: undefined, - helper: "load" as const, - expected: (file: string | undefined) => ({ - ok: false, - resolvedPath: file, - message: `Gateway password file at ${file} is empty.`, - }), - }, { name: "returns undefined from the non-throwing helper for rejected files", pathValue: async () => @@ -160,34 +124,25 @@ describe("readSecretFileSync", () => { }), label: "Telegram bot token", options: { rejectSymlink: true }, - helper: "try" as const, - expected: () => undefined, + expected: undefined, }, { name: "returns undefined from the non-throwing helper for blank file paths", pathValue: async () => " ", label: "Telegram bot token", options: undefined, - helper: "try" as const, - expected: () => undefined, + expected: undefined, }, { name: "returns undefined from the non-throwing helper for missing path values", pathValue: async () => undefined, label: "Telegram bot token", options: undefined, - helper: "try" as const, - expected: () => undefined, + expected: undefined, }, - ])("$name", async ({ pathValue, label, options, helper, expected }) => { + ])("$name", async ({ pathValue, label, options, expected }) => { const file = await pathValue(); - if (helper === "load") { - expect(loadSecretFileSync(file as string, label, options)).toMatchObject( - (expected as (file: string | undefined) => Record)(file), - ); - return; - } - expect(tryReadSecretFileSync(file, label, options)).toBe((expected as () => undefined)()); + expect(tryReadSecretFileSync(file, label, options)).toBe(expected); }); }); @@ -202,10 +157,7 @@ describe("writePrivateSecretFileAtomic", () => { content: '{"ok":true}\n', }); - expect(loadSecretFileSync(file, "Gateway password")).toMatchObject({ - ok: true, - secret: '{"ok":true}', - }); + expect(readSecretFileSync(file, "Gateway password")).toBe('{"ok":true}'); if (process.platform !== "win32") { const dirStat = await fsPromises.stat(path.dirname(file)); const fileStat = await fsPromises.stat(file); diff --git a/src/infra/secret-file.ts b/src/infra/secret-file.ts index 75a24998f50..4cf28da5f95 100644 --- a/src/infra/secret-file.ts +++ b/src/infra/secret-file.ts @@ -1,18 +1,16 @@ -import { randomBytes } from "node:crypto"; -import fs from "node:fs"; -import fsp from "node:fs/promises"; -import path from "node:path"; +import "./fs-safe-defaults.js"; +import { readSecretFileSync as readSecretFileSyncImpl } from "@openclaw/fs-safe/secret"; import { resolveUserPath } from "../utils.js"; -import { openVerifiedFileSync } from "./safe-open-sync.js"; -export const DEFAULT_SECRET_FILE_MAX_BYTES = 16 * 1024; -export const PRIVATE_SECRET_DIR_MODE = 0o700; -export const PRIVATE_SECRET_FILE_MODE = 0o600; - -export type SecretFileReadOptions = { - maxBytes?: number; - rejectSymlink?: boolean; -}; +export { + DEFAULT_SECRET_FILE_MAX_BYTES, + PRIVATE_SECRET_DIR_MODE, + PRIVATE_SECRET_FILE_MODE, + readSecretFileSync, + tryReadSecretFileSync, + type SecretFileReadOptions, +} from "@openclaw/fs-safe/secret"; +export { writeSecretFileAtomic as writePrivateSecretFileAtomic } from "@openclaw/fs-safe/secret"; export type SecretFileReadResult = | { @@ -27,14 +25,11 @@ export type SecretFileReadResult = error?: unknown; }; -function normalizeSecretReadError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); -} - +/** @deprecated Use readSecretFileSync() or tryReadSecretFileSync(). */ export function loadSecretFileSync( filePath: string, label: string, - options: SecretFileReadOptions = {}, + options: Parameters[2] = {}, ): SecretFileReadResult { const trimmedPath = filePath.trim(); const resolvedPath = resolveUserPath(trimmedPath); @@ -42,240 +37,18 @@ export function loadSecretFileSync( return { ok: false, message: `${label} file path is empty.` }; } - const maxBytes = options.maxBytes ?? DEFAULT_SECRET_FILE_MAX_BYTES; - - let previewStat: fs.Stats; try { - previewStat = fs.lstatSync(resolvedPath); + return { + ok: true, + secret: readSecretFileSyncImpl(filePath, label, options), + resolvedPath, + }; } catch (error) { - const normalized = normalizeSecretReadError(error); - return { - ok: false, - resolvedPath, - error: normalized, - message: `Failed to inspect ${label} file at ${resolvedPath}: ${String(normalized)}`, - }; - } - - if (options.rejectSymlink && previewStat.isSymbolicLink()) { - return { - ok: false, - resolvedPath, - message: `${label} file at ${resolvedPath} must not be a symlink.`, - }; - } - if (!previewStat.isFile()) { - return { - ok: false, - resolvedPath, - message: `${label} file at ${resolvedPath} must be a regular file.`, - }; - } - if (previewStat.size > maxBytes) { - return { - ok: false, - resolvedPath, - message: `${label} file at ${resolvedPath} exceeds ${maxBytes} bytes.`, - }; - } - - const opened = openVerifiedFileSync({ - filePath: resolvedPath, - rejectPathSymlink: options.rejectSymlink, - maxBytes, - }); - if (!opened.ok) { - const error = normalizeSecretReadError( - opened.reason === "validation" ? new Error("security validation failed") : opened.error, - ); return { ok: false, + message: error instanceof Error ? error.message : String(error), resolvedPath, error, - message: `Failed to read ${label} file at ${resolvedPath}: ${String(error)}`, }; } - - try { - const raw = fs.readFileSync(opened.fd, "utf8"); - const secret = raw.trim(); - if (!secret) { - return { - ok: false, - resolvedPath, - message: `${label} file at ${resolvedPath} is empty.`, - }; - } - return { ok: true, secret, resolvedPath }; - } catch (error) { - const normalized = normalizeSecretReadError(error); - return { - ok: false, - resolvedPath, - error: normalized, - message: `Failed to read ${label} file at ${resolvedPath}: ${String(normalized)}`, - }; - } finally { - fs.closeSync(opened.fd); - } -} - -export function readSecretFileSync( - filePath: string, - label: string, - options: SecretFileReadOptions = {}, -): string { - const result = loadSecretFileSync(filePath, label, options); - if (result.ok) { - return result.secret; - } - throw new Error(result.message, result.error ? { cause: result.error } : undefined); -} - -export function tryReadSecretFileSync( - filePath: string | undefined, - label: string, - options: SecretFileReadOptions = {}, -): string | undefined { - if (!filePath?.trim()) { - return undefined; - } - const result = loadSecretFileSync(filePath, label, options); - return result.ok ? result.secret : undefined; -} - -function assertPathWithinRoot(rootDir: string, targetPath: string): void { - const relative = path.relative(rootDir, targetPath); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error(`Private secret path must stay under ${rootDir}.`); - } -} - -function assertRealPathWithinRoot(rootDir: string, targetPath: string): void { - const relative = path.relative(rootDir, targetPath); - if (relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error(`Private secret path must stay under ${rootDir}.`); - } -} - -async function enforcePrivatePathMode( - resolvedPath: string, - expectedMode: number, - kind: "directory" | "file", -): Promise { - if (process.platform === "win32") { - return; - } - await fsp.chmod(resolvedPath, expectedMode); - const stat = await fsp.stat(resolvedPath); - const actualMode = stat.mode & 0o777; - if (actualMode !== expectedMode) { - throw new Error( - `Private secret ${kind} ${resolvedPath} has insecure permissions ${actualMode.toString(8)}.`, - ); - } -} - -async function ensurePrivateDirectory(rootDir: string, targetDir: string): Promise { - const resolvedRoot = path.resolve(rootDir); - const resolvedTarget = path.resolve(targetDir); - if (resolvedTarget === resolvedRoot) { - await fsp.mkdir(resolvedRoot, { recursive: true, mode: PRIVATE_SECRET_DIR_MODE }); - const rootStat = await fsp.lstat(resolvedRoot); - if (rootStat.isSymbolicLink()) { - throw new Error(`Private secret root ${resolvedRoot} must not be a symlink.`); - } - if (!rootStat.isDirectory()) { - throw new Error(`Private secret root ${resolvedRoot} must be a directory.`); - } - await enforcePrivatePathMode(resolvedRoot, PRIVATE_SECRET_DIR_MODE, "directory"); - return; - } - - assertPathWithinRoot(resolvedRoot, resolvedTarget); - await ensurePrivateDirectory(resolvedRoot, resolvedRoot); - const resolvedRootReal = await fsp.realpath(resolvedRoot); - - let current = resolvedRoot; - for (const segment of path - .relative(resolvedRoot, resolvedTarget) - .split(path.sep) - .filter(Boolean)) { - current = path.join(current, segment); - try { - const stat = await fsp.lstat(current); - if (stat.isSymbolicLink()) { - throw new Error(`Private secret directory component ${current} must not be a symlink.`); - } - if (!stat.isDirectory()) { - throw new Error(`Private secret directory component ${current} must be a directory.`); - } - } catch (error) { - if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") { - throw error; - } - await fsp.mkdir(current, { mode: PRIVATE_SECRET_DIR_MODE }); - } - const currentReal = await fsp.realpath(current); - assertRealPathWithinRoot(resolvedRootReal, currentReal); - await enforcePrivatePathMode(currentReal, PRIVATE_SECRET_DIR_MODE, "directory"); - } -} - -export async function writePrivateSecretFileAtomic(params: { - rootDir: string; - filePath: string; - content: string | Uint8Array; -}): Promise { - const resolvedRoot = path.resolve(params.rootDir); - const resolvedFile = path.resolve(params.filePath); - assertPathWithinRoot(resolvedRoot, resolvedFile); - const intendedParentDir = path.dirname(resolvedFile); - await ensurePrivateDirectory(resolvedRoot, intendedParentDir); - const resolvedRootReal = await fsp.realpath(resolvedRoot); - const parentDir = await fsp.realpath(intendedParentDir); - assertRealPathWithinRoot(resolvedRootReal, parentDir); - const fileName = path.basename(resolvedFile); - const finalFilePath = path.join(parentDir, fileName); - - try { - const stat = await fsp.lstat(finalFilePath); - if (stat.isSymbolicLink()) { - throw new Error(`Private secret file ${finalFilePath} must not be a symlink.`); - } - if (!stat.isFile()) { - throw new Error(`Private secret file ${finalFilePath} must be a regular file.`); - } - } catch (error) { - if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") { - throw error; - } - } - - const tempPath = path.join( - parentDir, - `.tmp-${process.pid}-${Date.now()}-${randomBytes(6).toString("hex")}`, - ); - let createdTemp = false; - try { - const handle = await fsp.open(tempPath, "wx", PRIVATE_SECRET_FILE_MODE); - createdTemp = true; - try { - await handle.writeFile(params.content); - } finally { - await handle.close(); - } - await enforcePrivatePathMode(tempPath, PRIVATE_SECRET_FILE_MODE, "file"); - const refreshedParentReal = await fsp.realpath(intendedParentDir); - if (refreshedParentReal !== parentDir) { - throw new Error(`Private secret parent directory changed during write for ${finalFilePath}.`); - } - await fsp.rename(tempPath, finalFilePath); - createdTemp = false; - await enforcePrivatePathMode(finalFilePath, PRIVATE_SECRET_FILE_MODE, "file"); - } finally { - if (createdTemp) { - await fsp.unlink(tempPath).catch(() => undefined); - } - } } diff --git a/src/infra/session-cost-usage.ts b/src/infra/session-cost-usage.ts index ce4434622a2..3a3f32e32e7 100644 --- a/src/infra/session-cost-usage.ts +++ b/src/infra/session-cost-usage.ts @@ -28,6 +28,7 @@ import { resolveModelCostConfigFingerprint, } from "../utils/usage-format.js"; import { formatErrorMessage } from "./errors.js"; +import { replaceFileAtomic } from "./replace-file.js"; import type { CostBreakdown, CostUsageTotals, @@ -337,10 +338,11 @@ async function readUsageCostCache(cachePath: string): Promise { - const tmpPath = `${cachePath}.${process.pid}.${Date.now()}.tmp`; - await fs.promises.mkdir(path.dirname(cachePath), { recursive: true }); - await fs.promises.writeFile(tmpPath, `${JSON.stringify(cache)}\n`, "utf-8"); - await fs.promises.rename(tmpPath, cachePath); + await replaceFileAtomic({ + filePath: cachePath, + content: `${JSON.stringify(cache)}\n`, + tempPrefix: ".usage-cost-cache", + }); } async function listUsageCountedTranscriptFiles( diff --git a/src/infra/session-delivery-queue-storage.ts b/src/infra/session-delivery-queue-storage.ts index dcbf629bd6a..0fc87752752 100644 --- a/src/infra/session-delivery-queue-storage.ts +++ b/src/infra/session-delivery-queue-storage.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import type { ChatType } from "../channels/chat-type.js"; import { resolveStateDir } from "../config/paths.js"; +import { replaceFileAtomic } from "./replace-file.js"; import { generateSecureUuid } from "./secure-random.js"; const QUEUE_DIRNAME = "session-delivery-queue"; @@ -86,12 +87,12 @@ async function unlinkStaleTmpBestEffort(filePath: string, now: number): Promise< } async function writeQueueEntry(filePath: string, entry: QueuedSessionDelivery): Promise { - const tmp = `${filePath}.${process.pid}.tmp`; - await fs.promises.writeFile(tmp, JSON.stringify(entry, null, 2), { - encoding: "utf-8", + await replaceFileAtomic({ + filePath, + content: JSON.stringify(entry, null, 2), mode: 0o600, + tempPrefix: ".session-delivery-queue", }); - await fs.promises.rename(tmp, filePath); } async function readQueueEntry(filePath: string): Promise { diff --git a/src/infra/sibling-temp-file.ts b/src/infra/sibling-temp-file.ts new file mode 100644 index 00000000000..a0845107221 --- /dev/null +++ b/src/infra/sibling-temp-file.ts @@ -0,0 +1,6 @@ +import "./fs-safe-defaults.js"; +export { + writeSiblingTempFile, + type WriteSiblingTempFileOptions, + type WriteSiblingTempFileResult, +} from "@openclaw/fs-safe/advanced"; diff --git a/src/infra/temp-download.ts b/src/infra/temp-download.ts index 89d10b63962..71f371232e2 100644 --- a/src/infra/temp-download.ts +++ b/src/infra/temp-download.ts @@ -1,7 +1,8 @@ +import "./fs-safe-defaults.js"; import crypto from "node:crypto"; -import { mkdtemp, rm } from "node:fs/promises"; import path from "node:path"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { tempWorkspace, type TempWorkspace } from "./private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js"; const logger = createSubsystemLogger("infra:temp-download"); @@ -11,15 +12,21 @@ export { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js"; type TempDownloadTarget = { dir: string; path: string; + file(fileName?: string): string; cleanup: () => Promise; + [Symbol.asyncDispose](): Promise; }; -function sanitizePrefix(prefix: string): string { +function resolveTempRoot(tmpDir?: string): string { + return tmpDir ?? resolvePreferredOpenClawTmpDir(); +} + +function sanitizeTempPrefix(prefix: string): string { const normalized = prefix.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, ""); return normalized || "tmp"; } -function sanitizeExtension(extension?: string): string { +function sanitizeTempExtension(extension?: string): string { if (!extension) { return ""; } @@ -35,29 +42,6 @@ export function sanitizeTempFileName(fileName: string): string { return normalized || "download.bin"; } -function resolveTempRoot(tmpDir?: string): string { - return tmpDir ?? resolvePreferredOpenClawTmpDir(); -} - -function isNodeErrorWithCode(err: unknown, code: string): boolean { - return ( - typeof err === "object" && - err !== null && - "code" in err && - (err as { code?: string }).code === code - ); -} - -async function cleanupTempDir(dir: string) { - try { - await rm(dir, { recursive: true, force: true }); - } catch (err) { - if (!isNodeErrorWithCode(err, "ENOENT")) { - logger.warn(`temp-path cleanup failed for ${dir}: ${String(err)}`, { dir, error: err }); - } - } -} - export function buildRandomTempFilePath(params: { prefix: string; extension?: string; @@ -65,15 +49,33 @@ export function buildRandomTempFilePath(params: { now?: number; uuid?: string; }): string { - const prefix = sanitizePrefix(params.prefix); - const extension = sanitizeExtension(params.extension); const nowCandidate = params.now; const now = typeof nowCandidate === "number" && Number.isFinite(nowCandidate) ? Math.trunc(nowCandidate) : Date.now(); const uuid = params.uuid?.trim() || crypto.randomUUID(); - return path.join(resolveTempRoot(params.tmpDir), `${prefix}-${now}-${uuid}${extension}`); + return path.join( + resolveTempRoot(params.tmpDir), + `${sanitizeTempPrefix(params.prefix)}-${now}-${uuid}${sanitizeTempExtension(params.extension)}`, + ); +} + +function buildTempDownloadTarget( + workspace: TempWorkspace, + fileName: string | undefined, +): TempDownloadTarget { + const file = (nextName?: string) => + workspace.path(sanitizeTempFileName(nextName ?? fileName ?? "download.bin")); + return { + dir: workspace.dir, + path: file(), + file, + cleanup: async () => { + await workspace.cleanup(); + }, + [Symbol.asyncDispose]: workspace[Symbol.asyncDispose].bind(workspace), + }; } export async function createTempDownloadTarget(params: { @@ -81,15 +83,22 @@ export async function createTempDownloadTarget(params: { fileName?: string; tmpDir?: string; }): Promise { - const tempRoot = resolveTempRoot(params.tmpDir); - const prefix = `${sanitizePrefix(params.prefix)}-`; - const dir = await mkdtemp(path.join(tempRoot, prefix)); + const workspace = await tempWorkspace({ + rootDir: resolveTempRoot(params.tmpDir), + prefix: sanitizeTempPrefix(params.prefix), + }); + const target = buildTempDownloadTarget(workspace, params.fileName); + const cleanup = async () => { + try { + await workspace.cleanup(); + } catch (err) { + logger.warn(`temp-path cleanup failed: ${String(err)}`, { error: err }); + } + }; return { - dir, - path: path.join(dir, sanitizeTempFileName(params.fileName ?? "download.bin")), - cleanup: async () => { - await cleanupTempDir(dir); - }, + ...target, + cleanup, + [Symbol.asyncDispose]: cleanup, }; } diff --git a/src/infra/tls/gateway.ts b/src/infra/tls/gateway.ts index d4ff5c2116d..bb04dd5ab3c 100644 --- a/src/infra/tls/gateway.ts +++ b/src/infra/tls/gateway.ts @@ -6,6 +6,7 @@ import tls from "node:tls"; import { promisify } from "node:util"; import type { GatewayTlsConfig } from "../../config/types.gateway.js"; import { CONFIG_DIR, ensureDir, resolveUserPath, shortenHomeInString } from "../../utils.js"; +import { pathExists } from "../fs-safe.js"; import { resolveSystemBin } from "../resolve-system-bin.js"; import { normalizeFingerprint } from "./fingerprint.js"; @@ -22,15 +23,6 @@ export type GatewayTlsRuntime = { error?: string; }; -async function fileExists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - async function generateSelfSignedCert(params: { certPath: string; keyPath: string; @@ -85,8 +77,8 @@ export async function loadGatewayTlsRuntime( const keyPath = resolveUserPath(cfg.keyPath ?? path.join(baseDir, "gateway-key.pem")); const caPath = cfg.caPath ? resolveUserPath(cfg.caPath) : undefined; - const hasCert = await fileExists(certPath); - const hasKey = await fileExists(keyPath); + const hasCert = await pathExists(certPath); + const hasKey = await pathExists(keyPath); if (!hasCert && !hasKey && autoGenerate) { try { @@ -102,7 +94,7 @@ export async function loadGatewayTlsRuntime( } } - if (!(await fileExists(certPath)) || !(await fileExists(keyPath))) { + if (!(await pathExists(certPath)) || !(await pathExists(keyPath))) { return { enabled: false, required: true, diff --git a/src/infra/tmp-openclaw-dir.ts b/src/infra/tmp-openclaw-dir.ts index a69a6eaf7a0..b88b2ad3871 100644 --- a/src/infra/tmp-openclaw-dir.ts +++ b/src/infra/tmp-openclaw-dir.ts @@ -1,205 +1,26 @@ -import fs from "node:fs"; -import { tmpdir as getOsTmpDir } from "node:os"; -import path from "node:path"; +import "./fs-safe-defaults.js"; +import { resolveSecureTempRoot, type ResolveSecureTempRootOptions } from "@openclaw/fs-safe/temp"; export const POSIX_OPENCLAW_TMP_DIR = "/tmp/openclaw"; -type ResolvePreferredOpenClawTmpDirOptions = { - accessSync?: (path: string, mode?: number) => void; - chmodSync?: (path: string, mode: number) => void; - lstatSync?: (path: string) => { - isDirectory(): boolean; - isSymbolicLink(): boolean; - mode?: number; - uid?: number; - }; - mkdirSync?: (path: string, opts: { recursive: boolean; mode?: number }) => void; - getuid?: () => number | undefined; - tmpdir?: () => string; - warn?: (message: string) => void; -}; - -type MaybeNodeError = { code?: string }; - -function isNodeErrorWithCode(err: unknown, code: string): err is MaybeNodeError { - return ( - typeof err === "object" && - err !== null && - "code" in err && - (err as MaybeNodeError).code === code - ); -} - -type ResolvePreferredOpenClawTmpDirInternalOptions = ResolvePreferredOpenClawTmpDirOptions & { - /** Test seam for the host platform; defaults to `process.platform`. */ - platform?: NodeJS.Platform; -}; +type ResolvePreferredOpenClawTmpDirOptions = Omit< + ResolveSecureTempRootOptions, + | "fallbackPrefix" + | "preferredDir" + | "skipPreferredOnWindows" + | "unsafeFallbackLabel" + | "warningPrefix" +>; export function resolvePreferredOpenClawTmpDir( - options: ResolvePreferredOpenClawTmpDirInternalOptions = {}, + options: ResolvePreferredOpenClawTmpDirOptions = {}, ): string { - // Evaluated here (not at module load) so this file is safe to import in browser bundles. - const TMP_DIR_ACCESS_MODE = fs.constants.W_OK | fs.constants.X_OK; - const accessSync = options.accessSync ?? fs.accessSync; - const chmodSync = options.chmodSync ?? fs.chmodSync; - const lstatSync = options.lstatSync ?? fs.lstatSync; - const mkdirSync = options.mkdirSync ?? fs.mkdirSync; - const warn = options.warn ?? ((message: string) => console.warn(message)); - const getuid = - options.getuid ?? - (() => { - try { - return typeof process.getuid === "function" ? process.getuid() : undefined; - } catch { - return undefined; - } - }); - const tmpdir = typeof options.tmpdir === "function" ? options.tmpdir : getOsTmpDir; - const platform = options.platform ?? process.platform; - const uid = getuid(); - - const isSecureDirForUser = (st: { mode?: number; uid?: number }): boolean => { - if (uid === undefined) { - return true; - } - if (typeof st.uid === "number" && st.uid !== uid) { - return false; - } - // Avoid group/other writable dirs when running on multi-user hosts. - if (typeof st.mode === "number" && (st.mode & 0o022) !== 0) { - return false; - } - return true; - }; - - const fallback = (): string => { - const base = tmpdir(); - const suffix = uid === undefined ? "openclaw" : `openclaw-${uid}`; - // Use the platform-specific joiner so Windows fallbacks stay in pure - // backslash form even when the host process is non-Windows (e.g. when - // tests inject `platform: "win32"` on a Linux runner). - const joiner = platform === "win32" ? path.win32.join : path.join; - return joiner(base, suffix); - }; - - const isTrustedTmpDir = (st: { - isDirectory(): boolean; - isSymbolicLink(): boolean; - mode?: number; - uid?: number; - }): boolean => { - return st.isDirectory() && !st.isSymbolicLink() && isSecureDirForUser(st); - }; - - const resolveDirState = (candidatePath: string): "available" | "missing" | "invalid" => { - try { - const candidate = lstatSync(candidatePath); - if (!isTrustedTmpDir(candidate)) { - return "invalid"; - } - accessSync(candidatePath, TMP_DIR_ACCESS_MODE); - return "available"; - } catch (err) { - if (isNodeErrorWithCode(err, "ENOENT")) { - return "missing"; - } - return "invalid"; - } - }; - - const tryRepairWritableBits = (candidatePath: string): boolean => { - try { - const st = lstatSync(candidatePath); - if (!st.isDirectory() || st.isSymbolicLink()) { - return false; - } - if (uid !== undefined && typeof st.uid === "number" && st.uid !== uid) { - return false; - } - if (typeof st.mode !== "number") { - return false; - } - if ((st.mode & 0o022) === 0) { - return resolveDirState(candidatePath) === "available"; - } - try { - chmodSync(candidatePath, 0o700); - } catch (chmodErr) { - if ( - isNodeErrorWithCode(chmodErr, "EPERM") || - isNodeErrorWithCode(chmodErr, "EACCES") || - isNodeErrorWithCode(chmodErr, "ENOENT") - ) { - return resolveDirState(candidatePath) === "available"; - } - throw chmodErr; - } - warn(`[openclaw] tightened permissions on temp dir: ${candidatePath}`); - return resolveDirState(candidatePath) === "available"; - } catch { - return false; - } - }; - - const ensureTrustedFallbackDir = (): string => { - const fallbackPath = fallback(); - const state = resolveDirState(fallbackPath); - if (state === "available") { - return fallbackPath; - } - if (state === "invalid") { - if (tryRepairWritableBits(fallbackPath)) { - return fallbackPath; - } - throw new Error(`Unsafe fallback OpenClaw temp dir: ${fallbackPath}`); - } - try { - mkdirSync(fallbackPath, { recursive: true, mode: 0o700 }); - chmodSync(fallbackPath, 0o700); - } catch { - throw new Error(`Unable to create fallback OpenClaw temp dir: ${fallbackPath}`); - } - if (resolveDirState(fallbackPath) !== "available" && !tryRepairWritableBits(fallbackPath)) { - throw new Error(`Unsafe fallback OpenClaw temp dir: ${fallbackPath}`); - } - return fallbackPath; - }; - - // On Windows, Node resolves the POSIX path `/tmp` to `C:\tmp` (relative to - // the current drive root). Many Windows hosts have `C:\tmp` because Git, - // MSYS2, and other Unix-compat tools create it; the existing logic then - // happily writes logs and TTS files to `C:\tmp\openclaw\` while every - // other code path expects `%TEMP%\openclaw\`. Skip the POSIX preferred - // path entirely on Windows so the function falls through to the - // os.tmpdir() fallback (#60713). - if (platform === "win32") { - return ensureTrustedFallbackDir(); - } - - const existingPreferredState = resolveDirState(POSIX_OPENCLAW_TMP_DIR); - if (existingPreferredState === "available") { - return POSIX_OPENCLAW_TMP_DIR; - } - if (existingPreferredState === "invalid") { - if (tryRepairWritableBits(POSIX_OPENCLAW_TMP_DIR)) { - return POSIX_OPENCLAW_TMP_DIR; - } - return ensureTrustedFallbackDir(); - } - - try { - accessSync("/tmp", TMP_DIR_ACCESS_MODE); - // Create with a safe default; subsequent callers expect it exists. - mkdirSync(POSIX_OPENCLAW_TMP_DIR, { recursive: true, mode: 0o700 }); - chmodSync(POSIX_OPENCLAW_TMP_DIR, 0o700); - if ( - resolveDirState(POSIX_OPENCLAW_TMP_DIR) !== "available" && - !tryRepairWritableBits(POSIX_OPENCLAW_TMP_DIR) - ) { - return ensureTrustedFallbackDir(); - } - return POSIX_OPENCLAW_TMP_DIR; - } catch { - return ensureTrustedFallbackDir(); - } + return resolveSecureTempRoot({ + ...options, + fallbackPrefix: "openclaw", + preferredDir: POSIX_OPENCLAW_TMP_DIR, + skipPreferredOnWindows: true, + unsafeFallbackLabel: "OpenClaw temp dir", + warningPrefix: "[openclaw]", + }); } diff --git a/src/infra/update-startup.ts b/src/infra/update-startup.ts index 97cdc6daa29..d808dc5e9fd 100644 --- a/src/infra/update-startup.ts +++ b/src/infra/update-startup.ts @@ -8,7 +8,7 @@ import { runCommandWithTimeout } from "../process/exec.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; import { VERSION } from "../version.js"; import { isTruthyEnvValue } from "./env.js"; -import { writeJsonAtomic } from "./json-files.js"; +import { writeJson } from "./json-files.js"; import { resolveOpenClawPackageRoot } from "./openclaw-root.js"; import { normalizeUpdateChannel, DEFAULT_PACKAGE_CHANNEL } from "./update-channels.js"; import { compareSemverStrings, resolveNpmChannelTag, checkUpdateStatus } from "./update-check.js"; @@ -127,7 +127,7 @@ async function readState(statePath: string): Promise { } async function writeState(statePath: string, state: UpdateCheckState): Promise { - await writeJsonAtomic(statePath, state); + await writeJson(statePath, state); } function sameUpdateAvailable(a: UpdateAvailable | null, b: UpdateAvailable | null): boolean { diff --git a/src/infra/voicewake-routing.ts b/src/infra/voicewake-routing.ts index aad5320bc61..ed68a50e38a 100644 --- a/src/infra/voicewake-routing.ts +++ b/src/infra/voicewake-routing.ts @@ -5,7 +5,7 @@ import { isValidAgentId, normalizeAgentId, } from "../routing/session-key.js"; -import { createAsyncLock, readJsonFile, writeJsonAtomic } from "./json-files.js"; +import { createAsyncLock, tryReadJson, writeJson } from "./json-files.js"; type VoiceWakeRouteTarget = | { mode: "current"; agentId?: undefined; sessionKey?: undefined } @@ -265,7 +265,7 @@ export async function loadVoiceWakeRoutingConfig( baseDir?: string, ): Promise { const filePath = resolvePath(baseDir); - const existing = await readJsonFile(filePath); + const existing = await tryReadJson(filePath); if (!existing) { return { ...DEFAULT_ROUTING }; } @@ -283,7 +283,7 @@ export async function setVoiceWakeRoutingConfig( ...normalized, updatedAtMs: Date.now(), }; - await writeJsonAtomic(filePath, next); + await writeJson(filePath, next); return next; }); } diff --git a/src/infra/voicewake.ts b/src/infra/voicewake.ts index a232a6beea3..f53da3f651d 100644 --- a/src/infra/voicewake.ts +++ b/src/infra/voicewake.ts @@ -1,7 +1,7 @@ import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; -import { createAsyncLock, readJsonFile, writeJsonAtomic } from "./json-files.js"; +import { createAsyncLock, tryReadJson, writeJson } from "./json-files.js"; type VoiceWakeConfig = { triggers: string[]; @@ -30,7 +30,7 @@ export function defaultVoiceWakeTriggers() { export async function loadVoiceWakeConfig(baseDir?: string): Promise { const filePath = resolvePath(baseDir); - const existing = await readJsonFile(filePath); + const existing = await tryReadJson(filePath); if (!existing) { return { triggers: defaultVoiceWakeTriggers(), updatedAtMs: 0 }; } @@ -54,7 +54,7 @@ export async function setVoiceWakeTriggers( triggers: sanitized, updatedAtMs: Date.now(), }; - await writeJsonAtomic(filePath, next); + await writeJson(filePath, next); return next; }); } diff --git a/src/logging/diagnostic-stability-bundle.ts b/src/logging/diagnostic-stability-bundle.ts index 7d9d96542a6..1e8d7cbf223 100644 --- a/src/logging/diagnostic-stability-bundle.ts +++ b/src/logging/diagnostic-stability-bundle.ts @@ -3,6 +3,7 @@ import path from "node:path"; import process from "node:process"; import { resolveStateDir } from "../config/paths.js"; import { registerFatalErrorHook } from "../infra/fatal-error-hooks.js"; +import { replaceFileAtomicSync } from "../infra/replace-file.js"; import { getDiagnosticStabilitySnapshot, MAX_DIAGNOSTIC_STABILITY_LIMIT, @@ -640,14 +641,14 @@ export function writeDiagnosticStabilityBundleSync( }; const dir = resolveDiagnosticStabilityBundleDir(options); - fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); const file = buildBundlePath(dir, now, reason); - const tmpFile = `${file}.${process.pid}.tmp`; - fs.writeFileSync(tmpFile, `${JSON.stringify(bundle, null, 2)}\n`, { - encoding: "utf8", + replaceFileAtomicSync({ + filePath: file, + content: `${JSON.stringify(bundle, null, 2)}\n`, + dirMode: 0o700, mode: 0o600, + tempPrefix: ".openclaw-stability", }); - fs.renameSync(tmpFile, file); pruneOldBundles(dir, options.retention ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION); return { status: "written", path: file, bundle }; } catch (error) { diff --git a/src/logging/diagnostic-support-bundle.ts b/src/logging/diagnostic-support-bundle.ts index 44fe13c80dc..4687834f337 100644 --- a/src/logging/diagnostic-support-bundle.ts +++ b/src/logging/diagnostic-support-bundle.ts @@ -1,5 +1,6 @@ import fsp from "node:fs/promises"; import path from "node:path"; +import { isPathInside } from "../infra/path-guards.js"; export type DiagnosticSupportBundleFile = { path: string; @@ -81,8 +82,7 @@ function resolveSupportBundleFilePath(outputDir: string, pathName: string): stri const safePath = assertSafeBundleRelativePath(pathName); const resolvedBase = path.resolve(outputDir); const resolvedFile = path.resolve(resolvedBase, safePath); - const relative = path.relative(resolvedBase, resolvedFile); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { + if (resolvedFile === resolvedBase || !isPathInside(resolvedBase, resolvedFile)) { throw new Error(`Bundle file path escaped output directory: ${pathName}`); } return resolvedFile; diff --git a/src/logging/logger.ts b/src/logging/logger.ts index abf1f8f5cb5..79c8ffb54fc 100644 --- a/src/logging/logger.ts +++ b/src/logging/logger.ts @@ -13,6 +13,7 @@ import { } from "../infra/diagnostic-trace-context.js"; import { expandHomePrefix } from "../infra/home-dir.js"; import { isBlockedObjectKey } from "../infra/prototype-keys.js"; +import { appendRegularFileSync } from "../infra/regular-file.js"; import { POSIX_OPENCLAW_TMP_DIR, resolvePreferredOpenClawTmpDir, @@ -597,7 +598,7 @@ function getCurrentLogFileBytes(file: string): number { function appendLogLine(file: string, line: string): boolean { try { - fs.appendFileSync(file, line, { encoding: "utf8" }); + appendRegularFileSync({ filePath: file, content: line }); return true; } catch { return false; diff --git a/src/media-understanding/attachments.cache.ts b/src/media-understanding/attachments.cache.ts index 194093d61d9..d5f1ceae852 100644 --- a/src/media-understanding/attachments.cache.ts +++ b/src/media-understanding/attachments.cache.ts @@ -1,7 +1,7 @@ -import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { logVerbose, shouldLogVerbose } from "../globals.js"; +import { FsSafeError, openLocalFileSafely } from "../infra/fs-safe.js"; import type { SsrFPolicy } from "../infra/net/ssrf.js"; import { isAbortError } from "../infra/unhandled-rejections.js"; import { fetchRemoteMedia, MediaFetchError } from "../media/fetch.js"; @@ -320,28 +320,18 @@ export class MediaAttachmentCache { } try { const currentPath = entry.resolvedPath; - const stat = await fs.stat(currentPath); - if (!stat.isFile()) { - entry.resolvedPath = undefined; - throw new MediaUnderstandingSkipError( - "empty", - `Attachment ${entry.attachment.index + 1} path is not a regular file.`, - ); + const opened = await openLocalFileSafely({ filePath: currentPath }); + let canonicalRoots: readonly string[]; + try { + canonicalRoots = await this.getCanonicalLocalPathRoots(); + } finally { + await opened.handle.close().catch(() => {}); } - const canonicalPath = await this.resolveCanonicalLocalPath(currentPath); - if (!canonicalPath) { - entry.resolvedPath = undefined; - throw new MediaUnderstandingSkipError( - "blocked", - `Attachment ${entry.attachment.index + 1} could not be canonicalized.`, - ); - } - const canonicalRoots = await this.getCanonicalLocalPathRoots(); - if (!isInboundPathAllowed({ filePath: canonicalPath, roots: canonicalRoots })) { + if (!isInboundPathAllowed({ filePath: opened.realPath, roots: canonicalRoots })) { entry.resolvedPath = undefined; if (shouldLogVerbose()) { logVerbose( - `Blocked canonicalized attachment path outside allowed roots: ${canonicalPath}`, + `Blocked canonicalized attachment path outside allowed roots: ${opened.realPath}`, ); } throw new MediaUnderstandingSkipError( @@ -349,13 +339,33 @@ export class MediaAttachmentCache { `Attachment ${entry.attachment.index + 1} path is outside allowed roots.`, ); } - entry.resolvedPath = canonicalPath; - entry.statSize = stat.size; - return stat.size; + entry.resolvedPath = opened.realPath; + entry.statSize = opened.stat.size; + return opened.stat.size; } catch (err) { if (err instanceof MediaUnderstandingSkipError) { throw err; } + if (err instanceof FsSafeError) { + entry.resolvedPath = undefined; + if (err.code === "not-file") { + throw new MediaUnderstandingSkipError( + "empty", + `Attachment ${entry.attachment.index + 1} path is not a regular file.`, + ); + } + if (err.code !== "not-found") { + throw new MediaUnderstandingSkipError( + "blocked", + `Attachment ${entry.attachment.index + 1} path is outside allowed roots.`, + ); + } + } else { + throw new MediaUnderstandingSkipError( + "blocked", + `Attachment ${entry.attachment.index + 1} could not be canonicalized.`, + ); + } entry.resolvedPath = undefined; if (shouldLogVerbose()) { logVerbose(`Failed to read attachment ${entry.attachment.index + 1}: ${String(err)}`); @@ -388,54 +398,52 @@ export class MediaAttachmentCache { filePath: string; maxBytes: number; }): Promise { - const flags = - fsConstants.O_RDONLY | (process.platform === "win32" ? 0 : fsConstants.O_NOFOLLOW); - const handle = await fs.open(params.filePath, flags); + let opened: Awaited> | undefined; try { - const stat = await handle.stat(); - if (!stat.isFile()) { + opened = await openLocalFileSafely({ filePath: params.filePath }); + if (opened.stat.size > params.maxBytes) { throw new MediaUnderstandingSkipError( - "empty", - `Attachment ${params.attachmentIndex + 1} path is not a regular file.`, - ); - } - const canonicalPath = await this.resolveCanonicalLocalPath(params.filePath); - if (!canonicalPath) { - throw new MediaUnderstandingSkipError( - "blocked", - `Attachment ${params.attachmentIndex + 1} could not be canonicalized.`, + "maxBytes", + `Attachment ${params.attachmentIndex + 1} exceeds maxBytes ${params.maxBytes}`, ); } const canonicalRoots = await this.getCanonicalLocalPathRoots(); - if (!isInboundPathAllowed({ filePath: canonicalPath, roots: canonicalRoots })) { + if (!isInboundPathAllowed({ filePath: opened.realPath, roots: canonicalRoots })) { throw new MediaUnderstandingSkipError( "blocked", `Attachment ${params.attachmentIndex + 1} path is outside allowed roots.`, ); } - const buffer = await handle.readFile(); + const buffer = await opened.handle.readFile(); if (buffer.length > params.maxBytes) { throw new MediaUnderstandingSkipError( "maxBytes", `Attachment ${params.attachmentIndex + 1} exceeds maxBytes ${params.maxBytes}`, ); } - return { buffer, filePath: canonicalPath }; - } finally { - await handle.close().catch(() => {}); - } - } - - private async resolveCanonicalLocalPath(filePath: string): Promise { - try { - return await fs.realpath(filePath); + return { buffer, filePath: opened.realPath }; } catch (err) { - if (shouldLogVerbose()) { - logVerbose( - `Blocked attachment path when canonicalization failed: ${filePath} (${String(err)})`, + if (err instanceof FsSafeError) { + if (err.code === "too-large") { + throw new MediaUnderstandingSkipError( + "maxBytes", + `Attachment ${params.attachmentIndex + 1} exceeds maxBytes ${params.maxBytes}`, + ); + } + if (err.code === "not-file" || err.code === "not-found") { + throw new MediaUnderstandingSkipError( + "empty", + `Attachment ${params.attachmentIndex + 1} path is not a regular file.`, + ); + } + throw new MediaUnderstandingSkipError( + "blocked", + `Attachment ${params.attachmentIndex + 1} path is outside allowed roots.`, ); } - return undefined; + throw err; + } finally { + await opened?.handle.close().catch(() => {}); } } } diff --git a/src/media-understanding/fs.ts b/src/media-understanding/fs.ts index 3bea43d0536..0d11b5880ab 100644 --- a/src/media-understanding/fs.ts +++ b/src/media-understanding/fs.ts @@ -1,13 +1,5 @@ -import fs from "node:fs/promises"; +import { pathExists } from "../infra/fs-safe.js"; export async function fileExists(filePath?: string | null): Promise { - if (!filePath) { - return false; - } - try { - await fs.stat(filePath); - return true; - } catch { - return false; - } + return filePath ? await pathExists(filePath) : false; } diff --git a/src/media-understanding/runtime.ts b/src/media-understanding/runtime.ts index 28cacc5ee24..74cd4c0b052 100644 --- a/src/media-understanding/runtime.ts +++ b/src/media-understanding/runtime.ts @@ -1,5 +1,5 @@ -import fs from "node:fs/promises"; import path from "node:path"; +import { readLocalFileSafely } from "../infra/fs-safe.js"; import { normalizeMediaProviderId } from "./provider-registry.js"; import { findDecisionReason, normalizeDecisionReason } from "./runner.entries.js"; import { @@ -156,7 +156,7 @@ export async function describeImageFileWithModel(params: DescribeImageFileWithMo if (!provider?.describeImage) { throw new Error(`Provider does not support image analysis: ${params.provider}`); } - const buffer = await fs.readFile(params.filePath); + const buffer = (await readLocalFileSafely({ filePath: params.filePath })).buffer; return await provider.describeImage({ buffer, fileName: path.basename(params.filePath), diff --git a/src/media/audio-transcode.test.ts b/src/media/audio-transcode.test.ts index 171e4fd4ac9..e0df3ac1911 100644 --- a/src/media/audio-transcode.test.ts +++ b/src/media/audio-transcode.test.ts @@ -1,4 +1,4 @@ -import { existsSync } from "node:fs"; +import { existsSync, realpathSync } from "node:fs"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -45,9 +45,8 @@ describe("transcodeAudioBufferToOpus", () => { expect.arrayContaining(["-c:a", "libopus", "-b:a", "64k", "-ar", "48000", "-ac", "1"]), { timeoutMs: 1234 }, ); - expect( - capturedInputPath?.startsWith(path.join(resolvePreferredOpenClawTmpDir(), "tts-test-")), - ).toBe(true); + const tempRoot = realpathSync(resolvePreferredOpenClawTmpDir()); + expect(capturedInputPath?.startsWith(path.join(tempRoot, "tts-test-"))).toBe(true); expect(capturedInputPath ? existsSync(capturedInputPath) : true).toBe(false); expect(capturedOutputPath ? existsSync(capturedOutputPath) : true).toBe(false); }); @@ -93,7 +92,7 @@ describe("transcodeAudioBufferToOpus", () => { tempPrefix: "../bad-prefix", }); - const tempRoot = resolvePreferredOpenClawTmpDir(); + const tempRoot = realpathSync(resolvePreferredOpenClawTmpDir()); expect(capturedInputPath?.startsWith(tempRoot)).toBe(true); expect(capturedOutputPath?.startsWith(tempRoot)).toBe(true); }); diff --git a/src/media/audio-transcode.ts b/src/media/audio-transcode.ts index e1631fa4a41..c512bb5b998 100644 --- a/src/media/audio-transcode.ts +++ b/src/media/audio-transcode.ts @@ -1,5 +1,5 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; +import { withTempWorkspace } from "../infra/private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { runFfmpeg } from "./ffmpeg-exec.js"; @@ -50,38 +50,41 @@ export async function transcodeAudioBufferToOpus(params: { bitrate?: string; channels?: number; }): Promise { - const tempRoot = resolvePreferredOpenClawTmpDir(); - await mkdir(tempRoot, { recursive: true, mode: 0o700 }); - const tempDir = await mkdtemp(path.join(tempRoot, normalizeTempPrefix(params.tempPrefix))); - try { - const inputPath = path.join(tempDir, `input${normalizeAudioExtension(params)}`); - const outputPath = path.join(tempDir, normalizeOutputFileName(params.outputFileName)); - await writeFile(inputPath, params.audioBuffer, { mode: 0o600 }); - await runFfmpeg( - [ - "-hide_banner", - "-loglevel", - "error", - "-y", - "-i", - inputPath, - "-vn", - "-sn", - "-dn", - "-c:a", - "libopus", - "-b:a", - params.bitrate ?? DEFAULT_OPUS_BITRATE, - "-ar", - String(params.sampleRateHz ?? DEFAULT_OPUS_SAMPLE_RATE_HZ), - "-ac", - String(params.channels ?? DEFAULT_OPUS_CHANNELS), - outputPath, - ], - { timeoutMs: params.timeoutMs }, - ); - return await readFile(outputPath); - } finally { - await rm(tempDir, { recursive: true, force: true }); - } + return await withTempWorkspace( + { + rootDir: resolvePreferredOpenClawTmpDir(), + prefix: normalizeTempPrefix(params.tempPrefix), + }, + async (workspace) => { + const inputPath = await workspace.write( + `input${normalizeAudioExtension(params)}`, + params.audioBuffer, + ); + const outputPath = workspace.path(normalizeOutputFileName(params.outputFileName)); + await runFfmpeg( + [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + inputPath, + "-vn", + "-sn", + "-dn", + "-c:a", + "libopus", + "-b:a", + params.bitrate ?? DEFAULT_OPUS_BITRATE, + "-ar", + String(params.sampleRateHz ?? DEFAULT_OPUS_SAMPLE_RATE_HZ), + "-ac", + String(params.channels ?? DEFAULT_OPUS_CHANNELS), + outputPath, + ], + { timeoutMs: params.timeoutMs }, + ); + return await workspace.read(normalizeOutputFileName(params.outputFileName)); + }, + ); } diff --git a/src/media/file-context.ts b/src/media/file-context.ts index 00629b88afe..0dab8b55f46 100644 --- a/src/media/file-context.ts +++ b/src/media/file-context.ts @@ -1,3 +1,4 @@ +import { sanitizeUntrustedFileName } from "../infra/fs-safe-advanced.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; const XML_ESCAPE_MAP: Record = { @@ -21,7 +22,7 @@ function sanitizeFileName(value: string | null | undefined, fallbackName: string normalizeOptionalString( typeof value === "string" ? value.replace(/[\r\n\t]+/g, " ") : undefined, ) ?? ""; - return normalized || fallbackName; + return sanitizeUntrustedFileName(normalized, fallbackName); } export function renderFileContextBlock(params: { diff --git a/src/media/image-ops.ts b/src/media/image-ops.ts index dc53c719085..4d08f04f9aa 100644 --- a/src/media/image-ops.ts +++ b/src/media/image-ops.ts @@ -1,5 +1,4 @@ -import fs from "node:fs/promises"; -import path from "node:path"; +import { withTempWorkspace, type TempWorkspace } from "../infra/private-temp-workspace.js"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { runExec } from "../process/exec.js"; import { createLazyPromiseLoader } from "../shared/lazy-promise.js"; @@ -357,19 +356,16 @@ function readJpegExifOrientation(buffer: Buffer): number | null { return null; } -async function withTempDir(fn: (dir: string) => Promise): Promise { - const dir = await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), "openclaw-img-")); - try { - return await fn(dir); - } finally { - await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); - } +async function withImageTemp(fn: (workspace: TempWorkspace) => Promise): Promise { + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "openclaw-img-" }, + fn, + ); } async function sipsMetadataFromBuffer(buffer: Buffer): Promise { - return await withTempDir(async (dir) => { - const input = path.join(dir, "in.img"); - await fs.writeFile(input, buffer); + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", buffer); const { stdout } = await runExec( "/usr/bin/sips", ["-g", "pixelWidth", "-g", "pixelHeight", input], @@ -400,10 +396,9 @@ async function sipsResizeToJpeg(params: { maxSide: number; quality: number; }): Promise { - return await withTempDir(async (dir) => { - const input = path.join(dir, "in.img"); - const output = path.join(dir, "out.jpg"); - await fs.writeFile(input, params.buffer); + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.img", params.buffer); + const output = workspace.path("out.jpg"); await runExec( "/usr/bin/sips", [ @@ -421,20 +416,19 @@ async function sipsResizeToJpeg(params: { ], { timeoutMs: 20_000, maxBuffer: 1024 * 1024 }, ); - return await fs.readFile(output); + return await workspace.read("out.jpg"); }); } async function sipsConvertToJpeg(buffer: Buffer): Promise { - return await withTempDir(async (dir) => { - const input = path.join(dir, "in.heic"); - const output = path.join(dir, "out.jpg"); - await fs.writeFile(input, buffer); + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.heic", buffer); + const output = workspace.path("out.jpg"); await runExec("/usr/bin/sips", ["-s", "format", "jpeg", input, "--out", output], { timeoutMs: 20_000, maxBuffer: 1024 * 1024, }); - return await fs.readFile(output); + return await workspace.read("out.jpg"); }); } @@ -495,15 +489,14 @@ async function sipsApplyOrientation(buffer: Buffer, orientation: number): Promis return buffer; } - return await withTempDir(async (dir) => { - const input = path.join(dir, "in.jpg"); - const output = path.join(dir, "out.jpg"); - await fs.writeFile(input, buffer); + return await withImageTemp(async (workspace) => { + const input = await workspace.write("in.jpg", buffer); + const output = workspace.path("out.jpg"); await runExec("/usr/bin/sips", [...ops, input, "--out", output], { timeoutMs: 20_000, maxBuffer: 1024 * 1024, }); - return await fs.readFile(output); + return await workspace.read("out.jpg"); }); } diff --git a/src/media/local-media-access.ts b/src/media/local-media-access.ts index cc3f7ac7f2d..d12fc404d25 100644 --- a/src/media/local-media-access.ts +++ b/src/media/local-media-access.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { assertNoWindowsNetworkPath } from "../infra/local-file-access.js"; +import { isPathInside } from "../infra/path-guards.js"; import { getDefaultMediaLocalRoots } from "./local-roots.js"; import { resolveInboundMediaReference } from "./media-reference.js"; @@ -59,7 +60,7 @@ export async function assertLocalMediaAllowed( if (workspaceRoot) { const stateDir = path.dirname(workspaceRoot); const rel = path.relative(stateDir, resolved); - if (rel && !rel.startsWith("..") && !path.isAbsolute(rel)) { + if (rel && isPathInside(stateDir, resolved)) { const firstSegment = rel.split(path.sep)[0] ?? ""; if (firstSegment.startsWith("workspace-")) { throw new LocalMediaAccessError( @@ -84,7 +85,7 @@ export async function assertLocalMediaAllowed( `Invalid localRoots entry (refuses filesystem root): ${root}. Pass a narrower directory.`, ); } - if (resolved === resolvedRoot || resolved.startsWith(resolvedRoot + path.sep)) { + if (isPathInside(resolvedRoot, resolved)) { return; } } diff --git a/src/media/qr-image.ts b/src/media/qr-image.ts index 6ab374b7240..5fd1dc212dd 100644 --- a/src/media/qr-image.ts +++ b/src/media/qr-image.ts @@ -1,5 +1,5 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; import path from "node:path"; +import { tempWorkspace } from "../infra/private-temp-workspace.js"; import { loadQrCodeRuntime, normalizeQrText } from "./qr-runtime.ts"; const DEFAULT_QR_PNG_SCALE = 6; @@ -102,17 +102,17 @@ export async function writeQrPngTempFile( const dirPrefix = resolveQrTempPathSegment("dirPrefix", opts.dirPrefix); const fileName = resolveQrTempPathSegment("fileName", opts.fileName ?? "qr.png"); const pngBase64 = await renderQrPngBase64(input, opts); - const dirPath = await mkdtemp(path.join(opts.tmpRoot, dirPrefix)); - const filePath = path.join(dirPath, fileName); + const workspace = await tempWorkspace({ rootDir: opts.tmpRoot, prefix: dirPrefix }); + const dirPath = workspace.dir; try { - await writeFile(filePath, Buffer.from(pngBase64, "base64")); + const filePath = await workspace.write(fileName, Buffer.from(pngBase64, "base64")); + return { + filePath, + dirPath, + mediaLocalRoots: [dirPath], + }; } catch (err) { - await rm(dirPath, { recursive: true, force: true }).catch(() => {}); + await workspace.cleanup(); throw err; } - return { - filePath, - dirPath, - mediaLocalRoots: [dirPath], - }; } diff --git a/src/media/store.outside-workspace.test.ts b/src/media/store.outside-workspace.test.ts index f76692af9c4..6190cf9d91c 100644 --- a/src/media/store.outside-workspace.test.ts +++ b/src/media/store.outside-workspace.test.ts @@ -5,7 +5,7 @@ import { createTempHomeEnv, type TempHomeEnv } from "../test-utils/temp-home.js" const mocks = vi.hoisted(() => ({ readLocalFileSafely: vi.fn(), - isSafeOpenError: vi.fn( + isFsSafeError: vi.fn( (error: unknown) => typeof error === "object" && error !== null && "code" in error, ), })); @@ -13,7 +13,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("./store.runtime.js", () => { return { readLocalFileSafely: mocks.readLocalFileSafely, - isSafeOpenError: mocks.isSafeOpenError, + isFsSafeError: mocks.isFsSafeError, }; }); diff --git a/src/media/store.runtime.ts b/src/media/store.runtime.ts index 7e030fcb5c1..7d0ade68957 100644 --- a/src/media/store.runtime.ts +++ b/src/media/store.runtime.ts @@ -1,16 +1,17 @@ +import "../infra/fs-safe-defaults.js"; import { + FsSafeError, readLocalFileSafely as readLocalFileSafelyImpl, - SafeOpenError, - type SafeOpenErrorCode, + type FsSafeErrorCode, } from "../infra/fs-safe.js"; -export type SafeOpenLikeError = { - code: SafeOpenErrorCode; +export type FsSafeLikeError = { + code: FsSafeErrorCode; message: string; }; export const readLocalFileSafely = readLocalFileSafelyImpl; -export function isSafeOpenError(error: unknown): error is SafeOpenLikeError { - return error instanceof SafeOpenError; +export function isFsSafeError(error: unknown): error is FsSafeLikeError { + return error instanceof FsSafeError; } diff --git a/src/media/store.test.ts b/src/media/store.test.ts index 44f84d01ebf..46caab52d12 100644 --- a/src/media/store.test.ts +++ b/src/media/store.test.ts @@ -400,6 +400,47 @@ describe("media store", () => { }); }, }, + { + name: "reads media IDs through the media root boundary", + run: async () => { + await withTempStore(async (store) => { + const saved = await store.saveMediaBuffer(Buffer.from("source bytes"), "text/plain"); + + const read = await store.readMediaBuffer(saved.id, "inbound"); + + await expect(fs.realpath(read.path)).resolves.toBe(await fs.realpath(saved.path)); + expect(read.size).toBe("source bytes".length); + expect(read.buffer.toString("utf8")).toBe("source bytes"); + }); + }, + }, + { + name: "rejects oversized media ID reads before materializing the file", + run: async () => { + await withTempStore(async (store) => { + const saved = await store.saveMediaBuffer(Buffer.from("too large"), "text/plain"); + + await expect(store.readMediaBuffer(saved.id, "inbound", 3)).rejects.toThrow( + "maximum is 3 bytes", + ); + }); + }, + }, + { + name: "rejects traversal media subdirs before reading IDs", + run: async () => { + await withTempStore(async (store, home) => { + const mediaDir = await store.ensureMediaDir(); + const outsideDir = path.join(home, "outside-media-read"); + await fs.mkdir(outsideDir, { recursive: true }); + await fs.writeFile(path.join(outsideDir, "passwd"), "not media"); + + await expect( + store.readMediaBuffer("passwd", path.relative(mediaDir, outsideDir)), + ).rejects.toThrow("unsafe media subdir"); + }); + }, + }, { name: "retries local-source writes when cleanup prunes the target directory", run: async () => { diff --git a/src/media/store.ts b/src/media/store.ts index 0aeb949c30a..a8b31c6bfef 100644 --- a/src/media/store.ts +++ b/src/media/store.ts @@ -1,3 +1,4 @@ +import "../infra/fs-safe-defaults.js"; import crypto from "node:crypto"; import { createWriteStream } from "node:fs"; import fs from "node:fs/promises"; @@ -5,12 +6,16 @@ import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; import path from "node:path"; import { pipeline } from "node:stream/promises"; +import { fileStore } from "../infra/file-store.js"; +import { sanitizeUntrustedFileName } from "../infra/fs-safe-advanced.js"; +import { isPathInside } from "../infra/fs-safe.js"; import { retainSafeHeadersForCrossOriginRedirect } from "../infra/net/redirect-headers.js"; import { resolvePinnedHostname } from "../infra/net/ssrf.js"; +import { writeSiblingTempFile } from "../infra/sibling-temp-file.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; import { resolveConfigDir } from "../utils.js"; import { detectMime, extensionForMime } from "./mime.js"; -import { isSafeOpenError, readLocalFileSafely, type SafeOpenLikeError } from "./store.runtime.js"; +import { isFsSafeError, readLocalFileSafely, type FsSafeLikeError } from "./store.runtime.js"; const resolveMediaDir = () => path.join(resolveConfigDir(), "media"); export const MEDIA_MAX_BYTES = 5 * 1024 * 1024; // 5MB default @@ -60,13 +65,29 @@ function resolveMediaScopedDir(subdir: string, caller: string): string { const mediaDir = resolveMediaDir(); const safeSubdir = resolveMediaSubdir(subdir, caller); const dir = safeSubdir ? path.join(mediaDir, safeSubdir) : mediaDir; - const relative = path.relative(mediaDir, dir); - if (relative && (relative === ".." || relative.startsWith(`..${path.sep}`))) { + if (!isPathInside(mediaDir, dir)) { throw new Error(`${caller}: media subdir escapes media directory: ${JSON.stringify(subdir)}`); } return dir; } +function resolveMediaRelativePath(id: string, subdir: string, caller: string): string { + if (!id || id.includes("/") || id.includes("\\") || id.includes("\0") || id === "..") { + throw new Error(`${caller}: unsafe media ID: ${JSON.stringify(id)}`); + } + const safeSubdir = resolveMediaSubdir(subdir, caller); + return safeSubdir ? path.join(safeSubdir, id) : id; +} + +function openMediaStore(maxBytes = MAX_BYTES) { + return fileStore({ + rootDir: resolveMediaDir(), + dirMode: 0o700, + maxBytes, + mode: MEDIA_FILE_MODE, + }); +} + let httpRequestImpl: RequestImpl = defaultHttpRequestImpl; let httpsRequestImpl: RequestImpl = defaultHttpsRequestImpl; let resolvePinnedHostnameImpl: ResolvePinnedHostnameImpl = defaultResolvePinnedHostnameImpl; @@ -87,11 +108,11 @@ export function setMediaStoreNetworkDepsForTest(deps?: { * Keeps: alphanumeric, dots, hyphens, underscores, Unicode letters/numbers. */ function sanitizeFilename(name: string): string { - const trimmed = name.trim(); - if (!trimmed) { + const base = sanitizeUntrustedFileName(name, ""); + if (!base) { return ""; } - const sanitized = trimmed.replace(/[^\p{L}\p{N}._-]+/gu, "_"); + const sanitized = base.replace(/[^\p{L}\p{N}._-]+/gu, "_"); // Collapse multiple underscores, trim leading/trailing, limit length return sanitized.replace(/_+/g, "_").replace(/^_|_$/g, "").slice(0, 60); } @@ -150,63 +171,12 @@ async function retryAfterRecreatingDir(dir: string, run: () => Promise): P } export async function cleanOldMedia(ttlMs = DEFAULT_TTL_MS, options: CleanOldMediaOptions = {}) { - const mediaDir = await ensureMediaDir(); - const now = Date.now(); - const recursive = options.recursive ?? false; - const pruneEmptyDirs = recursive && (options.pruneEmptyDirs ?? false); - - const removeExpiredFilesInDir = async (dir: string): Promise => { - const dirEntries = await fs.readdir(dir).catch(() => null); - if (!dirEntries) { - return false; - } - for (const entry of dirEntries) { - const fullPath = path.join(dir, entry); - const stat = await fs.lstat(fullPath).catch(() => null); - if (!stat || stat.isSymbolicLink()) { - continue; - } - if (stat.isDirectory()) { - if (recursive) { - const childIsEmpty = await removeExpiredFilesInDir(fullPath); - if (childIsEmpty) { - await fs.rmdir(fullPath).catch(() => {}); - } - } - continue; - } - if (!stat.isFile()) { - continue; - } - if (now - stat.mtimeMs > ttlMs) { - await fs.rm(fullPath, { force: true }).catch(() => {}); - } - } - if (!pruneEmptyDirs) { - return false; - } - const remainingEntries = await fs.readdir(dir).catch(() => null); - return remainingEntries !== null && remainingEntries.length === 0; - }; - - const entries = await fs.readdir(mediaDir).catch(() => []); - for (const file of entries) { - const full = path.join(mediaDir, file); - const stat = await fs.lstat(full).catch(() => null); - if (!stat || stat.isSymbolicLink()) { - continue; - } - if (stat.isDirectory()) { - const dirIsEmpty = await removeExpiredFilesInDir(full); - if (dirIsEmpty) { - await fs.rmdir(full).catch(() => {}); - } - continue; - } - if (stat.isFile() && now - stat.mtimeMs > ttlMs) { - await fs.rm(full, { force: true }).catch(() => {}); - } - } + await openMediaStore().pruneExpired({ + maxDepth: options.recursive ? undefined : 1, + ttlMs, + recursive: options.recursive ?? true, + pruneEmptyDirs: options.pruneEmptyDirs, + }); } function looksLikeUrl(src: string) { @@ -340,39 +310,19 @@ function buildSavedMediaResult(params: { } async function writeSavedMediaBuffer(params: { - dir: string; + subdir: string; id: string; buffer: Buffer; }): Promise { - const dest = path.join(params.dir, params.id); - await retryAfterRecreatingDir(params.dir, async () => { - const tempDest = path.join(params.dir, `.${params.id}.${crypto.randomUUID()}.tmp`); - try { - await fs.writeFile(tempDest, params.buffer, { mode: MEDIA_FILE_MODE }); - const handle = await fs.open(tempDest, "r+"); - try { - await syncSavedMediaHandle(handle); - } finally { - await handle.close(); - } - await fs.rename(tempDest, dest); - } catch (err) { - await fs.rm(tempDest, { force: true }).catch(() => {}); - throw err; - } - }); - return dest; -} - -async function syncSavedMediaHandle(handle: fs.FileHandle): Promise { - try { - await handle.sync(); - } catch (err) { - if ((err as NodeJS.ErrnoException | undefined)?.code === "EPERM") { - return; - } - throw err; - } + const dir = resolveMediaScopedDir(params.subdir, "writeSavedMediaBuffer"); + const relativePath = resolveMediaRelativePath(params.id, params.subdir, "writeSavedMediaBuffer"); + return await retryAfterRecreatingDir( + dir, + async () => + await openMediaStore(params.buffer.byteLength).write(relativePath, params.buffer, { + tempPrefix: `.${params.id}`, + }), + ); } export type SaveMediaSourceErrorCode = @@ -392,10 +342,7 @@ export class SaveMediaSourceError extends Error { } } -function toSaveMediaSourceError( - err: SafeOpenLikeError, - maxBytes = MAX_BYTES, -): SaveMediaSourceError { +function toSaveMediaSourceError(err: FsSafeLikeError, maxBytes = MAX_BYTES): SaveMediaSourceError { switch (err.code) { case "symlink": return new SaveMediaSourceError("invalid-path", "Media path must not be a symlink", { @@ -438,30 +385,47 @@ export async function saveMediaSource( await cleanOldMedia(DEFAULT_TTL_MS, { recursive: false }); const baseId = crypto.randomUUID(); if (looksLikeUrl(source)) { - const tempDest = path.join(dir, `${baseId}.tmp`); - const { headerMime, sniffBuffer, size } = await retryAfterRecreatingDir(dir, () => - downloadToFile(source, tempDest, headers, 5, maxBytes), + const saved = await retryAfterRecreatingDir(dir, () => + writeSiblingTempFile({ + dir, + mode: MEDIA_FILE_MODE, + tempPrefix: `.${baseId}`, + writeTemp: async (tempPath) => { + const { headerMime, sniffBuffer, size } = await downloadToFile( + source, + tempPath, + headers, + 5, + maxBytes, + ); + const mime = await detectMime({ + buffer: sniffBuffer, + headerMime, + filePath: source, + }); + const ext = extensionForMime(mime) ?? path.extname(new URL(source).pathname); + const id = buildSavedMediaId({ baseId, ext }); + return { id, size, contentType: mime }; + }, + resolveFinalPath: (result) => path.join(dir, result.id), + }), ); - const mime = await detectMime({ - buffer: sniffBuffer, - headerMime, - filePath: source, + return buildSavedMediaResult({ + dir, + id: saved.result.id, + size: saved.result.size, + contentType: saved.result.contentType, }); - const ext = extensionForMime(mime) ?? path.extname(new URL(source).pathname); - const id = buildSavedMediaId({ baseId, ext }); - const finalDest = path.join(dir, id); - await fs.rename(tempDest, finalDest); - return buildSavedMediaResult({ dir, id, size, contentType: mime }); } try { const { buffer, stat } = await readLocalFileSafely({ filePath: source, maxBytes }); const mime = await detectMime({ buffer, filePath: source }); const ext = extensionForMime(mime) ?? path.extname(source); const id = buildSavedMediaId({ baseId, ext }); - await writeSavedMediaBuffer({ dir, id, buffer }); + await writeSavedMediaBuffer({ subdir, id, buffer }); return buildSavedMediaResult({ dir, id, size: stat.size, contentType: mime }); } catch (err) { - if (isSafeOpenError(err)) { + if (isFsSafeError(err)) { throw toSaveMediaSourceError(err, maxBytes); } throw err; @@ -486,7 +450,7 @@ export async function saveMediaBuffer( const ext = headerExt ?? extensionForMime(mime) ?? safeOriginalFilenameExtension(originalFilename) ?? ""; const id = buildSavedMediaId({ baseId: uuid, ext, originalFilename }); - await writeSavedMediaBuffer({ dir, id, buffer }); + await writeSavedMediaBuffer({ subdir, id, buffer }); return buildSavedMediaResult({ dir, id, size: buffer.byteLength, contentType: mime }); } @@ -510,51 +474,62 @@ export async function saveMediaBuffer( * @returns Absolute path to the file on disk. * @throws If the ID is unsafe, the file does not exist, or is not a * regular file. + * + * Prefer readMediaBuffer when the caller needs the bytes; this path-returning + * helper is for channel surfaces that need a stable local attachment path. */ export async function resolveMediaBufferPath(id: string, subdir = "inbound"): Promise { - // Guard against path traversal and null-byte injection. - // - // - Separator checks: reject any ID containing "/" or "\" (covers all - // relative traversal sequences such as "../foo" or "..\\foo"). - // - Exact ".." check: reject the bare traversal operator in case a caller - // strips separators but keeps the dots. - // - Null-byte check: reject "\0" which can truncate paths on some platforms - // and cause the OS to open a different file than intended. - // - // We allow consecutive dots in legitimate filenames (e.g. "report..draft.png"), - // so we only reject the exact two-character string "..". - // - // JSON.stringify is used in the error message so that control characters - // (including \0) are rendered visibly in logs rather than silently dropped. - if (!id || id.includes("/") || id.includes("\\") || id.includes("\0") || id === "..") { - throw new Error(`resolveMediaBufferPath: unsafe media ID: ${JSON.stringify(id)}`); - } - - const dir = resolveMediaScopedDir(subdir, "resolveMediaBufferPath"); - const resolved = path.join(dir, id); - - // Double-check that path.join didn't escape the intended directory. - // This should be unreachable after the separator check above, but be - // explicit about the invariant. - if (!resolved.startsWith(dir + path.sep) && resolved !== dir) { - throw new Error(`resolveMediaBufferPath: path escapes media directory: ${JSON.stringify(id)}`); - } - - // lstat (not stat) so we see symlinks rather than following them. - const stat = await fs.lstat(resolved); - - if (stat.isSymbolicLink()) { - throw new Error( - `resolveMediaBufferPath: refusing to follow symlink for media ID: ${JSON.stringify(id)}`, - ); - } - if (!stat.isFile()) { + const relativePath = resolveMediaRelativePath(id, subdir, "resolveMediaBufferPath"); + const opened = await openMediaStore() + .open(relativePath) + .catch(() => null); + if (!opened?.stat.isFile()) { throw new Error( `resolveMediaBufferPath: media ID does not resolve to a file: ${JSON.stringify(id)}`, ); } + try { + return opened.realPath; + } finally { + await opened.handle.close().catch(() => undefined); + } +} - return resolved; +export type ReadMediaBufferResult = { + id: string; + path: string; + buffer: Buffer; + size: number; +}; + +export async function readMediaBuffer( + id: string, + subdir = "inbound", + maxBytes = MAX_BYTES, +): Promise { + const relativePath = resolveMediaRelativePath(id, subdir, "readMediaBuffer"); + const opened = await openMediaStore(maxBytes) + .open(relativePath) + .catch(() => null); + if (!opened?.stat.isFile()) { + throw new Error(`readMediaBuffer: media ID does not resolve to a file: ${JSON.stringify(id)}`); + } + try { + if (opened.stat.size > maxBytes) { + throw new Error( + `readMediaBuffer: media ID ${JSON.stringify(id)} is ${opened.stat.size} bytes; maximum is ${maxBytes} bytes`, + ); + } + const buffer = await opened.handle.readFile(); + if (buffer.byteLength > maxBytes) { + throw new Error( + `readMediaBuffer: media ID ${JSON.stringify(id)} read ${buffer.byteLength} bytes; maximum is ${maxBytes} bytes`, + ); + } + return { id, path: opened.realPath, buffer, size: buffer.byteLength }; + } finally { + await opened.handle.close().catch(() => undefined); + } } /** @@ -565,8 +540,8 @@ export async function resolveMediaBufferPath(id: string, subdir = "inbound"): Pr * fails validation and the entire parse is aborted, preventing orphaned files * from accumulating on disk ahead of the periodic TTL sweep. * - * Uses resolveMediaBufferPath to apply the same path-safety guards as the - * read path (separator checks, symlink rejection, etc.) before unlinking. + * Uses a media-root handle to apply the same path-safety guards as the read + * path while removing the file under the pinned media root. * * Errors are intentionally not suppressed — callers that want best-effort * cleanup should catch and discard exceptions themselves (e.g. via @@ -576,6 +551,6 @@ export async function resolveMediaBufferPath(id: string, subdir = "inbound"): Pr * @param subdir The subdirectory the file was saved into (default "inbound"). */ export async function deleteMediaBuffer(id: string, subdir: "inbound" = "inbound"): Promise { - const physicalPath = await resolveMediaBufferPath(id, subdir); - await fs.unlink(physicalPath); + const relativePath = resolveMediaRelativePath(id, subdir, "deleteMediaBuffer"); + await openMediaStore().remove(relativePath); } diff --git a/src/media/web-media.ts b/src/media/web-media.ts index edf37dbad35..962728a9b32 100644 --- a/src/media/web-media.ts +++ b/src/media/web-media.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { resolveCanvasHttpPathToLocalPath } from "../gateway/canvas-documents.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { SafeOpenError, readLocalFileSafely } from "../infra/fs-safe.js"; +import { FsSafeError, readLocalFileSafely } from "../infra/fs-safe.js"; import { assertNoWindowsNetworkPath, safeFileURLToPath } from "../infra/local-file-access.js"; import type { PinnedDispatcherPolicy, SsrFPolicy } from "../infra/net/ssrf.js"; import { resolveUserPath } from "../utils.js"; @@ -558,7 +558,7 @@ async function loadWebMediaInternal( try { data = (await readLocalFileSafely({ filePath: mediaUrl })).buffer; } catch (err) { - if (err instanceof SafeOpenError) { + if (err instanceof FsSafeError) { if (err.code === "not-found") { throw new LocalMediaAccessError("not-found", `Local media file not found: ${mediaUrl}`, { cause: err, diff --git a/src/memory-host-sdk/events.ts b/src/memory-host-sdk/events.ts index 0b961ad057d..752bb7d16f2 100644 --- a/src/memory-host-sdk/events.ts +++ b/src/memory-host-sdk/events.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { appendRegularFile } from "../infra/fs-safe.js"; import type { MemoryDreamingPhaseName } from "./dreaming.js"; export const MEMORY_HOST_EVENT_LOG_RELATIVE_PATH = path.join("memory", ".dreams", "events.jsonl"); @@ -57,7 +58,11 @@ export async function appendMemoryHostEvent( ): Promise { const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir); await fs.mkdir(path.dirname(eventLogPath), { recursive: true }); - await fs.appendFile(eventLogPath, `${JSON.stringify(event)}\n`, "utf8"); + await appendRegularFile({ + filePath: eventLogPath, + content: `${JSON.stringify(event)}\n`, + rejectSymlinkParents: true, + }); } export async function readMemoryHostEvents(params: { diff --git a/src/node-host/config.ts b/src/node-host/config.ts index a9ee950a4ba..df6f8d4d1aa 100644 --- a/src/node-host/config.ts +++ b/src/node-host/config.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; -import { writeJsonAtomic } from "../infra/json-files.js"; +import { writeJson } from "../infra/json-files.js"; export type NodeHostGatewayConfig = { host?: string; @@ -55,7 +55,7 @@ export async function loadNodeHostConfig(): Promise { export async function saveNodeHostConfig(config: NodeHostConfig): Promise { const filePath = resolveNodeHostConfigPath(); - await writeJsonAtomic(filePath, config, { mode: 0o600 }); + await writeJson(filePath, config, { mode: 0o600 }); } export async function ensureNodeHostConfig(): Promise { diff --git a/src/node-host/invoke-system-run-plan.ts b/src/node-host/invoke-system-run-plan.ts index ed653857882..0df201108d4 100644 --- a/src/node-host/invoke-system-run-plan.ts +++ b/src/node-host/invoke-system-run-plan.ts @@ -13,7 +13,7 @@ import { unwrapKnownDispatchWrapperInvocation, unwrapKnownShellMultiplexerInvocation, } from "../infra/exec-wrapper-resolution.js"; -import { sameFileIdentity } from "../infra/file-identity.js"; +import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; import { POSIX_INLINE_COMMAND_FLAGS, resolveInlineCommandMatch, diff --git a/src/node-host/invoke-system-run.test.ts b/src/node-host/invoke-system-run.test.ts index 7c0cc14b2a1..f213fc04ba0 100644 --- a/src/node-host/invoke-system-run.test.ts +++ b/src/node-host/invoke-system-run.test.ts @@ -802,11 +802,11 @@ describe("handleSystemRunInvoke mac app exec host routing", () => { label: "parent symlink", setup: () => { const tmp = createFixtureDir("openclaw-approval-cwd-parent-link-"); - const safeRoot = path.join(tmp, "safe-root"); - const safeSub = path.join(safeRoot, "sub"); + const safeSymlinkRoot = path.join(tmp, "safe-root"); + const safeSymlinkSub = path.join(safeSymlinkRoot, "sub"); const linkRoot = path.join(tmp, "approved-link"); - fs.mkdirSync(safeSub, { recursive: true }); - fs.symlinkSync(safeRoot, linkRoot, "dir"); + fs.mkdirSync(safeSymlinkSub, { recursive: true }); + fs.symlinkSync(safeSymlinkRoot, linkRoot, "dir"); return { cwd: path.join(linkRoot, "sub"), message: "no symlink path components", diff --git a/src/plugin-sdk/browser-trash.ts b/src/plugin-sdk/browser-trash.ts index 1057102b9c9..9610f687a88 100644 --- a/src/plugin-sdk/browser-trash.ts +++ b/src/plugin-sdk/browser-trash.ts @@ -1,148 +1,2 @@ -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -export type MovePathToTrashOptions = { - allowedRoots?: Iterable; -}; - -const TRASH_DESTINATION_COLLISION_CODES = new Set(["EEXIST", "ENOTEMPTY", "ERR_FS_CP_EEXIST"]); -const TRASH_DESTINATION_RETRY_LIMIT = 4; - -function getFsErrorCode(error: unknown): string | undefined { - if (!error || typeof error !== "object" || !("code" in error)) { - return undefined; - } - const code = (error as NodeJS.ErrnoException).code; - return typeof code === "string" ? code : undefined; -} - -function isTrashDestinationCollision(error: unknown): boolean { - const code = getFsErrorCode(error); - return Boolean(code && TRASH_DESTINATION_COLLISION_CODES.has(code)); -} - -function isSameOrChildPath(candidate: string, parent: string): boolean { - return candidate === parent || candidate.startsWith(`${parent}${path.sep}`); -} - -function resolveAllowedTrashRoots(allowedRoots?: Iterable): string[] { - const roots = [...(allowedRoots ?? [os.homedir(), os.tmpdir()])].map((root) => { - try { - return path.resolve(fs.realpathSync.native(root)); - } catch { - return path.resolve(root); - } - }); - return [...new Set(roots)]; -} - -function assertAllowedTrashTarget(targetPath: string, allowedRoots?: Iterable): void { - let resolvedTargetPath = path.resolve(targetPath); - try { - resolvedTargetPath = path.resolve(fs.realpathSync.native(targetPath)); - } catch { - // The subsequent move will surface missing or inaccessible targets. - } - const isAllowed = resolveAllowedTrashRoots(allowedRoots).some( - (root) => resolvedTargetPath !== root && isSameOrChildPath(resolvedTargetPath, root), - ); - if (!isAllowed) { - throw new Error(`Refusing to trash path outside allowed roots: ${targetPath}`); - } -} - -function resolveTrashDir(): string { - const homeDir = os.homedir(); - const trashDir = path.join(homeDir, ".Trash"); - fs.mkdirSync(trashDir, { recursive: true, mode: 0o700 }); - const trashDirStat = fs.lstatSync(trashDir); - if (!trashDirStat.isDirectory() || trashDirStat.isSymbolicLink()) { - throw new Error(`Refusing to use non-directory/symlink trash directory: ${trashDir}`); - } - const realHome = path.resolve(fs.realpathSync.native(homeDir)); - const resolvedTrashDir = path.resolve(fs.realpathSync.native(trashDir)); - if (resolvedTrashDir === realHome || !isSameOrChildPath(resolvedTrashDir, realHome)) { - throw new Error(`Trash directory escaped home directory: ${trashDir}`); - } - return resolvedTrashDir; -} - -function trashBaseName(targetPath: string): string { - const resolvedTargetPath = path.resolve(targetPath); - if (resolvedTargetPath === path.parse(resolvedTargetPath).root) { - throw new Error(`Refusing to trash root path: ${targetPath}`); - } - const base = path.basename(resolvedTargetPath).replace(/[\\/]+/g, ""); - if (!base) { - throw new Error(`Unable to derive safe trash basename for: ${targetPath}`); - } - return base; -} - -function resolveContainedPath(root: string, leaf: string): string { - const resolvedRoot = path.resolve(root); - const resolvedPath = path.resolve(resolvedRoot, leaf); - if (!isSameOrChildPath(resolvedPath, resolvedRoot) || resolvedPath === resolvedRoot) { - throw new Error(`Trash destination escaped trash directory: ${resolvedPath}`); - } - return resolvedPath; -} - -function reserveTrashDestination(trashDir: string, base: string, timestamp: number): string { - const containerPrefix = resolveContainedPath(trashDir, `${base}-${timestamp}-`); - const container = fs.mkdtempSync(containerPrefix); - const resolvedContainer = path.resolve(container); - const resolvedTrashDir = path.resolve(trashDir); - if ( - resolvedContainer === resolvedTrashDir || - !isSameOrChildPath(resolvedContainer, resolvedTrashDir) - ) { - throw new Error(`Trash destination escaped trash directory: ${container}`); - } - return resolveContainedPath(container, base); -} - -function movePathToDestination(targetPath: string, dest: string): boolean { - try { - fs.renameSync(targetPath, dest); - return true; - } catch (error) { - if (getFsErrorCode(error) !== "EXDEV") { - if (isTrashDestinationCollision(error)) { - return false; - } - throw error; - } - } - - try { - fs.cpSync(targetPath, dest, { recursive: true, force: false, errorOnExist: true }); - fs.rmSync(targetPath, { recursive: true, force: false }); - return true; - } catch (error) { - if (isTrashDestinationCollision(error)) { - return false; - } - throw error; - } -} - -export async function movePathToTrash( - targetPath: string, - options: MovePathToTrashOptions = {}, -): Promise { - // Avoid resolving external trash helpers through the service PATH during cleanup. - const base = trashBaseName(targetPath); - assertAllowedTrashTarget(targetPath, options.allowedRoots); - const trashDir = resolveTrashDir(); - const timestamp = Date.now(); - for (let attempt = 0; attempt < TRASH_DESTINATION_RETRY_LIMIT; attempt += 1) { - const dest = reserveTrashDestination(trashDir, base, timestamp); - if (movePathToDestination(targetPath, dest)) { - return dest; - } - } - - throw new Error(`Unable to choose a unique trash destination for ${targetPath}`); -} +import "../infra/fs-safe-defaults.js"; +export { movePathToTrash, type MovePathToTrashOptions } from "@openclaw/fs-safe/advanced"; diff --git a/src/plugin-sdk/channel-entry-contract.test.ts b/src/plugin-sdk/channel-entry-contract.test.ts index a9c9cf397f0..81c1cb7ffc5 100644 --- a/src/plugin-sdk/channel-entry-contract.test.ts +++ b/src/plugin-sdk/channel-entry-contract.test.ts @@ -324,7 +324,7 @@ describe("loadBundledEntryExportSync", () => { const jitiLoad = vi.fn(() => ({ load: 42 })); const createJiti = vi.fn(() => jitiLoad); vi.doMock("../infra/boundary-file-read.js", () => ({ - openBoundaryFileSync: () => ({ + openRootFileSync: () => ({ ok: true, path: "C:\\Users\\alice\\openclaw\\dist\\extensions\\feishu\\helper.ts", fd: fs.openSync(openedFdPath, "r"), diff --git a/src/plugin-sdk/channel-entry-contract.ts b/src/plugin-sdk/channel-entry-contract.ts index fffbde83816..cc3f6bdc7e4 100644 --- a/src/plugin-sdk/channel-entry-contract.ts +++ b/src/plugin-sdk/channel-entry-contract.ts @@ -7,7 +7,7 @@ import type { ChannelConfigSchema } from "../channels/plugins/types.config.js"; import type { ChannelLegacyStateMigrationPlan } from "../channels/plugins/types.core.js"; import type { ChannelPlugin } from "../channels/plugins/types.plugin.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { createProfiler, formatPluginLoadProfileLine, @@ -262,7 +262,7 @@ function formatBundledEntryModuleOpenFailure(params: { specifier: string; resolvedPath: string; boundaryRoot: string; - failure: Extract, { ok: false }>; + failure: Extract, { ok: false }>; }): string { const importerPath = fileURLToPath(params.importMetaUrl); const errorDetail = @@ -286,11 +286,11 @@ function resolveBundledEntryModulePath(importMetaUrl: string, specifier: string) let firstFailure: { candidate: BundledEntryModuleCandidate; - failure: Extract, { ok: false }>; + failure: Extract, { ok: false }>; } | null = null; for (const candidate of candidates) { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: candidate.path, rootPath: candidate.boundaryRoot, boundaryLabel: "plugin root", diff --git a/src/plugin-sdk/facade-loader.ts b/src/plugin-sdk/facade-loader.ts index f7672070fbf..46ed403f187 100644 --- a/src/plugin-sdk/facade-loader.ts +++ b/src/plugin-sdk/facade-loader.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { resolveBundledPluginsDir } from "../plugins/bundled-dir.js"; import { getCachedPluginModuleLoader, @@ -145,7 +145,7 @@ export function loadFacadeModuleAtLocationSync(params: { return cached as T; } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: location.modulePath, rootPath: location.boundaryRoot, boundaryLabel: @@ -224,7 +224,7 @@ export async function loadBundledPluginPublicSurfaceModule(par return cached as T; } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: preparedLocation.modulePath, rootPath: preparedLocation.boundaryRoot, boundaryLabel: diff --git a/src/plugin-sdk/file-access-runtime.ts b/src/plugin-sdk/file-access-runtime.ts index 3f1188e25b4..d1dc4ed0258 100644 --- a/src/plugin-sdk/file-access-runtime.ts +++ b/src/plugin-sdk/file-access-runtime.ts @@ -1,4 +1,9 @@ // Safe local-file helpers for plugin runtime media and bridge code. -export { readFileWithinRoot, writeFileWithinRoot } from "../infra/fs-safe.js"; +export { + readFileWithinRoot, + readLocalFileFromRoots, + root, + writeFileWithinRoot, +} from "../infra/fs-safe.js"; export { basenameFromMediaSource, safeFileURLToPath } from "../infra/local-file-access.js"; diff --git a/src/plugin-sdk/file-lock.ts b/src/plugin-sdk/file-lock.ts index 28d1f939d77..11130538957 100644 --- a/src/plugin-sdk/file-lock.ts +++ b/src/plugin-sdk/file-lock.ts @@ -1,8 +1,10 @@ -import fsSync from "node:fs"; -import fs from "node:fs/promises"; -import path from "node:path"; +import "../infra/fs-safe-defaults.js"; +import { + acquireFileLock as acquireFsSafeFileLock, + drainFileLockManagerForTest, + resetFileLockManagerForTest, +} from "@openclaw/fs-safe/file-lock"; import { isPidAlive } from "../shared/pid-alive.js"; -import { resolveProcessScopedMap } from "../shared/process-scoped-map.js"; export type FileLockOptions = { retries: { @@ -16,108 +18,10 @@ export type FileLockOptions = { }; type LockFilePayload = { - pid: number; - createdAt: string; + pid?: number; + createdAt?: string; }; -type HeldLock = { - count: number; - handle: fs.FileHandle; - lockPath: string; -}; - -const HELD_LOCKS_KEY = Symbol.for("openclaw.fileLockHeldLocks"); -const HELD_LOCKS = resolveProcessScopedMap(HELD_LOCKS_KEY); -const CLEANUP_REGISTERED_KEY = Symbol.for("openclaw.fileLockCleanupRegistered"); - -function releaseAllLocksSync(): void { - for (const [normalizedFile, held] of HELD_LOCKS) { - // Kick off best-effort async closes before dropping references so tests - // don't leave FileHandle objects for GC to close later. - void held.handle.close().catch(() => undefined); - rmLockPathSync(held.lockPath); - HELD_LOCKS.delete(normalizedFile); - } -} - -async function drainAllLocks(): Promise { - for (const [normalizedFile, held] of Array.from(HELD_LOCKS.entries())) { - HELD_LOCKS.delete(normalizedFile); - await held.handle.close().catch(() => undefined); - await fs.rm(held.lockPath, { force: true }).catch(() => undefined); - } -} - -function rmLockPathSync(lockPath: string): void { - try { - fsSync.rmSync(lockPath, { force: true }); - } catch { - // Best-effort exit cleanup only. - } -} - -function ensureExitCleanupRegistered(): void { - const proc = process as NodeJS.Process & { [CLEANUP_REGISTERED_KEY]?: boolean }; - if (proc[CLEANUP_REGISTERED_KEY]) { - return; - } - proc[CLEANUP_REGISTERED_KEY] = true; - process.on("exit", releaseAllLocksSync); -} - -function computeDelayMs(retries: FileLockOptions["retries"], attempt: number): number { - const base = Math.min( - retries.maxTimeout, - Math.max(retries.minTimeout, retries.minTimeout * retries.factor ** attempt), - ); - const jitter = retries.randomize ? 1 + Math.random() : 1; - return Math.min(retries.maxTimeout, Math.round(base * jitter)); -} - -async function readLockPayload(lockPath: string): Promise { - try { - const raw = await fs.readFile(lockPath, "utf8"); - const parsed = JSON.parse(raw) as Partial; - if (typeof parsed.pid !== "number" || typeof parsed.createdAt !== "string") { - return null; - } - return { pid: parsed.pid, createdAt: parsed.createdAt }; - } catch { - return null; - } -} - -async function resolveNormalizedFilePath(filePath: string): Promise { - const resolved = path.resolve(filePath); - const dir = path.dirname(resolved); - await fs.mkdir(dir, { recursive: true }); - try { - const realDir = await fs.realpath(dir); - return path.join(realDir, path.basename(resolved)); - } catch { - return resolved; - } -} - -async function isStaleLock(lockPath: string, staleMs: number): Promise { - const payload = await readLockPayload(lockPath); - if (payload?.pid && !isPidAlive(payload.pid)) { - return true; - } - if (payload?.createdAt) { - const createdAt = Date.parse(payload.createdAt); - if (!Number.isFinite(createdAt) || Date.now() - createdAt > staleMs) { - return true; - } - } - try { - const stat = await fs.stat(lockPath); - return Date.now() - stat.mtimeMs > staleMs; - } catch { - return true; - } -} - export type FileLockHandle = { lockPath: string; release: () => Promise; @@ -130,37 +34,51 @@ export type FileLockTimeoutError = Error & { lockPath: string; }; -function createFileLockTimeoutError( - normalizedFile: string, - lockPath: string, -): FileLockTimeoutError { - const error = new Error(`file lock timeout for ${normalizedFile}`); - return Object.assign(error, { - code: FILE_LOCK_TIMEOUT_ERROR_CODE, - lockPath, - }) as FileLockTimeoutError; +const FILE_LOCK_MANAGER_KEY = "openclaw.plugin-sdk.file-lock"; + +function readLockPayload(value: Record | null): LockFilePayload | null { + if (!value) { + return null; + } + return { + pid: typeof value.pid === "number" ? value.pid : undefined, + createdAt: typeof value.createdAt === "string" ? value.createdAt : undefined, + }; } -async function releaseHeldLock(normalizedFile: string): Promise { - const current = HELD_LOCKS.get(normalizedFile); - if (!current) { - return; +async function shouldReclaimPluginLock(params: { + lockPath: string; + payload: Record | null; + staleMs: number; + nowMs: number; +}): Promise { + const payload = readLockPayload(params.payload); + if (payload?.pid && !isPidAlive(payload.pid)) { + return true; } - current.count -= 1; - if (current.count > 0) { - return; + if (payload?.createdAt) { + const createdAt = Date.parse(payload.createdAt); + return !Number.isFinite(createdAt) || params.nowMs - createdAt > params.staleMs; } - HELD_LOCKS.delete(normalizedFile); - await current.handle.close().catch(() => undefined); - await fs.rm(current.lockPath, { force: true }).catch(() => undefined); + return true; +} + +function normalizeTimeoutError(err: unknown): never { + if ((err as { code?: unknown }).code === FILE_LOCK_TIMEOUT_ERROR_CODE) { + throw Object.assign(new Error((err as Error).message), { + code: FILE_LOCK_TIMEOUT_ERROR_CODE, + lockPath: (err as { lockPath?: string }).lockPath ?? "", + }) as FileLockTimeoutError; + } + throw err; } export function resetFileLockStateForTest(): void { - releaseAllLocksSync(); + resetFileLockManagerForTest(FILE_LOCK_MANAGER_KEY, FILE_LOCK_MANAGER_KEY); } export async function drainFileLockStateForTest(): Promise { - await drainAllLocks(); + await drainFileLockManagerForTest(FILE_LOCK_MANAGER_KEY, FILE_LOCK_MANAGER_KEY); } /** Acquire a re-entrant process-local file lock backed by a `.lock` sidecar file. */ @@ -168,53 +86,19 @@ export async function acquireFileLock( filePath: string, options: FileLockOptions, ): Promise { - ensureExitCleanupRegistered(); - const normalizedFile = await resolveNormalizedFilePath(filePath); - const lockPath = `${normalizedFile}.lock`; - const held = HELD_LOCKS.get(normalizedFile); - if (held) { - held.count += 1; - return { - lockPath, - release: () => releaseHeldLock(normalizedFile), - }; + try { + const lock = await acquireFsSafeFileLock(filePath, { + managerKey: FILE_LOCK_MANAGER_KEY, + staleMs: options.stale, + retry: options.retries, + allowReentrant: true, + payload: () => ({ pid: process.pid, createdAt: new Date().toISOString() }), + shouldReclaim: shouldReclaimPluginLock, + }); + return { lockPath: lock.lockPath, release: lock.release }; + } catch (err) { + return normalizeTimeoutError(err); } - - for (let attempt = 0; attempt <= options.retries.retries; attempt += 1) { - try { - const handle = await fs.open(lockPath, "wx"); - try { - await handle.writeFile( - JSON.stringify({ pid: process.pid, createdAt: new Date().toISOString() }, null, 2), - "utf8", - ); - } catch (writeError) { - await handle.close().catch(() => undefined); - await fs.rm(lockPath, { force: true }).catch(() => undefined); - throw writeError; - } - HELD_LOCKS.set(normalizedFile, { count: 1, handle, lockPath }); - return { - lockPath, - release: () => releaseHeldLock(normalizedFile), - }; - } catch (err) { - const code = (err as { code?: string }).code; - if (code !== "EEXIST") { - throw err; - } - if (await isStaleLock(lockPath, options.stale)) { - await fs.rm(lockPath, { force: true }).catch(() => undefined); - continue; - } - if (attempt >= options.retries.retries) { - break; - } - await new Promise((resolve) => setTimeout(resolve, computeDelayMs(options.retries, attempt))); - } - } - - throw createFileLockTimeoutError(normalizedFile, lockPath); } /** Run an async callback while holding a file lock, always releasing the lock afterward. */ diff --git a/src/plugin-sdk/fs-safe-compat.test.ts b/src/plugin-sdk/fs-safe-compat.test.ts new file mode 100644 index 00000000000..5fa6c9d142a --- /dev/null +++ b/src/plugin-sdk/fs-safe-compat.test.ts @@ -0,0 +1,49 @@ +import fs from "node:fs"; +import path from "node:path"; +import { loadSecretFileSync as loadSecretFileSyncFromCore } from "openclaw/plugin-sdk/core"; +import { readFileWithinRoot, writeFileWithinRoot } from "openclaw/plugin-sdk/file-access-runtime"; +import { + loadSecretFileSync, + type SecretFileReadResult, +} from "openclaw/plugin-sdk/secret-file-runtime"; +import { describe, expect, it } from "vitest"; +import { withTempDir } from "../test-helpers/temp-dir.js"; + +describe("plugin SDK fs-safe compatibility exports", () => { + it("keeps deprecated secret-file result helpers on public SDK subpaths", async () => { + await withTempDir({ prefix: "openclaw-sdk-secret-compat-" }, async (root) => { + const secretPath = path.join(root, "token.txt"); + fs.writeFileSync(secretPath, "secret\n", { mode: 0o600 }); + + const result: SecretFileReadResult = loadSecretFileSync(secretPath, "token"); + expect(result).toMatchObject({ + ok: true, + secret: "secret", + resolvedPath: secretPath, + }); + expect(loadSecretFileSyncFromCore(secretPath, "token")).toMatchObject({ + ok: true, + secret: "secret", + }); + }); + }); + + it("keeps deprecated root-bounded read/write helpers on file-access-runtime", async () => { + await withTempDir({ prefix: "openclaw-sdk-file-access-compat-" }, async (root) => { + await writeFileWithinRoot({ + rootDir: root, + relativePath: "nested/file.txt", + data: "hello", + mkdir: true, + }); + + const result = await readFileWithinRoot({ + rootDir: root, + relativePath: "nested/file.txt", + }); + + expect(result.buffer.toString("utf8")).toBe("hello"); + expect(result.realPath).toBe(fs.realpathSync(path.join(root, "nested", "file.txt"))); + }); + }); +}); diff --git a/src/plugin-sdk/json-store.ts b/src/plugin-sdk/json-store.ts index b95ee5b819b..858ff19e295 100644 --- a/src/plugin-sdk/json-store.ts +++ b/src/plugin-sdk/json-store.ts @@ -1,40 +1,33 @@ -import fs from "node:fs"; -import { loadJsonFile, saveJsonFile } from "../infra/json-file.js"; -import { writeJsonAtomic } from "../infra/json-files.js"; -import { safeParseJson } from "../utils.js"; +import "../infra/fs-safe-defaults.js"; +import { pathExists } from "../infra/fs-safe.js"; +import { tryReadJson, tryReadJsonSync, writeJson, writeJsonSync } from "../infra/json-files.js"; /** Read small JSON blobs synchronously for token/state caches. */ -export { loadJsonFile }; +// oxlint-disable-next-line typescript-eslint/no-unnecessary-type-parameters -- public SDK compatibility helper. +export function loadJsonFile(filePath: string): T | undefined { + return tryReadJsonSync(filePath) ?? undefined; +} /** Persist small JSON blobs synchronously with restrictive permissions. */ -export { saveJsonFile }; +export const saveJsonFile = writeJsonSync; /** Read JSON from disk and fall back cleanly when the file is missing or invalid. */ export async function readJsonFileWithFallback( filePath: string, fallback: T, ): Promise<{ value: T; exists: boolean }> { - try { - const raw = await fs.promises.readFile(filePath, "utf-8"); - const parsed = safeParseJson(raw); - if (parsed == null) { - return { value: fallback, exists: true }; - } + const parsed = await tryReadJson(filePath); + if (parsed != null) { return { value: parsed, exists: true }; - } catch (err) { - const code = (err as { code?: string }).code; - if (code === "ENOENT") { - return { value: fallback, exists: false }; - } - return { value: fallback, exists: false }; } + return { value: fallback, exists: await pathExists(filePath) }; } /** Write JSON with secure file permissions and atomic replacement semantics. */ export async function writeJsonFileAtomically(filePath: string, value: unknown): Promise { - await writeJsonAtomic(filePath, value, { + await writeJson(filePath, value, { mode: 0o600, + dirMode: 0o700, trailingNewline: true, - ensureDirMode: 0o700, }); } diff --git a/src/plugin-sdk/media-store.ts b/src/plugin-sdk/media-store.ts index b6fd6f58c5a..8b46f0f11ff 100644 --- a/src/plugin-sdk/media-store.ts +++ b/src/plugin-sdk/media-store.ts @@ -1,3 +1,3 @@ // Narrow media store helpers for channel runtimes that do not need the full media runtime. -export { resolveMediaBufferPath, saveMediaBuffer } from "../media/store.js"; +export { readMediaBuffer, resolveMediaBufferPath, saveMediaBuffer } from "../media/store.js"; diff --git a/src/plugin-sdk/memory-core-host-engine-foundation.ts b/src/plugin-sdk/memory-core-host-engine-foundation.ts index 914f3eb947f..dd926e61194 100644 --- a/src/plugin-sdk/memory-core-host-engine-foundation.ts +++ b/src/plugin-sdk/memory-core-host-engine-foundation.ts @@ -32,7 +32,7 @@ export type { MemoryQmdSearchMode, } from "../config/types.memory.js"; export type { MemorySearchConfig } from "../config/types.tools.js"; -export { writeFileWithinRoot } from "../infra/fs-safe.js"; +export { root } from "../infra/fs-safe.js"; export { createSubsystemLogger } from "../logging/subsystem.js"; export { detectMime } from "../media/mime.js"; export { onSessionTranscriptUpdate } from "../sessions/transcript-events.js"; diff --git a/src/plugin-sdk/migration-runtime.ts b/src/plugin-sdk/migration-runtime.ts index 2fa19df0596..b772992f61e 100644 --- a/src/plugin-sdk/migration-runtime.ts +++ b/src/plugin-sdk/migration-runtime.ts @@ -3,6 +3,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { pathExists } from "../infra/fs-safe.js"; import type { MigrationApplyResult, MigrationItem, @@ -67,12 +68,7 @@ export function withCachedMigrationConfigRuntime( } async function exists(filePath: string): Promise { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } + return await pathExists(filePath); } async function backupExistingMigrationTarget( diff --git a/src/plugin-sdk/sandbox.ts b/src/plugin-sdk/sandbox.ts index 1434033b37c..4a7212f7aff 100644 --- a/src/plugin-sdk/sandbox.ts +++ b/src/plugin-sdk/sandbox.ts @@ -48,3 +48,12 @@ export { type PluginCommandRunResult, } from "./run-command.js"; export { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; +export { + tempWorkspace, + tempWorkspaceSync, + type TempWorkspace, + type TempWorkspaceOptions, + type TempWorkspaceSync, + withTempWorkspace, + withTempWorkspaceSync, +} from "../infra/private-temp-workspace.js"; diff --git a/src/plugin-sdk/security-runtime.ts b/src/plugin-sdk/security-runtime.ts index 8a79ef7281c..72e959b6379 100644 --- a/src/plugin-sdk/security-runtime.ts +++ b/src/plugin-sdk/security-runtime.ts @@ -1,5 +1,7 @@ // Public security/policy helpers for plugins that need shared trust and DM gating logic. +import { root as fsRoot, type OpenResult } from "../infra/fs-safe.js"; + export * from "../secrets/channel-secret-collector-runtime.js"; export * from "../secrets/runtime-shared.js"; export * from "../secrets/shared.js"; @@ -17,10 +19,51 @@ export { export * from "../security/external-content.js"; export * from "../security/safe-regex.js"; export { - SafeOpenError, - openFileWithinRoot, - writeFileFromPathWithinRoot, + appendRegularFile, + appendRegularFileSync, + FsSafeError, + FsSafeError as SafeOpenError, + openLocalFileSafely, + pathExists, + pathExistsSync, + readRegularFile, + resolveLocalPathFromRootsSync, + readRegularFileSync, + resolveRegularFileAppendFlags, + root, + statRegularFileSync, + withTimeout, + type FsSafeErrorCode as SafeOpenErrorCode, } from "../infra/fs-safe.js"; + +export async function openFileWithinRoot(params: { + rootDir: string; + relativePath: string; + rejectHardlinks?: boolean; + nonBlockingRead?: boolean; + allowSymlinkTargetWithinRoot?: boolean; +}): Promise { + const root = await fsRoot(params.rootDir); + return await root.open(params.relativePath, { + hardlinks: params.rejectHardlinks === false ? "allow" : "reject", + nonBlockingRead: params.nonBlockingRead, + symlinks: params.allowSymlinkTargetWithinRoot === true ? "follow-within-root" : "reject", + }); +} + +export async function writeFileFromPathWithinRoot(params: { + rootDir: string; + relativePath: string; + sourcePath: string; + mkdir?: boolean; +}): Promise { + const root = await fsRoot(params.rootDir); + await root.copyIn(params.relativePath, params.sourcePath, { + mkdir: params.mkdir, + sourceHardlinks: "reject", + }); +} + export { extractErrorCode, formatErrorMessage } from "../infra/errors.js"; export { hasProxyEnvConfigured } from "../infra/net/proxy-env.js"; export { normalizeHostname } from "../infra/net/hostname.js"; @@ -34,8 +77,54 @@ export { type SsrFPolicy, } from "../infra/net/ssrf.js"; export { isNotFoundPathError, isPathInside } from "../infra/path-guards.js"; +export { + assertAbsolutePathInput, + canonicalPathFromExistingAncestor, + findExistingAncestor, + resolveAbsolutePathForRead, + resolveAbsolutePathForWrite, + type AbsolutePathSymlinkPolicy, + type ResolvedAbsolutePath, + type ResolvedWritableAbsolutePath, +} from "../infra/fs-safe.js"; +export { sanitizeUntrustedFileName } from "../infra/fs-safe-advanced.js"; +export { + privateFileStore, + privateFileStoreSync, + type PrivateFileStore, +} from "../infra/private-file-store.js"; +export { + movePathWithCopyFallback, + replaceFileAtomic, + replaceFileAtomicSync, + type MovePathWithCopyFallbackOptions, + type ReplaceFileAtomicFileSystem, + type ReplaceFileAtomicOptions, + type ReplaceFileAtomicResult, + type ReplaceFileAtomicSyncFileSystem, + type ReplaceFileAtomicSyncOptions, +} from "../infra/replace-file.js"; +export { + writeSiblingTempFile, + type WriteSiblingTempFileOptions, + type WriteSiblingTempFileResult, +} from "../infra/sibling-temp-file.js"; +export { + assertNoSymlinkParents, + assertNoSymlinkParentsSync, + type AssertNoSymlinkParentsOptions, +} from "../infra/fs-safe-advanced.js"; export { ensurePortAvailable } from "../infra/ports.js"; export { generateSecureToken } from "../infra/secure-random.js"; +export { + resolveExistingPathsWithinRoot, + pathScope, + resolvePathsWithinRoot, + resolvePathWithinRoot, + resolveStrictExistingPathsWithinRoot, + resolveWritablePathWithinRoot, +} from "../infra/root-paths.js"; +export { writeViaSiblingTempPath } from "../infra/fs-safe-advanced.js"; export { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; export { redactSensitiveText } from "../logging/redact.js"; export { safeEqualSecret } from "../security/secret-equal.js"; diff --git a/src/plugin-sdk/temp-path.test.ts b/src/plugin-sdk/temp-path.test.ts index cc51cc381ba..79e111c48cc 100644 --- a/src/plugin-sdk/temp-path.test.ts +++ b/src/plugin-sdk/temp-path.test.ts @@ -1,3 +1,4 @@ +import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; @@ -5,8 +6,13 @@ import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; import { buildRandomTempFilePath, withTempDownloadPath } from "./temp-path.js"; function expectPathInsideTmpRoot(resultPath: string) { - const tmpRoot = path.resolve(resolvePreferredOpenClawTmpDir()); - const resolved = path.resolve(resultPath); + const tmpRoot = fsSync.realpathSync(resolvePreferredOpenClawTmpDir()); + let resolved = path.resolve(resultPath); + try { + resolved = path.join(fsSync.realpathSync(path.dirname(resultPath)), path.basename(resultPath)); + } catch { + // The temp parent is intentionally gone after withTempDownloadPath cleanup. + } const rel = path.relative(tmpRoot, resolved); expect(rel === ".." || rel.startsWith(`..${path.sep}`)).toBe(false); expect(resultPath).not.toContain(".."); diff --git a/src/plugin-sdk/temp-path.ts b/src/plugin-sdk/temp-path.ts index 5deb49e021d..4ad38c8aa2f 100644 --- a/src/plugin-sdk/temp-path.ts +++ b/src/plugin-sdk/temp-path.ts @@ -5,3 +5,12 @@ export { sanitizeTempFileName, withTempDownloadPath, } from "../infra/temp-download.js"; +export { + tempWorkspace, + tempWorkspaceSync, + type TempWorkspace, + type TempWorkspaceOptions, + type TempWorkspaceSync, + withTempWorkspace, + withTempWorkspaceSync, +} from "../infra/private-temp-workspace.js"; diff --git a/src/plugins/bundle-commands.ts b/src/plugins/bundle-commands.ts index 02f9895cb8f..366ac656c4f 100644 --- a/src/plugins/bundle-commands.ts +++ b/src/plugins/bundle-commands.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { parseFrontmatterBlock } from "../markdown/frontmatter.js"; import { isPathInsideWithRealpath } from "../security/scan-paths.js"; import { @@ -56,7 +56,7 @@ function stripFrontmatter(content: string): string { function readClaudeBundleManifest(rootDir: string): Record { const manifestPath = path.join(rootDir, CLAUDE_BUNDLE_MANIFEST_RELATIVE_PATH); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: manifestPath, rootPath: rootDir, boundaryLabel: "plugin root", diff --git a/src/plugins/bundle-config-shared.ts b/src/plugins/bundle-config-shared.ts index 73f1cf188ab..8ec0ca4acc8 100644 --- a/src/plugins/bundle-config-shared.ts +++ b/src/plugins/bundle-config-shared.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { applyMergePatch } from "../config/merge-patch.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { matchBoundaryFileOpenFailure, openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { matchRootFileOpenFailure, openRootFileSync } from "../infra/boundary-file-read.js"; import { isRecord } from "../utils.js"; import { normalizePluginsConfig, resolveEffectivePluginActivationState } from "./config-state.js"; import type { PluginBundleFormat } from "./manifest-types.js"; @@ -23,11 +23,11 @@ export function readBundleJsonObject(params: { rootDir: string; relativePath: string; onOpenFailure?: ( - failure: Extract, { ok: false }>, + failure: Extract, { ok: false }>, ) => ReadBundleJsonResult; }): ReadBundleJsonResult { const absolutePath = path.join(params.rootDir, params.relativePath); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath, rootPath: params.rootDir, boundaryLabel: "plugin root", @@ -50,11 +50,11 @@ export function readBundleJsonObject(params: { } export function resolveBundleJsonOpenFailure(params: { - failure: Extract, { ok: false }>; + failure: Extract, { ok: false }>; relativePath: string; allowMissing?: boolean; }): ReadBundleJsonResult { - return matchBoundaryFileOpenFailure(params.failure, { + return matchRootFileOpenFailure(params.failure, { path: () => { if (params.allowMissing) { return { ok: true, raw: {} }; diff --git a/src/plugins/bundle-lsp.ts b/src/plugins/bundle-lsp.ts index 913b3f0835d..2e66f1c3f00 100644 --- a/src/plugins/bundle-lsp.ts +++ b/src/plugins/bundle-lsp.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { applyMergePatch } from "../config/merge-patch.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { isRecord } from "../utils.js"; import { inspectBundleServerRuntimeSupport, @@ -65,7 +65,7 @@ function loadBundleLspConfigFile(params: { relativePath: string; }): BundleLspConfig { const absolutePath = path.resolve(params.rootDir, params.relativePath); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath, rootPath: params.rootDir, boundaryLabel: "plugin root", diff --git a/src/plugins/bundle-manifest.ts b/src/plugins/bundle-manifest.ts index 5e56c6dc353..3234ddbe623 100644 --- a/src/plugins/bundle-manifest.ts +++ b/src/plugins/bundle-manifest.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import JSON5 from "json5"; -import { matchBoundaryFileOpenFailure, openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { matchRootFileOpenFailure, openRootFileSync } from "../infra/boundary-file-read.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, @@ -98,7 +98,7 @@ function loadBundleManifestFile(params: { allowMissing?: boolean; }): BundleManifestFileLoadResult { const manifestPath = path.join(params.rootDir, params.manifestRelativePath); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: manifestPath, rootPath: params.rootDir, ...(params.rootRealPath !== undefined ? { rootRealPath: params.rootRealPath } : {}), @@ -106,7 +106,7 @@ function loadBundleManifestFile(params: { rejectHardlinks: params.rejectHardlinks, }); if (!opened.ok) { - return matchBoundaryFileOpenFailure(opened, { + return matchRootFileOpenFailure(opened, { path: () => { if (params.allowMissing) { return { ok: true, raw: {}, manifestPath }; diff --git a/src/plugins/bundle-mcp.ts b/src/plugins/bundle-mcp.ts index 2992245d9e0..1530c1ae25e 100644 --- a/src/plugins/bundle-mcp.ts +++ b/src/plugins/bundle-mcp.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { applyMergePatch } from "../config/merge-patch.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { isRecord } from "../utils.js"; import { inspectBundleServerRuntimeSupport, @@ -168,7 +168,7 @@ function loadBundleFileBackedMcpConfig(params: { }): BundleMcpConfig { const rootDir = normalizeBundlePath(params.rootDir); const absolutePath = path.resolve(rootDir, params.relativePath); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath, rootPath: rootDir, boundaryLabel: "plugin root", diff --git a/src/plugins/bundled-capability-runtime.ts b/src/plugins/bundled-capability-runtime.ts index 1f45149d5f1..2b65a2eb44d 100644 --- a/src/plugins/bundled-capability-runtime.ts +++ b/src/plugins/bundled-capability-runtime.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import { fileURLToPath } from "node:url"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { withBundledPluginEnablementCompat, @@ -277,7 +277,7 @@ export function loadBundledCapabilityRuntimeRegistry(params: { workspaceDir: candidate.workspaceDir, }); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: record.source, rootPath: record.source === candidate.source ? candidate.rootDir : repoRoot, boundaryLabel: record.source === candidate.source ? "plugin root" : "repo root", diff --git a/src/plugins/bundled-dir.ts b/src/plugins/bundled-dir.ts index 3c6475f2ad0..c6b901eea0d 100644 --- a/src/plugins/bundled-dir.ts +++ b/src/plugins/bundled-dir.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; +import { isPathInside } from "../infra/path-guards.js"; import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; import { resolveUserPath } from "../utils.js"; @@ -78,8 +79,7 @@ function safeRealpathSync(targetPath: string): string | null { } function pathContains(parentDir: string, childPath: string): boolean { - const relative = path.relative(parentDir, childPath); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); + return isPathInside(parentDir, childPath); } function trustedBundledPluginRootsForPackageRoot(packageRoot: string): string[] { diff --git a/src/plugins/clawhub.ts b/src/plugins/clawhub.ts index 558016b083f..98839a41161 100644 --- a/src/plugins/clawhub.ts +++ b/src/plugins/clawhub.ts @@ -749,7 +749,7 @@ async function verifyClawHubArchiveFiles(params: { extractedBytes += bytes; return extractedBytes <= DEFAULT_MAX_EXTRACTED_BYTES; }; - for (const entry of Object.values(zip.files)) { + for (const entry of Object.values(zip.files as Record)) { entryCount += 1; if (entryCount > DEFAULT_MAX_ENTRIES) { return buildClawHubInstallFailure( diff --git a/src/plugins/conversation-binding.ts b/src/plugins/conversation-binding.ts index 558ce4a0e2f..93b1edde894 100644 --- a/src/plugins/conversation-binding.ts +++ b/src/plugins/conversation-binding.ts @@ -10,7 +10,7 @@ import { import { getChannelPlugin, normalizeChannelId } from "../channels/plugins/index.js"; import { formatErrorMessage } from "../infra/errors.js"; import { expandHomePrefix } from "../infra/home-dir.js"; -import { writeJsonAtomic } from "../infra/json-files.js"; +import { writeJson } from "../infra/json-files.js"; import { type ConversationRef } from "../infra/outbound/session-binding-service.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveGlobalMap, resolveGlobalSingleton } from "../shared/global-singleton.js"; @@ -379,7 +379,7 @@ async function saveApprovals(file: PluginBindingApprovalsFile): Promise { const state = getPluginBindingGlobalState(); state.approvalsCache = file; state.approvalsLoaded = true; - await writeJsonAtomic(filePath, file, { + await writeJson(filePath, file, { mode: 0o600, trailingNewline: true, }); diff --git a/src/plugins/discovery.ts b/src/plugins/discovery.ts index 2be659f1ed5..955eea09d93 100644 --- a/src/plugins/discovery.ts +++ b/src/plugins/discovery.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import type { PluginInstallRecord } from "../config/types.plugins.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, @@ -415,7 +415,7 @@ function readPackageManifest( rootRealPath?: string, ): PackageManifest | null { const manifestPath = path.join(dir, "package.json"); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: manifestPath, rootPath: dir, ...(rootRealPath !== undefined ? { rootRealPath } : {}), diff --git a/src/plugins/git-install.ts b/src/plugins/git-install.ts index 4298a1c6f8c..bb52f22ee25 100644 --- a/src/plugins/git-install.ts +++ b/src/plugins/git-install.ts @@ -1,7 +1,8 @@ +import "../infra/fs-safe-defaults.js"; import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; import path from "node:path"; import { withTempDir } from "../infra/install-source-utils.js"; +import { replaceDirectoryAtomic } from "../infra/replace-file.js"; import { createSafeNpmInstallArgs, createSafeNpmInstallEnv, @@ -192,34 +193,12 @@ async function replaceManagedGitRepo(params: { stagedRepoDir: string; persistentRepoDir: string; }): Promise<{ ok: true } | { ok: false; error: string }> { - const parentDir = path.dirname(params.persistentRepoDir); - const backupDir = path.join(parentDir, `.repo-backup-${process.pid}-${Date.now()}`); - let backupCreated = false; - try { - await fs.mkdir(parentDir, { recursive: true }); - try { - await fs.rename(params.persistentRepoDir, backupDir); - backupCreated = true; - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - } - - try { - await fs.rename(params.stagedRepoDir, params.persistentRepoDir); - } catch (err) { - if (backupCreated) { - await fs.rename(backupDir, params.persistentRepoDir); - backupCreated = false; - } - throw err; - } - - if (backupCreated) { - await fs.rm(backupDir, { recursive: true, force: true }); - } + await replaceDirectoryAtomic({ + stagedDir: params.stagedRepoDir, + targetDir: params.persistentRepoDir, + backupPrefix: ".repo-backup-", + }); return { ok: true }; } catch (err) { return { diff --git a/src/plugins/install.runtime.ts b/src/plugins/install.runtime.ts index 7d300ea1f23..a20d17c6833 100644 --- a/src/plugins/install.runtime.ts +++ b/src/plugins/install.runtime.ts @@ -1,5 +1,5 @@ -import { fileExists, readJsonFile, resolveArchiveKind } from "../infra/archive.js"; -import { writeFileFromPathWithinRoot } from "../infra/fs-safe.js"; +import { resolveArchiveKind } from "../infra/archive.js"; +import { pathExists, root } from "../infra/fs-safe.js"; import { resolveExistingInstallPath, withExtractedArchiveRoot } from "../infra/install-flow.js"; import { resolveInstallModeOptions, @@ -15,6 +15,7 @@ import { ensureInstallTargetAvailable, resolveCanonicalInstallTarget, } from "../infra/install-target.js"; +import { readJson } from "../infra/json-files.js"; import { finalizeNpmSpecArchiveInstall, installFromNpmSpecArchiveWithInstaller, @@ -40,9 +41,10 @@ export type { NpmIntegrityDrift, NpmSpecResolution }; export { checkMinHostVersion, + root, detectBundleManifestFormat, ensureInstallTargetAvailable, - fileExists, + pathExists as fileExists, finalizeNpmSpecArchiveInstall, getPackageManifestMetadata, installFromNpmSpecArchiveWithInstaller, @@ -50,7 +52,7 @@ export { isPathInside, loadBundleManifest, loadPluginManifest, - readJsonFile, + readJson as readJsonFile, resolveArchiveKind, resolveArchiveSourcePath, resolveCanonicalInstallTarget, @@ -66,5 +68,4 @@ export { scanPackageInstallSource, validateRegistryNpmSpec, withExtractedArchiveRoot, - writeFileFromPathWithinRoot, }; diff --git a/src/plugins/install.ts b/src/plugins/install.ts index 4c0e0a4a2d2..33399abcc68 100644 --- a/src/plugins/install.ts +++ b/src/plugins/install.ts @@ -1204,11 +1204,8 @@ export async function installPluginFromFile(params: { logger.info?.(`Installing to ${preparedTarget.targetPath}…`); try { - await runtime.writeFileFromPathWithinRoot({ - rootDir: extensionsDir, - relativePath: path.basename(preparedTarget.targetPath), - sourcePath: filePath, - }); + const root = await runtime.root(extensionsDir); + await root.copyIn(path.basename(preparedTarget.targetPath), filePath); } catch (err) { return { ok: false, error: String(err) }; } diff --git a/src/plugins/installed-plugin-index-record-reader.ts b/src/plugins/installed-plugin-index-record-reader.ts index 34a6a49e030..2046c1599cd 100644 --- a/src/plugins/installed-plugin-index-record-reader.ts +++ b/src/plugins/installed-plugin-index-record-reader.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import type { PluginInstallRecord } from "../config/types.plugins.js"; -import { readJsonFile, readJsonFileSync } from "../infra/json-files.js"; +import { tryReadJson, tryReadJsonSync } from "../infra/json-files.js"; import { resolveDefaultPluginNpmDir, validatePluginId } from "./install-paths.js"; import { resolveInstalledPluginIndexStorePath, @@ -163,14 +163,14 @@ function extractPluginInstallRecordsFromPersistedInstalledPluginIndex( export async function readPersistedInstalledPluginIndexInstallRecords( options: InstalledPluginIndexStoreOptions = {}, ): Promise | null> { - const parsed = await readJsonFile(resolveInstalledPluginIndexStorePath(options)); + const parsed = await tryReadJson(resolveInstalledPluginIndexStorePath(options)); return extractPluginInstallRecordsFromPersistedInstalledPluginIndex(parsed); } export function readPersistedInstalledPluginIndexInstallRecordsSync( options: InstalledPluginIndexStoreOptions = {}, ): Record | null { - const parsed = readJsonFileSync(resolveInstalledPluginIndexStorePath(options)); + const parsed = tryReadJsonSync(resolveInstalledPluginIndexStorePath(options)); return extractPluginInstallRecordsFromPersistedInstalledPluginIndex(parsed); } diff --git a/src/plugins/installed-plugin-index-store.ts b/src/plugins/installed-plugin-index-store.ts index 1044be8fc2e..3c3f9707956 100644 --- a/src/plugins/installed-plugin-index-store.ts +++ b/src/plugins/installed-plugin-index-store.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { saveJsonFile } from "../infra/json-file.js"; -import { readJsonFile, readJsonFileSync, writeJsonAtomic } from "../infra/json-files.js"; +import { tryReadJson, tryReadJsonSync, writeJson } from "../infra/json-files.js"; import { isBlockedObjectKey } from "../infra/prototype-keys.js"; import { safeParseWithSchema } from "../utils/zod-parse.js"; import { resolveCompatibilityHostVersion } from "../version.js"; @@ -161,14 +161,14 @@ function parseInstalledPluginIndex(value: unknown): InstalledPluginIndex | null export async function readPersistedInstalledPluginIndex( options: InstalledPluginIndexStoreOptions = {}, ): Promise { - const parsed = await readJsonFile(resolveInstalledPluginIndexStorePath(options)); + const parsed = await tryReadJson(resolveInstalledPluginIndexStorePath(options)); return parseInstalledPluginIndex(parsed); } export function readPersistedInstalledPluginIndexSync( options: InstalledPluginIndexStoreOptions = {}, ): InstalledPluginIndex | null { - const parsed = readJsonFileSync(resolveInstalledPluginIndexStorePath(options)); + const parsed = tryReadJsonSync(resolveInstalledPluginIndexStorePath(options)); return parseInstalledPluginIndex(parsed); } @@ -177,12 +177,12 @@ export async function writePersistedInstalledPluginIndex( options: InstalledPluginIndexStoreOptions = {}, ): Promise { const filePath = resolveInstalledPluginIndexStorePath(options); - await writeJsonAtomic( + await writeJson( filePath, { ...index, warning: INSTALLED_PLUGIN_INDEX_WARNING }, { trailingNewline: true, - ensureDirMode: 0o700, + dirMode: 0o700, mode: 0o600, }, ); diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index d3c4efff640..cb2cd627783 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -9,7 +9,7 @@ import { import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginInstallRecord } from "../config/types.plugins.js"; import type { GatewayRequestHandler } from "../gateway/server-methods/types.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { DEFAULT_MEMORY_DREAMING_PLUGIN_ID, @@ -2005,7 +2005,7 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi : runtimeCandidateEntry; const moduleLoadSource = resolveCanonicalDistRuntimeSource(loadEntry.source); const moduleRoot = resolveCanonicalDistRuntimeSource(loadEntry.rootDir); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: moduleLoadSource, rootPath: moduleRoot, boundaryLabel: "plugin root", @@ -2096,7 +2096,7 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi const runtimeModuleRoot = resolveCanonicalDistRuntimeSource( runtimeCandidateEntry.rootDir, ); - const runtimeOpened = openBoundaryFileSync({ + const runtimeOpened = openRootFileSync({ absolutePath: runtimeModuleSource, rootPath: runtimeModuleRoot, boundaryLabel: "plugin root", @@ -2677,7 +2677,7 @@ export async function loadOpenClawPluginCliRegistry( seenIds.set(pluginId, candidate.origin); continue; } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: sourceForCliMetadata, rootPath: pluginRoot, boundaryLabel: "plugin root", diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index b64f715a9e7..e3c73e84467 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import type { ChannelConfigRuntimeSchema } from "../channels/plugins/types.config.js"; import { MANIFEST_KEY } from "../compat/legacy-names.js"; -import { matchBoundaryFileOpenFailure, openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { matchRootFileOpenFailure, openRootFileSync } from "../infra/boundary-file-read.js"; import { isBlockedObjectKey } from "../infra/prototype-keys.js"; import { normalizeModelCatalog, @@ -1513,7 +1513,7 @@ export function loadPluginManifest( rootRealPath?: string, ): PluginManifestLoadResult { const manifestPath = resolvePluginManifestPath(rootDir); - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: manifestPath, rootPath: rootDir, ...(rootRealPath !== undefined ? { rootRealPath } : {}), @@ -1522,7 +1522,7 @@ export function loadPluginManifest( rejectHardlinks, }); if (!opened.ok) { - return matchBoundaryFileOpenFailure(opened, { + return matchRootFileOpenFailure(opened, { path: () => ({ ok: false, error: `plugin manifest not found: ${manifestPath}`, diff --git a/src/plugins/marketplace.ts b/src/plugins/marketplace.ts index 68776355ab9..fc4a25e564e 100644 --- a/src/plugins/marketplace.ts +++ b/src/plugins/marketplace.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { resolveArchiveKind } from "../infra/archive.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { pathExists } from "../infra/fs-safe.js"; import { resolveOsHomeRelativePath } from "../infra/home-dir.js"; import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js"; import { isPathInside } from "../infra/path-guards.js"; @@ -308,15 +309,6 @@ function parseMarketplaceManifest( }; } -async function pathExists(target: string): Promise { - try { - await fs.access(target); - return true; - } catch { - return false; - } -} - async function readClaudeKnownMarketplaces(): Promise> { const knownPath = resolveOsHomeRelativePath(CLAUDE_KNOWN_MARKETPLACES_PATH); if (!(await pathExists(knownPath))) { diff --git a/src/plugins/package-entry-resolution.ts b/src/plugins/package-entry-resolution.ts index 62274ee883d..147bc510635 100644 --- a/src/plugins/package-entry-resolution.ts +++ b/src/plugins/package-entry-resolution.ts @@ -1,11 +1,11 @@ import fs from "node:fs"; import path from "node:path"; import { - matchBoundaryFileOpenFailure, - openBoundaryFile, - openBoundaryFileSync, + matchRootFileOpenFailure, + openRootFile, + openRootFileSync, } from "../infra/boundary-file-read.js"; -import { resolveBoundaryPath, resolveBoundaryPathSync } from "../infra/boundary-path.js"; +import { resolveRootPath, resolveRootPathSync } from "../infra/boundary-path.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; import type { PluginDiagnostic } from "./manifest-types.js"; import { getPackageManifestMetadata, type PackageManifest } from "./manifest.js"; @@ -98,7 +98,7 @@ async function validatePackageExtensionEntry(params: { }): Promise { const absolutePath = path.resolve(params.packageDir, params.entry); try { - const resolved = await resolveBoundaryPath({ + const resolved = await resolveRootPath({ absolutePath, rootPath: params.packageDir, boundaryLabel: "plugin package directory", @@ -115,13 +115,13 @@ async function validatePackageExtensionEntry(params: { }; } - const opened = await openBoundaryFile({ + const opened = await openRootFile({ absolutePath, rootPath: params.packageDir, boundaryLabel: "plugin package directory", }); if (!opened.ok) { - return matchBoundaryFileOpenFailure(opened, { + return matchRootFileOpenFailure(opened, { path: () => ({ ok: false, error: `${params.label} not found: ${params.entry}` }), io: () => ({ ok: false, error: `${params.label} unreadable: ${params.entry}` }), validation: () => ({ @@ -326,7 +326,7 @@ function resolvePackageEntrySource(params: { const rejectHardlinks = params.rejectHardlinks ?? true; const candidates = [source]; const openCandidate = (absolutePath: string): string | null => { - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath, rootPath: params.packageDir, ...(params.packageRootRealPath !== undefined @@ -336,7 +336,7 @@ function resolvePackageEntrySource(params: { rejectHardlinks, }); if (!opened.ok) { - return matchBoundaryFileOpenFailure(opened, { + return matchRootFileOpenFailure(opened, { path: () => null, io: () => { params.diagnostics.push({ @@ -415,7 +415,7 @@ function resolveSafePackageEntry(params: { } try { - resolveBoundaryPathSync({ + resolveRootPathSync({ absolutePath, rootPath: params.packageDir, ...(params.packageRootRealPath !== undefined diff --git a/src/plugins/path-safety.ts b/src/plugins/path-safety.ts index bcc637fbc61..bfa2c2202f5 100644 --- a/src/plugins/path-safety.ts +++ b/src/plugins/path-safety.ts @@ -1,33 +1,16 @@ -import fs from "node:fs"; -import { isPathInside as isBoundaryPathInside } from "../infra/path-guards.js"; - -export function isPathInside(baseDir: string, targetPath: string): boolean { - return isBoundaryPathInside(baseDir, targetPath); -} - -export function safeRealpathSync(targetPath: string, cache?: Map): string | null { - const cached = cache?.get(targetPath); - if (cached) { - return cached; - } - try { - const resolved = fs.realpathSync(targetPath); - cache?.set(targetPath, resolved); - cache?.set(resolved, resolved); - return resolved; - } catch { - return null; - } -} - -export function safeStatSync(targetPath: string): fs.Stats | null { - try { - return fs.statSync(targetPath); - } catch { - return null; - } -} - -export function formatPosixMode(mode: number): string { - return (mode & 0o777).toString(8).padStart(3, "0"); -} +export { + isNotFoundPathError, + hasNodeErrorCode, + isNodeError, + isPathInside, + isPathInsideWithRealpath, + isSymlinkOpenError, + isWithinDir, + normalizeWindowsPathForComparison, + resolveSafeBaseDir, + resolveSafeRelativePath, + safeRealpathSync, + safeStatSync, + splitSafeRelativePath, + formatPosixMode, +} from "../infra/path-safety.js"; diff --git a/src/plugins/public-surface-loader.ts b/src/plugins/public-surface-loader.ts index f7e880e09e0..7b279f8dfb0 100644 --- a/src/plugins/public-surface-loader.ts +++ b/src/plugins/public-surface-loader.ts @@ -2,8 +2,8 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; -import { sameFileIdentity } from "../infra/file-identity.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; +import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; import { resolveBundledPluginsDir } from "./bundled-dir.js"; import { createPluginModuleLoaderCache, @@ -129,7 +129,7 @@ export function loadBundledPluginPublicArtifactModuleSync(para return cached as T; } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: location.modulePath, rootPath: location.boundaryRoot, boundaryLabel: diff --git a/src/plugins/source-display.ts b/src/plugins/source-display.ts index 8e955d08edc..ae33488712a 100644 --- a/src/plugins/source-display.ts +++ b/src/plugins/source-display.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { isPathInside } from "../infra/path-guards.js"; import { shortenHomeInString } from "../utils.js"; import type { PluginRecord } from "./registry.js"; import type { PluginSourceRoots } from "./roots.js"; @@ -6,19 +7,13 @@ export { resolvePluginSourceRoots } from "./roots.js"; export type { PluginSourceRoots } from "./roots.js"; function tryRelative(root: string, filePath: string): string | null { + if (!isPathInside(root, filePath)) { + return null; + } const rel = path.relative(root, filePath); if (!rel || rel === ".") { return null; } - if (rel === "..") { - return null; - } - if (rel.startsWith(`..${path.sep}`) || rel.startsWith("../") || rel.startsWith("..\\")) { - return null; - } - if (path.isAbsolute(rel)) { - return null; - } // Normalize to forward slashes for display (path.relative uses backslashes on Windows) return rel.replaceAll("\\", "/"); } diff --git a/src/secrets/channel-contract-api.ts b/src/secrets/channel-contract-api.ts index 908b2d48e5b..58741519fff 100644 --- a/src/secrets/channel-contract-api.ts +++ b/src/secrets/channel-contract-api.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { openBoundaryFileSync } from "../infra/boundary-file-read.js"; +import { openRootFileSync } from "../infra/boundary-file-read.js"; import type { PluginManifestRecord } from "../plugins/manifest-registry.js"; import { loadPluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { @@ -122,7 +122,7 @@ function loadExternalChannelSecretContractFromRecord( if (!contractPath) { return undefined; } - const opened = openBoundaryFileSync({ + const opened = openRootFileSync({ absolutePath: contractPath, rootPath: record.rootDir, boundaryLabel: "plugin root", diff --git a/src/secrets/resolve.test.ts b/src/secrets/resolve.test.ts index fc6f33fb389..31026818d20 100644 --- a/src/secrets/resolve.test.ts +++ b/src/secrets/resolve.test.ts @@ -400,16 +400,14 @@ describe("secret ref resolver", () => { }), ); - const originalReadFile = fs.readFile.bind(fs); - const readFileSpy = vi.spyOn(fs, "readFile").mockImplementation((( - targetPath: Parameters[0], - options?: Parameters[1], - ) => { - if (typeof targetPath === "string" && targetPath === filePath) { - return new Promise(() => {}); - } - return originalReadFile(targetPath, options); - }) as typeof fs.readFile); + const sampleHandle = await fs.open(filePath, "r"); + const fileHandlePrototype = Object.getPrototypeOf(sampleHandle) as { + readFile: typeof sampleHandle.readFile; + }; + await sampleHandle.close(); + const readFileSpy = vi + .spyOn(fileHandlePrototype, "readFile") + .mockImplementation(() => new Promise(() => {}) as never); try { await expect( diff --git a/src/secrets/resolve.ts b/src/secrets/resolve.ts index 02efffb168e..f97fe9f6ff0 100644 --- a/src/secrets/resolve.ts +++ b/src/secrets/resolve.ts @@ -10,6 +10,7 @@ import type { SecretRefSource, } from "../config/types.secrets.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { FsSafeError, readSecureFile } from "../infra/fs-safe.js"; import { inspectPathPermissions, safeStat } from "../security/audit-fs.js"; import { isPathInside } from "../security/scan-paths.js"; import { resolveUserPath } from "../utils.js"; @@ -283,33 +284,18 @@ async function readFileProviderPayload(params: { const filePath = resolveUserPath(params.providerConfig.path); const readPromise = (async () => { - const secureFilePath = await assertSecurePath({ - targetPath: filePath, - label: `secrets.providers.${params.providerName}.path`, - allowInsecurePath: params.providerConfig.allowInsecurePath, - }); const timeoutMs = normalizePositiveInt( params.providerConfig.timeoutMs, DEFAULT_FILE_TIMEOUT_MS, ); const maxBytes = normalizePositiveInt(params.providerConfig.maxBytes, DEFAULT_FILE_MAX_BYTES); - const abortController = new AbortController(); - const timeoutErrorMessage = `File provider "${params.providerName}" timed out after ${timeoutMs}ms.`; - let timeoutHandle: NodeJS.Timeout | null = null; - const timeoutPromise = new Promise((_resolve, reject) => { - timeoutHandle = setTimeout(() => { - abortController.abort(); - reject(new Error(timeoutErrorMessage)); - }, timeoutMs); - }); try { - const payload = await Promise.race([ - fs.readFile(secureFilePath, { signal: abortController.signal }), - timeoutPromise, - ]); - if (payload.byteLength > maxBytes) { - throw new Error(`File provider "${params.providerName}" exceeded maxBytes (${maxBytes}).`); - } + const { buffer: payload } = await readSecureFile({ + filePath, + label: `secrets.providers.${params.providerName}.path`, + io: { maxBytes, timeoutMs }, + permissions: { allowInsecure: params.providerConfig.allowInsecurePath }, + }); const text = payload.toString("utf8").replace(/^\uFEFF/, ""); if (params.providerConfig.mode === "singleValue") { return text.replace(/\r?\n$/, ""); @@ -320,14 +306,12 @@ async function readFileProviderPayload(params: { } return parsed; } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - throw new Error(timeoutErrorMessage, { cause: error }); + if (error instanceof FsSafeError && error.code === "timeout") { + throw new Error(`File provider "${params.providerName}" timed out after ${timeoutMs}ms.`, { + cause: error, + }); } throw error; - } finally { - if (timeoutHandle) { - clearTimeout(timeoutHandle); - } } })(); diff --git a/src/secrets/shared.ts b/src/secrets/shared.ts index 98635ac25f9..d23444663a2 100644 --- a/src/secrets/shared.ts +++ b/src/secrets/shared.ts @@ -1,5 +1,7 @@ import fs from "node:fs"; import path from "node:path"; +import { privateFileStoreSync } from "../infra/private-file-store.js"; +import { replaceFileAtomicSync } from "../infra/replace-file.js"; export { isRecord } from "../utils.js"; export function isNonEmptyString(value: unknown): value is string { @@ -40,9 +42,9 @@ export function ensureDirForFile(filePath: string): void { } export function writeJsonFileSecure(pathname: string, value: unknown): void { - ensureDirForFile(pathname); - fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8"); - fs.chmodSync(pathname, 0o600); + privateFileStoreSync(path.dirname(pathname)).writeJson(path.basename(pathname), value, { + trailingNewline: true, + }); } export function readTextFileIfExists(pathname: string): string | null { @@ -53,9 +55,14 @@ export function readTextFileIfExists(pathname: string): string | null { } export function writeTextFileAtomic(pathname: string, value: string, mode = 0o600): void { - ensureDirForFile(pathname); - const tempPath = `${pathname}.tmp-${process.pid}-${Date.now()}`; - fs.writeFileSync(tempPath, value, "utf8"); - fs.chmodSync(tempPath, mode); - fs.renameSync(tempPath, pathname); + if (mode !== 0o600) { + replaceFileAtomicSync({ + filePath: pathname, + content: value, + mode, + tempPrefix: ".openclaw-secrets", + }); + return; + } + privateFileStoreSync(path.dirname(pathname)).writeText(path.basename(pathname), value); } diff --git a/src/security/audit-fs.ts b/src/security/audit-fs.ts index 34b329de9d7..2ea52648ffb 100644 --- a/src/security/audit-fs.ts +++ b/src/security/audit-fs.ts @@ -1,206 +1,8 @@ -import fs from "node:fs/promises"; -import { - formatIcaclsResetCommand, - formatWindowsAclSummary, - inspectWindowsAcl, - type ExecFn, -} from "./windows-acl.js"; - -export type PermissionCheck = { - ok: boolean; - isSymlink: boolean; - isDir: boolean; - mode: number | null; - bits: number | null; - source: "posix" | "windows-acl" | "unknown"; - worldWritable: boolean; - groupWritable: boolean; - worldReadable: boolean; - groupReadable: boolean; - aclSummary?: string; - error?: string; -}; - -export type PermissionCheckOptions = { - platform?: NodeJS.Platform; - env?: NodeJS.ProcessEnv; - exec?: ExecFn; -}; - -export async function safeStat(targetPath: string): Promise<{ - ok: boolean; - isSymlink: boolean; - isDir: boolean; - mode: number | null; - uid: number | null; - gid: number | null; - error?: string; -}> { - try { - const lst = await fs.lstat(targetPath); - return { - ok: true, - isSymlink: lst.isSymbolicLink(), - isDir: lst.isDirectory(), - mode: typeof lst.mode === "number" ? lst.mode : null, - uid: typeof lst.uid === "number" ? lst.uid : null, - gid: typeof lst.gid === "number" ? lst.gid : null, - }; - } catch (err) { - return { - ok: false, - isSymlink: false, - isDir: false, - mode: null, - uid: null, - gid: null, - error: String(err), - }; - } -} - -export async function inspectPathPermissions( - targetPath: string, - opts?: PermissionCheckOptions, -): Promise { - const st = await safeStat(targetPath); - if (!st.ok) { - return { - ok: false, - isSymlink: false, - isDir: false, - mode: null, - bits: null, - source: "unknown", - worldWritable: false, - groupWritable: false, - worldReadable: false, - groupReadable: false, - error: st.error, - }; - } - - let effectiveMode = st.mode; - let effectiveIsDir = st.isDir; - if (st.isSymlink) { - try { - const target = await fs.stat(targetPath); - effectiveMode = typeof target.mode === "number" ? target.mode : st.mode; - effectiveIsDir = target.isDirectory(); - } catch { - // Keep lstat-derived metadata when target lookup fails. - } - } - - const bits = modeBits(effectiveMode); - const platform = opts?.platform ?? process.platform; - - if (platform === "win32") { - const acl = await inspectWindowsAcl(targetPath, { env: opts?.env, exec: opts?.exec }); - if (!acl.ok) { - return { - ok: true, - isSymlink: st.isSymlink, - isDir: effectiveIsDir, - mode: effectiveMode, - bits, - source: "unknown", - worldWritable: false, - groupWritable: false, - worldReadable: false, - groupReadable: false, - error: acl.error, - }; - } - return { - ok: true, - isSymlink: st.isSymlink, - isDir: effectiveIsDir, - mode: effectiveMode, - bits, - source: "windows-acl", - worldWritable: acl.untrustedWorld.some((entry) => entry.canWrite), - groupWritable: acl.untrustedGroup.some((entry) => entry.canWrite), - worldReadable: acl.untrustedWorld.some((entry) => entry.canRead), - groupReadable: acl.untrustedGroup.some((entry) => entry.canRead), - aclSummary: formatWindowsAclSummary(acl), - }; - } - - return { - ok: true, - isSymlink: st.isSymlink, - isDir: effectiveIsDir, - mode: effectiveMode, - bits, - source: "posix", - worldWritable: isWorldWritable(bits), - groupWritable: isGroupWritable(bits), - worldReadable: isWorldReadable(bits), - groupReadable: isGroupReadable(bits), - }; -} - -export function formatPermissionDetail(targetPath: string, perms: PermissionCheck): string { - if (perms.source === "windows-acl") { - const summary = perms.aclSummary ?? "unknown"; - return `${targetPath} acl=${summary}`; - } - return `${targetPath} mode=${formatOctal(perms.bits)}`; -} - -export function formatPermissionRemediation(params: { - targetPath: string; - perms: PermissionCheck; - isDir: boolean; - posixMode: number; - env?: NodeJS.ProcessEnv; -}): string { - if (params.perms.source === "windows-acl") { - return formatIcaclsResetCommand(params.targetPath, { isDir: params.isDir, env: params.env }); - } - const mode = params.posixMode.toString(8).padStart(3, "0"); - return `chmod ${mode} ${params.targetPath}`; -} - -export function modeBits(mode: number | null): number | null { - if (mode == null) { - return null; - } - return mode & 0o777; -} - -export function formatOctal(bits: number | null): string { - if (bits == null) { - return "unknown"; - } - return bits.toString(8).padStart(3, "0"); -} - -export function isWorldWritable(bits: number | null): boolean { - if (bits == null) { - return false; - } - return (bits & 0o002) !== 0; -} - -export function isGroupWritable(bits: number | null): boolean { - if (bits == null) { - return false; - } - return (bits & 0o020) !== 0; -} - -export function isWorldReadable(bits: number | null): boolean { - if (bits == null) { - return false; - } - return (bits & 0o004) !== 0; -} - -export function isGroupReadable(bits: number | null): boolean { - if (bits == null) { - return false; - } - return (bits & 0o040) !== 0; -} +export { + formatPermissionDetail, + formatPermissionRemediation, + inspectPathPermissions, + safeStat, + type PermissionCheck, + type PermissionCheckOptions, +} from "../infra/permissions.js"; diff --git a/src/security/scan-paths.ts b/src/security/scan-paths.ts index 43b4bc03e8e..b4fed326179 100644 --- a/src/security/scan-paths.ts +++ b/src/security/scan-paths.ts @@ -1,39 +1,4 @@ -import fs from "node:fs"; -import path from "node:path"; - -export function isPathInside(basePath: string, candidatePath: string): boolean { - const base = path.resolve(basePath); - const candidate = path.resolve(candidatePath); - const rel = path.relative(base, candidate); - return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel)); -} - -function safeRealpathSync(filePath: string): string | null { - try { - return fs.realpathSync(filePath); - } catch { - return null; - } -} - -export function isPathInsideWithRealpath( - basePath: string, - candidatePath: string, - opts?: { requireRealpath?: boolean }, -): boolean { - if (!isPathInside(basePath, candidatePath)) { - return false; - } - const baseReal = safeRealpathSync(basePath); - const candidateReal = safeRealpathSync(candidatePath); - if (!baseReal || !candidateReal) { - // Default to false (safe): only bypass the realpath check when the caller - // explicitly opts out with requireRealpath: false. All production callers - // already pass requireRealpath: true; this change makes the default secure. - return opts?.requireRealpath === false; - } - return isPathInside(baseReal, candidateReal); -} +export { isPathInside, isPathInsideWithRealpath } from "../infra/path-safety.js"; export function extensionUsesSkippedScannerPath(entry: string): boolean { const segments = entry.split(/[\\/]+/).filter(Boolean); diff --git a/src/security/windows-acl.ts b/src/security/windows-acl.ts index 03f739a93c6..26796f51e94 100644 --- a/src/security/windows-acl.ts +++ b/src/security/windows-acl.ts @@ -1,415 +1,12 @@ -import os from "node:os"; -import path from "node:path"; -import { getWindowsInstallRoots } from "../infra/windows-install-roots.js"; -import { createSubsystemLogger } from "../logging/subsystem.js"; -import { runExec } from "../process/exec.js"; -import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; - -const log = createSubsystemLogger("security/windows-acl"); - -export type ExecFn = typeof runExec; - -export type WindowsAclEntry = { - principal: string; - rights: string[]; - rawRights: string; - canRead: boolean; - canWrite: boolean; -}; - -export type WindowsAclSummary = { - ok: boolean; - entries: WindowsAclEntry[]; - untrustedWorld: WindowsAclEntry[]; - untrustedGroup: WindowsAclEntry[]; - trusted: WindowsAclEntry[]; - error?: string; -}; - -export type WindowsUserInfoProvider = () => { username?: string | null }; - -export type IcaclsResetCommandOptions = { - isDir: boolean; - env?: NodeJS.ProcessEnv; - userInfo?: WindowsUserInfoProvider; -}; - -const INHERIT_FLAGS = new Set(["I", "OI", "CI", "IO", "NP"]); -const WORLD_PRINCIPALS = new Set([ - "everyone", - "users", - "builtin\\users", - "authenticated users", - "nt authority\\authenticated users", -]); -const TRUSTED_BASE = new Set([ - "nt authority\\system", - "system", - "builtin\\administrators", - "creator owner", - // Localized SYSTEM account names (French, German, Spanish, Portuguese) - "autorite nt\\système", - "nt-autorität\\system", - "autoridad nt\\system", - "autoridade nt\\system", -]); -const WORLD_SUFFIXES = ["\\users", "\\authenticated users"]; -const TRUSTED_SUFFIXES = ["\\administrators", "\\system", "\\système"]; - -// Accept an optional leading * which icacls prefixes to SIDs when invoked with /sid -// (e.g. *S-1-5-18 instead of S-1-5-18). -const SID_RE = /^\*?s-\d+-\d+(-\d+)+$/i; -const TRUSTED_SIDS = new Set([ - "s-1-5-18", - "s-1-5-32-544", - "s-1-5-80-956008885-3418522649-1831038044-1853292631-2271478464", -]); -// SIDs for world-equivalent principals that icacls /sid emits as raw SIDs. -// Without this list these would be classified as "group" instead of "world". -// S-1-1-0 Everyone -// S-1-5-11 Authenticated Users -// S-1-5-32-545 BUILTIN\Users -const WORLD_SIDS = new Set(["s-1-1-0", "s-1-5-11", "s-1-5-32-545"]); -const STATUS_PREFIXES = [ - "successfully processed", - "processed", - "failed processing", - "no mapping between account names", -]; - -const normalize = (value: string) => normalizeLowercaseStringOrEmpty(value); -const defaultWindowsUserInfo: WindowsUserInfoProvider = () => os.userInfo(); - -function normalizeSid(value: string): string { - const normalized = normalize(value); - return normalized.startsWith("*") ? normalized.slice(1) : normalized; -} - -export function resolveWindowsUserPrincipal( - env?: NodeJS.ProcessEnv, - userInfo: WindowsUserInfoProvider = defaultWindowsUserInfo, -): string | null { - const username = env?.USERNAME?.trim() || userInfo().username?.trim(); - if (!username) { - return null; - } - const domain = env?.USERDOMAIN?.trim(); - return domain ? `${domain}\\${username}` : username; -} - -function buildTrustedPrincipals(env?: NodeJS.ProcessEnv): Set { - const trusted = new Set(TRUSTED_BASE); - const principal = resolveWindowsUserPrincipal(env); - if (principal) { - trusted.add(normalize(principal)); - const parts = principal.split("\\"); - const userOnly = parts.at(-1); - if (userOnly) { - trusted.add(normalize(userOnly)); - } - } - const userSid = normalizeSid(env?.USERSID ?? ""); - // Guard: never add world-equivalent SIDs (Everyone, Authenticated Users, BUILTIN\\Users) - // to the trusted set, even if USERSID is set to one of them by a malicious process. - if (userSid && SID_RE.test(userSid) && !WORLD_SIDS.has(userSid)) { - trusted.add(userSid); - } - return trusted; -} - -function resolveWindowsSystemCommand(command: string, env?: NodeJS.ProcessEnv): string { - // Never fall back to a bare helper name here; Windows command search can - // consult the current directory and PATH before the real System32 helper. - const root = getWindowsInstallRoots(env ?? process.env).systemRoot; - return path.win32.join(root, "System32", command); -} - -function classifyPrincipal( - principal: string, - trustedPrincipals: Set, -): "trusted" | "world" | "group" { - const normalized = normalize(principal); - - if (SID_RE.test(normalized)) { - // Strip the leading * that icacls /sid prefixes to SIDs before lookup. - const sid = normalizeSid(normalized); - // World-equivalent SIDs must be classified as "world", not "group", so - // that callers applying world-write policies catch everyone/authenticated- - // users entries the same way they would catch the human-readable names. - if (WORLD_SIDS.has(sid)) { - return "world"; - } - if (TRUSTED_SIDS.has(sid) || trustedPrincipals.has(sid)) { - return "trusted"; - } - return "group"; - } - - if ( - trustedPrincipals.has(normalized) || - TRUSTED_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) - ) { - return "trusted"; - } - if ( - WORLD_PRINCIPALS.has(normalized) || - WORLD_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) - ) { - return "world"; - } - - // Fallback: strip diacritics and re-check for localized SYSTEM variants - const stripped = normalized.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); - if ( - stripped !== normalized && - (TRUSTED_BASE.has(stripped) || - TRUSTED_SUFFIXES.some((suffix) => { - const strippedSuffix = suffix.normalize("NFD").replace(/[\u0300-\u036f]/g, ""); - return stripped.endsWith(strippedSuffix); - })) - ) { - return "trusted"; - } - - return "group"; -} - -function rightsFromTokens(tokens: string[]): { - canRead: boolean; - canWrite: boolean; -} { - const upper = tokens.join("").toUpperCase(); - const canWrite = - upper.includes("F") || upper.includes("M") || upper.includes("W") || upper.includes("D"); - const canRead = upper.includes("F") || upper.includes("M") || upper.includes("R"); - return { canRead, canWrite }; -} - -function isStatusLine(lowerLine: string): boolean { - return STATUS_PREFIXES.some((prefix) => lowerLine.startsWith(prefix)); -} - -function stripTargetPrefix(params: { - trimmedLine: string; - lowerLine: string; - normalizedTarget: string; - lowerTarget: string; - quotedTarget: string; - quotedLower: string; -}): string { - if (params.lowerLine.startsWith(params.lowerTarget)) { - return params.trimmedLine.slice(params.normalizedTarget.length).trim(); - } - if (params.lowerLine.startsWith(params.quotedLower)) { - return params.trimmedLine.slice(params.quotedTarget.length).trim(); - } - return params.trimmedLine; -} - -function parseAceEntry(entry: string): WindowsAclEntry | null { - if (!entry || !entry.includes("(")) { - return null; - } - - const idx = entry.indexOf(":"); - if (idx === -1) { - return null; - } - - const principal = entry.slice(0, idx).trim(); - const rawRights = entry.slice(idx + 1).trim(); - const tokens = - rawRights - .match(/\(([^)]+)\)/g) - ?.map((token) => token.slice(1, -1).trim()) - .filter(Boolean) ?? []; - - if (tokens.some((token) => token.toUpperCase() === "DENY")) { - return null; - } - - const rights = tokens.filter((token) => !INHERIT_FLAGS.has(token.toUpperCase())); - if (rights.length === 0) { - return null; - } - - const { canRead, canWrite } = rightsFromTokens(rights); - return { principal, rights, rawRights, canRead, canWrite }; -} - -export function parseIcaclsOutput(output: string, targetPath: string): WindowsAclEntry[] { - const entries: WindowsAclEntry[] = []; - const normalizedTarget = targetPath.trim(); - const lowerTarget = normalizedTarget.toLowerCase(); - const quotedTarget = `"${normalizedTarget}"`; - const quotedLower = quotedTarget.toLowerCase(); - - for (const rawLine of output.split(/\r?\n/)) { - const line = rawLine.trimEnd(); - if (!line.trim()) { - continue; - } - const trimmed = line.trim(); - const lower = trimmed.toLowerCase(); - if (isStatusLine(lower)) { - continue; - } - - const entry = stripTargetPrefix({ - trimmedLine: trimmed, - lowerLine: lower, - normalizedTarget, - lowerTarget, - quotedTarget, - quotedLower, - }); - const parsed = parseAceEntry(entry); - if (!parsed) { - continue; - } - entries.push(parsed); - } - - return entries; -} - -export function summarizeWindowsAcl( - entries: WindowsAclEntry[], - env?: NodeJS.ProcessEnv, -): Pick { - const trustedPrincipals = buildTrustedPrincipals(env); - const trusted: WindowsAclEntry[] = []; - const untrustedWorld: WindowsAclEntry[] = []; - const untrustedGroup: WindowsAclEntry[] = []; - for (const entry of entries) { - const classification = classifyPrincipal(entry.principal, trustedPrincipals); - if (classification === "trusted") { - trusted.push(entry); - } else if (classification === "world") { - untrustedWorld.push(entry); - } else { - untrustedGroup.push(entry); - } - } - return { trusted, untrustedWorld, untrustedGroup }; -} - -async function resolveCurrentUserSid( - exec: ExecFn, - env?: NodeJS.ProcessEnv, -): Promise { - try { - const { stdout, stderr } = await exec(resolveWindowsSystemCommand("whoami.exe", env), [ - "/user", - "/fo", - "csv", - "/nh", - ]); - const match = `${stdout}\n${stderr}`.match(/\*?S-\d+-\d+(?:-\d+)+/i); - return match ? normalizeSid(match[0]) : null; - } catch (err) { - // Log but do not propagate — SID resolution is best-effort. - // Callers fall back to env-based resolution when this returns null. - log.warn("resolveCurrentUserSid failed", { error: String(err) }); - return null; - } -} - -export async function inspectWindowsAcl( - targetPath: string, - opts?: { env?: NodeJS.ProcessEnv; exec?: ExecFn }, -): Promise { - const exec = opts?.exec ?? runExec; - try { - // /sid outputs security identifiers (e.g. *S-1-5-18) instead of locale- - // dependent account names so the audit works correctly on non-English - // Windows (Russian, Chinese, etc.) where icacls prints Cyrillic / CJK - // characters that may be garbled when Node reads them in the wrong code - // page. Fixes #35834. - const { stdout, stderr } = await exec(resolveWindowsSystemCommand("icacls.exe", opts?.env), [ - targetPath, - "/sid", - ]); - const output = `${stdout}\n${stderr}`.trim(); - const entries = parseIcaclsOutput(output, targetPath); - let effectiveEnv = opts?.env; - let { trusted, untrustedWorld, untrustedGroup } = summarizeWindowsAcl(entries, effectiveEnv); - - const needsUserSidResolution = - !effectiveEnv?.USERSID && - untrustedGroup.some((entry) => SID_RE.test(normalize(entry.principal))); - if (needsUserSidResolution) { - const currentUserSid = await resolveCurrentUserSid(exec, effectiveEnv); - if (currentUserSid) { - effectiveEnv = { ...effectiveEnv, USERSID: currentUserSid }; - ({ trusted, untrustedWorld, untrustedGroup } = summarizeWindowsAcl(entries, effectiveEnv)); - } - } - - return { ok: true, entries, trusted, untrustedWorld, untrustedGroup }; - } catch (err) { - return { - ok: false, - entries: [], - trusted: [], - untrustedWorld: [], - untrustedGroup: [], - error: String(err), - }; - } -} - -export function formatWindowsAclSummary(summary: WindowsAclSummary): string { - if (!summary.ok) { - return "unknown"; - } - const untrusted = [...summary.untrustedWorld, ...summary.untrustedGroup]; - if (untrusted.length === 0) { - return "trusted-only"; - } - return untrusted.map((entry) => `${entry.principal}:${entry.rawRights}`).join(", "); -} - -export function formatIcaclsResetCommand( - targetPath: string, - opts: IcaclsResetCommandOptions, -): string { - const command = resolveWindowsSystemCommand("icacls.exe", opts.env); - const user = resolveWindowsUserPrincipal(opts.env, opts.userInfo) ?? "%USERNAME%"; - const grant = opts.isDir ? "(OI)(CI)F" : "F"; - // Quoted executable paths need shell-specific handling in PowerShell; keep - // the resolved System32 helper as the command token and quote only arguments. - return [ - command, - `"${targetPath}"`, - "/inheritance:r", - "/grant:r", - `"${user}:${grant}"`, - "/grant:r", - `"*S-1-5-18:${grant}"`, - ].join(" "); -} - -export function createIcaclsResetCommand( - targetPath: string, - opts: IcaclsResetCommandOptions, -): { command: string; args: string[]; display: string } | null { - const user = resolveWindowsUserPrincipal(opts.env, opts.userInfo); - if (!user) { - return null; - } - const grant = opts.isDir ? "(OI)(CI)F" : "F"; - const args = [ - targetPath, - "/inheritance:r", - "/grant:r", - `${user}:${grant}`, - "/grant:r", - `*S-1-5-18:${grant}`, - ]; - return { - command: resolveWindowsSystemCommand("icacls.exe", opts.env), - args, - display: formatIcaclsResetCommand(targetPath, opts), - }; -} +export { + createIcaclsResetCommand, + formatIcaclsResetCommand, + formatWindowsAclSummary, + inspectWindowsAcl, + parseIcaclsOutput, + resolveWindowsUserPrincipal, + summarizeWindowsAcl, + type ExecFn, + type WindowsAclEntry, + type WindowsAclSummary, +} from "../infra/permissions.js"; diff --git a/src/shared/avatar-policy.ts b/src/shared/avatar-policy.ts index eff674c13ac..07a3665538b 100644 --- a/src/shared/avatar-policy.ts +++ b/src/shared/avatar-policy.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { isPathInside } from "../infra/path-guards.js"; import { normalizeLowercaseStringOrEmpty } from "./string-coerce.js"; export const AVATAR_MAX_BYTES = 2 * 1024 * 1024; @@ -64,11 +65,7 @@ export function isWorkspaceRelativeAvatarPath(value: string): boolean { } export function isPathWithinRoot(rootDir: string, targetPath: string): boolean { - const relative = path.relative(rootDir, targetPath); - if (relative === "") { - return true; - } - return !relative.startsWith("..") && !path.isAbsolute(relative); + return isPathInside(rootDir, targetPath); } export function looksLikeAvatarPath(value: string): boolean { diff --git a/src/trajectory/cleanup.ts b/src/trajectory/cleanup.ts index 43e0ea66836..0be83151021 100644 --- a/src/trajectory/cleanup.ts +++ b/src/trajectory/cleanup.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveSessionFilePath } from "../config/sessions/paths.js"; +import { isPathInside } from "../infra/path-guards.js"; import { resolveTrajectoryFilePath, resolveTrajectoryPointerFilePath, @@ -32,8 +33,7 @@ function canonicalizePathForComparison(filePath: string): string { function isPathWithinDir(parentDir: string, filePath: string): boolean { const resolvedParent = canonicalizePathForComparison(parentDir); const resolvedFile = canonicalizePathForComparison(filePath); - const relative = path.relative(resolvedParent, resolvedFile); - return Boolean(relative) && !relative.startsWith("..") && !path.isAbsolute(relative); + return resolvedFile !== resolvedParent && isPathInside(resolvedParent, resolvedFile); } function isRegularNonSymlinkFile(filePath: string): boolean { diff --git a/src/trajectory/command-export.ts b/src/trajectory/command-export.ts index 34bfdaf46aa..dc301540a33 100644 --- a/src/trajectory/command-export.ts +++ b/src/trajectory/command-export.ts @@ -1,5 +1,7 @@ import fsp from "node:fs/promises"; import path from "node:path"; +import { pathExists } from "../infra/fs-safe.js"; +import { isPathInside } from "../infra/path-guards.js"; import { exportTrajectoryBundle, resolveDefaultTrajectoryExportDir } from "./export.js"; export type TrajectoryCommandExportSummary = { @@ -12,11 +14,6 @@ export type TrajectoryCommandExportSummary = { files: string[]; }; -function isPathInsideOrEqual(baseDir: string, candidate: string): boolean { - const relative = path.relative(baseDir, candidate); - return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); -} - async function validateExistingExportDirectory(params: { dir: string; label: string; @@ -27,7 +24,7 @@ async function validateExistingExportDirectory(params: { throw new Error(`${params.label} must be a real directory inside the workspace`); } const realDir = await fsp.realpath(params.dir); - if (!isPathInsideOrEqual(params.realWorkspace, realDir)) { + if (!isPathInside(params.realWorkspace, realDir)) { throw new Error("Trajectory exports directory must stay inside the workspace"); } return realDir; @@ -69,15 +66,6 @@ async function resolveTrajectoryExportBaseDir(workspaceDir: string): Promise<{ return { baseDir: path.resolve(baseDir), realBase }; } -async function pathExists(pathName: string): Promise { - try { - await fsp.access(pathName); - return true; - } catch { - return false; - } -} - export async function resolveTrajectoryCommandOutputDir(params: { outputPath?: string; workspaceDir: string; @@ -110,7 +98,7 @@ export async function resolveTrajectoryCommandOutputDir(params: { existingParent = next; } const realExistingParent = await fsp.realpath(existingParent); - if (!isPathInsideOrEqual(realBase, realExistingParent)) { + if (!isPathInside(realBase, realExistingParent)) { throw new Error("Output path must stay inside the real trajectory exports directory"); } return outputDir; diff --git a/src/trajectory/paths.ts b/src/trajectory/paths.ts index d2b3033830a..64edd63ba6b 100644 --- a/src/trajectory/paths.ts +++ b/src/trajectory/paths.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolveHomeRelativePath } from "../infra/home-dir.js"; +import { isPathInside } from "../infra/path-guards.js"; export const TRAJECTORY_RUNTIME_CAPTURE_MAX_BYTES = 10 * 1024 * 1024; export const TRAJECTORY_RUNTIME_FILE_MAX_BYTES = 50 * 1024 * 1024; @@ -32,8 +33,7 @@ export function resolveTrajectoryPointerOpenFlags( function resolveContainedPath(baseDir: string, fileName: string): string { const resolvedBase = path.resolve(baseDir); const resolvedFile = path.resolve(resolvedBase, fileName); - const relative = path.relative(resolvedBase, resolvedFile); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { + if (resolvedFile === resolvedBase || !isPathInside(resolvedBase, resolvedFile)) { throw new Error("Trajectory file path escaped its configured directory"); } return resolvedFile; diff --git a/src/tui/tui-last-session.ts b/src/tui/tui-last-session.ts index a35494f45a5..622fd34ff66 100644 --- a/src/tui/tui-last-session.ts +++ b/src/tui/tui-last-session.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; +import { privateFileStore } from "../infra/private-file-store.js"; import { normalizeAgentId, parseAgentSessionKey } from "../routing/session-key.js"; import type { TuiSessionList } from "./tui-backend.js"; import type { SessionScope } from "./tui-types.js"; @@ -32,8 +32,9 @@ export function buildTuiLastSessionScopeKey(params: { async function readStore(filePath: string): Promise { try { - const raw = await fs.readFile(filePath, "utf8"); - const parsed = JSON.parse(raw) as unknown; + const parsed = await privateFileStore(path.dirname(filePath)).readJsonIfExists( + path.basename(filePath), + ); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? (parsed as LastSessionStore) : {}; @@ -90,10 +91,8 @@ export async function writeTuiLastSessionKey(params: { sessionKey, updatedAt: Date.now(), }; - await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }); - await fs.writeFile(filePath, `${JSON.stringify(store, null, 2)}\n`, { - encoding: "utf8", - mode: 0o600, + await privateFileStore(path.dirname(filePath)).writeJson(path.basename(filePath), store, { + trailingNewline: true, }); } diff --git a/src/utils.ts b/src/utils.ts index de63ef58974..997cf71306e 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { pathExists as fsSafePathExists } from "./infra/fs-safe.js"; import { resolveEffectiveHomeDir, resolveHomeRelativePath, @@ -13,18 +14,6 @@ export async function ensureDir(dir: string) { await fs.promises.mkdir(dir, { recursive: true }); } -/** - * Check if a file or directory exists at the given path. - */ -export async function pathExists(targetPath: string): Promise { - try { - await fs.promises.access(targetPath); - return true; - } catch { - return false; - } -} - export function clampNumber(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } @@ -206,3 +195,9 @@ export function displayString(input: string): string { // Configuration root; can be overridden via OPENCLAW_STATE_DIR. export const CONFIG_DIR = resolveConfigDir(); +/** + * Check if a file or directory exists at the given path. + */ +export async function pathExists(targetPath: string): Promise { + return await fsSafePathExists(targetPath); +} diff --git a/src/utils/with-timeout.ts b/src/utils/with-timeout.ts index 7a2d8361df1..225d2e965e0 100644 --- a/src/utils/with-timeout.ts +++ b/src/utils/with-timeout.ts @@ -1,14 +1 @@ -export function withTimeout(promise: Promise, timeoutMs: number): Promise { - if (!timeoutMs || timeoutMs <= 0) { - return promise; - } - let timer: NodeJS.Timeout | null = null; - const timeout = new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("timeout")), timeoutMs); - }); - return Promise.race([promise, timeout]).finally(() => { - if (timer) { - clearTimeout(timer); - } - }); -} +export { withTimeout } from "../infra/fs-safe.js"; diff --git a/test/openclaw-npm-release-check.test.ts b/test/openclaw-npm-release-check.test.ts index 98941946d3f..5d4b3117f6f 100644 --- a/test/openclaw-npm-release-check.test.ts +++ b/test/openclaw-npm-release-check.test.ts @@ -616,6 +616,21 @@ describe("collectReleasePackageMetadataErrors", () => { ).toContain('package.json dependencies["node-llama-cpp"] must be omitted; keep it optional.'); }); + it("rejects local fs-safe dependency specs for npm release", () => { + expect( + collectReleasePackageMetadataErrors({ + name: "openclaw", + description: "Multi-channel AI gateway with extensible messaging integrations", + license: "MIT", + repository: { url: "git+https://github.com/openclaw/openclaw.git" }, + bin: { openclaw: "openclaw.mjs" }, + dependencies: { "@openclaw/fs-safe": "link:../fs-safe" }, + }), + ).toContain( + 'package.json dependencies["@openclaw/fs-safe"] must use a published semver range before npm release; found "link:../fs-safe".', + ); + }); + it("rejects node-llama-cpp as an optional dependency", () => { expect( collectReleasePackageMetadataErrors({ diff --git a/tsdown.config.ts b/tsdown.config.ts index 9ee5a8f18f8..fc8406f1608 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -174,6 +174,10 @@ function shouldNeverBundleDependency(id: string): boolean { }); } +function shouldAlwaysBundleDependency(id: string): boolean { + return id === "@openclaw/fs-safe" || id.startsWith("@openclaw/fs-safe/"); +} + function listBundledPluginEntrySources( entries: Array<{ id: string; @@ -297,6 +301,7 @@ export default defineConfig([ clean: true, entry: buildUnifiedDistEntries(), deps: { + alwaysBundle: shouldAlwaysBundleDependency, neverBundle: shouldNeverBundleDependency, }, }),