mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 09:20:43 +00:00
[codex] Extract filesystem safety primitives (#77918)
* refactor: extract filesystem safety primitives * refactor: use fs-safe for file access helpers * refactor: reuse fs-safe for media reads * refactor: use fs-safe for image reads * refactor: reuse fs-safe in qqbot media opener * refactor: reuse fs-safe for local media checks * refactor: consume cleaner fs-safe api * refactor: align fs-safe json option names * fix: preserve fs-safe migration contracts * refactor: use fs-safe primitive subpaths * refactor: use grouped fs-safe subpaths * refactor: align fs-safe api usage * refactor: adapt private state store api * chore: refresh proof gate * refactor: follow fs-safe json api split * refactor: follow reduced fs-safe surface * build: default fs-safe python helper off * fix: preserve fs-safe plugin sdk aliases * refactor: consolidate fs-safe usage * refactor: unify fs-safe store usage * refactor: trim fs-safe temp workspace usage * refactor: hide low-level fs-safe primitives * build: use published fs-safe package * fix: preserve outbound recovery durability after rebase * chore: refresh pr checks
This commit is contained in:
committed by
GitHub
parent
61481eb34f
commit
538605ff44
@@ -13,6 +13,7 @@ import {
|
||||
import { resolveGlobalMap } from "openclaw/plugin-sdk/global-singleton";
|
||||
import { resolveStateDir } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { getRuntimeConfig } from "openclaw/plugin-sdk/runtime-config-snapshot";
|
||||
import { pathExists, replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime";
|
||||
import {
|
||||
loadSessionStore,
|
||||
resolveStorePath,
|
||||
@@ -488,29 +489,14 @@ async function assertSafeDreamsPath(dreamsPath: string): Promise<void> {
|
||||
|
||||
async function writeDreamsFileAtomic(dreamsPath: string, content: string): Promise<void> {
|
||||
await assertSafeDreamsPath(dreamsPath);
|
||||
const existing = await fs.stat(dreamsPath).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
await replaceFileAtomic({
|
||||
filePath: dreamsPath,
|
||||
content,
|
||||
mode: 0o600,
|
||||
preserveExistingMode: true,
|
||||
tempPrefix: `${path.basename(dreamsPath)}.dreams`,
|
||||
throwOnCleanupError: true,
|
||||
});
|
||||
const mode = existing?.mode ?? 0o600;
|
||||
const tempPath = `${dreamsPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tempPath, content, { encoding: "utf-8", flag: "wx", mode });
|
||||
await fs.chmod(tempPath, mode).catch(() => undefined);
|
||||
try {
|
||||
await fs.rename(tempPath, dreamsPath);
|
||||
await fs.chmod(dreamsPath, mode).catch(() => undefined);
|
||||
} catch (err) {
|
||||
const cleanupError = await fs.rm(tempPath, { force: true }).catch((rmErr) => rmErr);
|
||||
if (cleanupError) {
|
||||
throw new Error(
|
||||
`Atomic DREAMS.md write failed (${formatErrorMessage(err)}); cleanup also failed (${formatErrorMessage(cleanupError)})`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateDreamsFile<T>(params: {
|
||||
@@ -710,15 +696,6 @@ export async function appendNarrativeEntry(params: {
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────────────
|
||||
|
||||
async function safePathExists(pathname: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.stat(pathname);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeComparablePath(pathname: string): string {
|
||||
return process.platform === "win32" ? pathname.toLowerCase() : pathname;
|
||||
}
|
||||
@@ -814,7 +791,7 @@ async function scrubDreamingNarrativeArtifacts(logger: Logger): Promise<void> {
|
||||
if (!isDreamingSessionStoreKey(key)) {
|
||||
continue;
|
||||
}
|
||||
if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) {
|
||||
if (!normalizedSessionFile || !(await pathExists(normalizedSessionFile))) {
|
||||
needsStoreUpdate = true;
|
||||
}
|
||||
}
|
||||
@@ -834,7 +811,7 @@ async function scrubDreamingNarrativeArtifacts(logger: Logger): Promise<void> {
|
||||
if (!isDreamingSessionStoreKey(key)) {
|
||||
continue;
|
||||
}
|
||||
if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) {
|
||||
if (!normalizedSessionFile || !(await pathExists(normalizedSessionFile))) {
|
||||
delete lockedStore[key];
|
||||
prunedForAgent += 1;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
resolveMemoryRemDreamingConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { appendRegularFile, privateFileStore } from "openclaw/plugin-sdk/security-runtime";
|
||||
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
|
||||
import {
|
||||
generateAndAppendDreamNarrative,
|
||||
@@ -443,11 +444,11 @@ function normalizeMemoryDay(value: unknown): string | undefined {
|
||||
async function readDailyIngestionState(workspaceDir: string): Promise<DailyIngestionState> {
|
||||
const statePath = resolveDailyIngestionStatePath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(statePath, "utf-8");
|
||||
return normalizeDailyIngestionState(JSON.parse(raw) as unknown);
|
||||
return normalizeDailyIngestionState(
|
||||
await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, statePath)),
|
||||
);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code === "ENOENT" || err instanceof SyntaxError) {
|
||||
if (err instanceof SyntaxError) {
|
||||
return { version: 1, files: {} };
|
||||
}
|
||||
throw err;
|
||||
@@ -459,10 +460,9 @@ async function writeDailyIngestionState(
|
||||
state: DailyIngestionState,
|
||||
): Promise<void> {
|
||||
const statePath = resolveDailyIngestionStatePath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
const tmpPath = `${statePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, statePath);
|
||||
await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, statePath), state, {
|
||||
trailingNewline: true,
|
||||
});
|
||||
}
|
||||
|
||||
type SessionIngestionFileState = {
|
||||
@@ -556,11 +556,11 @@ function normalizeSessionIngestionState(raw: unknown): SessionIngestionState {
|
||||
async function readSessionIngestionState(workspaceDir: string): Promise<SessionIngestionState> {
|
||||
const statePath = resolveSessionIngestionStatePath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(statePath, "utf-8");
|
||||
return normalizeSessionIngestionState(JSON.parse(raw) as unknown);
|
||||
return normalizeSessionIngestionState(
|
||||
await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, statePath)),
|
||||
);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code === "ENOENT" || err instanceof SyntaxError) {
|
||||
if (err instanceof SyntaxError) {
|
||||
return { version: 3, files: {}, seenMessages: {} };
|
||||
}
|
||||
throw err;
|
||||
@@ -572,10 +572,9 @@ async function writeSessionIngestionState(
|
||||
state: SessionIngestionState,
|
||||
): Promise<void> {
|
||||
const statePath = resolveSessionIngestionStatePath(workspaceDir);
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
const tmpPath = `${statePath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(state, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, statePath);
|
||||
await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, statePath), state, {
|
||||
trailingNewline: true,
|
||||
});
|
||||
}
|
||||
|
||||
function trimTrackedSessionScopes(
|
||||
@@ -714,7 +713,11 @@ async function appendSessionCorpusLines(params: {
|
||||
? normalizedExisting.slice(0, -1).split("\n").length
|
||||
: normalizedExisting.split("\n").length;
|
||||
const payload = `${params.lines.map((entry) => entry.rendered).join("\n")}\n`;
|
||||
await fs.appendFile(absolutePath, payload, "utf-8");
|
||||
await appendRegularFile({
|
||||
filePath: absolutePath,
|
||||
content: payload,
|
||||
rejectSymlinkParents: true,
|
||||
});
|
||||
return params.lines.map((entry, index) => {
|
||||
const lineNumber = existingLineCount + index + 1;
|
||||
return {
|
||||
|
||||
@@ -9,12 +9,13 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { withFileLock } from "openclaw/plugin-sdk/file-lock";
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
isPathInside,
|
||||
root,
|
||||
resolveAgentContextLimits,
|
||||
resolveMemorySearchSyncConfig,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveGlobalSingleton,
|
||||
resolveStateDir,
|
||||
writeFileWithinRoot,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
@@ -1302,7 +1303,15 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const statResult = await statRegularFile(absPath);
|
||||
let statResult: Awaited<ReturnType<typeof statRegularFile>>;
|
||||
try {
|
||||
statResult = await statRegularFile(absPath);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message === "path must be a regular file") {
|
||||
throw new Error("path required", { cause: err });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (statResult.missing) {
|
||||
return { text: "", path: relPath };
|
||||
}
|
||||
@@ -2203,6 +2212,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
}
|
||||
const exportDir = this.sessionExporter.dir;
|
||||
await fs.mkdir(exportDir, { recursive: true });
|
||||
const exportRoot = await root(exportDir);
|
||||
const files = await listSessionFilesForAgent(this.agentId);
|
||||
const keep = new Set<string>();
|
||||
const tracked = new Set<string>();
|
||||
@@ -2222,10 +2232,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
tracked.add(sessionFile);
|
||||
const state = this.exportedSessionState.get(sessionFile);
|
||||
if (!state || state.hash !== entry.hash || state.mtimeMs !== entry.mtimeMs) {
|
||||
await writeFileWithinRoot({
|
||||
rootDir: exportDir,
|
||||
relativePath: targetName,
|
||||
data: this.renderSessionMarkdown(entry),
|
||||
await exportRoot.write(targetName, this.renderSessionMarkdown(entry), {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
}
|
||||
@@ -2236,18 +2243,18 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
});
|
||||
keep.add(target);
|
||||
}
|
||||
const exported = await fs.readdir(exportDir).catch(() => []);
|
||||
const exported = await exportRoot.list(".").catch(() => []);
|
||||
for (const name of exported) {
|
||||
if (!name.endsWith(".md")) {
|
||||
continue;
|
||||
}
|
||||
const full = path.join(exportDir, name);
|
||||
if (!keep.has(full)) {
|
||||
await fs.rm(full, { force: true });
|
||||
await exportRoot.remove(name).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
for (const [sessionFile, state] of this.exportedSessionState) {
|
||||
if (!tracked.has(sessionFile) || !state.target.startsWith(exportDir + path.sep)) {
|
||||
if (!tracked.has(sessionFile) || !isPathInside(exportDir, state.target)) {
|
||||
this.exportedSessionState.delete(sessionFile);
|
||||
}
|
||||
}
|
||||
@@ -2788,23 +2795,11 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
private isWithinWorkspace(absPath: string): boolean {
|
||||
const normalizedWorkspace = this.workspaceDir.endsWith(path.sep)
|
||||
? this.workspaceDir
|
||||
: `${this.workspaceDir}${path.sep}`;
|
||||
if (absPath === this.workspaceDir) {
|
||||
return true;
|
||||
}
|
||||
const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`;
|
||||
return candidate.startsWith(normalizedWorkspace);
|
||||
return isPathInside(this.workspaceDir, absPath);
|
||||
}
|
||||
|
||||
private isWithinRoot(root: string, candidate: string): boolean {
|
||||
const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`;
|
||||
if (candidate === root) {
|
||||
return true;
|
||||
}
|
||||
const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`;
|
||||
return next.startsWith(normalizedRoot);
|
||||
return isPathInside(root, candidate);
|
||||
}
|
||||
|
||||
private clampResultsByInjectedChars(results: MemorySearchResult[]): MemorySearchResult[] {
|
||||
|
||||
@@ -3,17 +3,9 @@ import path from "node:path";
|
||||
import { resolveMemoryHostEventLogPath } from "openclaw/plugin-sdk/memory-core-host-events";
|
||||
import { resolveMemoryDreamingWorkspaces } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import type { MemoryPluginPublicArtifact } from "openclaw/plugin-sdk/memory-host-core";
|
||||
import { pathExists } from "openclaw/plugin-sdk/security-runtime";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
|
||||
async function pathExists(inputPath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(inputPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function listMarkdownFilesRecursive(rootDir: string): Promise<string[]> {
|
||||
const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []);
|
||||
const files: string[] = [];
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
|
||||
import { privateFileStore } from "openclaw/plugin-sdk/security-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import {
|
||||
deriveConceptTags,
|
||||
@@ -758,9 +759,10 @@ async function withShortTermLock<T>(workspaceDir: string, task: () => Promise<T>
|
||||
async function readStore(workspaceDir: string, nowIso: string): Promise<ShortTermRecallStore> {
|
||||
const storePath = resolveStorePath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(storePath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return normalizeStore(parsed, nowIso);
|
||||
return normalizeStore(
|
||||
await privateFileStore(workspaceDir).readJsonIfExists(path.relative(workspaceDir, storePath)),
|
||||
nowIso,
|
||||
);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return emptyStore(nowIso);
|
||||
@@ -830,13 +832,13 @@ async function readPhaseSignalStore(
|
||||
): Promise<ShortTermPhaseSignalStore> {
|
||||
const phaseSignalPath = resolvePhaseSignalPath(workspaceDir);
|
||||
try {
|
||||
const raw = await fs.readFile(phaseSignalPath, "utf-8");
|
||||
return normalizePhaseSignalStore(JSON.parse(raw) as unknown, nowIso);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code === "ENOENT" || err instanceof SyntaxError) {
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
return normalizePhaseSignalStore(
|
||||
await privateFileStore(workspaceDir).readJsonIfExists(
|
||||
path.relative(workspaceDir, phaseSignalPath),
|
||||
),
|
||||
nowIso,
|
||||
);
|
||||
} catch {
|
||||
return emptyPhaseSignalStore(nowIso);
|
||||
}
|
||||
}
|
||||
@@ -847,17 +849,21 @@ async function writePhaseSignalStore(
|
||||
): Promise<void> {
|
||||
const phaseSignalPath = resolvePhaseSignalPath(workspaceDir);
|
||||
await ensureShortTermArtifactsDir(workspaceDir);
|
||||
const tmpPath = `${phaseSignalPath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(store, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, phaseSignalPath);
|
||||
await privateFileStore(workspaceDir).writeJson(
|
||||
path.relative(workspaceDir, phaseSignalPath),
|
||||
store,
|
||||
{
|
||||
trailingNewline: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function writeStore(workspaceDir: string, store: ShortTermRecallStore): Promise<void> {
|
||||
const storePath = resolveStorePath(workspaceDir);
|
||||
await ensureShortTermArtifactsDir(workspaceDir);
|
||||
const tmpPath = `${storePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
||||
await fs.writeFile(tmpPath, `${JSON.stringify(store, null, 2)}\n`, "utf-8");
|
||||
await fs.rename(tmpPath, storePath);
|
||||
await privateFileStore(workspaceDir).writeJson(path.relative(workspaceDir, storePath), store, {
|
||||
trailingNewline: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function isShortTermMemoryPath(filePath: string): boolean {
|
||||
|
||||
Reference in New Issue
Block a user