fix(qqbot): avoid eager storage directory creation

This commit is contained in:
Peter Steinberger
2026-04-22 01:42:10 +01:00
parent 540171ddbd
commit 5c74e9da01
7 changed files with 140 additions and 47 deletions

View File

@@ -26,6 +26,7 @@
*/
import fs from "node:fs";
import path from "node:path";
import { getCredentialBackupFile, getLegacyCredentialBackupFile } from "../utils/data-paths.js";
interface CredentialBackup {
@@ -42,6 +43,7 @@ export function saveCredentialBackup(accountId: string, appId: string, clientSec
}
try {
const backupPath = getCredentialBackupFile(accountId);
fs.mkdirSync(path.dirname(backupPath), { recursive: true });
const data: CredentialBackup = {
accountId,
appId,
@@ -86,9 +88,11 @@ export function loadCredentialBackup(accountId?: string): CredentialBackup | nul
}
if (data.accountId) {
try {
const tmpPath = `${getCredentialBackupFile(data.accountId)}.tmp`;
const backupPath = getCredentialBackupFile(data.accountId);
fs.mkdirSync(path.dirname(backupPath), { recursive: true });
const tmpPath = `${backupPath}.tmp`;
fs.writeFileSync(tmpPath, `${JSON.stringify(data, null, 2)}\n`, "utf8");
fs.renameSync(tmpPath, getCredentialBackupFile(data.accountId));
fs.renameSync(tmpPath, backupPath);
fs.unlinkSync(legacy);
} catch {
/* ignore migration errors */

View File

@@ -9,15 +9,13 @@ import fs from "node:fs";
import path from "node:path";
import { formatErrorMessage } from "../utils/format.js";
import { debugLog, debugError } from "../utils/log.js";
import { getQQBotDataDir } from "../utils/platform.js";
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
import type { RefIndexEntry } from "./types.js";
// Re-export types and format function for convenience.
export type { RefIndexEntry, RefAttachmentSummary } from "./types.js";
export { formatRefEntryForAgent } from "./format-ref-entry.js";
const STORAGE_DIR = getQQBotDataDir("data");
const REF_INDEX_FILE = path.join(STORAGE_DIR, "ref-index.jsonl");
const MAX_ENTRIES = 50000;
const TTL_MS = 7 * 24 * 60 * 60 * 1000;
const COMPACT_THRESHOLD_RATIO = 2;
@@ -31,6 +29,10 @@ interface RefIndexLine {
let cache: Map<string, RefIndexEntry & { _createdAt: number }> | null = null;
let totalLinesOnDisk = 0;
function getRefIndexFile(): string {
return path.join(getQQBotDataPath("data"), "ref-index.jsonl");
}
function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
if (cache !== null) {
return cache;
@@ -39,10 +41,11 @@ function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
totalLinesOnDisk = 0;
try {
if (!fs.existsSync(REF_INDEX_FILE)) {
const refIndexFile = getRefIndexFile();
if (!fs.existsSync(refIndexFile)) {
return cache;
}
const raw = fs.readFileSync(REF_INDEX_FILE, "utf-8");
const raw = fs.readFileSync(refIndexFile, "utf-8");
const lines = raw.split("\n");
const now = Date.now();
let expired = 0;
@@ -79,15 +82,13 @@ function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
}
function ensureDir(): void {
if (!fs.existsSync(STORAGE_DIR)) {
fs.mkdirSync(STORAGE_DIR, { recursive: true });
}
getQQBotDataDir("data");
}
function appendLine(line: RefIndexLine): void {
try {
ensureDir();
fs.appendFileSync(REF_INDEX_FILE, JSON.stringify(line) + "\n", "utf-8");
fs.appendFileSync(getRefIndexFile(), JSON.stringify(line) + "\n", "utf-8");
totalLinesOnDisk++;
} catch (err) {
debugError(`[ref-index-store] Failed to append: ${formatErrorMessage(err)}`);
@@ -107,7 +108,8 @@ function compactFile(): void {
const before = totalLinesOnDisk;
try {
ensureDir();
const tmpPath = REF_INDEX_FILE + ".tmp";
const refIndexFile = getRefIndexFile();
const tmpPath = refIndexFile + ".tmp";
const lines: string[] = [];
for (const [key, entry] of cache) {
lines.push(
@@ -126,7 +128,7 @@ function compactFile(): void {
);
}
fs.writeFileSync(tmpPath, lines.join("\n") + "\n", "utf-8");
fs.renameSync(tmpPath, REF_INDEX_FILE);
fs.renameSync(tmpPath, refIndexFile);
totalLinesOnDisk = cache.size;
debugLog(`[ref-index-store] Compacted: ${before} lines → ${totalLinesOnDisk} lines`);
} catch (err) {
@@ -213,5 +215,10 @@ export function getRefIndexStats(): {
filePath: string;
} {
const store = loadFromFile();
return { size: store.size, maxEntries: MAX_ENTRIES, totalLinesOnDisk, filePath: REF_INDEX_FILE };
return {
size: store.size,
maxEntries: MAX_ENTRIES,
totalLinesOnDisk,
filePath: getRefIndexFile(),
};
}

View File

@@ -10,7 +10,7 @@ import path from "node:path";
import type { ChatScope } from "../types.js";
import { formatErrorMessage } from "../utils/format.js";
import { debugLog, debugError } from "../utils/log.js";
import { getQQBotDataDir } from "../utils/platform.js";
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
/** Persisted record for a user who has interacted with the bot. */
export interface KnownUser {
@@ -24,18 +24,17 @@ export interface KnownUser {
interactionCount: number;
}
const KNOWN_USERS_DIR = getQQBotDataDir("data");
const KNOWN_USERS_FILE = path.join(KNOWN_USERS_DIR, "known-users.json");
let usersCache: Map<string, KnownUser> | null = null;
const SAVE_THROTTLE_MS = 5000;
let saveTimer: ReturnType<typeof setTimeout> | null = null;
let isDirty = false;
function ensureDir(): void {
if (!fs.existsSync(KNOWN_USERS_DIR)) {
fs.mkdirSync(KNOWN_USERS_DIR, { recursive: true });
}
getQQBotDataDir("data");
}
function getKnownUsersFile(): string {
return path.join(getQQBotDataPath("data"), "known-users.json");
}
function makeUserKey(user: Partial<KnownUser>): string {
@@ -49,8 +48,9 @@ function loadUsersFromFile(): Map<string, KnownUser> {
}
usersCache = new Map();
try {
if (fs.existsSync(KNOWN_USERS_FILE)) {
const data = fs.readFileSync(KNOWN_USERS_FILE, "utf-8");
const knownUsersFile = getKnownUsersFile();
if (fs.existsSync(knownUsersFile)) {
const data = fs.readFileSync(knownUsersFile, "utf-8");
const users = JSON.parse(data) as KnownUser[];
for (const user of users) {
usersCache.set(makeUserKey(user), user);
@@ -81,7 +81,7 @@ function doSaveUsersToFile(): void {
try {
ensureDir();
fs.writeFileSync(
KNOWN_USERS_FILE,
getKnownUsersFile(),
JSON.stringify(Array.from(usersCache.values()), null, 2),
"utf-8",
);

View File

@@ -9,7 +9,7 @@ import fs from "node:fs";
import path from "node:path";
import { formatErrorMessage } from "../utils/format.js";
import { debugLog, debugError } from "../utils/log.js";
import { getQQBotDataDir } from "../utils/platform.js";
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
/** Persisted gateway session state. */
export interface SessionState {
@@ -22,7 +22,6 @@ export interface SessionState {
appId?: string;
}
const SESSION_DIR = getQQBotDataDir("sessions");
const SESSION_EXPIRE_TIME = 5 * 60 * 1000;
const SAVE_THROTTLE_MS = 1000;
@@ -36,9 +35,11 @@ const throttleState = new Map<
>();
function ensureDir(): void {
if (!fs.existsSync(SESSION_DIR)) {
fs.mkdirSync(SESSION_DIR, { recursive: true });
}
getQQBotDataDir("sessions");
}
function getSessionDir(): string {
return getQQBotDataPath("sessions");
}
function encodeAccountIdForFileName(accountId: string): string {
@@ -47,12 +48,12 @@ function encodeAccountIdForFileName(accountId: string): string {
function getLegacySessionPath(accountId: string): string {
const safeId = accountId.replace(/[^a-zA-Z0-9_-]/g, "_");
return path.join(SESSION_DIR, `session-${safeId}.json`);
return path.join(getSessionDir(), `session-${safeId}.json`);
}
function getSessionPath(accountId: string): string {
const encodedId = encodeAccountIdForFileName(accountId);
return path.join(SESSION_DIR, `session-${encodedId}.json`);
return path.join(getSessionDir(), `session-${encodedId}.json`);
}
function getCandidateSessionPaths(accountId: string): string[] {
@@ -66,7 +67,7 @@ function isSessionFileName(file: string): boolean {
}
function readSessionStateFile(file: string): { filePath: string; state: SessionState } {
const filePath = path.join(SESSION_DIR, file);
const filePath = path.join(getSessionDir(), file);
const data = fs.readFileSync(filePath, "utf-8");
return { filePath, state: JSON.parse(data) as SessionState };
}
@@ -224,8 +225,11 @@ export function updateLastSeq(accountId: string, lastSeq: number): void {
export function getAllSessions(): SessionState[] {
const sessions = new Map<string, SessionState>();
try {
ensureDir();
const files = fs.readdirSync(SESSION_DIR);
const sessionDir = getSessionDir();
if (!fs.existsSync(sessionDir)) {
return [];
}
const files = fs.readdirSync(sessionDir);
for (const file of files) {
if (isSessionFileName(file)) {
@@ -249,13 +253,16 @@ export function getAllSessions(): SessionState[] {
export function cleanupExpiredSessions(): number {
let cleaned = 0;
try {
ensureDir();
const sessionDir = getSessionDir();
if (!fs.existsSync(sessionDir)) {
return 0;
}
const now = Date.now();
const files = fs.readdirSync(SESSION_DIR);
const files = fs.readdirSync(sessionDir);
for (const file of files) {
if (isSessionFileName(file)) {
const filePath = path.join(SESSION_DIR, file);
const filePath = path.join(sessionDir, file);
try {
const { state } = readSessionStateFile(file);

View File

@@ -11,7 +11,7 @@
*/
import path from "node:path";
import { getQQBotDataDir } from "./platform.js";
import { getQQBotDataPath } from "./platform.js";
/**
* Normalise an identifier so it is safe to embed in a filename.
@@ -29,10 +29,10 @@ export function safeName(id: string): string {
* missing from the live config.
*/
export function getCredentialBackupFile(accountId: string): string {
return path.join(getQQBotDataDir("data"), `credential-backup-${safeName(accountId)}.json`);
return path.join(getQQBotDataPath("data"), `credential-backup-${safeName(accountId)}.json`);
}
/** Legacy single-file credential backup (pre-multi-account-isolation). */
export function getLegacyCredentialBackupFile(): string {
return path.join(getQQBotDataDir("data"), "credential-backup.json");
return path.join(getQQBotDataPath("data"), "credential-backup.json");
}

View File

@@ -0,0 +1,65 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
const createdHomes: string[] = [];
async function useMockHome(homeDir: string): Promise<void> {
vi.resetModules();
vi.doMock("node:os", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:os")>();
return {
...actual,
default: { ...actual, homedir: () => homeDir },
homedir: () => homeDir,
};
});
}
function makeHome(): string {
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "qqbot-home-"));
createdHomes.push(homeDir);
return homeDir;
}
describe("qqbot storage laziness", () => {
afterEach(() => {
vi.doUnmock("node:os");
vi.resetModules();
for (const home of createdHomes.splice(0)) {
fs.rmSync(home, { recursive: true, force: true });
}
});
it("does not create ~/.openclaw/qqbot from module imports or read-only probes", async () => {
const homeDir = makeHome();
await useMockHome(homeDir);
const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot");
const sessionStore = await import("../session/session-store.js");
await import("../session/known-users.js");
await import("../ref/store.js");
const { loadCredentialBackup } = await import("../config/credential-backup.js");
expect(loadCredentialBackup("default")).toBeNull();
expect(sessionStore.getAllSessions()).toEqual([]);
expect(sessionStore.cleanupExpiredSessions()).toBe(0);
expect(fs.existsSync(qqbotRoot)).toBe(false);
});
it("creates storage when qqbot persists runtime state", async () => {
const homeDir = makeHome();
await useMockHome(homeDir);
const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot");
const { saveCredentialBackup } = await import("../config/credential-backup.js");
saveCredentialBackup("default", "123456", "secret");
expect(fs.existsSync(path.join(qqbotRoot, "data", "credential-backup-default.json"))).toBe(
true,
);
});
});

View File

@@ -41,9 +41,14 @@ export function getHomeDir(): string {
return getPlatformAdapter().getTempDir();
}
/** Return a path under `~/.openclaw/qqbot` without creating it. */
export function getQQBotDataPath(...subPaths: string[]): string {
return path.join(getHomeDir(), ".openclaw", "qqbot", ...subPaths);
}
/** Return a path under `~/.openclaw/qqbot`, creating it on demand. */
export function getQQBotDataDir(...subPaths: string[]): string {
const dir = path.join(getHomeDir(), ".openclaw", "qqbot", ...subPaths);
const dir = getQQBotDataPath(...subPaths);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
@@ -51,13 +56,18 @@ export function getQQBotDataDir(...subPaths: string[]): string {
}
/**
* Return a path under `~/.openclaw/media/qqbot`, creating it on demand.
* Return a path under `~/.openclaw/media/qqbot` without creating it.
*
* Unlike `getQQBotDataDir`, this lives under OpenClaw's core media allowlist so
* Unlike `getQQBotDataPath`, this lives under OpenClaw's core media allowlist so
* downloaded images and audio can be accessed by framework media tooling.
*/
export function getQQBotMediaPath(...subPaths: string[]): string {
return path.join(getHomeDir(), ".openclaw", "media", "qqbot", ...subPaths);
}
/** Return a path under `~/.openclaw/media/qqbot`, creating it on demand. */
export function getQQBotMediaDir(...subPaths: string[]): string {
const dir = path.join(getHomeDir(), ".openclaw", "media", "qqbot", ...subPaths);
const dir = getQQBotMediaPath(...subPaths);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
@@ -286,8 +296,8 @@ export function resolveQQBotLocalMediaPath(p: string): string {
}
const homeDir = getHomeDir();
const mediaRoot = getQQBotMediaDir();
const dataRoot = getQQBotDataDir();
const mediaRoot = getQQBotMediaPath();
const dataRoot = getQQBotDataPath();
const workspaceRoot = path.join(homeDir, ".openclaw", "workspace", "qqbot");
const candidateRoots = [
{ from: workspaceRoot, to: mediaRoot },
@@ -331,7 +341,7 @@ export function resolveQQBotPayloadLocalFilePath(p: string): string | null {
// attachments under sibling directories (e.g. `media/outbound/`) that are already
// part of the core media allowlist; we mirror that so auto-routed sends work
// without leaving the plugin's trust boundary.
const allowedRoots = [getOpenClawMediaDir(), getQQBotMediaDir()];
const allowedRoots = [getOpenClawMediaDir(), getQQBotMediaPath()];
for (const root of allowedRoots) {
const resolvedRoot = path.resolve(root);