mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:40:44 +00:00
fix(qqbot): avoid eager storage directory creation
This commit is contained in:
@@ -26,6 +26,7 @@
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { getCredentialBackupFile, getLegacyCredentialBackupFile } from "../utils/data-paths.js";
|
||||
|
||||
interface CredentialBackup {
|
||||
@@ -42,6 +43,7 @@ export function saveCredentialBackup(accountId: string, appId: string, clientSec
|
||||
}
|
||||
try {
|
||||
const backupPath = getCredentialBackupFile(accountId);
|
||||
fs.mkdirSync(path.dirname(backupPath), { recursive: true });
|
||||
const data: CredentialBackup = {
|
||||
accountId,
|
||||
appId,
|
||||
@@ -86,9 +88,11 @@ export function loadCredentialBackup(accountId?: string): CredentialBackup | nul
|
||||
}
|
||||
if (data.accountId) {
|
||||
try {
|
||||
const tmpPath = `${getCredentialBackupFile(data.accountId)}.tmp`;
|
||||
const backupPath = getCredentialBackupFile(data.accountId);
|
||||
fs.mkdirSync(path.dirname(backupPath), { recursive: true });
|
||||
const tmpPath = `${backupPath}.tmp`;
|
||||
fs.writeFileSync(tmpPath, `${JSON.stringify(data, null, 2)}\n`, "utf8");
|
||||
fs.renameSync(tmpPath, getCredentialBackupFile(data.accountId));
|
||||
fs.renameSync(tmpPath, backupPath);
|
||||
fs.unlinkSync(legacy);
|
||||
} catch {
|
||||
/* ignore migration errors */
|
||||
|
||||
@@ -9,15 +9,13 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "../utils/format.js";
|
||||
import { debugLog, debugError } from "../utils/log.js";
|
||||
import { getQQBotDataDir } from "../utils/platform.js";
|
||||
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
|
||||
import type { RefIndexEntry } from "./types.js";
|
||||
|
||||
// Re-export types and format function for convenience.
|
||||
export type { RefIndexEntry, RefAttachmentSummary } from "./types.js";
|
||||
export { formatRefEntryForAgent } from "./format-ref-entry.js";
|
||||
|
||||
const STORAGE_DIR = getQQBotDataDir("data");
|
||||
const REF_INDEX_FILE = path.join(STORAGE_DIR, "ref-index.jsonl");
|
||||
const MAX_ENTRIES = 50000;
|
||||
const TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
const COMPACT_THRESHOLD_RATIO = 2;
|
||||
@@ -31,6 +29,10 @@ interface RefIndexLine {
|
||||
let cache: Map<string, RefIndexEntry & { _createdAt: number }> | null = null;
|
||||
let totalLinesOnDisk = 0;
|
||||
|
||||
function getRefIndexFile(): string {
|
||||
return path.join(getQQBotDataPath("data"), "ref-index.jsonl");
|
||||
}
|
||||
|
||||
function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
|
||||
if (cache !== null) {
|
||||
return cache;
|
||||
@@ -39,10 +41,11 @@ function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
|
||||
totalLinesOnDisk = 0;
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(REF_INDEX_FILE)) {
|
||||
const refIndexFile = getRefIndexFile();
|
||||
if (!fs.existsSync(refIndexFile)) {
|
||||
return cache;
|
||||
}
|
||||
const raw = fs.readFileSync(REF_INDEX_FILE, "utf-8");
|
||||
const raw = fs.readFileSync(refIndexFile, "utf-8");
|
||||
const lines = raw.split("\n");
|
||||
const now = Date.now();
|
||||
let expired = 0;
|
||||
@@ -79,15 +82,13 @@ function loadFromFile(): Map<string, RefIndexEntry & { _createdAt: number }> {
|
||||
}
|
||||
|
||||
function ensureDir(): void {
|
||||
if (!fs.existsSync(STORAGE_DIR)) {
|
||||
fs.mkdirSync(STORAGE_DIR, { recursive: true });
|
||||
}
|
||||
getQQBotDataDir("data");
|
||||
}
|
||||
|
||||
function appendLine(line: RefIndexLine): void {
|
||||
try {
|
||||
ensureDir();
|
||||
fs.appendFileSync(REF_INDEX_FILE, JSON.stringify(line) + "\n", "utf-8");
|
||||
fs.appendFileSync(getRefIndexFile(), JSON.stringify(line) + "\n", "utf-8");
|
||||
totalLinesOnDisk++;
|
||||
} catch (err) {
|
||||
debugError(`[ref-index-store] Failed to append: ${formatErrorMessage(err)}`);
|
||||
@@ -107,7 +108,8 @@ function compactFile(): void {
|
||||
const before = totalLinesOnDisk;
|
||||
try {
|
||||
ensureDir();
|
||||
const tmpPath = REF_INDEX_FILE + ".tmp";
|
||||
const refIndexFile = getRefIndexFile();
|
||||
const tmpPath = refIndexFile + ".tmp";
|
||||
const lines: string[] = [];
|
||||
for (const [key, entry] of cache) {
|
||||
lines.push(
|
||||
@@ -126,7 +128,7 @@ function compactFile(): void {
|
||||
);
|
||||
}
|
||||
fs.writeFileSync(tmpPath, lines.join("\n") + "\n", "utf-8");
|
||||
fs.renameSync(tmpPath, REF_INDEX_FILE);
|
||||
fs.renameSync(tmpPath, refIndexFile);
|
||||
totalLinesOnDisk = cache.size;
|
||||
debugLog(`[ref-index-store] Compacted: ${before} lines → ${totalLinesOnDisk} lines`);
|
||||
} catch (err) {
|
||||
@@ -213,5 +215,10 @@ export function getRefIndexStats(): {
|
||||
filePath: string;
|
||||
} {
|
||||
const store = loadFromFile();
|
||||
return { size: store.size, maxEntries: MAX_ENTRIES, totalLinesOnDisk, filePath: REF_INDEX_FILE };
|
||||
return {
|
||||
size: store.size,
|
||||
maxEntries: MAX_ENTRIES,
|
||||
totalLinesOnDisk,
|
||||
filePath: getRefIndexFile(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import path from "node:path";
|
||||
import type { ChatScope } from "../types.js";
|
||||
import { formatErrorMessage } from "../utils/format.js";
|
||||
import { debugLog, debugError } from "../utils/log.js";
|
||||
import { getQQBotDataDir } from "../utils/platform.js";
|
||||
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
|
||||
|
||||
/** Persisted record for a user who has interacted with the bot. */
|
||||
export interface KnownUser {
|
||||
@@ -24,18 +24,17 @@ export interface KnownUser {
|
||||
interactionCount: number;
|
||||
}
|
||||
|
||||
const KNOWN_USERS_DIR = getQQBotDataDir("data");
|
||||
const KNOWN_USERS_FILE = path.join(KNOWN_USERS_DIR, "known-users.json");
|
||||
|
||||
let usersCache: Map<string, KnownUser> | null = null;
|
||||
const SAVE_THROTTLE_MS = 5000;
|
||||
let saveTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let isDirty = false;
|
||||
|
||||
function ensureDir(): void {
|
||||
if (!fs.existsSync(KNOWN_USERS_DIR)) {
|
||||
fs.mkdirSync(KNOWN_USERS_DIR, { recursive: true });
|
||||
}
|
||||
getQQBotDataDir("data");
|
||||
}
|
||||
|
||||
function getKnownUsersFile(): string {
|
||||
return path.join(getQQBotDataPath("data"), "known-users.json");
|
||||
}
|
||||
|
||||
function makeUserKey(user: Partial<KnownUser>): string {
|
||||
@@ -49,8 +48,9 @@ function loadUsersFromFile(): Map<string, KnownUser> {
|
||||
}
|
||||
usersCache = new Map();
|
||||
try {
|
||||
if (fs.existsSync(KNOWN_USERS_FILE)) {
|
||||
const data = fs.readFileSync(KNOWN_USERS_FILE, "utf-8");
|
||||
const knownUsersFile = getKnownUsersFile();
|
||||
if (fs.existsSync(knownUsersFile)) {
|
||||
const data = fs.readFileSync(knownUsersFile, "utf-8");
|
||||
const users = JSON.parse(data) as KnownUser[];
|
||||
for (const user of users) {
|
||||
usersCache.set(makeUserKey(user), user);
|
||||
@@ -81,7 +81,7 @@ function doSaveUsersToFile(): void {
|
||||
try {
|
||||
ensureDir();
|
||||
fs.writeFileSync(
|
||||
KNOWN_USERS_FILE,
|
||||
getKnownUsersFile(),
|
||||
JSON.stringify(Array.from(usersCache.values()), null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
@@ -9,7 +9,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "../utils/format.js";
|
||||
import { debugLog, debugError } from "../utils/log.js";
|
||||
import { getQQBotDataDir } from "../utils/platform.js";
|
||||
import { getQQBotDataDir, getQQBotDataPath } from "../utils/platform.js";
|
||||
|
||||
/** Persisted gateway session state. */
|
||||
export interface SessionState {
|
||||
@@ -22,7 +22,6 @@ export interface SessionState {
|
||||
appId?: string;
|
||||
}
|
||||
|
||||
const SESSION_DIR = getQQBotDataDir("sessions");
|
||||
const SESSION_EXPIRE_TIME = 5 * 60 * 1000;
|
||||
const SAVE_THROTTLE_MS = 1000;
|
||||
|
||||
@@ -36,9 +35,11 @@ const throttleState = new Map<
|
||||
>();
|
||||
|
||||
function ensureDir(): void {
|
||||
if (!fs.existsSync(SESSION_DIR)) {
|
||||
fs.mkdirSync(SESSION_DIR, { recursive: true });
|
||||
}
|
||||
getQQBotDataDir("sessions");
|
||||
}
|
||||
|
||||
function getSessionDir(): string {
|
||||
return getQQBotDataPath("sessions");
|
||||
}
|
||||
|
||||
function encodeAccountIdForFileName(accountId: string): string {
|
||||
@@ -47,12 +48,12 @@ function encodeAccountIdForFileName(accountId: string): string {
|
||||
|
||||
function getLegacySessionPath(accountId: string): string {
|
||||
const safeId = accountId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
return path.join(SESSION_DIR, `session-${safeId}.json`);
|
||||
return path.join(getSessionDir(), `session-${safeId}.json`);
|
||||
}
|
||||
|
||||
function getSessionPath(accountId: string): string {
|
||||
const encodedId = encodeAccountIdForFileName(accountId);
|
||||
return path.join(SESSION_DIR, `session-${encodedId}.json`);
|
||||
return path.join(getSessionDir(), `session-${encodedId}.json`);
|
||||
}
|
||||
|
||||
function getCandidateSessionPaths(accountId: string): string[] {
|
||||
@@ -66,7 +67,7 @@ function isSessionFileName(file: string): boolean {
|
||||
}
|
||||
|
||||
function readSessionStateFile(file: string): { filePath: string; state: SessionState } {
|
||||
const filePath = path.join(SESSION_DIR, file);
|
||||
const filePath = path.join(getSessionDir(), file);
|
||||
const data = fs.readFileSync(filePath, "utf-8");
|
||||
return { filePath, state: JSON.parse(data) as SessionState };
|
||||
}
|
||||
@@ -224,8 +225,11 @@ export function updateLastSeq(accountId: string, lastSeq: number): void {
|
||||
export function getAllSessions(): SessionState[] {
|
||||
const sessions = new Map<string, SessionState>();
|
||||
try {
|
||||
ensureDir();
|
||||
const files = fs.readdirSync(SESSION_DIR);
|
||||
const sessionDir = getSessionDir();
|
||||
if (!fs.existsSync(sessionDir)) {
|
||||
return [];
|
||||
}
|
||||
const files = fs.readdirSync(sessionDir);
|
||||
|
||||
for (const file of files) {
|
||||
if (isSessionFileName(file)) {
|
||||
@@ -249,13 +253,16 @@ export function getAllSessions(): SessionState[] {
|
||||
export function cleanupExpiredSessions(): number {
|
||||
let cleaned = 0;
|
||||
try {
|
||||
ensureDir();
|
||||
const sessionDir = getSessionDir();
|
||||
if (!fs.existsSync(sessionDir)) {
|
||||
return 0;
|
||||
}
|
||||
const now = Date.now();
|
||||
const files = fs.readdirSync(SESSION_DIR);
|
||||
const files = fs.readdirSync(sessionDir);
|
||||
|
||||
for (const file of files) {
|
||||
if (isSessionFileName(file)) {
|
||||
const filePath = path.join(SESSION_DIR, file);
|
||||
const filePath = path.join(sessionDir, file);
|
||||
try {
|
||||
const { state } = readSessionStateFile(file);
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
*/
|
||||
|
||||
import path from "node:path";
|
||||
import { getQQBotDataDir } from "./platform.js";
|
||||
import { getQQBotDataPath } from "./platform.js";
|
||||
|
||||
/**
|
||||
* Normalise an identifier so it is safe to embed in a filename.
|
||||
@@ -29,10 +29,10 @@ export function safeName(id: string): string {
|
||||
* missing from the live config.
|
||||
*/
|
||||
export function getCredentialBackupFile(accountId: string): string {
|
||||
return path.join(getQQBotDataDir("data"), `credential-backup-${safeName(accountId)}.json`);
|
||||
return path.join(getQQBotDataPath("data"), `credential-backup-${safeName(accountId)}.json`);
|
||||
}
|
||||
|
||||
/** Legacy single-file credential backup (pre-multi-account-isolation). */
|
||||
export function getLegacyCredentialBackupFile(): string {
|
||||
return path.join(getQQBotDataDir("data"), "credential-backup.json");
|
||||
return path.join(getQQBotDataPath("data"), "credential-backup.json");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const createdHomes: string[] = [];
|
||||
|
||||
async function useMockHome(homeDir: string): Promise<void> {
|
||||
vi.resetModules();
|
||||
vi.doMock("node:os", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:os")>();
|
||||
return {
|
||||
...actual,
|
||||
default: { ...actual, homedir: () => homeDir },
|
||||
homedir: () => homeDir,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function makeHome(): string {
|
||||
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "qqbot-home-"));
|
||||
createdHomes.push(homeDir);
|
||||
return homeDir;
|
||||
}
|
||||
|
||||
describe("qqbot storage laziness", () => {
|
||||
afterEach(() => {
|
||||
vi.doUnmock("node:os");
|
||||
vi.resetModules();
|
||||
for (const home of createdHomes.splice(0)) {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not create ~/.openclaw/qqbot from module imports or read-only probes", async () => {
|
||||
const homeDir = makeHome();
|
||||
await useMockHome(homeDir);
|
||||
|
||||
const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot");
|
||||
|
||||
const sessionStore = await import("../session/session-store.js");
|
||||
await import("../session/known-users.js");
|
||||
await import("../ref/store.js");
|
||||
const { loadCredentialBackup } = await import("../config/credential-backup.js");
|
||||
|
||||
expect(loadCredentialBackup("default")).toBeNull();
|
||||
expect(sessionStore.getAllSessions()).toEqual([]);
|
||||
expect(sessionStore.cleanupExpiredSessions()).toBe(0);
|
||||
expect(fs.existsSync(qqbotRoot)).toBe(false);
|
||||
});
|
||||
|
||||
it("creates storage when qqbot persists runtime state", async () => {
|
||||
const homeDir = makeHome();
|
||||
await useMockHome(homeDir);
|
||||
|
||||
const qqbotRoot = path.join(homeDir, ".openclaw", "qqbot");
|
||||
const { saveCredentialBackup } = await import("../config/credential-backup.js");
|
||||
|
||||
saveCredentialBackup("default", "123456", "secret");
|
||||
|
||||
expect(fs.existsSync(path.join(qqbotRoot, "data", "credential-backup-default.json"))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -41,9 +41,14 @@ export function getHomeDir(): string {
|
||||
return getPlatformAdapter().getTempDir();
|
||||
}
|
||||
|
||||
/** Return a path under `~/.openclaw/qqbot` without creating it. */
|
||||
export function getQQBotDataPath(...subPaths: string[]): string {
|
||||
return path.join(getHomeDir(), ".openclaw", "qqbot", ...subPaths);
|
||||
}
|
||||
|
||||
/** Return a path under `~/.openclaw/qqbot`, creating it on demand. */
|
||||
export function getQQBotDataDir(...subPaths: string[]): string {
|
||||
const dir = path.join(getHomeDir(), ".openclaw", "qqbot", ...subPaths);
|
||||
const dir = getQQBotDataPath(...subPaths);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
@@ -51,13 +56,18 @@ export function getQQBotDataDir(...subPaths: string[]): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a path under `~/.openclaw/media/qqbot`, creating it on demand.
|
||||
* Return a path under `~/.openclaw/media/qqbot` without creating it.
|
||||
*
|
||||
* Unlike `getQQBotDataDir`, this lives under OpenClaw's core media allowlist so
|
||||
* Unlike `getQQBotDataPath`, this lives under OpenClaw's core media allowlist so
|
||||
* downloaded images and audio can be accessed by framework media tooling.
|
||||
*/
|
||||
export function getQQBotMediaPath(...subPaths: string[]): string {
|
||||
return path.join(getHomeDir(), ".openclaw", "media", "qqbot", ...subPaths);
|
||||
}
|
||||
|
||||
/** Return a path under `~/.openclaw/media/qqbot`, creating it on demand. */
|
||||
export function getQQBotMediaDir(...subPaths: string[]): string {
|
||||
const dir = path.join(getHomeDir(), ".openclaw", "media", "qqbot", ...subPaths);
|
||||
const dir = getQQBotMediaPath(...subPaths);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
@@ -286,8 +296,8 @@ export function resolveQQBotLocalMediaPath(p: string): string {
|
||||
}
|
||||
|
||||
const homeDir = getHomeDir();
|
||||
const mediaRoot = getQQBotMediaDir();
|
||||
const dataRoot = getQQBotDataDir();
|
||||
const mediaRoot = getQQBotMediaPath();
|
||||
const dataRoot = getQQBotDataPath();
|
||||
const workspaceRoot = path.join(homeDir, ".openclaw", "workspace", "qqbot");
|
||||
const candidateRoots = [
|
||||
{ from: workspaceRoot, to: mediaRoot },
|
||||
@@ -331,7 +341,7 @@ export function resolveQQBotPayloadLocalFilePath(p: string): string | null {
|
||||
// attachments under sibling directories (e.g. `media/outbound/`) that are already
|
||||
// part of the core media allowlist; we mirror that so auto-routed sends work
|
||||
// without leaving the plugin's trust boundary.
|
||||
const allowedRoots = [getOpenClawMediaDir(), getQQBotMediaDir()];
|
||||
const allowedRoots = [getOpenClawMediaDir(), getQQBotMediaPath()];
|
||||
|
||||
for (const root of allowedRoots) {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
|
||||
Reference in New Issue
Block a user