diff --git a/src/gateway/server-methods/sessions-files.test-support.ts b/src/gateway/server-methods/sessions-files.test-support.ts new file mode 100644 index 000000000000..b69c892b1b0a --- /dev/null +++ b/src/gateway/server-methods/sessions-files.test-support.ts @@ -0,0 +1,144 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect } from "vitest"; +import type { GatewayRequestHandlers, RespondFn } from "./types.js"; + +type SessionFilesMethod = + | "sessions.files.list" + | "sessions.files.get" + | "sessions.files.set" + | "sessions.files.reveal"; + +type ResponderCall = { ok: boolean; payload?: unknown; error?: unknown }; +type ReturnValueMock = { mockReturnValue: (value: unknown) => unknown }; + +function createResponder() { + const calls: ResponderCall[] = []; + const respond: RespondFn = (ok, payload, error) => { + calls.push({ ok, payload, error }); + }; + return { calls, respond }; +} + +export function createSessionFilesHandlerInvoker(handlers: GatewayRequestHandlers) { + return async ( + method: SessionFilesMethod, + params: Record, + context: Record = {}, + ) => { + const responder = createResponder(); + await handlers[method]?.({ + req: { type: "req", id: method, method, params: {} }, + params, + client: null, + isWebchatConnect: () => false, + respond: responder.respond, + context: context as never, + }); + return responder.calls; + }; +} + +export function expectOkPayload(calls: ResponderCall[]): Record { + expect(calls).toHaveLength(1); + expect(calls[0]?.ok).toBe(true); + return calls[0]?.payload as Record; +} + +export function expectError(calls: ResponderCall[]): Record { + expect(calls).toHaveLength(1); + expect(calls[0]?.ok).toBe(false); + return calls[0]?.error as Record; +} + +export function assistantToolCall(name: string, args: Record) { + return { + role: "assistant", + content: [ + { + type: "toolCall", + name, + arguments: args, + }, + ], + }; +} + +export function visibleMessageEvent(message: unknown, seq: number) { + return { + event: { id: `event-${String(seq)}`, message }, + eventSeq: seq, + parentId: seq > 1 ? `event-${String(seq - 1)}` : null, + seq, + }; +} + +export function createVisibleMessagesMock(readVisibleMessageDelta: ReturnValueMock) { + return (messages: unknown[], cursor = "visible-messages-final"): void => { + readVisibleMessageDelta.mockReturnValue({ + kind: "page", + cursor, + events: messages.map(visibleMessageEvent), + hasMore: false, + serializedBytes: 100, + }); + }; +} + +export function writeWorkspaceFile(root: string, filePath: string, content: string): void { + const resolved = path.join(root, filePath); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, content, "utf8"); +} + +export function createWorkspaceFixture(prefix: string): string { + const tempRoot = fs.realpathSync(os.tmpdir()); + const workspaceRoot = fs.mkdtempSync(path.join(tempRoot, prefix)); + writeWorkspaceFile(workspaceRoot, "package.json", '{"name":"openclaw-test"}\n'); + writeWorkspaceFile(workspaceRoot, "src/readme.md", "# Read me\n"); + writeWorkspaceFile(workspaceRoot, "ui/chat.ts", "export const chat = true;\n"); + writeWorkspaceFile(workspaceRoot, "ui/vite.config.ts", "export default {};\n"); + return workspaceRoot; +} + +export function hashContent(content: string): string { + return createHash("sha256").update(content, "utf8").digest("hex"); +} + +export function createSessionEntryFixture( + workspaceRoot: string, + sessionId: string, + storePath = path.join(workspaceRoot, ".sessions.json"), +) { + return { + canonicalKey: "agent:main:main", + cfg: {}, + storePath, + entry: { + sessionId, + sessionFile: `${sessionId}.jsonl`, + spawnedCwd: workspaceRoot, + }, + }; +} + +export function useSqliteSession( + loadSessionEntry: ReturnValueMock, + workspaceRoot: string, + sessionId: string, + storePath = path.join(workspaceRoot, `${sessionId}.sqlite`), +): string { + loadSessionEntry.mockReturnValue({ + canonicalKey: "agent:main:main", + cfg: {}, + storePath, + entry: { + sessionId, + sessionFile: `sqlite:main:${sessionId}:${storePath}`, + spawnedCwd: workspaceRoot, + }, + }); + return storePath; +} diff --git a/src/gateway/server-methods/sessions-files.test.ts b/src/gateway/server-methods/sessions-files.test.ts index 6163093cafed..86aa4280780f 100644 --- a/src/gateway/server-methods/sessions-files.test.ts +++ b/src/gateway/server-methods/sessions-files.test.ts @@ -1,13 +1,22 @@ -import { execFileSync } from "node:child_process"; // Session file method tests cover transcript-linked files plus the workspace browser. import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { pathToFileURL } from "node:url"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveOpenPathCommand } from "./open-path.js"; import { sessionsFilesHandlers } from "./sessions-files.js"; +import { + assistantToolCall, + createSessionEntryFixture, + createSessionFilesHandlerInvoker, + createVisibleMessagesMock, + createWorkspaceFixture, + expectError, + expectOkPayload, + hashContent, + writeWorkspaceFile, +} from "./sessions-files.test-support.js"; import { updateWorkspaceFile } from "./workspace-fs.js"; const hoisted = vi.hoisted(() => ({ @@ -15,7 +24,7 @@ const hoisted = vi.hoisted(() => ({ loadSessionEntry: vi.fn(), resolveAgentWorkspaceDir: vi.fn(), resolveDefaultAgentId: vi.fn(), - visitSessionMessagesAsync: vi.fn(), + readSessionTranscriptVisibleMessageDelta: vi.fn(), })); vi.mock("./open-path.js", async () => { @@ -43,152 +52,39 @@ vi.mock("../session-transcript-readers.js", async () => { ); return { ...actual, - visitSessionMessagesAsync: hoisted.visitSessionMessagesAsync, + readSessionTranscriptVisibleMessageDelta: hoisted.readSessionTranscriptVisibleMessageDelta, }; }); -function createResponder() { - const calls: Array<{ ok: boolean; payload?: unknown; error?: unknown }> = []; - return { - calls, - respond: (ok: boolean, payload?: unknown, error?: unknown) => { - calls.push({ ok, payload, error }); - }, - }; -} - -type SessionFilesMethod = - | "sessions.files.list" - | "sessions.files.get" - | "sessions.files.set" - | "sessions.files.reveal"; - -async function invokeSessionFilesHandler( - method: SessionFilesMethod, - params: Record, - context: Record = {}, -) { - const responder = createResponder(); - await sessionsFilesHandlers[method]?.({ - req: { type: "req", id: method, method, params: {} }, - params, - client: null, - isWebchatConnect: () => false, - respond: responder.respond, - context: context as never, - }); - return responder.calls; -} - -function expectOkPayload(calls: ReturnType["calls"]): Record { - expect(calls).toHaveLength(1); - expect(calls[0]?.ok).toBe(true); - return calls[0]?.payload as Record; -} - -function expectError(calls: ReturnType["calls"]): Record { - expect(calls).toHaveLength(1); - expect(calls[0]?.ok).toBe(false); - return calls[0]?.error as Record; -} - -function assistantToolCall(name: string, args: Record) { - return { - role: "assistant", - content: [ - { - type: "toolCall", - name, - arguments: args, - }, - ], - }; -} - -function writeWorkspaceFile(root: string, filePath: string, content: string) { - const resolved = path.join(root, filePath); - fs.mkdirSync(path.dirname(resolved), { recursive: true }); - fs.writeFileSync(resolved, content, "utf8"); -} - -function hashContent(content: string): string { - return createHash("sha256").update(content, "utf8").digest("hex"); -} +const invokeSessionFilesHandler = createSessionFilesHandlerInvoker(sessionsFilesHandlers); +const mockVisibleMessages = createVisibleMessagesMock( + hoisted.readSessionTranscriptVisibleMessageDelta, +); describe("sessions.files RPC handlers", () => { let workspaceRoot: string; beforeEach(() => { vi.clearAllMocks(); - const tempRoot = fs.realpathSync(os.tmpdir()); - workspaceRoot = fs.mkdtempSync(path.join(tempRoot, "openclaw-session-files-test-")); + hoisted.readSessionTranscriptVisibleMessageDelta.mockReset(); + workspaceRoot = createWorkspaceFixture("openclaw-session-files-test-"); hoisted.resolveDefaultAgentId.mockReturnValue("main"); hoisted.resolveAgentWorkspaceDir.mockReturnValue(workspaceRoot); hoisted.execOpenPath.mockResolvedValue(undefined); - writeWorkspaceFile(workspaceRoot, "package.json", '{"name":"openclaw-test"}\n'); - writeWorkspaceFile(workspaceRoot, "src/readme.md", "# Read me\n"); - writeWorkspaceFile(workspaceRoot, "ui/chat.ts", "export const chat = true;\n"); - writeWorkspaceFile(workspaceRoot, "ui/vite.config.ts", "export default {};\n"); - - hoisted.loadSessionEntry.mockReturnValue({ - canonicalKey: "agent:main:main", - cfg: {}, - storePath: path.join(workspaceRoot, ".sessions.json"), - entry: { - sessionId: "sess-main", - sessionFile: "sess-main.jsonl", - spawnedCwd: workspaceRoot, - }, - }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - [ - assistantToolCall("edit", { path: "ui/chat.ts" }), - assistantToolCall("read", { path: "src/readme.md" }), - assistantToolCall("apply_patch", { - input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", - }), - ].forEach((message, index) => visit(message, index + 1)); - return 3; - }); + hoisted.loadSessionEntry.mockReturnValue(createSessionEntryFixture(workspaceRoot, "sess-main")); + mockVisibleMessages([ + assistantToolCall("edit", { path: "ui/chat.ts" }), + assistantToolCall("read", { path: "src/readme.md" }), + assistantToolCall("apply_patch", { + input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", + }), + ]); }); afterEach(() => { fs.rmSync(workspaceRoot, { recursive: true, force: true }); }); - it("lists session-touched files with a browser rooted at the session workspace", async () => { - const payload = expectOkPayload( - await invokeSessionFilesHandler("sessions.files.list", { - sessionKey: "agent:main:main", - }), - ); - - expect(payload.root).toBe(workspaceRoot); - expect(payload.gitCheckout).toBe(false); - expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ - ["package.json", "modified"], - ["ui/chat.ts", "modified"], - ["src/readme.md", "read"], - ]); - expect(payload.browser.path).toBe(""); - expect( - payload.browser.entries.map((entry: Record) => [ - entry.path, - entry.kind, - entry.sessionKind, - ]), - ).toEqual([ - ["src", "directory", "read"], - ["ui", "directory", "modified"], - ["package.json", "file", "modified"], - ]); - execFileSync("git", ["-C", workspaceRoot, "init", "-q", "-b", "main"]); - const gitPayload = expectOkPayload( - await invokeSessionFilesHandler("sessions.files.list", { sessionKey: "agent:main:main" }), - ); - expect(gitPayload.gitCheckout).toBe(true); - }); - it("reveals the same workspace root returned by sessions.files.list", async () => { const listPayload = expectOkPayload( await invokeSessionFilesHandler("sessions.files.list", { @@ -288,72 +184,6 @@ describe("sessions.files RPC handlers", () => { ); }); - it("collects touched files from existing transcript tool-call spellings", async () => { - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit( - { - role: "assistant", - content: [ - { type: "tool_use", name: "read", input: { path: "src/readme.md" } }, - { type: "toolcall", name: "edit", arguments: { path: "ui/vite.config.ts" } }, - { type: "tool_use", name: "read", args: { path: "ui/chat.ts" } }, - { - type: "tool_call", - name: "apply_patch", - input: { - input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", - }, - }, - ], - }, - 1, - ); - return 1; - }); - - const payload = expectOkPayload( - await invokeSessionFilesHandler("sessions.files.list", { - sessionKey: "agent:main:main", - }), - ); - - expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ - ["package.json", "modified"], - ["ui/vite.config.ts", "modified"], - ["src/readme.md", "read"], - ["ui/chat.ts", "read"], - ]); - }); - - it("collects changed files from structured apply_patch changes", async () => { - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit( - assistantToolCall("apply_patch", { - changes: [ - { path: "ui/chat.ts", kind: "update" }, - { path: "src/readme.md", kind: "delete" }, - { path: "old-name.md", kind: { type: "update", move_path: "package.json" } }, - ], - }), - 1, - ); - return 1; - }); - - const payload = expectOkPayload( - await invokeSessionFilesHandler("sessions.files.list", { - sessionKey: "agent:main:main", - }), - ); - - expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ - ["old-name.md", "modified"], - ["package.json", "modified"], - ["src/readme.md", "modified"], - ["ui/chat.ts", "modified"], - ]); - }); - it("prefers the spawned workspace root over a nested spawned cwd", async () => { const nestedCwd = path.join(workspaceRoot, "packages/app"); fs.mkdirSync(nestedCwd, { recursive: true }); @@ -370,11 +200,10 @@ describe("sessions.files RPC handlers", () => { spawnedWorkspaceDir: workspaceRoot, }, }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: "src/readme.md" }), 1); - visit(assistantToolCall("read", { path: "../shared/config.ts" }), 2); - return 2; - }); + mockVisibleMessages([ + assistantToolCall("read", { path: "src/readme.md" }), + assistantToolCall("read", { path: "../shared/config.ts" }), + ]); const payload = expectOkPayload( await invokeSessionFilesHandler("sessions.files.list", { @@ -467,10 +296,7 @@ describe("sessions.files RPC handlers", () => { sessionFile: "sess-main.jsonl", }, }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: "src/readme.md" }), 1); - return 1; - }); + mockVisibleMessages([assistantToolCall("read", { path: "src/readme.md" })]); const payload = expectOkPayload( await invokeSessionFilesHandler("sessions.files.list", { @@ -499,10 +325,7 @@ describe("sessions.files RPC handlers", () => { sessionFile: "sess-main.jsonl", }, }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: "src/readme.md" }), 1); - return 1; - }); + mockVisibleMessages([assistantToolCall("read", { path: "src/readme.md" })]); const payload = expectOkPayload( await invokeSessionFilesHandler("sessions.files.list", { @@ -600,10 +423,7 @@ describe("sessions.files RPC handlers", () => { sessionFile: "missing-session.jsonl", }, }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: outsidePath }), 1); - return 1; - }); + mockVisibleMessages([assistantToolCall("read", { path: outsidePath })]); try { for (const requestedPath of [outsidePath, "../outside.txt"]) { @@ -624,49 +444,6 @@ describe("sessions.files RPC handlers", () => { } }); - it("omits transcript paths outside the workspace without hiding missing workspace files", async () => { - const outsidePath = path.join(os.tmpdir(), `openclaw-outside-list-${Date.now()}.txt`); - fs.writeFileSync(outsidePath, "outside\n", "utf8"); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - [ - outsidePath, - "../outside.txt", - "~/.openclaw-external.txt", - pathToFileURL(outsidePath).href, - `@${outsidePath}`, - "..cache/missing.txt", - "missing.txt", - "src/readme.md", - ].forEach((filePath, index) => - visit(assistantToolCall("read", { path: filePath }), index + 1), - ); - return 8; - }); - - try { - const payload = expectOkPayload( - await invokeSessionFilesHandler("sessions.files.list", { - sessionKey: "agent:main:main", - }), - ); - - expect(payload.files).toHaveLength(3); - expect(payload.files).toEqual( - expect.arrayContaining([ - expect.objectContaining({ path: "..cache/missing.txt", missing: true }), - expect.objectContaining({ path: "missing.txt", missing: true }), - expect.objectContaining({ - path: "src/readme.md", - missing: false, - workspacePath: "src/readme.md", - }), - ]), - ); - } finally { - fs.rmSync(outsidePath, { force: true }); - } - }); - it("does not follow workspace symlinks for file previews", async () => { const outsidePath = path.join(os.tmpdir(), `openclaw-linked-${Date.now()}.txt`); fs.writeFileSync(outsidePath, "linked outside\n", "utf8"); @@ -759,10 +536,7 @@ describe("sessions.files RPC handlers", () => { sessionFile: "sess-main.jsonl", }, }); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: "secret.txt" }), 1); - return 1; - }); + mockVisibleMessages([assistantToolCall("read", { path: "secret.txt" })]); try { const listPayload = expectOkPayload( await invokeSessionFilesHandler("sessions.files.list", { @@ -796,10 +570,7 @@ describe("sessions.files RPC handlers", () => { it("reports oversized existing files without marking them missing", async () => { writeWorkspaceFile(workspaceRoot, "large.log", "x".repeat(260 * 1024)); - hoisted.visitSessionMessagesAsync.mockImplementation(async (_scope, visit) => { - visit(assistantToolCall("read", { path: "large.log" }), 1); - return 1; - }); + mockVisibleMessages([assistantToolCall("read", { path: "large.log" })]); const error = expectError( await invokeSessionFilesHandler("sessions.files.get", { @@ -841,7 +612,6 @@ describe("sessions.files RPC handlers", () => { }); expect(Number.isInteger(payload.file.updatedAtMs)).toBe(true); expect(payload.file.content).toBeUndefined(); - expect(hoisted.visitSessionMessagesAsync).not.toHaveBeenCalled(); }); it("rejects a stale file hash with the current hash", async () => { diff --git a/src/gateway/server-methods/sessions-files.touched-files.test.ts b/src/gateway/server-methods/sessions-files.touched-files.test.ts new file mode 100644 index 000000000000..42f3eb6cf0ad --- /dev/null +++ b/src/gateway/server-methods/sessions-files.touched-files.test.ts @@ -0,0 +1,376 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { sessionsFilesHandlers } from "./sessions-files.js"; +import { + assistantToolCall, + createSessionFilesHandlerInvoker, + createVisibleMessagesMock, + createWorkspaceFixture, + expectOkPayload, + useSqliteSession, + visibleMessageEvent, +} from "./sessions-files.test-support.js"; + +const hoisted = vi.hoisted(() => ({ + loadSessionEntry: vi.fn(), + resolveAgentWorkspaceDir: vi.fn(), + resolveDefaultAgentId: vi.fn(), + readSessionTranscriptVisibleMessageDelta: vi.fn(), +})); + +vi.mock("../../agents/agent-scope.js", () => ({ + resolveAgentWorkspaceDir: hoisted.resolveAgentWorkspaceDir, + resolveDefaultAgentId: hoisted.resolveDefaultAgentId, +})); + +vi.mock("../session-utils.js", async () => { + const actual = await vi.importActual("../session-utils.js"); + return { + ...actual, + loadSessionEntry: hoisted.loadSessionEntry, + loadSessionEntryReadOnly: hoisted.loadSessionEntry, + }; +}); + +vi.mock("../session-transcript-readers.js", async () => { + const actual = await vi.importActual( + "../session-transcript-readers.js", + ); + return { + ...actual, + readSessionTranscriptVisibleMessageDelta: hoisted.readSessionTranscriptVisibleMessageDelta, + }; +}); + +const invokeSessionFilesHandler = createSessionFilesHandlerInvoker(sessionsFilesHandlers); +const mockVisibleMessages = createVisibleMessagesMock( + hoisted.readSessionTranscriptVisibleMessageDelta, +); + +describe("sessions.files touched-file folds", () => { + let workspaceRoot: string; + + beforeEach(() => { + vi.clearAllMocks(); + hoisted.readSessionTranscriptVisibleMessageDelta.mockReset(); + workspaceRoot = createWorkspaceFixture("openclaw-session-touched-files-test-"); + hoisted.resolveDefaultAgentId.mockReturnValue("main"); + hoisted.resolveAgentWorkspaceDir.mockReturnValue(workspaceRoot); + }); + + afterEach(() => { + fs.rmSync(workspaceRoot, { recursive: true, force: true }); + }); + + it("lists session-touched files with a browser rooted at the session workspace", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-list"); + mockVisibleMessages([ + assistantToolCall("edit", { path: "ui/chat.ts" }), + assistantToolCall("read", { path: "src/readme.md" }), + assistantToolCall("apply_patch", { + input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", + }), + ]); + + const payload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(payload.root).toBe(workspaceRoot); + expect(payload.gitCheckout).toBe(false); + expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ + ["package.json", "modified"], + ["ui/chat.ts", "modified"], + ["src/readme.md", "read"], + ]); + expect(payload.browser.path).toBe(""); + expect( + payload.browser.entries.map((entry: Record) => [ + entry.path, + entry.kind, + entry.sessionKind, + ]), + ).toEqual([ + ["src", "directory", "read"], + ["ui", "directory", "modified"], + ["package.json", "file", "modified"], + ]); + execFileSync("git", ["-C", workspaceRoot, "init", "-q", "-b", "main"]); + const gitPayload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { sessionKey: "agent:main:main" }), + ); + expect(gitPayload.gitCheckout).toBe(true); + }); + + it("folds only appended SQLite messages after the cached cursor", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-incremental"); + hoisted.readSessionTranscriptVisibleMessageDelta.mockImplementation((_scope, limits) => { + if (limits.cursor === undefined) { + return { + kind: "page", + cursor: "cursor-1", + events: [visibleMessageEvent(assistantToolCall("read", { path: "ui/chat.ts" }), 1)], + hasMore: false, + serializedBytes: 100, + }; + } + if (limits.cursor === "cursor-1") { + return { + kind: "page", + cursor: "cursor-2", + events: [visibleMessageEvent(assistantToolCall("edit", { path: "ui/chat.ts" }), 2)], + hasMore: false, + serializedBytes: 100, + }; + } + throw new Error(`unexpected cursor: ${String(limits.cursor)}`); + }); + + const first = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + const second = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(first.files).toEqual([expect.objectContaining({ path: "ui/chat.ts", kind: "read" })]); + expect(second.files).toEqual([ + expect.objectContaining({ path: "ui/chat.ts", kind: "modified" }), + ]); + expect(hoisted.readSessionTranscriptVisibleMessageDelta).toHaveBeenCalledTimes(2); + expect(hoisted.readSessionTranscriptVisibleMessageDelta.mock.calls[1]?.[1]).toMatchObject({ + cursor: "cursor-1", + }); + }); + + it("isolates touched-file folds for the same session across stores", async () => { + const sessionId = "sess-touched-multi-store"; + const firstStorePath = path.join(workspaceRoot, "store-a.sqlite"); + const secondStorePath = path.join(workspaceRoot, "store-b.sqlite"); + hoisted.readSessionTranscriptVisibleMessageDelta.mockImplementation((scope, limits) => { + expect(limits.cursor).toBeUndefined(); + const message = + scope.storePath === firstStorePath + ? assistantToolCall("read", { path: "src/readme.md" }) + : assistantToolCall("edit", { path: "ui/chat.ts" }); + return { + kind: "page", + cursor: `${scope.storePath}-final`, + events: [visibleMessageEvent(message, 1)], + hasMore: false, + serializedBytes: 100, + }; + }); + + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, sessionId, firstStorePath); + const first = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, sessionId, secondStorePath); + const second = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(first.files).toEqual([expect.objectContaining({ path: "src/readme.md", kind: "read" })]); + expect(second.files).toEqual([ + expect.objectContaining({ path: "ui/chat.ts", kind: "modified" }), + ]); + }); + + it("rebuilds the SQLite fold from the bootstrap cursor after a reset", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-reset"); + hoisted.readSessionTranscriptVisibleMessageDelta.mockImplementation((_scope, limits) => { + if (limits.cursor === undefined) { + return { + kind: "page", + cursor: "generation-1", + events: [visibleMessageEvent(assistantToolCall("read", { path: "src/readme.md" }), 1)], + hasMore: false, + serializedBytes: 100, + }; + } + if (limits.cursor === "generation-1") { + return { + kind: "reset", + cursor: "generation-2-bootstrap", + reason: "generation_mismatch", + }; + } + if (limits.cursor === "generation-2-bootstrap") { + return { + kind: "page", + cursor: "generation-2-final", + events: [visibleMessageEvent(assistantToolCall("edit", { path: "ui/chat.ts" }), 1)], + hasMore: false, + serializedBytes: 100, + }; + } + throw new Error(`unexpected cursor: ${String(limits.cursor)}`); + }); + + const beforeReset = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + const afterReset = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(beforeReset.files).toEqual([ + expect.objectContaining({ path: "src/readme.md", kind: "read" }), + ]); + expect(afterReset.files).toEqual([ + expect.objectContaining({ path: "ui/chat.ts", kind: "modified" }), + ]); + expect(hoisted.readSessionTranscriptVisibleMessageDelta).toHaveBeenCalledTimes(3); + }); + + it("collects the expected files and kinds from the SQLite fold", async () => { + const messages = [ + assistantToolCall("read", { path: "ui/chat.ts" }), + assistantToolCall("edit", { path: "ui/chat.ts" }), + assistantToolCall("read", { path: "src/readme.md" }), + assistantToolCall("apply_patch", { + input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", + }), + ]; + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-parity"); + hoisted.readSessionTranscriptVisibleMessageDelta.mockReturnValue({ + kind: "page", + cursor: "parity-final", + events: messages.map(visibleMessageEvent), + hasMore: false, + serializedBytes: 400, + }); + + const payload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ + ["package.json", "modified"], + ["ui/chat.ts", "modified"], + ["src/readme.md", "read"], + ]); + }); + + it("collects touched files from existing transcript tool-call spellings", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-spellings"); + mockVisibleMessages([ + { + role: "assistant", + content: [ + { type: "tool_use", name: "read", input: { path: "src/readme.md" } }, + { type: "toolcall", name: "edit", arguments: { path: "ui/vite.config.ts" } }, + { type: "tool_use", name: "read", args: { path: "ui/chat.ts" } }, + { + type: "tool_call", + name: "apply_patch", + input: { + input: "*** Begin Patch\n*** Update File: package.json\n*** End Patch\n", + }, + }, + ], + }, + ]); + + const payload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ + ["package.json", "modified"], + ["ui/vite.config.ts", "modified"], + ["src/readme.md", "read"], + ["ui/chat.ts", "read"], + ]); + }); + + it("collects changed files from structured apply_patch changes", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-structured-patch"); + mockVisibleMessages([ + assistantToolCall("apply_patch", { + changes: [ + { path: "ui/chat.ts", kind: "update" }, + { path: "src/readme.md", kind: "delete" }, + { path: "old-name.md", kind: { type: "update", move_path: "package.json" } }, + ], + }), + ]); + + const payload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(payload.files.map((file: Record) => [file.path, file.kind])).toEqual([ + ["old-name.md", "modified"], + ["package.json", "modified"], + ["src/readme.md", "modified"], + ["ui/chat.ts", "modified"], + ]); + }); + + it("omits transcript paths outside the workspace without hiding missing workspace files", async () => { + useSqliteSession(hoisted.loadSessionEntry, workspaceRoot, "sess-touched-outside-paths"); + const outsidePath = path.join(os.tmpdir(), `${path.basename(workspaceRoot)}-outside-list.txt`); + fs.writeFileSync(outsidePath, "outside\n", "utf8"); + mockVisibleMessages( + [ + outsidePath, + "../outside.txt", + "~/.openclaw-external.txt", + pathToFileURL(outsidePath).href, + `@${outsidePath}`, + "..cache/missing.txt", + "missing.txt", + "src/readme.md", + ].map((filePath) => assistantToolCall("read", { path: filePath })), + ); + + try { + const payload = expectOkPayload( + await invokeSessionFilesHandler("sessions.files.list", { + sessionKey: "agent:main:main", + }), + ); + + expect(payload.files).toHaveLength(3); + expect(payload.files).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "..cache/missing.txt", missing: true }), + expect.objectContaining({ path: "missing.txt", missing: true }), + expect.objectContaining({ + path: "src/readme.md", + missing: false, + workspacePath: "src/readme.md", + }), + ]), + ); + } finally { + fs.rmSync(outsidePath, { force: true }); + } + }); +}); diff --git a/src/gateway/server-methods/sessions-files.ts b/src/gateway/server-methods/sessions-files.ts index 2b153d95a614..85040bdf8664 100644 --- a/src/gateway/server-methods/sessions-files.ts +++ b/src/gateway/server-methods/sessions-files.ts @@ -22,7 +22,13 @@ import { resolveToCwd as resolveSessionToolPathToCwd } from "../../agents/sessio import { runGit } from "../../agents/worktrees/git.js"; import { FsSafeError } from "../../infra/fs-safe.js"; import { normalizeAgentId, parseAgentSessionKey } from "../../routing/session-key.js"; -import { visitSessionMessagesAsync } from "../session-transcript-readers.js"; +import { + readSessionTranscriptVisibleMessageDelta, + resolveTranscriptReadTarget, + sqliteMessageEventWithSeq, + toTranscriptReadScope, + type SessionTranscriptReadScope, +} from "../session-transcript-readers.js"; import { loadSessionEntryReadOnly } from "../session-utils.js"; import { execOpenPath, @@ -64,10 +70,18 @@ type LoadedSessionFiles = { files: TouchedFile[]; }; +type TouchedFilesCacheEntry = { + cursor: string; + files: Map; +}; + const MAX_PREVIEW_BYTES = WORKSPACE_PREVIEW_MAX_BYTES; const MAX_BROWSER_ENTRIES = 250; const MAX_SEARCH_ENTRIES = 500; const MAX_SEARCH_VISITED_ENTRIES = 5_000; +const TOUCHED_FILES_CACHE_LIMIT = 16; +const TOUCHED_FILES_DELTA_MAX_MESSAGES = 1_000; +const TOUCHED_FILES_DELTA_MAX_BYTES = 1_000_000; const SEARCH_SKIP_DIRS = new Set([ ".git", ".hg", @@ -79,6 +93,31 @@ const SEARCH_SKIP_DIRS = new Set([ "node_modules", ]); +// Request latency must not scale with transcript size: delta resets rebuild the +// fold, while this process-local LRU cap bounds retained session state. +const touchedFilesCache = new Map(); + +function readTouchedFilesCache(key: string): TouchedFilesCacheEntry | undefined { + const cached = touchedFilesCache.get(key); + if (cached) { + touchedFilesCache.delete(key); + touchedFilesCache.set(key, cached); + } + return cached; +} + +function writeTouchedFilesCache(key: string, entry: TouchedFilesCacheEntry): void { + touchedFilesCache.delete(key); + touchedFilesCache.set(key, entry); + while (touchedFilesCache.size > TOUCHED_FILES_CACHE_LIMIT) { + const oldestKey = touchedFilesCache.keys().next().value; + if (oldestKey === undefined) { + break; + } + touchedFilesCache.delete(oldestKey); + } +} + function sessionFilesError(type: string, message: string, details?: Record) { return errorShape(ErrorCodes.INVALID_REQUEST, message, { details: { @@ -191,6 +230,50 @@ function collectTouchedFilesFromMessage(message: unknown, files: Map { + let cached = readTouchedFilesCache(cacheKey); + let cursor = cached?.cursor; + let files = cached?.files ?? new Map(); + let maxBytes = TOUCHED_FILES_DELTA_MAX_BYTES; + + while (true) { + const delta = readSessionTranscriptVisibleMessageDelta(scope, { + ...(cursor ? { cursor } : {}), + maxBytes, + maxMessages: TOUCHED_FILES_DELTA_MAX_MESSAGES, + }); + if (delta.kind === "missing") { + touchedFilesCache.delete(cacheKey); + return new Map(); + } + if (delta.kind === "reset") { + cached = { cursor: delta.cursor, files: new Map() }; + cursor = cached.cursor; + files = cached.files; + writeTouchedFilesCache(cacheKey, cached); + continue; + } + for (const event of delta.events) { + const message = sqliteMessageEventWithSeq(event); + if (message !== undefined) { + collectTouchedFilesFromMessage(message, files); + } + } + cached = { cursor: delta.cursor, files }; + cursor = cached.cursor; + writeTouchedFilesCache(cacheKey, cached); + if (!delta.hasMore) { + return files; + } + if (delta.requiredBytes !== undefined) { + maxBytes = delta.requiredBytes; + } + } +} + function toDisplayPath(root: string, resolved: string): string { const relative = path.relative(root, resolved); if (!relative) { @@ -531,21 +614,19 @@ async function loadSessionFiles(params: { if (!entry?.sessionId || !storePath || !agentId) { return { files: [] }; } - const files = new Map(); - await visitSessionMessagesAsync( - { - agentId, - sessionEntry: entry, - sessionId: entry.sessionId, - sessionKey: canonicalKey, - storePath, - }, - (message) => collectTouchedFilesFromMessage(message, files), - { - mode: "full", - reason: "session files transcript scan", - cache: "reuse", - }, + const scope = { + agentId, + sessionEntry: entry, + sessionId: entry.sessionId, + sessionKey: canonicalKey, + storePath, + } satisfies SessionTranscriptReadScope; + const target = resolveTranscriptReadTarget(scope); + // Entry-scoped reads without an explicit sessionFile always resolve to a canonical SQLite marker. + // Legacy transcript files are doctor-owned migration debt, not a runtime read path. + const files = loadSqliteTouchedFiles( + toTranscriptReadScope(target), + `${agentId}\0${entry.sessionId}\0${target.storePath ?? ""}`, ); return { root: loaded.root, diff --git a/src/gateway/session-transcript-readers.ts b/src/gateway/session-transcript-readers.ts index e6665fdf83dc..75d24397d484 100644 --- a/src/gateway/session-transcript-readers.ts +++ b/src/gateway/session-transcript-readers.ts @@ -38,6 +38,7 @@ import type { SessionPreviewItem } from "./session-utils.types.js"; export type { ReadSessionMessagesAsyncOptions }; export { attachOpenClawTranscriptMeta, capArrayByJsonBytes } from "./session-utils.fs.js"; +export { readSessionTranscriptVisibleMessageDelta } from "../config/sessions/session-accessor.js"; export type { SessionTranscriptReadScope };