vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-03-29 02:41:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: backfill claude cli chat history

Browse Source
This commit is contained in:
Peter Steinberger
2026-03-26 21:25:24 +00:00
parent 6aa9bec8d7
commit 98ea8e244f
6 changed files with 723 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/auto-reply/reply/strip-inbound-meta.test.ts
View File

@@ -144,6 +144,16 @@ describe("timestamp prefix stripping", () => {
Hello`;
expect(stripInboundMetadata(input)).toBe("Hello");
});
it("strips a timestamp prefix that remains after removing metadata blocks", () => {
const input = `Sender (untrusted metadata):
\`\`\`json
{"label":"OpenClaw UI"}
\`\`\`
[Thu 2026-03-12 07:00 UTC] what time is it?`;
expect(stripInboundMetadata(input)).toBe("what time is it?");
});
});
describe("extractInboundSenderLabel", () => {

6
src/auto-reply/reply/strip-inbound-meta.ts
View File

@@ -184,7 +184,11 @@ export function stripInboundMetadata(text: string): string {
result.push(line);
}
return result.join("\n").replace(/^\n+/, "").replace(/\n+$/, "");
return result
.join("\n")
.replace(/^\n+/, "")
.replace(/\n+$/, "")
.replace(LEADING_TIMESTAMP_PREFIX_RE, "");
}
export function stripLeadingInboundMetadata(text: string): string {

242
src/gateway/cli-session-history.test.ts Normal file
View File

@@ -0,0 +1,242 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
augmentChatHistoryWithCliSessionImports,
mergeImportedChatHistoryMessages,
readClaudeCliSessionMessages,
resolveClaudeCliSessionFilePath,
} from "./cli-session-history.js";
const ORIGINAL_HOME = process.env.HOME;
function createClaudeHistoryLines(sessionId: string) {
return [
JSON.stringify({
type: "queue-operation",
operation: "enqueue",
timestamp: "2026-03-26T16:29:54.722Z",
sessionId,
content: "[Thu 2026-03-26 16:29 GMT] Reply with exactly: AGENT CLI OK.",
}),
JSON.stringify({
type: "user",
uuid: "user-1",
timestamp: "2026-03-26T16:29:54.800Z",
message: {
role: "user",
content:
'Sender (untrusted metadata):\n```json\n{"label":"openclaw-control-ui"}\n```\n\n[Thu 2026-03-26 16:29 GMT] hi',
},
}),
JSON.stringify({
type: "assistant",
uuid: "assistant-1",
timestamp: "2026-03-26T16:29:55.500Z",
message: {
role: "assistant",
model: "claude-sonnet-4-6",
content: [{ type: "text", text: "hello from Claude" }],
stop_reason: "end_turn",
usage: {
input_tokens: 11,
output_tokens: 7,
cache_read_input_tokens: 22,
},
},
}),
JSON.stringify({
type: "last-prompt",
sessionId,
lastPrompt: "ignored",
}),
].join("\n");
}
async function withClaudeProjectsDir<T>(
run: (params: { homeDir: string; sessionId: string; filePath: string }) => Promise<T>,
): Promise<T> {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-claude-history-"));
const homeDir = path.join(root, "home");
const sessionId = "5b8b202c-f6bb-4046-9475-d2f15fd07530";
const projectsDir = path.join(homeDir, ".claude", "projects", "demo-workspace");
const filePath = path.join(projectsDir, `${sessionId}.jsonl`);
await fs.mkdir(projectsDir, { recursive: true });
await fs.writeFile(filePath, createClaudeHistoryLines(sessionId), "utf-8");
process.env.HOME = homeDir;
try {
return await run({ homeDir, sessionId, filePath });
} finally {
if (ORIGINAL_HOME === undefined) {
delete process.env.HOME;
} else {
process.env.HOME = ORIGINAL_HOME;
}
await fs.rm(root, { recursive: true, force: true });
}
}
describe("cli session history", () => {
afterEach(() => {
if (ORIGINAL_HOME === undefined) {
delete process.env.HOME;
} else {
process.env.HOME = ORIGINAL_HOME;
}
});
it("reads claude-cli session messages from the Claude projects store", async () => {
await withClaudeProjectsDir(async ({ homeDir, sessionId, filePath }) => {
expect(resolveClaudeCliSessionFilePath({ cliSessionId: sessionId, homeDir })).toBe(filePath);
const messages = readClaudeCliSessionMessages({ cliSessionId: sessionId, homeDir });
expect(messages).toHaveLength(2);
expect(messages[0]).toMatchObject({
role: "user",
content: expect.stringContaining("[Thu 2026-03-26 16:29 GMT] hi"),
__openclaw: {
importedFrom: "claude-cli",
externalId: "user-1",
cliSessionId: sessionId,
},
});
expect(messages[1]).toMatchObject({
role: "assistant",
provider: "claude-cli",
model: "claude-sonnet-4-6",
stopReason: "end_turn",
usage: {
input: 11,
output: 7,
cacheRead: 22,
},
__openclaw: {
importedFrom: "claude-cli",
externalId: "assistant-1",
cliSessionId: sessionId,
},
});
});
});
it("deduplicates imported messages against similar local transcript entries", () => {
const localMessages = [
{
role: "user",
content: "hi",
timestamp: Date.parse("2026-03-26T16:29:54.900Z"),
},
{
role: "assistant",
content: [{ type: "text", text: "hello from Claude" }],
timestamp: Date.parse("2026-03-26T16:29:55.700Z"),
},
];
const importedMessages = [
{
role: "user",
content:
'Sender (untrusted metadata):\n```json\n{"label":"openclaw-control-ui"}\n```\n\n[Thu 2026-03-26 16:29 GMT] hi',
timestamp: Date.parse("2026-03-26T16:29:54.800Z"),
__openclaw: {
importedFrom: "claude-cli",
externalId: "user-1",
cliSessionId: "session-1",
},
},
{
role: "assistant",
content: [{ type: "text", text: "hello from Claude" }],
timestamp: Date.parse("2026-03-26T16:29:55.500Z"),
__openclaw: {
importedFrom: "claude-cli",
externalId: "assistant-1",
cliSessionId: "session-1",
},
},
{
role: "user",
content: "[Thu 2026-03-26 16:31 GMT] follow-up",
timestamp: Date.parse("2026-03-26T16:31:00.000Z"),
__openclaw: {
importedFrom: "claude-cli",
externalId: "user-2",
cliSessionId: "session-1",
},
},
];
const merged = mergeImportedChatHistoryMessages({ localMessages, importedMessages });
expect(merged).toHaveLength(3);
expect(merged[2]).toMatchObject({
role: "user",
__openclaw: {
importedFrom: "claude-cli",
externalId: "user-2",
},
});
});
it("augments chat history when a session has a claude-cli binding", async () => {
await withClaudeProjectsDir(async ({ homeDir, sessionId }) => {
const messages = augmentChatHistoryWithCliSessionImports({
entry: {
sessionId: "openclaw-session",
cliSessionBindings: {
"claude-cli": {
sessionId,
},
},
},
provider: "claude-cli",
localMessages: [],
homeDir,
});
expect(messages).toHaveLength(2);
expect(messages[0]).toMatchObject({
role: "user",
__openclaw: { cliSessionId: sessionId },
});
});
});
it("falls back to legacy cliSessionIds when bindings are absent", async () => {
await withClaudeProjectsDir(async ({ homeDir, sessionId }) => {
const messages = augmentChatHistoryWithCliSessionImports({
entry: {
sessionId: "openclaw-session",
cliSessionIds: {
"claude-cli": sessionId,
},
},
provider: "claude-cli",
localMessages: [],
homeDir,
});
expect(messages).toHaveLength(2);
expect(messages[1]).toMatchObject({
role: "assistant",
__openclaw: { cliSessionId: sessionId },
});
});
});
it("falls back to legacy claudeCliSessionId when newer fields are absent", async () => {
await withClaudeProjectsDir(async ({ homeDir, sessionId }) => {
const messages = augmentChatHistoryWithCliSessionImports({
entry: {
sessionId: "openclaw-session",
claudeCliSessionId: sessionId,
},
provider: "claude-cli",
localMessages: [],
homeDir,
});
expect(messages).toHaveLength(2);
expect(messages[0]).toMatchObject({
role: "user",
__openclaw: { cliSessionId: sessionId },
});
});
});
});

375
src/gateway/cli-session-history.ts Normal file
View File

@@ -0,0 +1,375 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { normalizeProviderId } from "../agents/model-selection.js";
import { stripInboundMetadata } from "../auto-reply/reply/strip-inbound-meta.js";
import type { SessionEntry } from "../config/sessions.js";
import { attachOpenClawTranscriptMeta } from "./session-utils.fs.js";
const CLAUDE_CLI_PROVIDER = "claude-cli";
const CLAUDE_PROJECTS_RELATIVE_DIR = path.join(".claude", "projects");
const DEDUPE_TIMESTAMP_WINDOW_MS = 5 * 60 * 1000;
type ClaudeCliProjectEntry = {
type?: unknown;
timestamp?: unknown;
uuid?: unknown;
isSidechain?: unknown;
message?: {
role?: unknown;
content?: unknown;
model?: unknown;
stop_reason?: unknown;
usage?: {
input_tokens?: unknown;
output_tokens?: unknown;
cache_read_input_tokens?: unknown;
cache_creation_input_tokens?: unknown;
};
};
};
type TranscriptLikeMessage = Record<string, unknown>;
function resolveHistoryHomeDir(homeDir?: string): string {
return homeDir?.trim() || process.env.HOME || os.homedir();
}
function resolveClaudeProjectsDir(homeDir?: string): string {
return path.join(resolveHistoryHomeDir(homeDir), CLAUDE_PROJECTS_RELATIVE_DIR);
}
function resolveClaudeCliBindingSessionId(entry: SessionEntry | undefined): string | undefined {
const bindingSessionId = entry?.cliSessionBindings?.[CLAUDE_CLI_PROVIDER]?.sessionId?.trim();
if (bindingSessionId) {
return bindingSessionId;
}
const legacyMapSessionId = entry?.cliSessionIds?.[CLAUDE_CLI_PROVIDER]?.trim();
if (legacyMapSessionId) {
return legacyMapSessionId;
}
const legacyClaudeSessionId = entry?.claudeCliSessionId?.trim();
return legacyClaudeSessionId || undefined;
}
function resolveFiniteNumber(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}
function resolveTimestampMs(value: unknown): number | undefined {
if (typeof value !== "string") {
return undefined;
}
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
function resolveClaudeCliUsage(raw: ClaudeCliProjectEntry["message"]["usage"]) {
if (!raw || typeof raw !== "object") {
return undefined;
}
const input = resolveFiniteNumber(raw.input_tokens);
const output = resolveFiniteNumber(raw.output_tokens);
const cacheRead = resolveFiniteNumber(raw.cache_read_input_tokens);
const cacheWrite = resolveFiniteNumber(raw.cache_creation_input_tokens);
if (
input === undefined &&
output === undefined &&
cacheRead === undefined &&
cacheWrite === undefined
) {
return undefined;
}
return {
...(input !== undefined ? { input } : {}),
...(output !== undefined ? { output } : {}),
...(cacheRead !== undefined ? { cacheRead } : {}),
...(cacheWrite !== undefined ? { cacheWrite } : {}),
};
}
function cloneJsonValue<T>(value: T): T {
return JSON.parse(JSON.stringify(value)) as T;
}
function extractComparableText(message: unknown): string | undefined {
if (!message || typeof message !== "object") {
return undefined;
}
const record = message as { role?: unknown; text?: unknown; content?: unknown };
const role = typeof record.role === "string" ? record.role : undefined;
const parts: string[] = [];
if (typeof record.text === "string") {
parts.push(record.text);
}
if (typeof record.content === "string") {
parts.push(record.content);
} else if (Array.isArray(record.content)) {
for (const block of record.content) {
if (block && typeof block === "object" && "text" in block && typeof block.text === "string") {
parts.push(block.text);
}
}
}
if (parts.length === 0) {
return undefined;
}
const joined = parts.join("\n").trim();
if (!joined) {
return undefined;
}
const visible = role === "user" ? stripInboundMetadata(joined) : joined;
const normalized = visible.replace(/\s+/g, " ").trim();
return normalized || undefined;
}
function resolveComparableTimestamp(message: unknown): number | undefined {
if (!message || typeof message !== "object") {
return undefined;
}
return resolveFiniteNumber((message as { timestamp?: unknown }).timestamp);
}
function resolveComparableRole(message: unknown): string | undefined {
if (!message || typeof message !== "object") {
return undefined;
}
const role = (message as { role?: unknown }).role;
return typeof role === "string" ? role : undefined;
}
function resolveImportedExternalId(message: unknown): string | undefined {
if (!message || typeof message !== "object") {
return undefined;
}
const meta =
"__openclaw" in message &&
(message as { __openclaw?: unknown }).__openclaw &&
typeof (message as { __openclaw?: unknown }).__openclaw === "object"
? ((message as { __openclaw?: Record<string, unknown> }).__openclaw ?? {})
: undefined;
const externalId = meta?.externalId;
return typeof externalId === "string" && externalId.trim() ? externalId : undefined;
}
function isEquivalentImportedMessage(existing: unknown, imported: unknown): boolean {
const importedExternalId = resolveImportedExternalId(imported);
if (importedExternalId && resolveImportedExternalId(existing) === importedExternalId) {
return true;
}
const existingRole = resolveComparableRole(existing);
const importedRole = resolveComparableRole(imported);
if (!existingRole || existingRole !== importedRole) {
return false;
}
const existingText = extractComparableText(existing);
const importedText = extractComparableText(imported);
if (!existingText || !importedText || existingText !== importedText) {
return false;
}
const existingTimestamp = resolveComparableTimestamp(existing);
const importedTimestamp = resolveComparableTimestamp(imported);
if (existingTimestamp === undefined || importedTimestamp === undefined) {
return true;
}
return Math.abs(existingTimestamp - importedTimestamp) <= DEDUPE_TIMESTAMP_WINDOW_MS;
}
function compareHistoryMessages(
a: { message: unknown; order: number },
b: { message: unknown; order: number },
): number {
const aTimestamp = resolveComparableTimestamp(a.message);
const bTimestamp = resolveComparableTimestamp(b.message);
if (aTimestamp !== undefined && bTimestamp !== undefined && aTimestamp !== bTimestamp) {
return aTimestamp - bTimestamp;
}
if (aTimestamp !== undefined && bTimestamp === undefined) {
return -1;
}
if (aTimestamp === undefined && bTimestamp !== undefined) {
return 1;
}
return a.order - b.order;
}
function parseClaudeCliHistoryEntry(
entry: ClaudeCliProjectEntry,
cliSessionId: string,
): TranscriptLikeMessage | null {
if (entry.isSidechain === true || !entry.message || typeof entry.message !== "object") {
return null;
}
const type = typeof entry.type === "string" ? entry.type : undefined;
const role = typeof entry.message.role === "string" ? entry.message.role : undefined;
if (type !== "user" && type !== "assistant") {
return null;
}
if (role !== type) {
return null;
}
const timestamp = resolveTimestampMs(entry.timestamp);
const baseMeta = {
importedFrom: CLAUDE_CLI_PROVIDER,
cliSessionId,
...(typeof entry.uuid === "string" && entry.uuid.trim() ? { externalId: entry.uuid } : {}),
};
if (type === "user") {
const content =
typeof entry.message.content === "string" || Array.isArray(entry.message.content)
? cloneJsonValue(entry.message.content)
: undefined;
if (content === undefined) {
return null;
}
return attachOpenClawTranscriptMeta(
{
role: "user",
content,
...(timestamp !== undefined ? { timestamp } : {}),
},
baseMeta,
) as TranscriptLikeMessage;
}
const content =
typeof entry.message.content === "string" || Array.isArray(entry.message.content)
? cloneJsonValue(entry.message.content)
: undefined;
if (content === undefined) {
return null;
}
return attachOpenClawTranscriptMeta(
{
role: "assistant",
content,
api: "anthropic-messages",
provider: CLAUDE_CLI_PROVIDER,
...(typeof entry.message.model === "string" && entry.message.model.trim()
? { model: entry.message.model }
: {}),
...(typeof entry.message.stop_reason === "string" && entry.message.stop_reason.trim()
? { stopReason: entry.message.stop_reason }
: {}),
...(resolveClaudeCliUsage(entry.message.usage)
? { usage: resolveClaudeCliUsage(entry.message.usage) }
: {}),
...(timestamp !== undefined ? { timestamp } : {}),
},
baseMeta,
) as TranscriptLikeMessage;
}
export function resolveClaudeCliSessionFilePath(params: {
cliSessionId: string;
homeDir?: string;
}): string | undefined {
const projectsDir = resolveClaudeProjectsDir(params.homeDir);
let projectEntries: fs.Dirent[];
try {
projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
} catch {
return undefined;
}
for (const entry of projectEntries) {
if (!entry.isDirectory()) {
continue;
}
const candidate = path.join(projectsDir, entry.name, `${params.cliSessionId}.jsonl`);
if (fs.existsSync(candidate)) {
return candidate;
}
}
return undefined;
}
export function readClaudeCliSessionMessages(params: {
cliSessionId: string;
homeDir?: string;
}): TranscriptLikeMessage[] {
const filePath = resolveClaudeCliSessionFilePath(params);
if (!filePath) {
return [];
}
let content: string;
try {
content = fs.readFileSync(filePath, "utf-8");
} catch {
return [];
}
const messages: TranscriptLikeMessage[] = [];
for (const line of content.split(/\r?\n/)) {
if (!line.trim()) {
continue;
}
try {
const parsed = JSON.parse(line) as ClaudeCliProjectEntry;
const message = parseClaudeCliHistoryEntry(parsed, params.cliSessionId);
if (message) {
messages.push(message);
}
} catch {
// Ignore malformed external history entries.
}
}
return messages;
}
export function mergeImportedChatHistoryMessages(params: {
localMessages: unknown[];
importedMessages: unknown[];
}): unknown[] {
if (params.importedMessages.length === 0) {
return params.localMessages;
}
const merged = params.localMessages.map((message, index) => ({ message, order: index }));
let nextOrder = merged.length;
for (const imported of params.importedMessages) {
if (merged.some((existing) => isEquivalentImportedMessage(existing.message, imported))) {
continue;
}
merged.push({ message: imported, order: nextOrder });
nextOrder += 1;
}
merged.sort(compareHistoryMessages);
return merged.map((entry) => entry.message);
}
export function augmentChatHistoryWithCliSessionImports(params: {
entry: SessionEntry | undefined;
provider?: string;
localMessages: unknown[];
homeDir?: string;
}): unknown[] {
const cliSessionId = resolveClaudeCliBindingSessionId(params.entry);
if (!cliSessionId) {
return params.localMessages;
}
const normalizedProvider = normalizeProviderId(params.provider ?? "");
if (
normalizedProvider &&
normalizedProvider !== CLAUDE_CLI_PROVIDER &&
params.localMessages.length > 0
) {
return params.localMessages;
}
const importedMessages = readClaudeCliSessionMessages({
cliSessionId,
homeDir: params.homeDir,
});
return mergeImportedChatHistoryMessages({
localMessages: params.localMessages,
importedMessages,
});
}

16
src/gateway/server-methods/chat.ts
View File

@@ -37,6 +37,7 @@ import {
} from "../chat-abort.js";
import { type ChatImageContent, parseMessageWithAttachments } from "../chat-attachments.js";
import { stripEnvelopeFromMessage, stripEnvelopeFromMessages } from "../chat-sanitize.js";
import { augmentChatHistoryWithCliSessionImports } from "../cli-session-history.js";
import { ADMIN_SCOPE } from "../method-scopes.js";
import {
GATEWAY_CLIENT_CAPS,
@@ -1095,8 +1096,15 @@ export const chatHandlers: GatewayRequestHandlers = {
};
const { cfg, storePath, entry } = loadSessionEntry(sessionKey);
const sessionId = entry?.sessionId;
const rawMessages =
const sessionAgentId = resolveSessionAgentId({ sessionKey, config: cfg });
const resolvedSessionModel = resolveSessionModelRef(cfg, entry, sessionAgentId);
const localMessages =
sessionId && storePath ? readSessionMessages(sessionId, storePath, entry?.sessionFile) : [];
const rawMessages = augmentChatHistoryWithCliSessionImports({
entry,
provider: resolvedSessionModel.provider,
localMessages,
});
const hardMax = 1000;
const defaultLimit = 200;
const requested = typeof limit === "number" ? limit : defaultLimit;
@@ -1121,13 +1129,11 @@ export const chatHandlers: GatewayRequestHandlers = {
}
let thinkingLevel = entry?.thinkingLevel;
if (!thinkingLevel) {
const sessionAgentId = resolveSessionAgentId({ sessionKey, config: cfg });
const { provider, model } = resolveSessionModelRef(cfg, entry, sessionAgentId);
const catalog = await context.loadGatewayModelCatalog();
thinkingLevel = resolveThinkingDefault({
cfg,
provider,
model,
provider: resolvedSessionModel.provider,
model: resolvedSessionModel.model,
catalog,
});
}

80
src/gateway/server.chat.gateway-server-chat-b.test.ts
View File

@@ -99,6 +99,86 @@ async function prepareMainHistoryHarness(params: {
}
describe("gateway server chat", () => {
test("chat.history backfills claude-cli sessions from Claude project files", async () => {
await withGatewayChatHarness(async ({ ws, createSessionDir }) => {
await connectOk(ws);
const sessionDir = await createSessionDir();
const originalHome = process.env.HOME;
const homeDir = path.join(sessionDir, "home");
const cliSessionId = "5b8b202c-f6bb-4046-9475-d2f15fd07530";
const claudeProjectsDir = path.join(homeDir, ".claude", "projects", "workspace");
await fs.mkdir(claudeProjectsDir, { recursive: true });
await fs.writeFile(
path.join(claudeProjectsDir, `${cliSessionId}.jsonl`),
[
JSON.stringify({
type: "queue-operation",
operation: "enqueue",
timestamp: "2026-03-26T16:29:54.722Z",
sessionId: cliSessionId,
content: "[Thu 2026-03-26 16:29 GMT] hi",
}),
JSON.stringify({
type: "user",
uuid: "user-1",
timestamp: "2026-03-26T16:29:54.800Z",
message: {
role: "user",
content:
'Sender (untrusted metadata):\n```json\n{"label":"openclaw-control-ui"}\n```\n\n[Thu 2026-03-26 16:29 GMT] hi',
},
}),
JSON.stringify({
type: "assistant",
uuid: "assistant-1",
timestamp: "2026-03-26T16:29:55.500Z",
message: {
role: "assistant",
model: "claude-sonnet-4-6",
content: [{ type: "text", text: "hello from Claude" }],
},
}),
].join("\n"),
"utf-8",
);
process.env.HOME = homeDir;
try {
await writeSessionStore({
entries: {
main: {
sessionId: "sess-main",
updatedAt: Date.now(),
modelProvider: "claude-cli",
model: "claude-sonnet-4-6",
cliSessionBindings: {
"claude-cli": {
sessionId: cliSessionId,
},
},
},
},
});
const messages = await fetchHistoryMessages(ws);
expect(messages).toHaveLength(2);
expect(messages[0]).toMatchObject({
role: "user",
content: "hi",
});
expect(messages[1]).toMatchObject({
role: "assistant",
provider: "claude-cli",
});
} finally {
if (originalHome === undefined) {
delete process.env.HOME;
} else {
process.env.HOME = originalHome;
}
}
});
});
test("smoke: caps history payload and preserves routing metadata", async () => {
await withGatewayChatHarness(async ({ ws, createSessionDir }) => {
const historyMaxBytes = 64 * 1024;