mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:30:42 +00:00
chore: delete stale memory host bridges
This commit is contained in:
Peter Steinberger
@@ -1,8 +1,122 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { DEFAULT_LOCAL_MODEL } from "./embeddings.js";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./embeddings.js";
|
||||
|
||||
const nodeLlamaMock = vi.hoisted(() => ({
|
||||
importNodeLlamaCpp: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./node-llama.js", () => ({
|
||||
importNodeLlamaCpp: nodeLlamaMock.importNodeLlamaCpp,
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
function mockLocalEmbeddingRuntime(vector = new Float32Array([2.35, 3.45, 0.63, 4.3])) {
|
||||
const getEmbeddingFor = vi.fn().mockResolvedValue({ vector });
|
||||
const createEmbeddingContext = vi.fn().mockResolvedValue({ getEmbeddingFor });
|
||||
const loadModel = vi.fn().mockResolvedValue({ createEmbeddingContext });
|
||||
const resolveModelFile = vi.fn(async (modelPath: string) => `/resolved/${modelPath}`);
|
||||
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => ({ loadModel }),
|
||||
resolveModelFile,
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
|
||||
return { createEmbeddingContext, getEmbeddingFor, loadModel, resolveModelFile };
|
||||
}
|
||||
|
||||
describe("local embedding provider", () => {
|
||||
it("normalizes local embeddings and resolves the default local model", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedding = await provider.embedQuery("test query");
|
||||
const magnitude = Math.sqrt(embedding.reduce((sum, value) => sum + value * value, 0));
|
||||
|
||||
describe("package embeddings barrel", () => {
|
||||
it("re-exports the source local embedding contract", () => {
|
||||
expect(DEFAULT_LOCAL_MODEL).toContain("embeddinggemma");
|
||||
expect(magnitude).toBeCloseTo(1, 5);
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith(DEFAULT_LOCAL_MODEL, undefined);
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledWith("test query");
|
||||
});
|
||||
|
||||
it("passes default contextSize (4096) to createEmbeddingContext when not configured", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size default test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: 4096 });
|
||||
});
|
||||
|
||||
it("passes configured contextSize to createEmbeddingContext", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: 2048 },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size custom test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: 2048 });
|
||||
});
|
||||
|
||||
it('passes "auto" contextSize to createEmbeddingContext when explicitly set', async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: "auto" },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size auto test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: "auto" });
|
||||
});
|
||||
|
||||
it("trims explicit local model paths and cache directories", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime(new Float32Array([1, 0]));
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: {
|
||||
modelPath: " /models/embed.gguf ",
|
||||
modelCacheDir: " /cache/models ",
|
||||
},
|
||||
});
|
||||
|
||||
await provider.embedBatch(["a", "b"]);
|
||||
|
||||
expect(provider.model).toBe("/models/embed.gguf");
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith("/models/embed.gguf", "/cache/models");
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,45 +20,6 @@ export const KNIP_UNUSED_FILE_ALLOWLIST = [
|
||||
"src/mcp/plugin-tools-handlers.ts",
|
||||
"src/mcp/plugin-tools-serve.ts",
|
||||
"src/mcp/tools-stdio-server.ts",
|
||||
"src/memory-host-sdk/host/batch-error-utils.ts",
|
||||
"src/memory-host-sdk/host/batch-http.ts",
|
||||
"src/memory-host-sdk/host/batch-output.ts",
|
||||
"src/memory-host-sdk/host/batch-provider-common.ts",
|
||||
"src/memory-host-sdk/host/batch-runner.ts",
|
||||
"src/memory-host-sdk/host/batch-status.ts",
|
||||
"src/memory-host-sdk/host/batch-upload.ts",
|
||||
"src/memory-host-sdk/host/batch-utils.ts",
|
||||
"src/memory-host-sdk/host/embedding-chunk-limits.ts",
|
||||
"src/memory-host-sdk/host/embedding-input-limits.ts",
|
||||
"src/memory-host-sdk/host/embedding-model-limits.ts",
|
||||
"src/memory-host-sdk/host/embedding-provider-adapter-utils.ts",
|
||||
"src/memory-host-sdk/host/embedding-vectors.ts",
|
||||
"src/memory-host-sdk/host/embeddings-debug.ts",
|
||||
"src/memory-host-sdk/host/embeddings-model-normalize.ts",
|
||||
"src/memory-host-sdk/host/embeddings-remote-client.ts",
|
||||
"src/memory-host-sdk/host/embeddings-remote-fetch.ts",
|
||||
"src/memory-host-sdk/host/embeddings-remote-provider.ts",
|
||||
"src/memory-host-sdk/host/embeddings.ts",
|
||||
"src/memory-host-sdk/host/embeddings.types.ts",
|
||||
"src/memory-host-sdk/host/fs-utils.ts",
|
||||
"src/memory-host-sdk/host/hash.ts",
|
||||
"src/memory-host-sdk/host/internal.ts",
|
||||
"src/memory-host-sdk/host/memory-schema.ts",
|
||||
"src/memory-host-sdk/host/multimodal.ts",
|
||||
"src/memory-host-sdk/host/node-llama.ts",
|
||||
"src/memory-host-sdk/host/post-json.ts",
|
||||
"src/memory-host-sdk/host/qmd-process.ts",
|
||||
"src/memory-host-sdk/host/qmd-query-parser.ts",
|
||||
"src/memory-host-sdk/host/qmd-scope.ts",
|
||||
"src/memory-host-sdk/host/query-expansion.ts",
|
||||
"src/memory-host-sdk/host/read-file-shared.ts",
|
||||
"src/memory-host-sdk/host/read-file.ts",
|
||||
"src/memory-host-sdk/host/remote-http.ts",
|
||||
"src/memory-host-sdk/host/secret-input.ts",
|
||||
"src/memory-host-sdk/host/session-files.ts",
|
||||
"src/memory-host-sdk/host/sqlite-vec.ts",
|
||||
"src/memory-host-sdk/host/sqlite.ts",
|
||||
"src/memory-host-sdk/host/status-format.ts",
|
||||
"src/plugins/build-smoke-entry.ts",
|
||||
"src/plugins/contracts/host-hook-fixture.ts",
|
||||
"src/plugins/contracts/rootdir-boundary-canary.ts",
|
||||
|
||||
@@ -361,9 +361,8 @@ const SOURCE_TEST_TARGETS = new Map([
|
||||
],
|
||||
[
|
||||
"src/memory-host-sdk/host/embedding-defaults.ts",
|
||||
["src/memory-host-sdk/host/embeddings.test.ts"],
|
||||
["packages/memory-host-sdk/src/host/embeddings.test.ts"],
|
||||
],
|
||||
["src/memory-host-sdk/host/embeddings.ts", ["src/memory-host-sdk/host/embeddings.test.ts"]],
|
||||
[
|
||||
"src/plugin-sdk/test-helpers/directory-ids.ts",
|
||||
[
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/batch-error-utils.js";
|
||||
@@ -1,86 +0,0 @@
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../../infra/retry.js", () => ({
|
||||
retryAsync: vi.fn(async (run: () => Promise<unknown>) => await run()),
|
||||
}));
|
||||
|
||||
vi.mock("./post-json.js", () => ({
|
||||
postJson: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("postJsonWithRetry", () => {
|
||||
let retryAsyncMock: ReturnType<
|
||||
typeof vi.mocked<typeof import("../../infra/retry.js").retryAsync>
|
||||
>;
|
||||
let postJsonMock: ReturnType<typeof vi.mocked<typeof import("./post-json.js").postJson>>;
|
||||
let postJsonWithRetry: typeof import("./batch-http.js").postJsonWithRetry;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ postJsonWithRetry } = await import("./batch-http.js"));
|
||||
const retryModule = await import("../../infra/retry.js");
|
||||
const postJsonModule = await import("./post-json.js");
|
||||
retryAsyncMock = vi.mocked(retryModule.retryAsync);
|
||||
postJsonMock = vi.mocked(postJsonModule.postJson);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("posts JSON and returns parsed response payload", async () => {
|
||||
postJsonMock.mockImplementationOnce(async (params) => {
|
||||
return await params.parse({ ok: true, ids: [1, 2] });
|
||||
});
|
||||
|
||||
const result = await postJsonWithRetry<{ ok: boolean; ids: number[] }>({
|
||||
url: "https://memory.example/v1/batch",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
body: { chunks: ["a", "b"] },
|
||||
errorPrefix: "memory batch failed",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ ok: true, ids: [1, 2] });
|
||||
expect(postJsonMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "https://memory.example/v1/batch",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
body: { chunks: ["a", "b"] },
|
||||
errorPrefix: "memory batch failed",
|
||||
attachStatus: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const retryOptions = retryAsyncMock.mock.calls[0]?.[1] as
|
||||
| {
|
||||
attempts: number;
|
||||
minDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
shouldRetry: (err: unknown) => boolean;
|
||||
}
|
||||
| undefined;
|
||||
expect(retryOptions?.attempts).toBe(3);
|
||||
expect(retryOptions?.minDelayMs).toBe(300);
|
||||
expect(retryOptions?.maxDelayMs).toBe(2000);
|
||||
expect(retryOptions?.shouldRetry({ status: 429 })).toBe(true);
|
||||
expect(retryOptions?.shouldRetry({ status: 503 })).toBe(true);
|
||||
expect(retryOptions?.shouldRetry({ status: 400 })).toBe(false);
|
||||
});
|
||||
|
||||
it("attaches status to non-ok errors", async () => {
|
||||
postJsonMock.mockRejectedValueOnce(
|
||||
Object.assign(new Error("memory batch failed: 503 backend down"), { status: 503 }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
postJsonWithRetry({
|
||||
url: "https://memory.example/v1/batch",
|
||||
headers: {},
|
||||
body: { chunks: [] },
|
||||
errorPrefix: "memory batch failed",
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
message: expect.stringContaining("memory batch failed: 503 backend down"),
|
||||
status: 503,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,37 +0,0 @@
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import { retryAsync } from "../../infra/retry.js";
|
||||
import { postJson } from "./post-json.js";
|
||||
|
||||
export async function postJsonWithRetry<T>(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
}): Promise<T> {
|
||||
return await retryAsync(
|
||||
async () => {
|
||||
return await postJson<T>({
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
body: params.body,
|
||||
errorPrefix: params.errorPrefix,
|
||||
attachStatus: true,
|
||||
parse: async (payload) => payload as T,
|
||||
});
|
||||
},
|
||||
{
|
||||
attempts: 3,
|
||||
minDelayMs: 300,
|
||||
maxDelayMs: 2000,
|
||||
jitter: 0.2,
|
||||
shouldRetry: (err) => {
|
||||
const status = (err as { status?: number }).status;
|
||||
return status === 429 || (typeof status === "number" && status >= 500);
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/batch-output.js";
|
||||
@@ -1,12 +0,0 @@
|
||||
import type { EmbeddingBatchOutputLine } from "./batch-output.js";
|
||||
|
||||
export type EmbeddingBatchStatus = {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output_file_id?: string | null;
|
||||
error_file_id?: string | null;
|
||||
};
|
||||
|
||||
export type ProviderBatchOutputLine = EmbeddingBatchOutputLine;
|
||||
|
||||
export const EMBEDDING_BATCH_ENDPOINT = "/v1/embeddings";
|
||||
@@ -1,71 +0,0 @@
|
||||
import { runTasksWithConcurrency } from "../../utils/run-with-concurrency.js";
|
||||
import { splitBatchRequests } from "./batch-utils.js";
|
||||
|
||||
export type EmbeddingBatchExecutionParams = {
|
||||
wait: boolean;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
concurrency: number;
|
||||
debug?: (message: string, data?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
export async function runEmbeddingBatchGroups<TRequest>(params: {
|
||||
requests: TRequest[];
|
||||
maxRequests: number;
|
||||
wait: EmbeddingBatchExecutionParams["wait"];
|
||||
pollIntervalMs: EmbeddingBatchExecutionParams["pollIntervalMs"];
|
||||
timeoutMs: EmbeddingBatchExecutionParams["timeoutMs"];
|
||||
concurrency: EmbeddingBatchExecutionParams["concurrency"];
|
||||
debugLabel: string;
|
||||
debug?: EmbeddingBatchExecutionParams["debug"];
|
||||
runGroup: (args: {
|
||||
group: TRequest[];
|
||||
groupIndex: number;
|
||||
groups: number;
|
||||
byCustomId: Map<string, number[]>;
|
||||
}) => Promise<void>;
|
||||
}): Promise<Map<string, number[]>> {
|
||||
if (params.requests.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const groups = splitBatchRequests(params.requests, params.maxRequests);
|
||||
const byCustomId = new Map<string, number[]>();
|
||||
const tasks = groups.map((group, groupIndex) => async () => {
|
||||
await params.runGroup({ group, groupIndex, groups: groups.length, byCustomId });
|
||||
});
|
||||
|
||||
params.debug?.(params.debugLabel, {
|
||||
requests: params.requests.length,
|
||||
groups: groups.length,
|
||||
wait: params.wait,
|
||||
concurrency: params.concurrency,
|
||||
pollIntervalMs: params.pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
|
||||
const { firstError, hasError } = await runTasksWithConcurrency({
|
||||
tasks,
|
||||
limit: params.concurrency,
|
||||
errorMode: "stop",
|
||||
});
|
||||
if (hasError) {
|
||||
throw firstError;
|
||||
}
|
||||
return byCustomId;
|
||||
}
|
||||
|
||||
export function buildEmbeddingBatchGroupOptions<TRequest>(
|
||||
params: { requests: TRequest[] } & EmbeddingBatchExecutionParams,
|
||||
options: { maxRequests: number; debugLabel: string },
|
||||
) {
|
||||
return {
|
||||
requests: params.requests,
|
||||
maxRequests: options.maxRequests,
|
||||
wait: params.wait,
|
||||
pollIntervalMs: params.pollIntervalMs,
|
||||
timeoutMs: params.timeoutMs,
|
||||
concurrency: params.concurrency,
|
||||
debug: params.debug,
|
||||
debugLabel: options.debugLabel,
|
||||
};
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/batch-status.js";
|
||||
@@ -1,45 +0,0 @@
|
||||
import {
|
||||
buildBatchHeaders,
|
||||
normalizeBatchBaseUrl,
|
||||
type BatchHttpClientConfig,
|
||||
} from "./batch-utils.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
export async function uploadBatchJsonlFile(params: {
|
||||
client: BatchHttpClientConfig;
|
||||
requests: unknown[];
|
||||
errorPrefix: string;
|
||||
}): Promise<string> {
|
||||
const baseUrl = normalizeBatchBaseUrl(params.client);
|
||||
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
|
||||
const form = new FormData();
|
||||
form.append("purpose", "batch");
|
||||
form.append(
|
||||
"file",
|
||||
new Blob([jsonl], { type: "application/jsonl" }),
|
||||
`memory-embeddings.${hashText(String(Date.now()))}.jsonl`,
|
||||
);
|
||||
|
||||
const filePayload = await withRemoteHttpResponse({
|
||||
url: `${baseUrl}/files`,
|
||||
ssrfPolicy: params.client.ssrfPolicy,
|
||||
fetchImpl: params.client.fetchImpl,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: buildBatchHeaders(params.client, { json: false }),
|
||||
body: form,
|
||||
},
|
||||
onResponse: async (fileRes) => {
|
||||
if (!fileRes.ok) {
|
||||
const text = await fileRes.text();
|
||||
throw new Error(`${params.errorPrefix}: ${fileRes.status} ${text}`);
|
||||
}
|
||||
return (await fileRes.json()) as { id?: string };
|
||||
},
|
||||
});
|
||||
if (!filePayload.id) {
|
||||
throw new Error(`${params.errorPrefix}: missing file id`);
|
||||
}
|
||||
return filePayload.id;
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
|
||||
export type BatchHttpClientConfig = {
|
||||
baseUrl?: string;
|
||||
headers?: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
};
|
||||
|
||||
export function normalizeBatchBaseUrl(client: BatchHttpClientConfig): string {
|
||||
return client.baseUrl?.replace(/\/$/, "") ?? "";
|
||||
}
|
||||
|
||||
export function buildBatchHeaders(
|
||||
client: Pick<BatchHttpClientConfig, "headers">,
|
||||
params: { json: boolean },
|
||||
): Record<string, string> {
|
||||
const headers = client.headers ? { ...client.headers } : {};
|
||||
if (params.json) {
|
||||
if (!headers["Content-Type"] && !headers["content-type"]) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
} else {
|
||||
delete headers["Content-Type"];
|
||||
delete headers["content-type"];
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function splitBatchRequests<T>(requests: T[], maxRequests: number): T[][] {
|
||||
if (requests.length <= maxRequests) {
|
||||
return [requests];
|
||||
}
|
||||
const groups: T[][] = [];
|
||||
for (let i = 0; i < requests.length; i += maxRequests) {
|
||||
groups.push(requests.slice(i, i + maxRequests));
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-limits.js";
|
||||
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
|
||||
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import type { MemoryChunk } from "./internal.js";
|
||||
|
||||
export function enforceEmbeddingMaxInputTokens(
|
||||
provider: EmbeddingProvider,
|
||||
chunks: MemoryChunk[],
|
||||
hardMaxInputTokens?: number,
|
||||
): MemoryChunk[] {
|
||||
const providerMaxInputTokens = resolveEmbeddingMaxInputTokens(provider);
|
||||
const maxInputTokens =
|
||||
typeof hardMaxInputTokens === "number" && hardMaxInputTokens > 0
|
||||
? Math.min(providerMaxInputTokens, hardMaxInputTokens)
|
||||
: providerMaxInputTokens;
|
||||
const out: MemoryChunk[] = [];
|
||||
|
||||
for (const chunk of chunks) {
|
||||
if (hasNonTextEmbeddingParts(chunk.embeddingInput)) {
|
||||
out.push(chunk);
|
||||
continue;
|
||||
}
|
||||
if (estimateUtf8Bytes(chunk.text) <= maxInputTokens) {
|
||||
out.push(chunk);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const text of splitTextToUtf8ByteLimit(chunk.text, maxInputTokens)) {
|
||||
out.push({
|
||||
startLine: chunk.startLine,
|
||||
endLine: chunk.endLine,
|
||||
text,
|
||||
hash: hashText(text),
|
||||
embeddingInput: { text },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/embedding-input-limits.js";
|
||||
@@ -1,16 +0,0 @@
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
|
||||
const DEFAULT_EMBEDDING_MAX_INPUT_TOKENS = 8192;
|
||||
const DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS = 2048;
|
||||
|
||||
export function resolveEmbeddingMaxInputTokens(provider: EmbeddingProvider): number {
|
||||
if (typeof provider.maxInputTokens === "number") {
|
||||
return provider.maxInputTokens;
|
||||
}
|
||||
|
||||
if (provider.id === "local") {
|
||||
return DEFAULT_LOCAL_EMBEDDING_MAX_INPUT_TOKENS;
|
||||
}
|
||||
|
||||
return DEFAULT_EMBEDDING_MAX_INPUT_TOKENS;
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js";
|
||||
|
||||
export function isMissingEmbeddingApiKeyError(err: unknown): boolean {
|
||||
return err instanceof Error && err.message.includes("No API key found for provider");
|
||||
}
|
||||
|
||||
export function sanitizeEmbeddingCacheHeaders(
|
||||
headers: Record<string, string>,
|
||||
excludedHeaderNames: string[],
|
||||
): Array<[string, string]> {
|
||||
const excluded = new Set(
|
||||
excludedHeaderNames.map((name) => normalizeLowercaseStringOrEmpty(name)),
|
||||
);
|
||||
return Object.entries(headers)
|
||||
.filter(([key]) => !excluded.has(normalizeLowercaseStringOrEmpty(key)))
|
||||
.toSorted(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, value]) => [key, value]);
|
||||
}
|
||||
|
||||
export function mapBatchEmbeddingsByIndex(
|
||||
byCustomId: Map<string, number[]>,
|
||||
count: number,
|
||||
): number[][] {
|
||||
const embeddings: number[][] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
embeddings.push(byCustomId.get(String(index)) ?? []);
|
||||
}
|
||||
return embeddings;
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
export function sanitizeAndNormalizeEmbedding(vec: number[]): number[] {
|
||||
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
||||
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return sanitized;
|
||||
}
|
||||
return sanitized.map((value) => value / magnitude);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { isTruthyEnvValue } from "../../infra/env.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
|
||||
const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
|
||||
const log = createSubsystemLogger("memory/embeddings");
|
||||
|
||||
export function debugEmbeddingsLog(message: string, meta?: Record<string, unknown>): void {
|
||||
if (!debugEmbeddings) {
|
||||
return;
|
||||
}
|
||||
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
|
||||
log.raw(`${message}${suffix}`);
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
export function normalizeEmbeddingModelWithPrefixes(params: {
|
||||
model: string;
|
||||
defaultModel: string;
|
||||
prefixes: string[];
|
||||
}): string {
|
||||
const trimmed = params.model.trim();
|
||||
if (!trimmed) {
|
||||
return params.defaultModel;
|
||||
}
|
||||
for (const prefix of params.prefixes) {
|
||||
if (trimmed.startsWith(prefix)) {
|
||||
return trimmed.slice(prefix.length);
|
||||
}
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { requireApiKey, resolveApiKeyForProvider } from "../../agents/model-auth.js";
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import { normalizeOptionalString } from "../../shared/string-coerce.js";
|
||||
import type { EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
import { buildRemoteBaseUrlPolicy } from "./remote-http.js";
|
||||
import { resolveMemorySecretInputString } from "./secret-input.js";
|
||||
|
||||
export type RemoteEmbeddingProviderId = string;
|
||||
|
||||
export async function resolveRemoteEmbeddingBearerClient(params: {
|
||||
provider: RemoteEmbeddingProviderId;
|
||||
options: EmbeddingProviderOptions;
|
||||
defaultBaseUrl: string;
|
||||
}): Promise<{ baseUrl: string; headers: Record<string, string>; ssrfPolicy?: SsrFPolicy }> {
|
||||
const remote = params.options.remote;
|
||||
const remoteApiKey = resolveMemorySecretInputString({
|
||||
value: remote?.apiKey,
|
||||
path: "agents.*.memorySearch.remote.apiKey",
|
||||
});
|
||||
const remoteBaseUrl = normalizeOptionalString(remote?.baseUrl);
|
||||
const providerConfig = params.options.config.models?.providers?.[params.provider];
|
||||
const apiKey = remoteApiKey
|
||||
? remoteApiKey
|
||||
: requireApiKey(
|
||||
await resolveApiKeyForProvider({
|
||||
provider: params.provider,
|
||||
cfg: params.options.config,
|
||||
agentDir: params.options.agentDir,
|
||||
}),
|
||||
params.provider,
|
||||
);
|
||||
const baseUrl =
|
||||
remoteBaseUrl || normalizeOptionalString(providerConfig?.baseUrl) || params.defaultBaseUrl;
|
||||
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
...headerOverrides,
|
||||
};
|
||||
return { baseUrl, headers, ssrfPolicy: buildRemoteBaseUrlPolicy(baseUrl) };
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fetchRemoteEmbeddingVectors } from "./embeddings-remote-fetch.js";
|
||||
|
||||
describe("fetchRemoteEmbeddingVectors", () => {
|
||||
it("maps remote embedding response data through an injected fetch", async () => {
|
||||
const fetchImpl = vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({ data: [{ embedding: [0.1, 0.2] }, {}, { embedding: [0.3] }] }),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
) as typeof fetch;
|
||||
|
||||
const vectors = await fetchRemoteEmbeddingVectors({
|
||||
url: "https://example.com/v1/embeddings",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
ssrfPolicy: { allowedHostnames: ["example.com"] },
|
||||
fetchImpl,
|
||||
body: { input: ["one", "two", "three"] },
|
||||
errorPrefix: "embedding fetch failed",
|
||||
});
|
||||
|
||||
expect(vectors).toEqual([[0.1, 0.2], [], [0.3]]);
|
||||
});
|
||||
});
|
||||
@@ -1,27 +0,0 @@
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import { postJson } from "./post-json.js";
|
||||
|
||||
export async function fetchRemoteEmbeddingVectors(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
}): Promise<number[][]> {
|
||||
return await postJson({
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
body: params.body,
|
||||
errorPrefix: params.errorPrefix,
|
||||
parse: (payload) => {
|
||||
const typedPayload = payload as {
|
||||
data?: Array<{ embedding?: number[] }>;
|
||||
};
|
||||
const data = typedPayload.data ?? [];
|
||||
return data.map((entry) => entry.embedding ?? []);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/embeddings-remote-provider.js";
|
||||
@@ -1,124 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./embeddings.js";
|
||||
|
||||
const nodeLlamaMock = vi.hoisted(() => ({
|
||||
importNodeLlamaCpp: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../../packages/memory-host-sdk/src/host/node-llama.js", () => ({
|
||||
importNodeLlamaCpp: nodeLlamaMock.importNodeLlamaCpp,
|
||||
}));
|
||||
vi.mock("./node-llama.js", () => ({
|
||||
importNodeLlamaCpp: nodeLlamaMock.importNodeLlamaCpp,
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
function mockLocalEmbeddingRuntime(vector = new Float32Array([2.35, 3.45, 0.63, 4.3])) {
|
||||
const getEmbeddingFor = vi.fn().mockResolvedValue({ vector });
|
||||
const createEmbeddingContext = vi.fn().mockResolvedValue({ getEmbeddingFor });
|
||||
const loadModel = vi.fn().mockResolvedValue({ createEmbeddingContext });
|
||||
const resolveModelFile = vi.fn(async (modelPath: string) => `/resolved/${modelPath}`);
|
||||
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => ({ loadModel }),
|
||||
resolveModelFile,
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
|
||||
return { createEmbeddingContext, getEmbeddingFor, loadModel, resolveModelFile };
|
||||
}
|
||||
|
||||
describe("local embedding provider", () => {
|
||||
it("normalizes local embeddings and resolves the default local model", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedding = await provider.embedQuery("test query");
|
||||
const magnitude = Math.sqrt(embedding.reduce((sum, value) => sum + value * value, 0));
|
||||
|
||||
expect(magnitude).toBeCloseTo(1, 5);
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith(DEFAULT_LOCAL_MODEL, undefined);
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledWith("test query");
|
||||
});
|
||||
|
||||
it("passes default contextSize (4096) to createEmbeddingContext when not configured", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size default test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: 4096 });
|
||||
});
|
||||
|
||||
it("passes configured contextSize to createEmbeddingContext", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: 2048 },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size custom test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: 2048 });
|
||||
});
|
||||
|
||||
it('passes "auto" contextSize to createEmbeddingContext when explicitly set', async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: { contextSize: "auto" },
|
||||
});
|
||||
|
||||
await provider.embedQuery("context size auto test");
|
||||
|
||||
expect(runtime.createEmbeddingContext).toHaveBeenCalledWith({ contextSize: "auto" });
|
||||
});
|
||||
|
||||
it("trims explicit local model paths and cache directories", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime(new Float32Array([1, 0]));
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
local: {
|
||||
modelPath: " /models/embed.gguf ",
|
||||
modelCacheDir: " /cache/models ",
|
||||
},
|
||||
});
|
||||
|
||||
await provider.embedBatch(["a", "b"]);
|
||||
|
||||
expect(provider.model).toBe("/models/embed.gguf");
|
||||
expect(runtime.resolveModelFile).toHaveBeenCalledWith("/models/embed.gguf", "/cache/models");
|
||||
expect(runtime.getEmbeddingFor).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/embeddings.js";
|
||||
@@ -1,57 +0,0 @@
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import type { SecretInput } from "../../config/types.secrets.js";
|
||||
import type { EmbeddingInput } from "./embedding-inputs.js";
|
||||
|
||||
export type EmbeddingProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
maxInputTokens?: number;
|
||||
embedQuery: (text: string) => Promise<number[]>;
|
||||
embedBatch: (texts: string[]) => Promise<number[][]>;
|
||||
embedBatchInputs?: (inputs: EmbeddingInput[]) => Promise<number[][]>;
|
||||
};
|
||||
|
||||
export type EmbeddingProviderId = string;
|
||||
export type EmbeddingProviderRequest = string;
|
||||
export type EmbeddingProviderFallback = string;
|
||||
|
||||
export type GeminiTaskType =
|
||||
| "RETRIEVAL_QUERY"
|
||||
| "RETRIEVAL_DOCUMENT"
|
||||
| "SEMANTIC_SIMILARITY"
|
||||
| "CLASSIFICATION"
|
||||
| "CLUSTERING"
|
||||
| "QUESTION_ANSWERING"
|
||||
| "FACT_VERIFICATION";
|
||||
|
||||
export type EmbeddingProviderOptions = {
|
||||
config: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
provider?: EmbeddingProviderRequest;
|
||||
remote?: {
|
||||
baseUrl?: string;
|
||||
apiKey?: SecretInput;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
model: string;
|
||||
inputType?: string;
|
||||
queryInputType?: string;
|
||||
documentInputType?: string;
|
||||
fallback?: EmbeddingProviderFallback;
|
||||
local?: {
|
||||
modelPath?: string;
|
||||
modelCacheDir?: string;
|
||||
/**
|
||||
* Context size passed to node-llama-cpp `createEmbeddingContext`.
|
||||
* Default: 4096, chosen to cover typical memory-search chunks (128–512 tokens)
|
||||
* while keeping non-weight VRAM bounded.
|
||||
* Set `"auto"` to let node-llama-cpp use the model's trained maximum — not
|
||||
* recommended for 8B+ models (e.g. Qwen3-Embedding-8B: up to 40 960 tokens → ~32 GB VRAM).
|
||||
*/
|
||||
contextSize?: number | "auto";
|
||||
};
|
||||
/** Provider-specific output vector dimensions for supported embedding families. */
|
||||
outputDimensionality?: number;
|
||||
/** Gemini: override the default task type sent with embedding requests. */
|
||||
taskType?: GeminiTaskType;
|
||||
};
|
||||
@@ -1,31 +0,0 @@
|
||||
import type { Stats } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
export type RegularFileStatResult = { missing: true } | { missing: false; stat: Stats };
|
||||
|
||||
export function isFileMissingError(
|
||||
err: unknown,
|
||||
): err is NodeJS.ErrnoException & { code: "ENOENT" } {
|
||||
return Boolean(
|
||||
err &&
|
||||
typeof err === "object" &&
|
||||
"code" in err &&
|
||||
(err as Partial<NodeJS.ErrnoException>).code === "ENOENT",
|
||||
);
|
||||
}
|
||||
|
||||
export async function statRegularFile(absPath: string): Promise<RegularFileStatResult> {
|
||||
let stat: Stats;
|
||||
try {
|
||||
stat = await fs.lstat(absPath);
|
||||
} catch (err) {
|
||||
if (isFileMissingError(err)) {
|
||||
return { missing: true };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
return { missing: false, stat };
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
@@ -1,546 +0,0 @@
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("../../media/mime.js", () => ({
|
||||
detectMime: async (opts: { filePath?: string }) => {
|
||||
if (opts.filePath?.endsWith(".png")) {
|
||||
return "image/png";
|
||||
}
|
||||
if (opts.filePath?.endsWith(".wav")) {
|
||||
return "audio/wav";
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
}));
|
||||
|
||||
import {
|
||||
buildMultimodalChunkForIndexing,
|
||||
buildFileEntry,
|
||||
chunkMarkdown,
|
||||
isMemoryPath,
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
remapChunkLines,
|
||||
} from "./internal.js";
|
||||
import {
|
||||
DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES,
|
||||
type MemoryMultimodalSettings,
|
||||
} from "./multimodal.js";
|
||||
|
||||
const sharedTempRoot = fsSync.mkdtempSync(path.join(os.tmpdir(), "memory-host-sdk-tests-"));
|
||||
let sharedTempId = 0;
|
||||
|
||||
afterAll(() => {
|
||||
fsSync.rmSync(sharedTempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function setupTempDirLifecycle(prefix: string): () => string {
|
||||
let tmpDir = "";
|
||||
beforeEach(() => {
|
||||
tmpDir = path.join(sharedTempRoot, `${prefix}${sharedTempId++}`);
|
||||
fsSync.mkdirSync(tmpDir, { recursive: true });
|
||||
});
|
||||
return () => tmpDir;
|
||||
}
|
||||
|
||||
describe("normalizeExtraMemoryPaths", () => {
|
||||
it("trims, resolves, and dedupes paths", () => {
|
||||
const workspaceDir = path.join(os.tmpdir(), "memory-test-workspace");
|
||||
const absPath = path.resolve(path.sep, "shared-notes");
|
||||
const result = normalizeExtraMemoryPaths(workspaceDir, [
|
||||
" notes ",
|
||||
"./notes",
|
||||
absPath,
|
||||
absPath,
|
||||
"",
|
||||
]);
|
||||
expect(result).toEqual([path.resolve(workspaceDir, "notes"), absPath]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("listMemoryFiles", () => {
|
||||
const getTmpDir = setupTempDirLifecycle("memory-test-");
|
||||
const multimodal: MemoryMultimodalSettings = {
|
||||
enabled: true,
|
||||
modalities: ["image", "audio"],
|
||||
maxFileBytes: DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES,
|
||||
};
|
||||
|
||||
it("includes files from additional paths (directory)", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "extra-notes");
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "note1.md"), "# Note 1");
|
||||
fsSync.writeFileSync(path.join(extraDir, "note2.md"), "# Note 2");
|
||||
fsSync.writeFileSync(path.join(extraDir, "ignore.txt"), "Not a markdown file");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir]);
|
||||
expect(files).toHaveLength(3);
|
||||
expect(files.some((file) => file.endsWith("MEMORY.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("note1.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("note2.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("ignore.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
it("includes files from additional paths (single file)", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const singleFile = path.join(tmpDir, "standalone.md");
|
||||
fsSync.writeFileSync(singleFile, "# Standalone");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [singleFile]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((file) => file.endsWith("standalone.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores lowercase root memory.md when canonical MEMORY.md is absent", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "memory.md"), "# Legacy memory");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [path.join(tmpDir, "memory.md")]);
|
||||
|
||||
expect(files).toEqual([]);
|
||||
});
|
||||
|
||||
it("prefers canonical MEMORY.md over legacy root memory.md even through extra paths", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const canonicalPath = path.join(tmpDir, "MEMORY.md");
|
||||
const legacyPath = path.join(tmpDir, "memory.md");
|
||||
const actualLstat = fs.lstat.bind(fs);
|
||||
const actualReaddir = fs.readdir.bind(fs);
|
||||
const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (target) => {
|
||||
if (target === canonicalPath || target === legacyPath) {
|
||||
return {
|
||||
isSymbolicLink: () => false,
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
} as Awaited<ReturnType<typeof fs.lstat>>;
|
||||
}
|
||||
return actualLstat(target);
|
||||
});
|
||||
const readdirSpy = vi.spyOn(fs, "readdir").mockImplementation((async (
|
||||
target: unknown,
|
||||
options: unknown,
|
||||
) => {
|
||||
if (
|
||||
target === tmpDir &&
|
||||
typeof options === "object" &&
|
||||
options !== null &&
|
||||
"withFileTypes" in options &&
|
||||
options.withFileTypes
|
||||
) {
|
||||
return [
|
||||
{
|
||||
name: "MEMORY.md",
|
||||
isSymbolicLink: () => false,
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
},
|
||||
{
|
||||
name: "memory.md",
|
||||
isSymbolicLink: () => false,
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
},
|
||||
] as unknown as Awaited<ReturnType<typeof fs.readdir>>;
|
||||
}
|
||||
return actualReaddir(target as never, options as never);
|
||||
}) as never);
|
||||
|
||||
try {
|
||||
const files = await listMemoryFiles(tmpDir, [legacyPath, path.join(tmpDir, ".")]);
|
||||
expect(files).toEqual([canonicalPath]);
|
||||
} finally {
|
||||
lstatSpy.mockRestore();
|
||||
readdirSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("skips root-memory repair backups from workspace and explicit extra paths", async () => {
|
||||
for (const testCase of [
|
||||
{
|
||||
name: "workspace extra path",
|
||||
extraPaths: (tmpDir: string) => [tmpDir],
|
||||
},
|
||||
{
|
||||
name: "explicit repair root",
|
||||
extraPaths: (tmpDir: string) => [path.join(tmpDir, ".openclaw-repair", "root-memory")],
|
||||
},
|
||||
] as const) {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const repairDir = path.join(tmpDir, ".openclaw-repair", "root-memory", "2026-04-23");
|
||||
fsSync.mkdirSync(repairDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(repairDir, "memory.md"), "# Archived legacy memory");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, testCase.extraPaths(tmpDir));
|
||||
|
||||
expect(files, testCase.name).toHaveLength(1);
|
||||
expect(files[0], testCase.name).toBe(path.join(tmpDir, "MEMORY.md"));
|
||||
}
|
||||
});
|
||||
|
||||
it("handles relative paths in additional paths", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "subdir");
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "nested.md"), "# Nested");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, ["subdir"]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((file) => file.endsWith("nested.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores non-existent additional paths", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, ["/does/not/exist"]);
|
||||
expect(files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("ignores symlinked files and directories", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "extra");
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "note.md"), "# Note");
|
||||
|
||||
const targetFile = path.join(tmpDir, "target.md");
|
||||
fsSync.writeFileSync(targetFile, "# Target");
|
||||
const linkFile = path.join(extraDir, "linked.md");
|
||||
|
||||
const targetDir = path.join(tmpDir, "target-dir");
|
||||
fsSync.mkdirSync(targetDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(targetDir, "nested.md"), "# Nested");
|
||||
const linkDir = path.join(tmpDir, "linked-dir");
|
||||
|
||||
let symlinksOk = true;
|
||||
try {
|
||||
fsSync.symlinkSync(targetFile, linkFile, "file");
|
||||
fsSync.symlinkSync(targetDir, linkDir, "dir");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinksOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir, linkDir]);
|
||||
expect(files.some((file) => file.endsWith("note.md"))).toBe(true);
|
||||
if (symlinksOk) {
|
||||
expect(files.some((file) => file.endsWith("linked.md"))).toBe(false);
|
||||
expect(files.some((file) => file.endsWith("nested.md"))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("dedupes overlapping extra paths that resolve to the same file", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
fsSync.writeFileSync(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const files = await listMemoryFiles(tmpDir, [tmpDir, ".", path.join(tmpDir, "MEMORY.md")]);
|
||||
const memoryMatches = files.filter((file) => file.endsWith("MEMORY.md"));
|
||||
expect(memoryMatches).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("includes image and audio files from extra paths when multimodal is enabled", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const extraDir = path.join(tmpDir, "media");
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "diagram.png"), Buffer.from("png"));
|
||||
fsSync.writeFileSync(path.join(extraDir, "note.wav"), Buffer.from("wav"));
|
||||
fsSync.writeFileSync(path.join(extraDir, "ignore.bin"), Buffer.from("bin"));
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir], multimodal);
|
||||
expect(files.some((file) => file.endsWith("diagram.png"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("note.wav"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("ignore.bin"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isMemoryPath", () => {
|
||||
it("allows explicit access to top-level DREAMS.md", () => {
|
||||
expect(isMemoryPath("DREAMS.md")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFileEntry", () => {
|
||||
const getTmpDir = setupTempDirLifecycle("memory-build-entry-");
|
||||
const multimodal: MemoryMultimodalSettings = {
|
||||
enabled: true,
|
||||
modalities: ["image", "audio"],
|
||||
maxFileBytes: DEFAULT_MEMORY_MULTIMODAL_MAX_FILE_BYTES,
|
||||
};
|
||||
|
||||
it("returns null when the file disappears before reading", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const target = path.join(tmpDir, "ghost.md");
|
||||
fsSync.writeFileSync(target, "ghost", "utf-8");
|
||||
fsSync.rmSync(target);
|
||||
const entry = await buildFileEntry(target, tmpDir);
|
||||
expect(entry).toBeNull();
|
||||
});
|
||||
|
||||
it("returns metadata when the file exists", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const target = path.join(tmpDir, "note.md");
|
||||
fsSync.writeFileSync(target, "hello", "utf-8");
|
||||
const entry = await buildFileEntry(target, tmpDir);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.path).toBe("note.md");
|
||||
expect(entry?.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns multimodal metadata for eligible image files", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const target = path.join(tmpDir, "diagram.png");
|
||||
fsSync.writeFileSync(target, Buffer.from("png"));
|
||||
|
||||
const entry = await buildFileEntry(target, tmpDir, multimodal);
|
||||
|
||||
expect(entry).toMatchObject({
|
||||
path: "diagram.png",
|
||||
kind: "multimodal",
|
||||
modality: "image",
|
||||
mimeType: "image/png",
|
||||
contentText: "Image file: diagram.png",
|
||||
});
|
||||
});
|
||||
|
||||
it("builds a multimodal chunk lazily for indexing", async () => {
|
||||
const tmpDir = getTmpDir();
|
||||
const target = path.join(tmpDir, "diagram.png");
|
||||
fsSync.writeFileSync(target, Buffer.from("png"));
|
||||
|
||||
const entry = await buildFileEntry(target, tmpDir, multimodal);
|
||||
const built = await buildMultimodalChunkForIndexing(entry!);
|
||||
|
||||
expect(built?.chunk.embeddingInput?.parts).toEqual([
|
||||
{ type: "text", text: "Image file: diagram.png" },
|
||||
expect.objectContaining({ type: "inline-data", mimeType: "image/png" }),
|
||||
]);
|
||||
expect(built?.structuredInputBytes).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("skips lazy multimodal indexing when file state changes after discovery", async () => {
|
||||
for (const testCase of [
|
||||
{
|
||||
name: "grows",
|
||||
mutate: (target: string, entrySize: number) => {
|
||||
fsSync.writeFileSync(target, Buffer.alloc(entrySize + 32, 1));
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bytes change",
|
||||
mutate: (target: string) => {
|
||||
fsSync.writeFileSync(target, Buffer.from("gif"));
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "disappears",
|
||||
mutate: (target: string) => {
|
||||
fsSync.rmSync(target);
|
||||
},
|
||||
},
|
||||
] as const) {
|
||||
const tmpDir = getTmpDir();
|
||||
const target = path.join(tmpDir, `${testCase.name}.png`);
|
||||
fsSync.writeFileSync(target, Buffer.from("png"));
|
||||
|
||||
const entry = await buildFileEntry(target, tmpDir, multimodal);
|
||||
expect(entry, testCase.name).not.toBeNull();
|
||||
testCase.mutate(target, entry!.size);
|
||||
|
||||
await expect(buildMultimodalChunkForIndexing(entry!), testCase.name).resolves.toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkMarkdown", () => {
|
||||
it("splits overly long lines into max-sized chunks", () => {
|
||||
const chunkTokens = 400;
|
||||
const maxChars = chunkTokens * 4;
|
||||
const content = "a".repeat(maxChars * 3 + 25);
|
||||
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.text.length).toBeLessThanOrEqual(maxChars);
|
||||
}
|
||||
});
|
||||
|
||||
it("produces more chunks for CJK text than for equal-length ASCII text", () => {
|
||||
// CJK chars ≈ 1 token each; ASCII chars ≈ 0.25 tokens each.
|
||||
// For the same raw character count, CJK content should produce more chunks
|
||||
// because each character "weighs" ~4× more in token estimation.
|
||||
const chunkTokens = 50;
|
||||
|
||||
// 400 ASCII chars → ~100 tokens → fits in ~2 chunks
|
||||
const asciiLines = Array.from({ length: 20 }, () => "a".repeat(20)).join("\n");
|
||||
const asciiChunks = chunkMarkdown(asciiLines, { tokens: chunkTokens, overlap: 0 });
|
||||
|
||||
// 400 CJK chars → ~400 tokens → needs ~8 chunks
|
||||
const cjkLines = Array.from({ length: 20 }, () => "你".repeat(20)).join("\n");
|
||||
const cjkChunks = chunkMarkdown(cjkLines, { tokens: chunkTokens, overlap: 0 });
|
||||
|
||||
expect(cjkChunks.length).toBeGreaterThan(asciiChunks.length);
|
||||
});
|
||||
|
||||
it("respects token budget for Chinese text", () => {
|
||||
// With tokens=100, each CJK char ≈ 1 token, so chunks should hold ~100 CJK chars.
|
||||
const chunkTokens = 100;
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 50; i++) {
|
||||
lines.push("这是一个测试句子用来验证分块逻辑是否正确处理中文文本内容");
|
||||
}
|
||||
const content = lines.join("\n");
|
||||
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
// Each chunk's CJK content should not vastly exceed the token budget.
|
||||
// With CJK-aware estimation, each char ≈ 1 token, so chunk text length
|
||||
// (in CJK chars) should be roughly <= tokens budget (with some tolerance
|
||||
// for line boundaries).
|
||||
for (const chunk of chunks) {
|
||||
// Count actual CJK characters in the chunk
|
||||
const cjkCount = (chunk.text.match(/[\u4e00-\u9fff]/g) ?? []).length;
|
||||
// Allow 2× tolerance for line-boundary rounding
|
||||
expect(cjkCount).toBeLessThanOrEqual(chunkTokens * 2);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps English chunking behavior unchanged", () => {
|
||||
const chunkTokens = 100;
|
||||
const maxChars = chunkTokens * 4; // 400 chars
|
||||
const content = "hello world this is a test. ".repeat(50);
|
||||
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.text.length).toBeLessThanOrEqual(maxChars);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles mixed CJK and ASCII content correctly", () => {
|
||||
const chunkTokens = 50;
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < 30; i++) {
|
||||
lines.push(`Line ${i}: 这是中英文混合的测试内容 with some English text`);
|
||||
}
|
||||
const content = lines.join("\n");
|
||||
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
||||
// Should produce multiple chunks and not crash
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
// Verify all content is preserved
|
||||
const reconstructed = chunks.map((c) => c.text).join("\n");
|
||||
// Due to overlap=0, the concatenated chunks should cover all lines
|
||||
expect(reconstructed).toContain("Line 0");
|
||||
expect(reconstructed).toContain("Line 29");
|
||||
});
|
||||
|
||||
it("splits very long CJK lines into budget-sized segments", () => {
|
||||
// A single line of 2000 CJK characters (no newlines).
|
||||
// With tokens=200, each CJK char ≈ 1 token.
|
||||
const longCjkLine = "中".repeat(2000);
|
||||
const chunks = chunkMarkdown(longCjkLine, { tokens: 200, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThanOrEqual(8);
|
||||
for (const chunk of chunks) {
|
||||
const cjkCount = (chunk.text.match(/[\u4E00-\u9FFF]/g) ?? []).length;
|
||||
expect(cjkCount).toBeLessThanOrEqual(200 * 2);
|
||||
}
|
||||
});
|
||||
it("does not break surrogate pairs when splitting long CJK lines", () => {
|
||||
// "𠀀" (U+20000) is a surrogate pair: 2 UTF-16 code units per character.
|
||||
// With an odd token budget, the fine-split must not cut inside a pair.
|
||||
const surrogateChar = "\u{20000}"; // 𠀀
|
||||
const longLine = surrogateChar.repeat(120);
|
||||
const chunks = chunkMarkdown(longLine, { tokens: 31, overlap: 0 });
|
||||
for (const chunk of chunks) {
|
||||
// No chunk should contain the Unicode replacement character U+FFFD,
|
||||
// which would indicate a broken surrogate pair.
|
||||
expect(chunk.text).not.toContain("\uFFFD");
|
||||
// Every character in the chunk should be a valid string (no lone surrogates).
|
||||
for (let i = 0; i < chunk.text.length; i += 1) {
|
||||
const code = chunk.text.charCodeAt(i);
|
||||
if (code >= 0xd800 && code <= 0xdbff) {
|
||||
// High surrogate must be followed by a low surrogate
|
||||
const next = chunk.text.charCodeAt(i + 1);
|
||||
expect(next).toBeGreaterThanOrEqual(0xdc00);
|
||||
expect(next).toBeLessThanOrEqual(0xdfff);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
it("does not over-split long Latin lines (backward compat)", () => {
|
||||
// 2000 ASCII chars / 800 maxChars -> about 3 segments, not 10 tiny ones.
|
||||
const longLatinLine = "a".repeat(2000);
|
||||
const chunks = chunkMarkdown(longLatinLine, { tokens: 200, overlap: 0 });
|
||||
expect(chunks.length).toBeLessThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("remapChunkLines", () => {
|
||||
it("remaps chunk line numbers using a lineMap", () => {
|
||||
// Simulate 5 content lines that came from JSONL lines [4, 6, 7, 10, 13] (1-indexed)
|
||||
const lineMap = [4, 6, 7, 10, 13];
|
||||
|
||||
// Create chunks from content that has 5 lines
|
||||
const content = "User: Hello\nAssistant: Hi\nUser: Question\nAssistant: Answer\nUser: Thanks";
|
||||
const chunks = chunkMarkdown(content, { tokens: 400, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
|
||||
// Before remapping, startLine/endLine reference content line numbers (1-indexed)
|
||||
expect(chunks[0].startLine).toBe(1);
|
||||
|
||||
// Remap
|
||||
remapChunkLines(chunks, lineMap);
|
||||
|
||||
// After remapping, line numbers should reference original JSONL lines
|
||||
// Content line 1 → JSONL line 4, content line 5 → JSONL line 13
|
||||
expect(chunks[0].startLine).toBe(4);
|
||||
const lastChunk = chunks[chunks.length - 1];
|
||||
expect(lastChunk.endLine).toBe(13);
|
||||
});
|
||||
|
||||
it("preserves original line numbers when lineMap is undefined", () => {
|
||||
const content = "Line one\nLine two\nLine three";
|
||||
const chunks = chunkMarkdown(content, { tokens: 400, overlap: 0 });
|
||||
const originalStart = chunks[0].startLine;
|
||||
const originalEnd = chunks[chunks.length - 1].endLine;
|
||||
|
||||
remapChunkLines(chunks, undefined);
|
||||
|
||||
expect(chunks[0].startLine).toBe(originalStart);
|
||||
expect(chunks[chunks.length - 1].endLine).toBe(originalEnd);
|
||||
});
|
||||
|
||||
it("handles multi-chunk content with correct remapping", () => {
|
||||
// Use small chunk size to force multiple chunks
|
||||
// lineMap: 10 content lines from JSONL lines [2, 5, 8, 11, 14, 17, 20, 23, 26, 29]
|
||||
const lineMap = [2, 5, 8, 11, 14, 17, 20, 23, 26, 29];
|
||||
const contentLines = lineMap.map((_, i) =>
|
||||
i % 2 === 0 ? `User: Message ${i}` : `Assistant: Reply ${i}`,
|
||||
);
|
||||
const content = contentLines.join("\n");
|
||||
|
||||
// Use very small chunk size to force splitting
|
||||
const chunks = chunkMarkdown(content, { tokens: 10, overlap: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
|
||||
remapChunkLines(chunks, lineMap);
|
||||
|
||||
// First chunk should start at JSONL line 2
|
||||
expect(chunks[0].startLine).toBe(2);
|
||||
// Last chunk should end at JSONL line 29
|
||||
expect(chunks[chunks.length - 1].endLine).toBe(29);
|
||||
|
||||
// Each chunk's startLine should be ≤ its endLine
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.startLine).toBeLessThanOrEqual(chunk.endLine);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/internal.js";
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/memory-schema.js";
|
||||
@@ -1,40 +0,0 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const HOST_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const PACKAGE_BRIDGE_FILES = [
|
||||
"backend-config.ts",
|
||||
"batch-error-utils.ts",
|
||||
"batch-output.ts",
|
||||
"batch-status.ts",
|
||||
"embedding-input-limits.ts",
|
||||
"embeddings-remote-provider.ts",
|
||||
"embeddings.ts",
|
||||
"internal.ts",
|
||||
"memory-schema.ts",
|
||||
"multimodal.ts",
|
||||
"qmd-process.ts",
|
||||
"qmd-scope.ts",
|
||||
"query-expansion.ts",
|
||||
"read-file-shared.ts",
|
||||
"read-file.ts",
|
||||
"session-files.ts",
|
||||
"types.ts",
|
||||
] as const;
|
||||
|
||||
describe("memory-host-sdk host package bridges", () => {
|
||||
it("keeps package-owned source bridges thin", () => {
|
||||
for (const fileName of PACKAGE_BRIDGE_FILES) {
|
||||
const source = fs.readFileSync(path.join(HOST_DIR, fileName), "utf8");
|
||||
expect(source, fileName).toBe(
|
||||
`export * from "../../../packages/memory-host-sdk/src/host/${fileName.replace(
|
||||
/\.ts$/u,
|
||||
".js",
|
||||
)}";\n`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/multimodal.js";
|
||||
@@ -1,31 +0,0 @@
|
||||
export type LlamaEmbedding = {
|
||||
vector: Float32Array | number[];
|
||||
};
|
||||
|
||||
export type LlamaEmbeddingContext = {
|
||||
getEmbeddingFor: (text: string) => Promise<LlamaEmbedding>;
|
||||
};
|
||||
|
||||
export type LlamaModel = {
|
||||
createEmbeddingContext: (options?: {
|
||||
contextSize?: number | "auto";
|
||||
}) => Promise<LlamaEmbeddingContext>;
|
||||
};
|
||||
|
||||
export type Llama = {
|
||||
loadModel: (params: { modelPath: string }) => Promise<LlamaModel>;
|
||||
};
|
||||
|
||||
export type NodeLlamaCppModule = {
|
||||
LlamaLogLevel: {
|
||||
error: number;
|
||||
};
|
||||
getLlama: (params: { logLevel: number }) => Promise<Llama>;
|
||||
resolveModelFile: (modelPath: string, cacheDir?: string) => Promise<string>;
|
||||
};
|
||||
|
||||
const NODE_LLAMA_CPP_MODULE = "node-llama-cpp";
|
||||
|
||||
export async function importNodeLlamaCpp() {
|
||||
return import(NODE_LLAMA_CPP_MODULE) as Promise<NodeLlamaCppModule>;
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { postJson } from "./post-json.js";
|
||||
|
||||
describe("postJson", () => {
|
||||
it("parses JSON from an injected fetch response", async () => {
|
||||
const fetchImpl = vi.fn(
|
||||
async () =>
|
||||
new Response(JSON.stringify({ ok: true }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
) as typeof fetch;
|
||||
|
||||
const result = await postJson({
|
||||
url: "https://example.com/v1/post",
|
||||
headers: { Authorization: "Bearer test" },
|
||||
ssrfPolicy: { allowedHostnames: ["example.com"] },
|
||||
fetchImpl,
|
||||
body: { input: ["x"] },
|
||||
errorPrefix: "post failed",
|
||||
parse: (payload) => payload,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ ok: true });
|
||||
});
|
||||
});
|
||||
@@ -1,37 +0,0 @@
|
||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
export async function postJson<T>(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
body: unknown;
|
||||
errorPrefix: string;
|
||||
attachStatus?: boolean;
|
||||
parse: (payload: unknown) => T | Promise<T>;
|
||||
}): Promise<T> {
|
||||
return await withRemoteHttpResponse({
|
||||
url: params.url,
|
||||
ssrfPolicy: params.ssrfPolicy,
|
||||
fetchImpl: params.fetchImpl,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: params.headers,
|
||||
body: JSON.stringify(params.body),
|
||||
},
|
||||
onResponse: async (res) => {
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
const err = new Error(`${params.errorPrefix}: ${res.status} ${text}`) as Error & {
|
||||
status?: number;
|
||||
};
|
||||
if (params.attachStatus) {
|
||||
err.status = res.status;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return await params.parse(await res.json());
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/qmd-process.js";
|
||||
@@ -1,76 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseQmdQueryJson } from "./qmd-query-parser.js";
|
||||
|
||||
describe("parseQmdQueryJson", () => {
|
||||
it("parses clean qmd JSON output", () => {
|
||||
const results = parseQmdQueryJson('[{"docid":"abc","score":1,"snippet":"@@ -1,1\\none"}]', "");
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 1,
|
||||
snippet: "@@ -1,1\none",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts embedded result arrays from noisy stdout", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
`initializing
|
||||
{"payload":"ok"}
|
||||
[{"docid":"abc","score":0.5}]
|
||||
complete`,
|
||||
"",
|
||||
);
|
||||
expect(results).toEqual([{ docid: "abc", score: 0.5 }]);
|
||||
});
|
||||
|
||||
it("preserves explicit qmd line metadata when present", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
'[{"docid":"abc","score":0.5,"start_line":4,"end_line":6,"snippet":"@@ -10,1\\nignored"}]',
|
||||
"",
|
||||
);
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 0.5,
|
||||
snippet: "@@ -10,1\nignored",
|
||||
startLine: 4,
|
||||
endLine: 6,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps single-line qmd line metadata onto both line bounds", () => {
|
||||
const results = parseQmdQueryJson('[{"docid":"abc","score":0.5,"line":9}]', "");
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 0.5,
|
||||
startLine: 9,
|
||||
endLine: 9,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats plain-text no-results from stderr as an empty result set", () => {
|
||||
const results = parseQmdQueryJson("", "No results found\n");
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats prefixed no-results marker output as an empty result set", () => {
|
||||
expect(parseQmdQueryJson("warning: no results found", "")).toEqual([]);
|
||||
expect(parseQmdQueryJson("", "[qmd] warning: no results found\n")).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not treat arbitrary non-marker text as no-results output", () => {
|
||||
expect(() =>
|
||||
parseQmdQueryJson("warning: search completed; no results found for this query", ""),
|
||||
).toThrow(/qmd query returned invalid JSON/i);
|
||||
});
|
||||
|
||||
it("throws when stdout cannot be interpreted as qmd JSON", () => {
|
||||
expect(() => parseQmdQueryJson("this is not json", "")).toThrow(
|
||||
/qmd query returned invalid JSON/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,116 +0,0 @@
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { extractBalancedJsonPrefix } from "../../shared/balanced-json.js";
|
||||
import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
export type QmdQueryResult = {
|
||||
docid?: string;
|
||||
score?: number;
|
||||
collection?: string;
|
||||
file?: string;
|
||||
snippet?: string;
|
||||
body?: string;
|
||||
startLine?: number;
|
||||
endLine?: number;
|
||||
};
|
||||
|
||||
export function parseQmdQueryJson(stdout: string, stderr: string): QmdQueryResult[] {
|
||||
const trimmedStdout = stdout.trim();
|
||||
const trimmedStderr = stderr.trim();
|
||||
const stdoutIsMarker = trimmedStdout.length > 0 && isQmdNoResultsOutput(trimmedStdout);
|
||||
const stderrIsMarker = trimmedStderr.length > 0 && isQmdNoResultsOutput(trimmedStderr);
|
||||
if (stdoutIsMarker || (!trimmedStdout && stderrIsMarker)) {
|
||||
return [];
|
||||
}
|
||||
if (!trimmedStdout) {
|
||||
const context = trimmedStderr ? ` (stderr: ${summarizeQmdStderr(trimmedStderr)})` : "";
|
||||
const message = `stdout empty${context}`;
|
||||
log.warn(`qmd query returned invalid JSON: ${message}`);
|
||||
throw new Error(`qmd query returned invalid JSON: ${message}`);
|
||||
}
|
||||
try {
|
||||
const parsed = parseQmdQueryResultArray(trimmedStdout);
|
||||
if (parsed !== null) {
|
||||
return parsed;
|
||||
}
|
||||
const noisyPayload = extractBalancedJsonPrefix(trimmedStdout, { openers: ["["] })?.json;
|
||||
if (!noisyPayload) {
|
||||
throw new Error("qmd query JSON response was not an array");
|
||||
}
|
||||
const fallback = parseQmdQueryResultArray(noisyPayload);
|
||||
if (fallback !== null) {
|
||||
return fallback;
|
||||
}
|
||||
throw new Error("qmd query JSON response was not an array");
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
log.warn(`qmd query returned invalid JSON: ${message}`);
|
||||
throw new Error(`qmd query returned invalid JSON: ${message}`, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
function isQmdNoResultsOutput(raw: string): boolean {
|
||||
const lines = raw
|
||||
.split(/\r?\n/)
|
||||
.map((line) => normalizeLowercaseStringOrEmpty(line).replace(/\s+/g, " "))
|
||||
.filter((line) => line.length > 0);
|
||||
return lines.some((line) => isQmdNoResultsLine(line));
|
||||
}
|
||||
|
||||
function isQmdNoResultsLine(line: string): boolean {
|
||||
if (line === "no results found" || line === "no results found.") {
|
||||
return true;
|
||||
}
|
||||
return /^(?:\[[^\]]+\]\s*)?(?:(?:warn(?:ing)?|info|error|qmd)\s*:\s*)+no results found\.?$/.test(
|
||||
line,
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeQmdStderr(raw: string): string {
|
||||
return raw.length <= 120 ? raw : `${raw.slice(0, 117)}...`;
|
||||
}
|
||||
|
||||
function parseQmdQueryResultArray(raw: string): QmdQueryResult[] | null {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return null;
|
||||
}
|
||||
return parsed.map((item) => {
|
||||
if (typeof item !== "object" || item === null) {
|
||||
return item as QmdQueryResult;
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const docid = typeof record.docid === "string" ? record.docid : undefined;
|
||||
const score =
|
||||
typeof record.score === "number" && Number.isFinite(record.score)
|
||||
? record.score
|
||||
: undefined;
|
||||
const collection = typeof record.collection === "string" ? record.collection : undefined;
|
||||
const file = typeof record.file === "string" ? record.file : undefined;
|
||||
const snippet = typeof record.snippet === "string" ? record.snippet : undefined;
|
||||
const body = typeof record.body === "string" ? record.body : undefined;
|
||||
const line = parseQmdLineNumber(record.line);
|
||||
const startLine = parseQmdLineNumber(record.start_line ?? record.startLine) ?? line;
|
||||
const endLine = parseQmdLineNumber(record.end_line ?? record.endLine) ?? line;
|
||||
return {
|
||||
docid,
|
||||
score,
|
||||
collection,
|
||||
file,
|
||||
snippet,
|
||||
body,
|
||||
startLine,
|
||||
endLine,
|
||||
} as QmdQueryResult;
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseQmdLineNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ResolvedQmdConfig } from "./backend-config.js";
|
||||
import { deriveQmdScopeChannel, deriveQmdScopeChatType, isQmdScopeAllowed } from "./qmd-scope.js";
|
||||
|
||||
describe("qmd scope", () => {
|
||||
const allowDirect: ResolvedQmdConfig["scope"] = {
|
||||
default: "deny",
|
||||
rules: [{ action: "allow", match: { chatType: "direct" } }],
|
||||
};
|
||||
|
||||
it("derives channel and chat type from canonical keys once", () => {
|
||||
expect(deriveQmdScopeChannel("Workspace:group:123")).toBe("workspace");
|
||||
expect(deriveQmdScopeChatType("Workspace:group:123")).toBe("group");
|
||||
});
|
||||
|
||||
it("derives channel and chat type from stored key suffixes", () => {
|
||||
expect(deriveQmdScopeChannel("agent:agent-1:workspace:channel:chan-123")).toBe("workspace");
|
||||
expect(deriveQmdScopeChatType("agent:agent-1:workspace:channel:chan-123")).toBe("channel");
|
||||
});
|
||||
|
||||
it("treats parsed keys with no chat prefix as direct", () => {
|
||||
expect(deriveQmdScopeChannel("agent:agent-1:peer-direct")).toBeUndefined();
|
||||
expect(deriveQmdScopeChatType("agent:agent-1:peer-direct")).toBe("direct");
|
||||
expect(isQmdScopeAllowed(allowDirect, "agent:agent-1:peer-direct")).toBe(true);
|
||||
expect(isQmdScopeAllowed(allowDirect, "agent:agent-1:peer:group:abc")).toBe(false);
|
||||
});
|
||||
|
||||
it("applies scoped key-prefix checks against normalized key", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "deny",
|
||||
rules: [{ action: "allow", match: { keyPrefix: "workspace:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:agent-1:workspace:group:123")).toBe(true);
|
||||
expect(isQmdScopeAllowed(scope, "agent:agent-1:other:group:123")).toBe(false);
|
||||
});
|
||||
|
||||
it("supports rawKeyPrefix matches for agent-prefixed keys", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "allow",
|
||||
rules: [{ action: "deny", match: { rawKeyPrefix: "agent:main:discord:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:discord:channel:c123")).toBe(false);
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:slack:channel:c123")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps legacy agent-prefixed keyPrefix rules working", () => {
|
||||
const scope: ResolvedQmdConfig["scope"] = {
|
||||
default: "allow",
|
||||
rules: [{ action: "deny", match: { keyPrefix: "agent:main:discord:" } }],
|
||||
};
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:discord:channel:c123")).toBe(false);
|
||||
expect(isQmdScopeAllowed(scope, "agent:main:slack:channel:c123")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/qmd-scope.js";
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/query-expansion.js";
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/read-file-shared.js";
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/read-file.js";
|
||||
@@ -1,51 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GUARDED_FETCH_MODE } from "../../infra/net/fetch-guard.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
describe("withRemoteHttpResponse", () => {
|
||||
function makeFetchDeps({ useEnvProxy = false }: { useEnvProxy?: boolean } = {}) {
|
||||
const calls: unknown[] = [];
|
||||
return {
|
||||
calls,
|
||||
fetchWithSsrFGuardImpl: async (params: unknown) => {
|
||||
calls.push(params);
|
||||
return {
|
||||
response: new Response("ok", { status: 200 }),
|
||||
finalUrl: "https://memory.example/v1",
|
||||
release: async () => {},
|
||||
};
|
||||
},
|
||||
shouldUseEnvHttpProxyForUrlImpl: () => useEnvProxy,
|
||||
};
|
||||
}
|
||||
|
||||
it("uses trusted env proxy mode when the target will use EnvHttpProxyAgent", async () => {
|
||||
const deps = makeFetchDeps({ useEnvProxy: true });
|
||||
|
||||
await withRemoteHttpResponse({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
onResponse: async () => undefined,
|
||||
...deps,
|
||||
});
|
||||
|
||||
expect(deps.calls[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
url: "https://memory.example/v1/embeddings",
|
||||
mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps strict guarded fetch mode when proxy env would not proxy the target", async () => {
|
||||
const deps = makeFetchDeps();
|
||||
|
||||
await withRemoteHttpResponse({
|
||||
url: "https://internal.corp.example/v1/embeddings",
|
||||
onResponse: async () => undefined,
|
||||
...deps,
|
||||
});
|
||||
|
||||
expect(deps.calls[0]).toBeDefined();
|
||||
expect(deps.calls[0]).not.toHaveProperty("mode");
|
||||
});
|
||||
});
|
||||
@@ -1,32 +0,0 @@
|
||||
import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../../infra/net/fetch-guard.js";
|
||||
import { shouldUseEnvHttpProxyForUrl } from "../../infra/net/proxy-env.js";
|
||||
import { ssrfPolicyFromHttpBaseUrlAllowedHostname, type SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||
|
||||
export const buildRemoteBaseUrlPolicy = ssrfPolicyFromHttpBaseUrlAllowedHostname;
|
||||
|
||||
export async function withRemoteHttpResponse<T>(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
ssrfPolicy?: SsrFPolicy;
|
||||
fetchImpl?: typeof fetch;
|
||||
fetchWithSsrFGuardImpl?: typeof fetchWithSsrFGuard;
|
||||
shouldUseEnvHttpProxyForUrlImpl?: typeof shouldUseEnvHttpProxyForUrl;
|
||||
auditContext?: string;
|
||||
onResponse: (response: Response) => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const guardedFetch = params.fetchWithSsrFGuardImpl ?? fetchWithSsrFGuard;
|
||||
const shouldUseEnvProxy = params.shouldUseEnvHttpProxyForUrlImpl ?? shouldUseEnvHttpProxyForUrl;
|
||||
const { response, release } = await guardedFetch({
|
||||
url: params.url,
|
||||
fetchImpl: params.fetchImpl,
|
||||
init: params.init,
|
||||
policy: params.ssrfPolicy,
|
||||
auditContext: params.auditContext ?? "memory-remote",
|
||||
...(shouldUseEnvProxy(params.url) ? { mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY } : {}),
|
||||
});
|
||||
try {
|
||||
return await params.onResponse(response);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMemorySecretInputString } from "./secret-input.js";
|
||||
|
||||
describe("resolveMemorySecretInputString", () => {
|
||||
const googleApiKeyRef = {
|
||||
source: "env",
|
||||
provider: "default",
|
||||
id: "GOOGLE_API_KEY",
|
||||
};
|
||||
|
||||
it("uses the daemon env for env-backed SecretRefs", () => {
|
||||
expect(
|
||||
resolveMemorySecretInputString({
|
||||
value: googleApiKeyRef,
|
||||
path: "agents.main.memorySearch.remote.apiKey",
|
||||
env: { GOOGLE_API_KEY: "resolved-key" },
|
||||
}),
|
||||
).toBe("resolved-key");
|
||||
});
|
||||
|
||||
it("still throws when an env-backed SecretRef is missing from the daemon env", () => {
|
||||
expect(() =>
|
||||
resolveMemorySecretInputString({
|
||||
value: googleApiKeyRef,
|
||||
path: "agents.main.memorySearch.remote.apiKey",
|
||||
env: {},
|
||||
}),
|
||||
).toThrow(/unresolved SecretRef/);
|
||||
});
|
||||
});
|
||||
@@ -1,28 +0,0 @@
|
||||
import {
|
||||
hasConfiguredSecretInput,
|
||||
normalizeResolvedSecretInputString,
|
||||
normalizeSecretInputString,
|
||||
resolveSecretInputRef,
|
||||
} from "../../config/types.secrets.js";
|
||||
|
||||
export function hasConfiguredMemorySecretInput(value: unknown): boolean {
|
||||
return hasConfiguredSecretInput(value);
|
||||
}
|
||||
|
||||
export function resolveMemorySecretInputString(params: {
|
||||
value: unknown;
|
||||
path: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): string | undefined {
|
||||
const { ref } = resolveSecretInputRef({ value: params.value });
|
||||
if (ref?.source === "env") {
|
||||
const envValue = normalizeSecretInputString((params.env ?? process.env)[ref.id]);
|
||||
if (envValue) {
|
||||
return envValue;
|
||||
}
|
||||
}
|
||||
return normalizeResolvedSecretInputString({
|
||||
value: params.value,
|
||||
path: params.path,
|
||||
});
|
||||
}
|
||||
@@ -1,710 +0,0 @@
|
||||
import fsSync from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { buildSessionEntry, listSessionFilesForAgent } from "./session-files.js";
|
||||
|
||||
let fixtureRoot: string;
|
||||
let tmpDir: string;
|
||||
let originalStateDir: string | undefined;
|
||||
let fixtureId = 0;
|
||||
|
||||
beforeAll(() => {
|
||||
fixtureRoot = fsSync.mkdtempSync(path.join(os.tmpdir(), "session-entry-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fsSync.rmSync(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = path.join(fixtureRoot, `case-${fixtureId++}`);
|
||||
fsSync.mkdirSync(tmpDir, { recursive: true });
|
||||
originalStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
process.env.OPENCLAW_STATE_DIR = tmpDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalStateDir === undefined) {
|
||||
delete process.env.OPENCLAW_STATE_DIR;
|
||||
} else {
|
||||
process.env.OPENCLAW_STATE_DIR = originalStateDir;
|
||||
}
|
||||
});
|
||||
|
||||
function expectNoUnpairedSurrogates(value: string): void {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
if (code >= 0xd800 && code <= 0xdbff) {
|
||||
expect(index + 1).toBeLessThan(value.length);
|
||||
const next = value.charCodeAt(index + 1);
|
||||
expect(next).toBeGreaterThanOrEqual(0xdc00);
|
||||
expect(next).toBeLessThanOrEqual(0xdfff);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
expect(code < 0xdc00 || code > 0xdfff).toBe(true);
|
||||
}
|
||||
}
|
||||
|
||||
function writeSessionJsonl(fileName: string, records: readonly unknown[]): string {
|
||||
const filePath = path.join(tmpDir, fileName);
|
||||
fsSync.writeFileSync(filePath, records.map((record) => JSON.stringify(record)).join("\n"));
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function buildSessionEntryWithoutStoreClassification(filePath: string) {
|
||||
return buildSessionEntry(filePath, {
|
||||
generatedByCronRun: false,
|
||||
generatedByDreamingNarrative: false,
|
||||
});
|
||||
}
|
||||
|
||||
describe("listSessionFilesForAgent", () => {
|
||||
it("includes reset and deleted transcripts in session file listing", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(path.join(sessionsDir, "archive"), { recursive: true });
|
||||
|
||||
const included = [
|
||||
"active.jsonl",
|
||||
"active.jsonl.reset.2026-02-16T22-26-33.000Z",
|
||||
"active.jsonl.deleted.2026-02-16T22-27-33.000Z",
|
||||
];
|
||||
const excluded = ["active.jsonl.bak.2026-02-16T22-28-33.000Z", "sessions.json", "notes.md"];
|
||||
|
||||
for (const fileName of [...included, ...excluded]) {
|
||||
fsSync.writeFileSync(path.join(sessionsDir, fileName), "");
|
||||
}
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "archive", "nested.jsonl.deleted.2026-02-16T22-29-33.000Z"),
|
||||
"",
|
||||
);
|
||||
|
||||
const files = await listSessionFilesForAgent("main");
|
||||
|
||||
expect(files.map((filePath) => path.basename(filePath)).toSorted()).toEqual(
|
||||
included.toSorted(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildSessionEntry", () => {
|
||||
it("returns lineMap tracking original JSONL line numbers", async () => {
|
||||
// Simulate a real session JSONL file with metadata records interspersed
|
||||
// Lines 1-3: non-message metadata records
|
||||
// Line 4: user message
|
||||
// Line 5: metadata
|
||||
// Line 6: assistant message
|
||||
// Line 7: user message
|
||||
const jsonlLines = [
|
||||
JSON.stringify({ type: "custom", customType: "model-snapshot", data: {} }),
|
||||
JSON.stringify({ type: "custom", customType: "openclaw.cache-ttl", data: {} }),
|
||||
JSON.stringify({ type: "session-meta", agentId: "test" }),
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "Hello world" } }),
|
||||
JSON.stringify({ type: "custom", customType: "tool-result", data: {} }),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Hi there, how can I help?" },
|
||||
}),
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "Tell me a joke" } }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
|
||||
// The content should have 3 lines (3 message records)
|
||||
const contentLines = entry!.content.split("\n");
|
||||
expect(contentLines).toHaveLength(3);
|
||||
expect(contentLines[0]).toContain("User: Hello world");
|
||||
expect(contentLines[1]).toContain("Assistant: Hi there");
|
||||
expect(contentLines[2]).toContain("User: Tell me a joke");
|
||||
|
||||
// lineMap should map each content line to its original JSONL line (1-indexed)
|
||||
// Content line 0 → JSONL line 4 (the first user message)
|
||||
// Content line 1 → JSONL line 6 (the assistant message)
|
||||
// Content line 2 → JSONL line 7 (the second user message)
|
||||
expect(entry!.lineMap).toBeDefined();
|
||||
expect(entry!.lineMap).toEqual([4, 6, 7]);
|
||||
expect(entry!.messageTimestampsMs).toEqual([0, 0, 0]);
|
||||
});
|
||||
|
||||
it("returns empty lineMap when no messages are found", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({ type: "custom", customType: "model-snapshot", data: {} }),
|
||||
JSON.stringify({ type: "session-meta", agentId: "test" }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "empty-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.content).toBe("");
|
||||
expect(entry!.lineMap).toEqual([]);
|
||||
expect(entry!.messageTimestampsMs).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips blank lines and invalid JSON without breaking lineMap", async () => {
|
||||
const jsonlLines = [
|
||||
"",
|
||||
"not valid json",
|
||||
JSON.stringify({ type: "message", message: { role: "user", content: "First" } }),
|
||||
"",
|
||||
JSON.stringify({ type: "message", message: { role: "assistant", content: "Second" } }),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "gaps.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.lineMap).toEqual([3, 5]);
|
||||
expect(entry!.messageTimestampsMs).toEqual([0, 0]);
|
||||
});
|
||||
|
||||
it("captures message timestamps when present", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
timestamp: "2026-04-05T10:00:00.000Z",
|
||||
message: { role: "user", content: "First" },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
timestamp: "2026-04-05T10:01:00.000Z",
|
||||
content: "Second",
|
||||
},
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "timestamps.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.messageTimestampsMs).toEqual([
|
||||
Date.parse("2026-04-05T10:00:00.000Z"),
|
||||
Date.parse("2026-04-05T10:01:00.000Z"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("strips inbound metadata envelope from user messages before normalization", async () => {
|
||||
// Representative inbound envelope: Conversation info + Sender blocks prepended
|
||||
// to the actual user text. Without stripping, the JSON envelope dominates
|
||||
// the corpus entry and the user's real words get truncated by the
|
||||
// SESSION_INGESTION_MAX_SNIPPET_CHARS cap downstream.
|
||||
// See: https://github.com/openclaw/openclaw/issues/63921
|
||||
const envelopedUserText = [
|
||||
"Conversation info (untrusted metadata):",
|
||||
"```json",
|
||||
'{"message_id":"msg-100","chat_id":"-100123","sender":"Chris"}',
|
||||
"```",
|
||||
"",
|
||||
"Sender (untrusted metadata):",
|
||||
"```json",
|
||||
'{"label":"Chris","name":"Chris","id":"42"}',
|
||||
"```",
|
||||
"",
|
||||
"帮我看看今天的 Oura 数据",
|
||||
].join("\n");
|
||||
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: envelopedUserText },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "好的,我来查一下" },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "enveloped-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
|
||||
const contentLines = entry!.content.split("\n");
|
||||
expect(contentLines).toHaveLength(2);
|
||||
// User line should contain ONLY the real user text, not the JSON envelope.
|
||||
expect(contentLines[0]).toBe("User: 帮我看看今天的 Oura 数据");
|
||||
expect(contentLines[0]).not.toContain("untrusted metadata");
|
||||
expect(contentLines[0]).not.toContain("message_id");
|
||||
expect(contentLines[0]).not.toContain("```json");
|
||||
expect(contentLines[1]).toBe("Assistant: 好的,我来查一下");
|
||||
});
|
||||
|
||||
it("strips inbound metadata when a user envelope is split across text blocks", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Conversation info (untrusted metadata):" },
|
||||
{ type: "text", text: "```json" },
|
||||
{ type: "text", text: '{"message_id":"msg-100","chat_id":"-100123"}' },
|
||||
{ type: "text", text: "```" },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "Sender (untrusted metadata):" },
|
||||
{ type: "text", text: "```json" },
|
||||
{ type: "text", text: '{"label":"Chris","id":"42"}' },
|
||||
{ type: "text", text: "```" },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "Actual user text" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "enveloped-session-array.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.content).toBe("User: Actual user text");
|
||||
});
|
||||
|
||||
it("wraps pathological long messages into multiple exported lines and repeats mappings", async () => {
|
||||
const longWordyLine = Array.from({ length: 260 }, (_, idx) => `segment-${idx}`).join(" ");
|
||||
const timestamp = Date.parse("2026-04-05T10:00:00.000Z");
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
timestamp: "2026-04-05T10:00:00.000Z",
|
||||
message: { role: "user", content: longWordyLine },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "wrapped-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
|
||||
const contentLines = entry!.content.split("\n");
|
||||
expect(contentLines.length).toBeGreaterThan(1);
|
||||
expect(contentLines.every((line) => line.startsWith("User: "))).toBe(true);
|
||||
expect(contentLines.every((line) => line.length <= 810)).toBe(true);
|
||||
expect(entry!.lineMap).toEqual(contentLines.map(() => 1));
|
||||
expect(entry!.messageTimestampsMs).toEqual(contentLines.map(() => timestamp));
|
||||
});
|
||||
|
||||
it("hard-wraps pathological long tokens without spaces", async () => {
|
||||
const giantToken = "x".repeat(1800);
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: giantToken },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "hard-wrapped-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
|
||||
const contentLines = entry!.content.split("\n");
|
||||
expect(contentLines.length).toBe(3);
|
||||
expect(contentLines.every((line) => line.startsWith("Assistant: "))).toBe(true);
|
||||
expect(contentLines[0].length).toBeLessThanOrEqual(811);
|
||||
expect(contentLines[1].length).toBeLessThanOrEqual(811);
|
||||
expect(entry!.lineMap).toEqual([1, 1, 1]);
|
||||
expect(entry!.messageTimestampsMs).toEqual([0, 0, 0]);
|
||||
});
|
||||
|
||||
it("does not split surrogate pairs when hard-wrapping astral unicode without spaces", async () => {
|
||||
const astralChar = "\u{20000}";
|
||||
const giantToken = astralChar.repeat(410);
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: giantToken },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "surrogate-safe-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
|
||||
const contentLines = entry!.content.split("\n");
|
||||
expect(contentLines.length).toBeGreaterThan(1);
|
||||
expect(entry!.lineMap).toEqual(contentLines.map(() => 1));
|
||||
expect(entry!.messageTimestampsMs).toEqual(contentLines.map(() => 0));
|
||||
for (const line of contentLines) {
|
||||
expect(line.startsWith("Assistant: ")).toBe(true);
|
||||
expectNoUnpairedSurrogates(line);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves assistant messages that happen to contain sentinel-like text", async () => {
|
||||
// Assistant role must NOT be stripped — only user messages carry inbound
|
||||
// envelopes, and assistants may legitimately discuss metadata formats.
|
||||
const assistantText =
|
||||
"The envelope format uses 'Conversation info (untrusted metadata):' as a sentinel";
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: assistantText },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "assistant-sentinel.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.content).toBe(`Assistant: ${assistantText}`);
|
||||
});
|
||||
|
||||
it("flags dreaming narrative transcripts from bootstrap metadata", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "custom",
|
||||
customType: "openclaw:bootstrap-context:full",
|
||||
data: {
|
||||
runId: "dreaming-narrative-light-1775894400455",
|
||||
sessionId: "sid-1",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "Write a dream diary entry from these memory fragments" },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "dreaming-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.generatedByDreamingNarrative).toBe(true);
|
||||
});
|
||||
|
||||
it("flags cron run transcripts from the sibling session store and skips their content", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const filePath = path.join(sessionsDir, "cron-run-session.jsonl");
|
||||
fsSync.writeFileSync(
|
||||
filePath,
|
||||
[
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: "[cron:job-1 Example] Run the nightly sync",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "Running the nightly sync now.",
|
||||
},
|
||||
}),
|
||||
].join("\n"),
|
||||
);
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:cron:job-1:run:run-1": {
|
||||
sessionId: "cron-run-session",
|
||||
sessionFile: filePath,
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const entry = await buildSessionEntry(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.generatedByCronRun).toBe(true);
|
||||
expect(entry?.content).toBe("");
|
||||
expect(entry?.lineMap).toEqual([]);
|
||||
});
|
||||
|
||||
it("flags dreaming narrative transcripts from the sibling session store before bootstrap lands", async () => {
|
||||
const sessionsDir = path.join(tmpDir, "agents", "main", "sessions");
|
||||
fsSync.mkdirSync(sessionsDir, { recursive: true });
|
||||
const filePath = path.join(sessionsDir, "dreaming-session.jsonl");
|
||||
fsSync.writeFileSync(
|
||||
filePath,
|
||||
[
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content:
|
||||
"Write a dream diary entry from these memory fragments:\n- Candidate: durable note",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "A drifting archive breathed in moonlight.",
|
||||
},
|
||||
}),
|
||||
].join("\n"),
|
||||
);
|
||||
fsSync.writeFileSync(
|
||||
path.join(sessionsDir, "sessions.json"),
|
||||
JSON.stringify({
|
||||
"agent:main:dreaming-narrative-light-1775894400455": {
|
||||
sessionId: "dreaming-session",
|
||||
sessionFile: filePath,
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
}),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const entry = await buildSessionEntry(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.generatedByDreamingNarrative).toBe(true);
|
||||
expect(entry?.content).toBe("");
|
||||
expect(entry?.lineMap).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not flag ordinary transcripts that quote the dream-diary prompt", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content:
|
||||
"Write a dream diary entry from these memory fragments:\n- Candidate: durable note",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "A drifting archive breathed in moonlight." },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "dreaming-prompt-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.generatedByDreamingNarrative).toBeUndefined();
|
||||
expect(entry?.content).toContain(
|
||||
"User: Write a dream diary entry from these memory fragments:",
|
||||
);
|
||||
expect(entry?.content).toContain("Assistant: A drifting archive breathed in moonlight.");
|
||||
expect(entry?.lineMap).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it("drops generated runtime chatter while preserving real follow-up content", async () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "system wrapper",
|
||||
fileName: "system-wrapper-session.jsonl",
|
||||
records: [
|
||||
{
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content:
|
||||
"System (untrusted): [2026-04-15 14:45:20 PDT] Exec completed (quiet-fo, code 0) :: Converted: 1",
|
||||
},
|
||||
},
|
||||
{ type: "message", message: { role: "assistant", content: "Handled internally." } },
|
||||
{ type: "message", message: { role: "user", content: "What changed in the sync?" } },
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "One new session was converted." },
|
||||
},
|
||||
],
|
||||
content: [
|
||||
"Assistant: Handled internally.",
|
||||
"User: What changed in the sync?",
|
||||
"Assistant: One new session was converted.",
|
||||
].join("\n"),
|
||||
lineMap: [2, 3, 4],
|
||||
},
|
||||
{
|
||||
name: "cron prompt",
|
||||
fileName: "cron-prompt-session.jsonl",
|
||||
records: [
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "user", content: "[cron:job-1 Example] Run the nightly sync" },
|
||||
},
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "Running the nightly sync now." },
|
||||
},
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "user", content: "Did the nightly sync actually change anything?" },
|
||||
},
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "assistant", content: "No, everything was already current." },
|
||||
},
|
||||
],
|
||||
content: [
|
||||
"Assistant: Running the nightly sync now.",
|
||||
"User: Did the nightly sync actually change anything?",
|
||||
"Assistant: No, everything was already current.",
|
||||
].join("\n"),
|
||||
lineMap: [2, 3, 4],
|
||||
},
|
||||
{
|
||||
name: "heartbeat ack",
|
||||
fileName: "heartbeat-session.jsonl",
|
||||
records: [
|
||||
{
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content:
|
||||
"Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.",
|
||||
},
|
||||
},
|
||||
{ type: "message", message: { role: "assistant", content: "HEARTBEAT_OK" } },
|
||||
{
|
||||
type: "message",
|
||||
message: { role: "user", content: "Summarize what changed in the inbox today." },
|
||||
},
|
||||
],
|
||||
content: "User: Summarize what changed in the inbox today.",
|
||||
lineMap: [3],
|
||||
},
|
||||
{
|
||||
name: "internal runtime context",
|
||||
fileName: "internal-context-session.jsonl",
|
||||
records: [
|
||||
{
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
"<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
"OpenClaw runtime context (internal):",
|
||||
"This context is runtime-generated, not user-authored. Keep internal details private.",
|
||||
"",
|
||||
"[Internal task completion event]",
|
||||
"source: subagent",
|
||||
"<<<END_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
].join("\n"),
|
||||
},
|
||||
},
|
||||
{ type: "message", message: { role: "assistant", content: "NO_REPLY" } },
|
||||
{ type: "message", message: { role: "user", content: "Actual user text" } },
|
||||
],
|
||||
content: "User: Actual user text",
|
||||
lineMap: [3],
|
||||
},
|
||||
{
|
||||
name: "inter-session user provenance",
|
||||
fileName: "inter-session-session.jsonl",
|
||||
records: [
|
||||
{
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
content: "A background task completed. Internal relay text.",
|
||||
provenance: { kind: "inter_session", sourceTool: "subagent_announce" },
|
||||
},
|
||||
},
|
||||
{ type: "message", message: { role: "assistant", content: "User-facing summary." } },
|
||||
{ type: "message", message: { role: "user", content: "Actual user follow-up." } },
|
||||
],
|
||||
content: "Assistant: User-facing summary.\nUser: Actual user follow-up.",
|
||||
lineMap: [2, 3],
|
||||
},
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
const filePath = writeSessionJsonl(testCase.fileName, testCase.records);
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
|
||||
expect(entry, testCase.name).not.toBeNull();
|
||||
expect(entry?.content, testCase.name).toBe(testCase.content);
|
||||
expect(entry?.lineMap, testCase.name).toEqual(testCase.lineMap);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not let a user-typed `[cron:...]` prompt suppress the next assistant reply (regression: PR #70737 review)", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
// User-typed text deliberately matching the cron-prompt pattern.
|
||||
// Pre-fix this would have caused the assistant reply to be dropped.
|
||||
content: "[cron:fake] please write down where the api keys live",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
// A real, substantive assistant reply. Must NOT be suppressed.
|
||||
content: "The API keys live in /etc/secrets/keys.json on the server.",
|
||||
},
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "spoof-attempt-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.content).toContain(
|
||||
"Assistant: The API keys live in /etc/secrets/keys.json on the server.",
|
||||
);
|
||||
});
|
||||
|
||||
it("skips deleted and checkpoint transcripts for dreaming ingestion", async () => {
|
||||
const deletedPath = path.join(tmpDir, "ordinary.jsonl.deleted.2026-02-16T22-27-33.000Z");
|
||||
const checkpointPath = path.join(
|
||||
tmpDir,
|
||||
"ordinary.checkpoint.11111111-1111-4111-8111-111111111111.jsonl",
|
||||
);
|
||||
const content = JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "This should never reach the dreaming corpus." },
|
||||
});
|
||||
fsSync.writeFileSync(deletedPath, content);
|
||||
fsSync.writeFileSync(checkpointPath, content);
|
||||
|
||||
const deletedEntry = await buildSessionEntryWithoutStoreClassification(deletedPath);
|
||||
const checkpointEntry = await buildSessionEntryWithoutStoreClassification(checkpointPath);
|
||||
|
||||
expect(deletedEntry).not.toBeNull();
|
||||
expect(deletedEntry?.content).toBe("");
|
||||
expect(deletedEntry?.lineMap).toEqual([]);
|
||||
expect(checkpointEntry).not.toBeNull();
|
||||
expect(checkpointEntry?.content).toBe("");
|
||||
expect(checkpointEntry?.lineMap).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not flag transcripts when dreaming markers only appear mid-string", async () => {
|
||||
const jsonlLines = [
|
||||
JSON.stringify({
|
||||
type: "custom",
|
||||
customType: "note",
|
||||
data: {
|
||||
runId: "user-context-dreaming-narrative-light-1775894400455",
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: { role: "user", content: "Keep the archive index updated." },
|
||||
}),
|
||||
];
|
||||
const filePath = path.join(tmpDir, "substring-marker-session.jsonl");
|
||||
fsSync.writeFileSync(filePath, jsonlLines.join("\n"));
|
||||
|
||||
const entry = await buildSessionEntryWithoutStoreClassification(filePath);
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry?.generatedByDreamingNarrative).toBeUndefined();
|
||||
expect(entry?.content).toContain("User: Keep the archive index updated.");
|
||||
expect(entry?.lineMap).toEqual([2]);
|
||||
});
|
||||
});
|
||||
@@ -1 +0,0 @@
|
||||
export * from "../../../packages/memory-host-sdk/src/host/session-files.js";
|
||||
@@ -1,39 +0,0 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { normalizeOptionalString } from "../../shared/string-coerce.js";
|
||||
|
||||
type SqliteVecModule = {
|
||||
getLoadablePath: () => string;
|
||||
load: (db: DatabaseSync) => void;
|
||||
};
|
||||
|
||||
const SQLITE_VEC_MODULE_ID = "sqlite-vec";
|
||||
let sqliteVecModulePromise: Promise<SqliteVecModule> | null = null;
|
||||
|
||||
async function loadSqliteVecModule(): Promise<SqliteVecModule> {
|
||||
sqliteVecModulePromise ??= import(SQLITE_VEC_MODULE_ID) as Promise<SqliteVecModule>;
|
||||
return sqliteVecModulePromise;
|
||||
}
|
||||
|
||||
export async function loadSqliteVecExtension(params: {
|
||||
db: DatabaseSync;
|
||||
extensionPath?: string;
|
||||
}): Promise<{ ok: boolean; extensionPath?: string; error?: string }> {
|
||||
try {
|
||||
const sqliteVec = await loadSqliteVecModule();
|
||||
const resolvedPath = normalizeOptionalString(params.extensionPath);
|
||||
const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
|
||||
|
||||
params.db.enableLoadExtension(true);
|
||||
if (resolvedPath) {
|
||||
params.db.loadExtension(extensionPath);
|
||||
} else {
|
||||
sqliteVec.load(params.db);
|
||||
}
|
||||
|
||||
return { ok: true, extensionPath };
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
import { createRequire } from "node:module";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import {
|
||||
configureSqliteWalMaintenance,
|
||||
type SqliteWalMaintenance,
|
||||
type SqliteWalMaintenanceOptions,
|
||||
} from "../../infra/sqlite-wal.js";
|
||||
import { installProcessWarningFilter } from "../../infra/warning-filter.js";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const sqliteWalMaintenanceByDb = new WeakMap<DatabaseSync, SqliteWalMaintenance>();
|
||||
|
||||
export function requireNodeSqlite(): typeof import("node:sqlite") {
|
||||
installProcessWarningFilter();
|
||||
try {
|
||||
return require("node:sqlite") as typeof import("node:sqlite");
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
// Node distributions can ship without the experimental builtin SQLite module.
|
||||
// Surface an actionable error instead of the generic "unknown builtin module".
|
||||
throw new Error(
|
||||
`SQLite support is unavailable in this Node runtime (missing node:sqlite). ${message}`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function configureMemorySqliteWalMaintenance(
|
||||
db: DatabaseSync,
|
||||
options?: SqliteWalMaintenanceOptions,
|
||||
): SqliteWalMaintenance {
|
||||
const existing = sqliteWalMaintenanceByDb.get(db);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const maintenance = configureSqliteWalMaintenance(db, options);
|
||||
sqliteWalMaintenanceByDb.set(db, maintenance);
|
||||
return maintenance;
|
||||
}
|
||||
|
||||
export function closeMemorySqliteWalMaintenance(db: DatabaseSync): boolean {
|
||||
const maintenance = sqliteWalMaintenanceByDb.get(db);
|
||||
if (!maintenance) {
|
||||
return true;
|
||||
}
|
||||
sqliteWalMaintenanceByDb.delete(db);
|
||||
return maintenance.close();
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
export type Tone = "ok" | "warn" | "muted";
|
||||
|
||||
export function resolveMemoryVectorState(vector: { enabled: boolean; available?: boolean }): {
|
||||
tone: Tone;
|
||||
state: "ready" | "unavailable" | "disabled" | "unknown";
|
||||
} {
|
||||
if (!vector.enabled) {
|
||||
return { tone: "muted", state: "disabled" };
|
||||
}
|
||||
if (vector.available === true) {
|
||||
return { tone: "ok", state: "ready" };
|
||||
}
|
||||
if (vector.available === false) {
|
||||
return { tone: "warn", state: "unavailable" };
|
||||
}
|
||||
return { tone: "muted", state: "unknown" };
|
||||
}
|
||||
|
||||
export function resolveMemoryFtsState(fts: { enabled: boolean; available: boolean }): {
|
||||
tone: Tone;
|
||||
state: "ready" | "unavailable" | "disabled";
|
||||
} {
|
||||
if (!fts.enabled) {
|
||||
return { tone: "muted", state: "disabled" };
|
||||
}
|
||||
return fts.available ? { tone: "ok", state: "ready" } : { tone: "warn", state: "unavailable" };
|
||||
}
|
||||
|
||||
export function resolveMemoryCacheSummary(cache: { enabled: boolean; entries?: number }): {
|
||||
tone: Tone;
|
||||
text: string;
|
||||
} {
|
||||
if (!cache.enabled) {
|
||||
return { tone: "muted", text: "cache off" };
|
||||
}
|
||||
const suffix = typeof cache.entries === "number" ? ` (${cache.entries})` : "";
|
||||
return { tone: "ok", text: `cache on${suffix}` };
|
||||
}
|
||||
|
||||
export function resolveMemoryCacheState(cache: { enabled: boolean }): {
|
||||
tone: Tone;
|
||||
state: "enabled" | "disabled";
|
||||
} {
|
||||
return cache.enabled ? { tone: "ok", state: "enabled" } : { tone: "muted", state: "disabled" };
|
||||
}
|
||||
@@ -591,13 +591,12 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
resolveChangedTestTargetPlan([
|
||||
"src/commands/doctor-memory-search.ts",
|
||||
"src/memory-host-sdk/host/embedding-defaults.ts",
|
||||
"src/memory-host-sdk/host/embeddings.ts",
|
||||
]),
|
||||
).toEqual({
|
||||
mode: "targets",
|
||||
targets: [
|
||||
"src/commands/doctor-memory-search.test.ts",
|
||||
"src/memory-host-sdk/host/embeddings.test.ts",
|
||||
"packages/memory-host-sdk/src/host/embeddings.test.ts",
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,7 +32,7 @@ describe("unit-fast vitest lane", () => {
|
||||
expect(config.test?.include).toContain("src/crestodian/rescue-policy.test.ts");
|
||||
expect(config.test?.include).toContain("src/crestodian/assistant.configured.test.ts");
|
||||
expect(config.test?.include).toContain("src/flows/search-setup.test.ts");
|
||||
expect(config.test?.include).toContain("src/memory-host-sdk/host/mirror.test.ts");
|
||||
expect(config.test?.include).toContain("src/memory-host-sdk/host/backend-config.test.ts");
|
||||
expect(config.test?.include).toContain("src/plugins/config-policy.test.ts");
|
||||
expect(config.test?.include).toContain("src/proxy-capture/proxy-server.test.ts");
|
||||
expect(config.test?.include).toContain("src/realtime-voice/agent-consult-tool.test.ts");
|
||||
|
||||
@@ -108,14 +108,7 @@ export const forcedUnitFastTestFiles = [
|
||||
"src/install-sh-version.test.ts",
|
||||
"src/logger.test.ts",
|
||||
"src/library.test.ts",
|
||||
"src/memory-host-sdk/host/embeddings.test.ts",
|
||||
"src/memory-host-sdk/host/internal.test.ts",
|
||||
"src/memory-host-sdk/host/batch-http.test.ts",
|
||||
"src/memory-host-sdk/host/backend-config.test.ts",
|
||||
"src/memory-host-sdk/host/embeddings-remote-fetch.test.ts",
|
||||
"src/memory-host-sdk/host/mirror.test.ts",
|
||||
"src/memory-host-sdk/host/post-json.test.ts",
|
||||
"src/memory-host-sdk/host/session-files.test.ts",
|
||||
"src/media-generation/provider-capabilities.contract.test.ts",
|
||||
"src/music-generation/runtime.test.ts",
|
||||
"src/mcp/channel-server.shutdown-unhandled-rejection.test.ts",
|
||||
|
||||
Reference in New Issue
Block a user