From f7d139dfef960edec5c7efc6feeb9be602bff448 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 06:00:43 +0100 Subject: [PATCH] refactor(memory-host): localize host utilities --- .../src/host/batch-error-utils.ts | 2 +- .../src/host/batch-http.test.ts | 13 +- .../memory-host-sdk/src/host/batch-http.ts | 8 +- .../memory-host-sdk/src/host/batch-utils.ts | 2 +- .../src/host/embeddings-debug.ts | 18 +- .../src/host/embeddings-remote-client.ts | 4 +- .../src/host/embeddings-remote-fetch.ts | 2 +- .../src/host/embeddings-remote-provider.ts | 2 +- .../memory-host-sdk/src/host/error-utils.ts | 89 ++++++ .../memory-host-sdk/src/host/memory-schema.ts | 2 +- .../memory-host-sdk/src/host/post-json.ts | 2 +- .../memory-host-sdk/src/host/qmd-process.ts | 5 +- .../src/host/qmd-query-parser.ts | 16 +- .../memory-host-sdk/src/host/qmd-scope.ts | 14 +- .../src/host/remote-http.test.ts | 5 +- .../memory-host-sdk/src/host/remote-http.ts | 15 +- .../memory-host-sdk/src/host/retry-utils.ts | 141 +++++++++ .../src/host/secret-input-utils.ts | 142 +++++++++ .../memory-host-sdk/src/host/secret-input.ts | 8 +- .../memory-host-sdk/src/host/sqlite-vec.ts | 2 +- .../memory-host-sdk/src/host/sqlite-wal.ts | 74 +++++ packages/memory-host-sdk/src/host/sqlite.ts | 6 +- .../memory-host-sdk/src/host/ssrf-policy.ts | 7 + .../src/host/warning-filter.ts | 105 +++++++ .../memory-host-sdk/src/host/windows-spawn.ts | 285 ++++++++++++++++++ 25 files changed, 916 insertions(+), 53 deletions(-) create mode 100644 packages/memory-host-sdk/src/host/error-utils.ts create mode 100644 packages/memory-host-sdk/src/host/retry-utils.ts create mode 100644 packages/memory-host-sdk/src/host/secret-input-utils.ts create mode 100644 packages/memory-host-sdk/src/host/sqlite-wal.ts create mode 100644 packages/memory-host-sdk/src/host/ssrf-policy.ts create mode 100644 packages/memory-host-sdk/src/host/warning-filter.ts create mode 100644 packages/memory-host-sdk/src/host/windows-spawn.ts diff --git a/packages/memory-host-sdk/src/host/batch-error-utils.ts b/packages/memory-host-sdk/src/host/batch-error-utils.ts index 02a1000ec9c..8d4b9ba3302 100644 --- a/packages/memory-host-sdk/src/host/batch-error-utils.ts +++ b/packages/memory-host-sdk/src/host/batch-error-utils.ts @@ -1,4 +1,4 @@ -import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { formatErrorMessage } from "./error-utils.js"; type BatchOutputErrorLike = { error?: { message?: string }; diff --git a/packages/memory-host-sdk/src/host/batch-http.test.ts b/packages/memory-host-sdk/src/host/batch-http.test.ts index 70ba31bd137..c558a00035c 100644 --- a/packages/memory-host-sdk/src/host/batch-http.test.ts +++ b/packages/memory-host-sdk/src/host/batch-http.test.ts @@ -1,30 +1,23 @@ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -vi.mock("../../../../src/infra/retry.js", () => ({ - retryAsync: vi.fn(async (run: () => Promise) => await run()), -})); - vi.mock("./post-json.js", () => ({ postJson: vi.fn(), })); describe("postJsonWithRetry", () => { - let retryAsyncMock: ReturnType< - typeof vi.mocked - >; let postJsonMock: ReturnType>; let postJsonWithRetry: typeof import("./batch-http.js").postJsonWithRetry; + let retryAsyncMock: ReturnType; beforeAll(async () => { ({ postJsonWithRetry } = await import("./batch-http.js")); - const retryModule = await import("../../../../src/infra/retry.js"); const postJsonModule = await import("./post-json.js"); - retryAsyncMock = vi.mocked(retryModule.retryAsync); postJsonMock = vi.mocked(postJsonModule.postJson); }); beforeEach(() => { vi.clearAllMocks(); + retryAsyncMock = vi.fn(async (run: () => Promise) => await run()); }); it("posts JSON and returns parsed response payload", async () => { @@ -37,6 +30,7 @@ describe("postJsonWithRetry", () => { headers: { Authorization: "Bearer test" }, body: { chunks: ["a", "b"] }, errorPrefix: "memory batch failed", + retryImpl: retryAsyncMock as typeof import("./retry-utils.js").retryAsync, }); expect(result).toEqual({ ok: true, ids: [1, 2] }); @@ -77,6 +71,7 @@ describe("postJsonWithRetry", () => { headers: {}, body: { chunks: [] }, errorPrefix: "memory batch failed", + retryImpl: retryAsyncMock as typeof import("./retry-utils.js").retryAsync, }), ).rejects.toMatchObject({ message: expect.stringContaining("memory batch failed: 503 backend down"), diff --git a/packages/memory-host-sdk/src/host/batch-http.ts b/packages/memory-host-sdk/src/host/batch-http.ts index d890d039a97..30b6543baa6 100644 --- a/packages/memory-host-sdk/src/host/batch-http.ts +++ b/packages/memory-host-sdk/src/host/batch-http.ts @@ -1,16 +1,18 @@ -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; -import { retryAsync } from "../../../../src/infra/retry.js"; import { postJson } from "./post-json.js"; +import { retryAsync } from "./retry-utils.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; export async function postJsonWithRetry(params: { url: string; headers: Record; ssrfPolicy?: SsrFPolicy; fetchImpl?: typeof fetch; + retryImpl?: typeof retryAsync; body: unknown; errorPrefix: string; }): Promise { - return await retryAsync( + const retry = params.retryImpl ?? retryAsync; + return await retry( async () => { return await postJson({ url: params.url, diff --git a/packages/memory-host-sdk/src/host/batch-utils.ts b/packages/memory-host-sdk/src/host/batch-utils.ts index 64b7caffc5c..51c51015a9f 100644 --- a/packages/memory-host-sdk/src/host/batch-utils.ts +++ b/packages/memory-host-sdk/src/host/batch-utils.ts @@ -1,4 +1,4 @@ -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; export type BatchHttpClientConfig = { baseUrl?: string; diff --git a/packages/memory-host-sdk/src/host/embeddings-debug.ts b/packages/memory-host-sdk/src/host/embeddings-debug.ts index 283813ff51e..2109aad70ae 100644 --- a/packages/memory-host-sdk/src/host/embeddings-debug.ts +++ b/packages/memory-host-sdk/src/host/embeddings-debug.ts @@ -1,13 +1,23 @@ -import { isTruthyEnvValue } from "../../../../src/infra/env.js"; -import { createSubsystemLogger } from "../../../../src/logging/subsystem.js"; +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS); -const log = createSubsystemLogger("memory/embeddings"); export function debugEmbeddingsLog(message: string, meta?: Record): void { if (!debugEmbeddings) { return; } const suffix = meta ? ` ${JSON.stringify(meta)}` : ""; - log.raw(`${message}${suffix}`); + process.stderr.write(`${message}${suffix}\n`); +} + +function isTruthyEnvValue(value?: string): boolean { + switch (normalizeLowercaseStringOrEmpty(value)) { + case "1": + case "on": + case "true": + case "yes": + return true; + default: + return false; + } } diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-client.ts b/packages/memory-host-sdk/src/host/embeddings-remote-client.ts index c32bcd31a4e..4968b8ddd07 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-client.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-client.ts @@ -1,9 +1,9 @@ import { requireApiKey, resolveApiKeyForProvider } from "../../../../src/agents/model-auth.js"; -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; -import { normalizeOptionalString } from "../../../../src/shared/string-coerce.js"; import type { EmbeddingProviderOptions } from "./embeddings.types.js"; import { buildRemoteBaseUrlPolicy } from "./remote-http.js"; import { resolveMemorySecretInputString } from "./secret-input.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; +import { normalizeOptionalString } from "./string-utils.js"; export type RemoteEmbeddingProviderId = string; diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts b/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts index cbe2081a69c..a93fbd80d5a 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts @@ -1,5 +1,5 @@ -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; import { postJson } from "./post-json.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; export async function fetchRemoteEmbeddingVectors(params: { url: string; diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts b/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts index f58e41d3d29..0565a7e9651 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts @@ -1,10 +1,10 @@ -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; import { resolveRemoteEmbeddingBearerClient, type RemoteEmbeddingProviderId, } from "./embeddings-remote-client.js"; import { fetchRemoteEmbeddingVectors } from "./embeddings-remote-fetch.js"; import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; export type RemoteEmbeddingClient = { baseUrl: string; diff --git a/packages/memory-host-sdk/src/host/error-utils.ts b/packages/memory-host-sdk/src/host/error-utils.ts new file mode 100644 index 00000000000..eb0af0b0d73 --- /dev/null +++ b/packages/memory-host-sdk/src/host/error-utils.ts @@ -0,0 +1,89 @@ +const SECRET_PATTERNS: RegExp[] = [ + /\b[A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD)\b\s*[=:]\s*(["']?)([^\s"'\\]+)\1/g, + /[?&](?:access[-_]?token|auth[-_]?token|hook[-_]?token|refresh[-_]?token|api[-_]?key|client[-_]?secret|token|key|secret|password|pass|passwd|auth|signature)=([^&\s"'<>]+)/gi, + /"(?:apiKey|token|secret|password|passwd|accessToken|refreshToken)"\s*:\s*"([^"]+)"/g, + /--(?:api[-_]?key|hook[-_]?token|token|secret|password|passwd)\s+(["']?)([^\s"']+)\1/g, + /Authorization\s*[:=]\s*Bearer\s+([A-Za-z0-9._\-+=]+)/g, + /\bBearer\s+([A-Za-z0-9._\-+=]{18,})\b/g, + /(^|[\s,;])(?:access_token|refresh_token|api[-_]?key|token|secret|password|passwd)=([^\s&#]+)/g, + /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----/g, + /\b(sk-[A-Za-z0-9_-]{8,})\b/g, + /\b(ghp_[A-Za-z0-9]{20,})\b/g, + /\b(github_pat_[A-Za-z0-9_]{20,})\b/g, + /\b(xox[baprs]-[A-Za-z0-9-]{10,})\b/g, + /\b(xapp-[A-Za-z0-9-]{10,})\b/g, + /\b(gsk_[A-Za-z0-9_-]{10,})\b/g, + /\b(AIza[0-9A-Za-z\-_]{20,})\b/g, + /\b(pplx-[A-Za-z0-9_-]{10,})\b/g, + /\b(npm_[A-Za-z0-9]{10,})\b/g, + /\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b/g, + /\b(\d{6,}:[A-Za-z0-9_-]{20,})\b/g, +]; + +function maskToken(token: string): string { + if (token.length < 18) { + return "***"; + } + return `${token.slice(0, 6)}...${token.slice(-4)}`; +} + +function redactPemBlock(block: string): string { + const lines = block.split(/\r?\n/).filter(Boolean); + if (lines.length < 2) { + return "***"; + } + return `${lines[0]}\n...redacted...\n${lines[lines.length - 1]}`; +} + +function redactMatch(match: string, groups: string[]): string { + if (match.includes("PRIVATE KEY-----")) { + return redactPemBlock(match); + } + const token = groups.findLast((value) => typeof value === "string" && value.length > 0) ?? match; + const masked = maskToken(token); + return token === match ? masked : match.replace(token, masked); +} + +function redactSensitiveText(text: string): string { + let next = text; + for (const pattern of SECRET_PATTERNS) { + next = next.replace(pattern, (...args: string[]) => + redactMatch(args[0] ?? "", args.slice(1, -2)), + ); + } + return next; +} + +export function formatErrorMessage(err: unknown): string { + let formatted: string; + if (err instanceof Error) { + formatted = err.message || err.name || "Error"; + let cause: unknown = err.cause; + const seen = new Set([err]); + while (cause && !seen.has(cause)) { + seen.add(cause); + if (cause instanceof Error) { + if (cause.message) { + formatted += ` | ${cause.message}`; + } + cause = cause.cause; + } else if (typeof cause === "string") { + formatted += ` | ${cause}`; + break; + } else { + break; + } + } + } else if (typeof err === "string") { + formatted = err; + } else if (typeof err === "number" || typeof err === "boolean" || typeof err === "bigint") { + formatted = String(err); + } else { + try { + formatted = JSON.stringify(err); + } catch { + formatted = Object.prototype.toString.call(err); + } + } + return redactSensitiveText(formatted); +} diff --git a/packages/memory-host-sdk/src/host/memory-schema.ts b/packages/memory-host-sdk/src/host/memory-schema.ts index 9cb239c5cbc..1913c46f9db 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.ts @@ -1,5 +1,5 @@ import type { DatabaseSync } from "node:sqlite"; -import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { formatErrorMessage } from "./error-utils.js"; export function ensureMemoryIndexSchema(params: { db: DatabaseSync; diff --git a/packages/memory-host-sdk/src/host/post-json.ts b/packages/memory-host-sdk/src/host/post-json.ts index e53a833caf8..1e71eec4920 100644 --- a/packages/memory-host-sdk/src/host/post-json.ts +++ b/packages/memory-host-sdk/src/host/post-json.ts @@ -1,5 +1,5 @@ -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; import { withRemoteHttpResponse } from "./remote-http.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; export async function postJson(params: { url: string; diff --git a/packages/memory-host-sdk/src/host/qmd-process.ts b/packages/memory-host-sdk/src/host/qmd-process.ts index a07634a11cf..1342970a615 100644 --- a/packages/memory-host-sdk/src/host/qmd-process.ts +++ b/packages/memory-host-sdk/src/host/qmd-process.ts @@ -1,8 +1,5 @@ import { spawn } from "node:child_process"; -import { - materializeWindowsSpawnProgram, - resolveWindowsSpawnProgram, -} from "../../../../src/plugin-sdk/windows-spawn.js"; +import { materializeWindowsSpawnProgram, resolveWindowsSpawnProgram } from "./windows-spawn.js"; export type CliSpawnInvocation = { command: string; diff --git a/packages/memory-host-sdk/src/host/qmd-query-parser.ts b/packages/memory-host-sdk/src/host/qmd-query-parser.ts index a4c9c550df7..ed0c88748ca 100644 --- a/packages/memory-host-sdk/src/host/qmd-query-parser.ts +++ b/packages/memory-host-sdk/src/host/qmd-query-parser.ts @@ -1,9 +1,6 @@ -import { formatErrorMessage } from "../../../../src/infra/errors.js"; -import { createSubsystemLogger } from "../../../../src/logging/subsystem.js"; +import { formatErrorMessage } from "./error-utils.js"; import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; -const log = createSubsystemLogger("memory"); - export type QmdQueryResult = { docid?: string; score?: number; @@ -26,7 +23,7 @@ export function parseQmdQueryJson(stdout: string, stderr: string): QmdQueryResul if (!trimmedStdout) { const context = trimmedStderr ? ` (stderr: ${summarizeQmdStderr(trimmedStderr)})` : ""; const message = `stdout empty${context}`; - log.warn(`qmd query returned invalid JSON: ${message}`); + warnQmdQueryParseError(message); throw new Error(`qmd query returned invalid JSON: ${message}`); } try { @@ -45,11 +42,18 @@ export function parseQmdQueryJson(stdout: string, stderr: string): QmdQueryResul throw new Error("qmd query JSON response was not an array"); } catch (err) { const message = formatErrorMessage(err); - log.warn(`qmd query returned invalid JSON: ${message}`); + warnQmdQueryParseError(message); throw new Error(`qmd query returned invalid JSON: ${message}`, { cause: err }); } } +function warnQmdQueryParseError(message: string): void { + if (process.env.VITEST || process.env.NODE_ENV === "test") { + return; + } + process.stderr.write(`qmd query returned invalid JSON: ${message}\n`); +} + function isQmdNoResultsOutput(raw: string): boolean { const lines = raw .split(/\r?\n/) diff --git a/packages/memory-host-sdk/src/host/qmd-scope.ts b/packages/memory-host-sdk/src/host/qmd-scope.ts index 3edb490fb68..d9e2df84324 100644 --- a/packages/memory-host-sdk/src/host/qmd-scope.ts +++ b/packages/memory-host-sdk/src/host/qmd-scope.ts @@ -1,4 +1,3 @@ -import { parseAgentSessionKey } from "../../../../src/sessions/session-key-utils.js"; import type { ResolvedQmdConfig } from "./backend-config.js"; import { normalizeLowercaseStringOrEmpty, @@ -108,3 +107,16 @@ function normalizeQmdSessionKey(key?: string): string | undefined { } return normalized; } + +function parseAgentSessionKey(sessionKey: string | undefined | null): { rest: string } | null { + const raw = normalizeOptionalLowercaseString(sessionKey); + if (!raw) { + return null; + } + const parts = raw.split(":").filter(Boolean); + if (parts.length < 3 || parts[0] !== "agent") { + return null; + } + const rest = parts.slice(2).join(":"); + return rest ? { rest } : null; +} diff --git a/packages/memory-host-sdk/src/host/remote-http.test.ts b/packages/memory-host-sdk/src/host/remote-http.test.ts index 7559f0165df..6c1eb5e88f6 100644 --- a/packages/memory-host-sdk/src/host/remote-http.test.ts +++ b/packages/memory-host-sdk/src/host/remote-http.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; -import { GUARDED_FETCH_MODE } from "../../../../src/infra/net/fetch-guard.js"; -import { withRemoteHttpResponse } from "./remote-http.js"; +import { MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE, withRemoteHttpResponse } from "./remote-http.js"; describe("package withRemoteHttpResponse", () => { function makeFetchDeps({ useEnvProxy = false }: { useEnvProxy?: boolean } = {}) { @@ -31,7 +30,7 @@ describe("package withRemoteHttpResponse", () => { expect(deps.calls[0]).toEqual( expect.objectContaining({ url: "https://memory.example/v1/embeddings", - mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, + mode: MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE, }), ); }); diff --git a/packages/memory-host-sdk/src/host/remote-http.ts b/packages/memory-host-sdk/src/host/remote-http.ts index bf6537c3b1a..c89ef2ffd05 100644 --- a/packages/memory-host-sdk/src/host/remote-http.ts +++ b/packages/memory-host-sdk/src/host/remote-http.ts @@ -1,11 +1,12 @@ -import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../../../../src/infra/net/fetch-guard.js"; +import { fetchWithSsrFGuard } from "../../../../src/infra/net/fetch-guard.js"; import { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js"; -import { - ssrfPolicyFromHttpBaseUrlAllowedHostname, - type SsrFPolicy, -} from "../../../../src/infra/net/ssrf.js"; +import { ssrfPolicyFromHttpBaseUrlAllowedHostname } from "../../../../src/infra/net/ssrf.js"; +import type { SsrFPolicy } from "./ssrf-policy.js"; -export const buildRemoteBaseUrlPolicy = ssrfPolicyFromHttpBaseUrlAllowedHostname; +export const MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE = "trusted_env_proxy"; + +export const buildRemoteBaseUrlPolicy: (baseUrl: string) => SsrFPolicy | undefined = + ssrfPolicyFromHttpBaseUrlAllowedHostname; export async function withRemoteHttpResponse(params: { url: string; @@ -25,7 +26,7 @@ export async function withRemoteHttpResponse(params: { init: params.init, policy: params.ssrfPolicy, auditContext: params.auditContext ?? "memory-remote", - ...(shouldUseEnvProxy(params.url) ? { mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY } : {}), + ...(shouldUseEnvProxy(params.url) ? { mode: MEMORY_REMOTE_TRUSTED_ENV_PROXY_MODE } : {}), }); try { return await params.onResponse(response); diff --git a/packages/memory-host-sdk/src/host/retry-utils.ts b/packages/memory-host-sdk/src/host/retry-utils.ts new file mode 100644 index 00000000000..75945622c78 --- /dev/null +++ b/packages/memory-host-sdk/src/host/retry-utils.ts @@ -0,0 +1,141 @@ +export type RetryConfig = { + attempts?: number; + minDelayMs?: number; + maxDelayMs?: number; + jitter?: number; +}; + +export type RetryInfo = { + attempt: number; + maxAttempts: number; + delayMs: number; + err: unknown; + label?: string; +}; + +export type RetryOptions = RetryConfig & { + label?: string; + shouldRetry?: (err: unknown, attempt: number) => boolean; + retryAfterMs?: (err: unknown) => number | undefined; + onRetry?: (info: RetryInfo) => void; +}; + +const DEFAULT_RETRY_CONFIG = { + attempts: 3, + minDelayMs: 300, + maxDelayMs: 30_000, + jitter: 0, +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function asFiniteNumber(value: unknown): number | undefined { + if (typeof value !== "number" || !Number.isFinite(value)) { + return undefined; + } + return value; +} + +function clampNumber(value: unknown, fallback: number, min?: number, max?: number): number { + const next = asFiniteNumber(value); + if (next === undefined) { + return fallback; + } + const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY; + const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY; + return Math.min(Math.max(next, floor), ceiling); +} + +export function resolveRetryConfig( + defaults: Required = DEFAULT_RETRY_CONFIG, + overrides?: RetryConfig, +): Required { + const attempts = Math.max(1, Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1))); + const minDelayMs = Math.max( + 0, + Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0)), + ); + const maxDelayMs = Math.max( + minDelayMs, + Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0)), + ); + const jitter = clampNumber(overrides?.jitter, defaults.jitter, 0, 1); + return { attempts, minDelayMs, maxDelayMs, jitter }; +} + +function applyJitter(delayMs: number, jitter: number): number { + if (jitter <= 0) { + return delayMs; + } + const offset = (Math.random() * 2 - 1) * jitter; + return Math.max(0, Math.round(delayMs * (1 + offset))); +} + +export async function retryAsync( + fn: () => Promise, + attemptsOrOptions: number | RetryOptions = 3, + initialDelayMs = 300, +): Promise { + if (typeof attemptsOrOptions === "number") { + const attempts = Math.max(1, Math.round(attemptsOrOptions)); + let lastErr: unknown; + for (let i = 0; i < attempts; i += 1) { + try { + return await fn(); + } catch (err) { + lastErr = err; + if (i === attempts - 1) { + break; + } + await sleep(initialDelayMs * 2 ** i); + } + } + throw lastErr ?? new Error("Retry failed"); + } + + const options = attemptsOrOptions; + const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options); + const maxAttempts = resolved.attempts; + const minDelayMs = resolved.minDelayMs; + const maxDelayMs = + Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0 + ? resolved.maxDelayMs + : Number.POSITIVE_INFINITY; + const shouldRetry = options.shouldRetry ?? (() => true); + let lastErr: unknown; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + return await fn(); + } catch (err) { + lastErr = err; + if (attempt >= maxAttempts || !shouldRetry(err, attempt)) { + break; + } + + const retryAfterMs = options.retryAfterMs?.(err); + const hasRetryAfter = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs); + const baseDelay = hasRetryAfter + ? Math.max(retryAfterMs, minDelayMs) + : minDelayMs * 2 ** (attempt - 1); + let delay = Math.min(baseDelay, maxDelayMs); + delay = applyJitter(delay, resolved.jitter); + delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs); + + options.onRetry?.({ + attempt, + maxAttempts, + delayMs: delay, + err, + label: options.label, + }); + if (delay > 0) { + await sleep(delay); + } + } + } + + throw lastErr ?? new Error("Retry failed"); +} diff --git a/packages/memory-host-sdk/src/host/secret-input-utils.ts b/packages/memory-host-sdk/src/host/secret-input-utils.ts new file mode 100644 index 00000000000..4b140ff37c2 --- /dev/null +++ b/packages/memory-host-sdk/src/host/secret-input-utils.ts @@ -0,0 +1,142 @@ +export type SecretRefSource = "env" | "file" | "exec"; + +export type SecretRef = { + source: SecretRefSource; + provider: string; + id: string; +}; + +const DEFAULT_SECRET_PROVIDER_ALIAS = "default"; +const ENV_SECRET_REF_ID_RE = /^[A-Z][A-Z0-9_]{0,127}$/; +const LEGACY_SECRETREF_ENV_MARKER_PREFIX = "secretref-env:"; +const ENV_SECRET_TEMPLATE_RE = /^\$\{([A-Z][A-Z0-9_]{0,127})\}$/; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeSecretInputString(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function isSecretRef(value: unknown): value is SecretRef { + if (!isRecord(value)) { + return false; + } + if (Object.keys(value).length !== 3) { + return false; + } + return ( + (value.source === "env" || value.source === "file" || value.source === "exec") && + typeof value.provider === "string" && + value.provider.trim().length > 0 && + typeof value.id === "string" && + value.id.trim().length > 0 + ); +} + +function isLegacySecretRefWithoutProvider( + value: unknown, +): value is { source: SecretRefSource; id: string } { + if (!isRecord(value)) { + return false; + } + return ( + (value.source === "env" || value.source === "file" || value.source === "exec") && + typeof value.id === "string" && + value.id.trim().length > 0 && + value.provider === undefined + ); +} + +function parseEnvTemplateSecretRef(value: unknown): SecretRef | null { + if (typeof value !== "string") { + return null; + } + const match = ENV_SECRET_TEMPLATE_RE.exec(value.trim()); + if (!match) { + return null; + } + return { + source: "env", + provider: DEFAULT_SECRET_PROVIDER_ALIAS, + id: match[1] ?? "", + }; +} + +function parseLegacySecretRefEnvMarker(value: unknown): SecretRef | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + if (!trimmed.startsWith(LEGACY_SECRETREF_ENV_MARKER_PREFIX)) { + return null; + } + const id = trimmed.slice(LEGACY_SECRETREF_ENV_MARKER_PREFIX.length); + if (!ENV_SECRET_REF_ID_RE.test(id)) { + return null; + } + return { + source: "env", + provider: DEFAULT_SECRET_PROVIDER_ALIAS, + id, + }; +} + +function coerceSecretRef(value: unknown): SecretRef | null { + if (isSecretRef(value)) { + return value; + } + if (isLegacySecretRefWithoutProvider(value)) { + return { + source: value.source, + provider: DEFAULT_SECRET_PROVIDER_ALIAS, + id: value.id, + }; + } + return parseEnvTemplateSecretRef(value) ?? parseLegacySecretRefEnvMarker(value); +} + +export function hasConfiguredSecretInput(value: unknown): boolean { + if (normalizeSecretInputString(value)) { + return true; + } + return coerceSecretRef(value) !== null; +} + +function formatSecretRefLabel(ref: SecretRef): string { + return `${ref.source}:${ref.provider}:${ref.id}`; +} + +function createUnresolvedSecretInputError(params: { path: string; ref: SecretRef }): Error { + return new Error( + `${params.path}: unresolved SecretRef "${formatSecretRefLabel(params.ref)}". Resolve this command against an active gateway runtime snapshot before reading it.`, + ); +} + +export function resolveSecretInputRef(value: unknown): SecretRef | null { + return coerceSecretRef(value); +} + +export function normalizeResolvedSecretInputString(params: { + value: unknown; + path: string; +}): string | undefined { + const normalized = normalizeSecretInputString(params.value); + if (normalized) { + return normalized; + } + const ref = resolveSecretInputRef(params.value); + if (!ref) { + return undefined; + } + throw createUnresolvedSecretInputError({ path: params.path, ref }); +} + +export function normalizeEnvSecretInputString(value: unknown): string | undefined { + return normalizeSecretInputString(value); +} diff --git a/packages/memory-host-sdk/src/host/secret-input.ts b/packages/memory-host-sdk/src/host/secret-input.ts index 4a995d4eba6..35894f9b9f6 100644 --- a/packages/memory-host-sdk/src/host/secret-input.ts +++ b/packages/memory-host-sdk/src/host/secret-input.ts @@ -1,9 +1,9 @@ import { hasConfiguredSecretInput, + normalizeEnvSecretInputString, normalizeResolvedSecretInputString, - normalizeSecretInputString, resolveSecretInputRef, -} from "../../../../src/config/types.secrets.js"; +} from "./secret-input-utils.js"; export function hasConfiguredMemorySecretInput(value: unknown): boolean { return hasConfiguredSecretInput(value); @@ -13,9 +13,9 @@ export function resolveMemorySecretInputString(params: { value: unknown; path: string; }): string | undefined { - const { ref } = resolveSecretInputRef({ value: params.value }); + const ref = resolveSecretInputRef(params.value); if (ref?.source === "env") { - const envValue = normalizeSecretInputString(process.env[ref.id]); + const envValue = normalizeEnvSecretInputString(process.env[ref.id]); if (envValue) { return envValue; } diff --git a/packages/memory-host-sdk/src/host/sqlite-vec.ts b/packages/memory-host-sdk/src/host/sqlite-vec.ts index b887f1cc309..6a835a01814 100644 --- a/packages/memory-host-sdk/src/host/sqlite-vec.ts +++ b/packages/memory-host-sdk/src/host/sqlite-vec.ts @@ -1,5 +1,5 @@ import type { DatabaseSync } from "node:sqlite"; -import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { formatErrorMessage } from "./error-utils.js"; import { normalizeOptionalString } from "./string-utils.js"; type SqliteVecModule = { diff --git a/packages/memory-host-sdk/src/host/sqlite-wal.ts b/packages/memory-host-sdk/src/host/sqlite-wal.ts new file mode 100644 index 00000000000..1d29b1f0ba5 --- /dev/null +++ b/packages/memory-host-sdk/src/host/sqlite-wal.ts @@ -0,0 +1,74 @@ +import type { DatabaseSync } from "node:sqlite"; + +const DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000; +const DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS = 30 * 60 * 1000; + +type IntervalHandle = ReturnType & { + unref?: () => void; +}; + +type SqliteWalCheckpointMode = "PASSIVE" | "FULL" | "RESTART" | "TRUNCATE"; + +export type SqliteWalMaintenance = { + checkpoint: () => boolean; + close: () => boolean; +}; + +export type SqliteWalMaintenanceOptions = { + autoCheckpointPages?: number; + checkpointIntervalMs?: number; + checkpointMode?: SqliteWalCheckpointMode; + onCheckpointError?: (error: unknown) => void; +}; + +function normalizeNonNegativeInteger(value: number, label: string): number { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${label} must be a non-negative integer`); + } + return value; +} + +export function configureSqliteWalMaintenance( + db: DatabaseSync, + options: SqliteWalMaintenanceOptions = {}, +): SqliteWalMaintenance { + const autoCheckpointPages = normalizeNonNegativeInteger( + options.autoCheckpointPages ?? DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES, + "autoCheckpointPages", + ); + const checkpointIntervalMs = normalizeNonNegativeInteger( + options.checkpointIntervalMs ?? DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS, + "checkpointIntervalMs", + ); + const checkpointMode = options.checkpointMode ?? "TRUNCATE"; + + db.exec("PRAGMA journal_mode = WAL;"); + db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`); + + const checkpoint = (): boolean => { + try { + db.exec(`PRAGMA wal_checkpoint(${checkpointMode});`); + return true; + } catch (error) { + options.onCheckpointError?.(error); + return false; + } + }; + + let timer: IntervalHandle | null = null; + if (checkpointIntervalMs > 0) { + timer = setInterval(checkpoint, checkpointIntervalMs) as IntervalHandle; + timer.unref?.(); + } + + return { + checkpoint, + close: () => { + if (timer) { + clearInterval(timer); + timer = null; + } + return checkpoint(); + }, + }; +} diff --git a/packages/memory-host-sdk/src/host/sqlite.ts b/packages/memory-host-sdk/src/host/sqlite.ts index 9a2ec1695a3..6cc8e86dcaa 100644 --- a/packages/memory-host-sdk/src/host/sqlite.ts +++ b/packages/memory-host-sdk/src/host/sqlite.ts @@ -1,12 +1,12 @@ import { createRequire } from "node:module"; import type { DatabaseSync } from "node:sqlite"; -import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { formatErrorMessage } from "./error-utils.js"; import { configureSqliteWalMaintenance, type SqliteWalMaintenance, type SqliteWalMaintenanceOptions, -} from "../../../../src/infra/sqlite-wal.js"; -import { installProcessWarningFilter } from "../../../../src/infra/warning-filter.js"; +} from "./sqlite-wal.js"; +import { installProcessWarningFilter } from "./warning-filter.js"; const require = createRequire(import.meta.url); const sqliteWalMaintenanceByDb = new WeakMap(); diff --git a/packages/memory-host-sdk/src/host/ssrf-policy.ts b/packages/memory-host-sdk/src/host/ssrf-policy.ts new file mode 100644 index 00000000000..23d81905930 --- /dev/null +++ b/packages/memory-host-sdk/src/host/ssrf-policy.ts @@ -0,0 +1,7 @@ +export type SsrFPolicy = { + allowPrivateNetwork?: boolean; + dangerouslyAllowPrivateNetwork?: boolean; + allowRfc2544BenchmarkRange?: boolean; + allowedHostnames?: string[]; + hostnameAllowlist?: string[]; +}; diff --git a/packages/memory-host-sdk/src/host/warning-filter.ts b/packages/memory-host-sdk/src/host/warning-filter.ts new file mode 100644 index 00000000000..788459c6e21 --- /dev/null +++ b/packages/memory-host-sdk/src/host/warning-filter.ts @@ -0,0 +1,105 @@ +const warningFilterKey = Symbol.for("openclaw.warning-filter"); + +export type ProcessWarning = { + code?: string; + name?: string; + message?: string; +}; + +type ProcessWarningInstallState = { + installed: boolean; +}; + +function resolveWarningFilterState(): ProcessWarningInstallState { + const globalStore = globalThis as Record; + if (Object.prototype.hasOwnProperty.call(globalStore, warningFilterKey)) { + return globalStore[warningFilterKey] as ProcessWarningInstallState; + } + const state = { installed: false }; + globalStore[warningFilterKey] = state; + return state; +} + +export function shouldIgnoreWarning(warning: ProcessWarning): boolean { + if (warning.code === "DEP0040" && warning.message?.includes("punycode")) { + return true; + } + if (warning.code === "DEP0060" && warning.message?.includes("util._extend")) { + return true; + } + if ( + warning.name === "ExperimentalWarning" && + warning.message?.includes("SQLite is an experimental feature") + ) { + return true; + } + return false; +} + +function normalizeWarningArgs(args: unknown[]): ProcessWarning { + const warningArg = args[0]; + const secondArg = args[1]; + const thirdArg = args[2]; + let name: string | undefined; + let code: string | undefined; + let message: string | undefined; + + if (warningArg instanceof Error) { + name = warningArg.name; + message = warningArg.message; + code = (warningArg as Error & { code?: string }).code; + } else if (typeof warningArg === "string") { + message = warningArg; + } + + if (secondArg && typeof secondArg === "object" && !Array.isArray(secondArg)) { + const options = secondArg as { type?: unknown; code?: unknown }; + if (typeof options.type === "string") { + name = options.type; + } + if (typeof options.code === "string") { + code = options.code; + } + } else { + if (typeof secondArg === "string") { + name = secondArg; + } + if (typeof thirdArg === "string") { + code = thirdArg; + } + } + + return { name, code, message }; +} + +export function installProcessWarningFilter(): void { + const state = resolveWarningFilterState(); + if (state.installed) { + return; + } + + const originalEmitWarning = process.emitWarning.bind(process); + const wrappedEmitWarning: typeof process.emitWarning = ((...args: unknown[]) => { + if (shouldIgnoreWarning(normalizeWarningArgs(args))) { + return; + } + if ( + args[0] instanceof Error && + args[1] && + typeof args[1] === "object" && + !Array.isArray(args[1]) + ) { + const warning = args[0]; + const emitted = Object.assign(new Error(warning.message), { + name: warning.name, + code: (warning as Error & { code?: string }).code, + }); + process.emit("warning", emitted); + return; + } + Reflect.apply(originalEmitWarning, process, args); + }) as typeof process.emitWarning; + + process.emitWarning = wrappedEmitWarning; + state.installed = true; +} diff --git a/packages/memory-host-sdk/src/host/windows-spawn.ts b/packages/memory-host-sdk/src/host/windows-spawn.ts new file mode 100644 index 00000000000..06acce8cbcf --- /dev/null +++ b/packages/memory-host-sdk/src/host/windows-spawn.ts @@ -0,0 +1,285 @@ +import { readFileSync, statSync } from "node:fs"; +import path from "node:path"; +import { normalizeLowercaseStringOrEmpty, normalizeOptionalString } from "./string-utils.js"; + +type WindowsSpawnResolution = "direct" | "node-entrypoint" | "exe-entrypoint" | "shell-fallback"; + +type WindowsSpawnCandidateResolution = Exclude; + +type WindowsSpawnProgramCandidate = { + command: string; + leadingArgv: string[]; + resolution: WindowsSpawnCandidateResolution | "unresolved-wrapper"; + windowsHide?: boolean; +}; + +export type WindowsSpawnProgram = { + command: string; + leadingArgv: string[]; + resolution: WindowsSpawnResolution; + shell?: boolean; + windowsHide?: boolean; +}; + +export type WindowsSpawnInvocation = { + command: string; + argv: string[]; + resolution: WindowsSpawnResolution; + shell?: boolean; + windowsHide?: boolean; +}; + +export type ResolveWindowsSpawnProgramParams = { + command: string; + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + execPath?: string; + packageName?: string; + allowShellFallback?: boolean; +}; + +function isFilePath(candidate: string): boolean { + try { + return statSync(candidate).isFile(); + } catch { + return false; + } +} + +function resolveWindowsExecutablePath(command: string, env: NodeJS.ProcessEnv): string { + if (command.includes("/") || command.includes("\\") || path.isAbsolute(command)) { + return command; + } + + const pathValue = env.PATH ?? env.Path ?? process.env.PATH ?? process.env.Path ?? ""; + const pathEntries = pathValue + .split(";") + .map((entry) => entry.trim()) + .filter(Boolean); + const hasExtension = path.extname(command).length > 0; + const pathExtRaw = + env.PATHEXT ?? + env.Pathext ?? + process.env.PATHEXT ?? + process.env.Pathext ?? + ".EXE;.CMD;.BAT;.COM"; + const pathExt = hasExtension + ? [""] + : pathExtRaw + .split(";") + .map((ext) => ext.trim()) + .filter(Boolean) + .map((ext) => (ext.startsWith(".") ? ext : `.${ext}`)); + + for (const dir of pathEntries) { + for (const ext of pathExt) { + const normalizedExt = normalizeLowercaseStringOrEmpty(ext); + const uppercaseExt = ext.toUpperCase(); + for (const candidateExt of [ext, normalizedExt, uppercaseExt]) { + const candidate = path.join(dir, `${command}${candidateExt}`); + if (isFilePath(candidate)) { + return candidate; + } + } + } + } + + return command; +} + +function resolveEntrypointFromCmdShim(wrapperPath: string): string | null { + if (!isFilePath(wrapperPath)) { + return null; + } + + try { + const content = readFileSync(wrapperPath, "utf8"); + const candidates: string[] = []; + for (const match of content.matchAll(/"([^"\r\n]*)"/g)) { + const token = match[1] ?? ""; + const relMatch = token.match(/%~?dp0%?\s*[\\/]*(.*)$/i); + const relative = relMatch?.[1]?.trim(); + if (!relative) { + continue; + } + const normalizedRelative = relative.replace(/[\\/]+/g, path.sep).replace(/^[\\/]+/, ""); + const candidate = path.resolve(path.dirname(wrapperPath), normalizedRelative); + if (isFilePath(candidate)) { + candidates.push(candidate); + } + } + const nonNode = candidates.find((candidate) => { + const base = normalizeLowercaseStringOrEmpty(path.basename(candidate)); + return base !== "node.exe" && base !== "node"; + }); + return nonNode ?? null; + } catch { + return null; + } +} + +function resolveBinEntry( + packageName: string | undefined, + binField: string | Record | undefined, +): string | null { + if (typeof binField === "string") { + return normalizeOptionalString(binField) ?? null; + } + if (!binField || typeof binField !== "object") { + return null; + } + if (packageName) { + const preferred = binField[packageName]; + const normalizedPreferred = + typeof preferred === "string" ? normalizeOptionalString(preferred) : undefined; + if (normalizedPreferred) { + return normalizedPreferred; + } + } + for (const value of Object.values(binField)) { + const normalizedValue = typeof value === "string" ? normalizeOptionalString(value) : undefined; + if (normalizedValue) { + return normalizedValue; + } + } + return null; +} + +function resolveEntrypointFromPackageJson( + wrapperPath: string, + packageName?: string, +): string | null { + if (!packageName) { + return null; + } + + const wrapperDir = path.dirname(wrapperPath); + const packageDirs = [ + path.resolve(wrapperDir, "..", packageName), + path.resolve(wrapperDir, "node_modules", packageName), + ]; + + for (const packageDir of packageDirs) { + const packageJsonPath = path.join(packageDir, "package.json"); + if (!isFilePath(packageJsonPath)) { + continue; + } + try { + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { + bin?: string | Record; + }; + const entryRel = resolveBinEntry(packageName, packageJson.bin); + if (!entryRel) { + continue; + } + const entryPath = path.resolve(packageDir, entryRel); + if (isFilePath(entryPath)) { + return entryPath; + } + } catch { + // Ignore malformed package metadata. + } + } + + return null; +} + +function resolveWindowsSpawnProgramCandidate( + params: ResolveWindowsSpawnProgramParams, +): WindowsSpawnProgramCandidate { + const platform = params.platform ?? process.platform; + const env = params.env ?? process.env; + const execPath = params.execPath ?? process.execPath; + + if (platform !== "win32") { + return { + command: params.command, + leadingArgv: [], + resolution: "direct", + }; + } + + const resolvedCommand = resolveWindowsExecutablePath(params.command, env); + const ext = normalizeLowercaseStringOrEmpty(path.extname(resolvedCommand)); + if (ext === ".js" || ext === ".cjs" || ext === ".mjs") { + return { + command: execPath, + leadingArgv: [resolvedCommand], + resolution: "node-entrypoint", + windowsHide: true, + }; + } + + if (ext === ".cmd" || ext === ".bat") { + const entrypoint = + resolveEntrypointFromCmdShim(resolvedCommand) ?? + resolveEntrypointFromPackageJson(resolvedCommand, params.packageName); + if (entrypoint) { + const entryExt = normalizeLowercaseStringOrEmpty(path.extname(entrypoint)); + if (entryExt === ".exe") { + return { + command: entrypoint, + leadingArgv: [], + resolution: "exe-entrypoint", + windowsHide: true, + }; + } + return { + command: execPath, + leadingArgv: [entrypoint], + resolution: "node-entrypoint", + windowsHide: true, + }; + } + + return { + command: resolvedCommand, + leadingArgv: [], + resolution: "unresolved-wrapper", + }; + } + + return { + command: resolvedCommand, + leadingArgv: [], + resolution: "direct", + }; +} + +export function resolveWindowsSpawnProgram( + params: ResolveWindowsSpawnProgramParams, +): WindowsSpawnProgram { + const candidate = resolveWindowsSpawnProgramCandidate(params); + if (candidate.resolution !== "unresolved-wrapper") { + return { + command: candidate.command, + leadingArgv: candidate.leadingArgv, + resolution: candidate.resolution, + windowsHide: candidate.windowsHide, + }; + } + if (params.allowShellFallback === true) { + return { + command: candidate.command, + leadingArgv: [], + resolution: "shell-fallback", + shell: true, + }; + } + throw new Error( + `${path.basename(candidate.command)} wrapper resolved, but no executable/Node entrypoint could be resolved without shell execution.`, + ); +} + +export function materializeWindowsSpawnProgram( + program: WindowsSpawnProgram, + argv: string[], +): WindowsSpawnInvocation { + return { + command: program.command, + argv: [...program.leadingArgv, ...argv], + resolution: program.resolution, + shell: program.shell, + windowsHide: program.windowsHide, + }; +}