diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f6e38fc551..1eba08595a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Docs: https://docs.openclaw.ai - Exec/child commands: mark child command environments with `OPENCLAW_CLI` so subprocesses can detect when they were launched from the OpenClaw CLI. (#41411) Thanks @vincentkoc. - iOS/Home canvas: add a bundled welcome screen with a live agent overview that refreshes on connect, reconnect, and foreground return, and move the compact connection pill off the top-left canvas overlay. (#42456) Thanks @ngutman. - iOS/Home canvas: replace floating controls with a docked toolbar, make the bundled home scaffold adapt to smaller phones, and open chat in the resolved main session instead of a synthetic `ios` session. (#42456) Thanks @ngutman. -- Memory/Gemini: add `gemini-embedding-2-preview` memory-search support with configurable output dimensions and automatic reindexing when the configured dimensions change. (#42501) thanks @BillChirico. +- Memory/Gemini: add `gemini-embedding-2-preview` memory-search support with configurable output dimensions and automatic reindexing when the configured dimensions change. (#42501) Thanks @BillChirico and @gumadeiras. - Discord/auto threads: add `autoArchiveDuration` channel config for auto-created threads so Discord thread archiving can stay at 1 hour, 1 day, 3 days, or 1 week instead of always using the 1-hour default. (#35065) Thanks @davidguttman. - OpenCode/onboarding: add new OpenCode Go provider, treat Zen and Go as one OpenCode setup in the wizard/docs while keeping the runtime providers split, store one shared OpenCode key for both profiles, and stop overriding the built-in `opencode-go` catalog routing. (#42313) Thanks @ImLukeF and @vincentkoc. - macOS/chat UI: add a chat model picker, persist explicit thinking-level selections across relaunch, and harden provider-aware session model sync for the shared chat composer. (#42314) Thanks @ImLukeF. @@ -181,6 +181,7 @@ Docs: https://docs.openclaw.ai - SecretRef/models: harden custom/provider secret persistence and reuse across models.json snapshots, merge behavior, runtime headers, and secret audits. (#42554) Thanks @joshavant. - macOS/browser proxy: serialize non-GET browser proxy request bodies through `AnyCodable.foundationValue` so nested JSON bodies no longer crash the macOS app with `Invalid type in JSON write (__SwiftValue)`. (#43069) Thanks @Effet. - CLI/skills tables: keep terminal table borders aligned for wide graphemes, use full reported terminal width, and switch a few ambiguous skill icons to Terminal-safe emoji so `openclaw skills` renders more consistently in Terminal.app and iTerm. Thanks @vincentkoc. +- Memory/Gemini: normalize returned Gemini embeddings across direct query, direct batch, and async batch paths so memory search uses consistent vector handling for Gemini too. (#43409) Thanks @gumadeiras. ## 2026.3.7 diff --git a/src/memory/batch-gemini.test.ts b/src/memory/batch-gemini.test.ts index 57bc71291b9..0cbada7293b 100644 --- a/src/memory/batch-gemini.test.ts +++ b/src/memory/batch-gemini.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { GeminiEmbeddingClient } from "./embeddings-gemini.js"; +function magnitude(values: number[]) { + return Math.sqrt(values.reduce((sum, value) => sum + value * value, 0)); +} + describe("runGeminiEmbeddingBatches", () => { let runGeminiEmbeddingBatches: typeof import("./batch-gemini.js").runGeminiEmbeddingBatches; @@ -56,7 +60,7 @@ describe("runGeminiEmbeddingBatches", () => { return new Response( JSON.stringify({ key: "req-1", - response: { embedding: { values: [0.1, 0.2, 0.3] } }, + response: { embedding: { values: [3, 4] } }, }), { status: 200, @@ -88,7 +92,11 @@ describe("runGeminiEmbeddingBatches", () => { concurrency: 1, }); - expect(results.get("req-1")).toEqual([0.1, 0.2, 0.3]); + const embedding = results.get("req-1"); + expect(embedding).toBeDefined(); + expect(embedding?.[0]).toBeCloseTo(0.6, 5); + expect(embedding?.[1]).toBeCloseTo(0.8, 5); + expect(magnitude(embedding ?? [])).toBeCloseTo(1, 5); expect(fetchMock).toHaveBeenCalledTimes(3); }); }); diff --git a/src/memory/batch-gemini.ts b/src/memory/batch-gemini.ts index 3afb5121ff7..4bdc9fa055e 100644 --- a/src/memory/batch-gemini.ts +++ b/src/memory/batch-gemini.ts @@ -4,6 +4,7 @@ import { type EmbeddingBatchExecutionParams, } from "./batch-runner.js"; import { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js"; +import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; import { debugEmbeddingsLog } from "./embeddings-debug.js"; import type { GeminiEmbeddingClient, GeminiTextEmbeddingRequest } from "./embeddings-gemini.js"; import { hashText } from "./internal.js"; @@ -346,7 +347,9 @@ export async function runGeminiEmbeddingBatches( errors.push(`${customId}: ${line.response.error.message}`); continue; } - const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? []; + const embedding = sanitizeAndNormalizeEmbedding( + line.embedding?.values ?? line.response?.embedding?.values ?? [], + ); if (embedding.length === 0) { errors.push(`${customId}: empty embedding`); continue; diff --git a/src/memory/embedding-vectors.ts b/src/memory/embedding-vectors.ts new file mode 100644 index 00000000000..d589f61390d --- /dev/null +++ b/src/memory/embedding-vectors.ts @@ -0,0 +1,8 @@ +export function sanitizeAndNormalizeEmbedding(vec: number[]): number[] { + const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0)); + const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0)); + if (magnitude < 1e-10) { + return sanitized; + } + return sanitized.map((value) => value / magnitude); +} diff --git a/src/memory/embeddings-gemini.test.ts b/src/memory/embeddings-gemini.test.ts index 36cb6bfd111..ae65c8d72b8 100644 --- a/src/memory/embeddings-gemini.test.ts +++ b/src/memory/embeddings-gemini.test.ts @@ -44,6 +44,10 @@ function parseFetchBody(fetchMock: { mock: { calls: unknown[][] } }, callIndex = return JSON.parse((init?.body as string) ?? "{}") as Record; } +function magnitude(values: number[]) { + return Math.sqrt(values.reduce((sum, value) => sum + value * value, 0)); +} + afterEach(() => { vi.resetAllMocks(); vi.unstubAllGlobals(); @@ -224,6 +228,25 @@ describe("gemini-embedding-2-preview provider", () => { expect(body.content).toEqual({ parts: [{ text: "test query" }] }); }); + it("normalizes embedQuery response vectors", async () => { + const fetchMock = createGeminiFetchMock([3, 4]); + vi.stubGlobal("fetch", fetchMock); + mockResolvedProviderKey(); + + const { provider } = await createGeminiEmbeddingProvider({ + config: {} as never, + provider: "gemini", + model: "gemini-embedding-2-preview", + fallback: "none", + }); + + const embedding = await provider.embedQuery("test query"); + + expect(embedding[0]).toBeCloseTo(0.6, 5); + expect(embedding[1]).toBeCloseTo(0.8, 5); + expect(magnitude(embedding)).toBeCloseTo(1, 5); + }); + it("includes outputDimensionality in embedBatch request", async () => { const fetchMock = createGeminiBatchFetchMock(2); vi.stubGlobal("fetch", fetchMock); @@ -255,6 +278,28 @@ describe("gemini-embedding-2-preview provider", () => { ]); }); + it("normalizes embedBatch response vectors", async () => { + const fetchMock = createGeminiBatchFetchMock(2, [3, 4]); + vi.stubGlobal("fetch", fetchMock); + mockResolvedProviderKey(); + + const { provider } = await createGeminiEmbeddingProvider({ + config: {} as never, + provider: "gemini", + model: "gemini-embedding-2-preview", + fallback: "none", + }); + + const embeddings = await provider.embedBatch(["text1", "text2"]); + + expect(embeddings).toHaveLength(2); + for (const embedding of embeddings) { + expect(embedding[0]).toBeCloseTo(0.6, 5); + expect(embedding[1]).toBeCloseTo(0.8, 5); + expect(magnitude(embedding)).toBeCloseTo(1, 5); + } + }); + it("respects custom outputDimensionality", async () => { const fetchMock = createGeminiFetchMock(); vi.stubGlobal("fetch", fetchMock); @@ -310,6 +355,28 @@ describe("gemini-embedding-2-preview provider", () => { ).rejects.toThrow(/Invalid outputDimensionality 512/); }); + it("sanitizes non-finite values before normalization", async () => { + const fetchMock = createGeminiFetchMock([ + 1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]); + vi.stubGlobal("fetch", fetchMock); + mockResolvedProviderKey(); + + const { provider } = await createGeminiEmbeddingProvider({ + config: {} as never, + provider: "gemini", + model: "gemini-embedding-2-preview", + fallback: "none", + }); + + const embedding = await provider.embedQuery("test"); + + expect(embedding).toEqual([1, 0, 0, 0]); + }); + it("uses correct endpoint URL", async () => { const fetchMock = createGeminiFetchMock(); vi.stubGlobal("fetch", fetchMock); diff --git a/src/memory/embeddings-gemini.ts b/src/memory/embeddings-gemini.ts index f8c3d3f4a06..71c8b67fb1a 100644 --- a/src/memory/embeddings-gemini.ts +++ b/src/memory/embeddings-gemini.ts @@ -5,6 +5,7 @@ import { import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js"; import { parseGeminiAuth } from "../infra/gemini-auth.js"; import type { SsrFPolicy } from "../infra/net/ssrf.js"; +import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; import { debugEmbeddingsLog } from "./embeddings-debug.js"; import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js"; import { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./remote-http.js"; @@ -222,7 +223,7 @@ export async function createGeminiEmbeddingProvider( apiKeys: client.apiKeys, execute: (apiKey) => fetchWithGeminiAuth(apiKey, embedUrl, body), }); - return payload.embedding?.values ?? []; + return sanitizeAndNormalizeEmbedding(payload.embedding?.values ?? []); }; const embedBatch = async (texts: string[]): Promise => { @@ -244,7 +245,7 @@ export async function createGeminiEmbeddingProvider( execute: (apiKey) => fetchWithGeminiAuth(apiKey, batchUrl, batchBody), }); const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : []; - return texts.map((_, index) => embeddings[index]?.values ?? []); + return texts.map((_, index) => sanitizeAndNormalizeEmbedding(embeddings[index]?.values ?? [])); }; return { diff --git a/src/memory/embeddings-ollama.ts b/src/memory/embeddings-ollama.ts index 7ccdff6560d..7bd2bcf7428 100644 --- a/src/memory/embeddings-ollama.ts +++ b/src/memory/embeddings-ollama.ts @@ -3,6 +3,7 @@ import { resolveOllamaApiBase } from "../agents/ollama-models.js"; import { formatErrorMessage } from "../infra/errors.js"; import type { SsrFPolicy } from "../infra/net/ssrf.js"; import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js"; +import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; import { normalizeEmbeddingModelWithPrefixes } from "./embeddings-model-normalize.js"; import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js"; import { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./remote-http.js"; @@ -19,15 +20,6 @@ type OllamaEmbeddingClientConfig = Omit; export const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text"; -function sanitizeAndNormalizeEmbedding(vec: number[]): number[] { - const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0)); - const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0)); - if (magnitude < 1e-10) { - return sanitized; - } - return sanitized.map((value) => value / magnitude); -} - function normalizeOllamaModel(model: string): string { return normalizeEmbeddingModelWithPrefixes({ model, diff --git a/src/memory/embeddings.ts b/src/memory/embeddings.ts index d91807c54c8..a5da5222542 100644 --- a/src/memory/embeddings.ts +++ b/src/memory/embeddings.ts @@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/config.js"; import type { SecretInput } from "../config/types.secrets.js"; import { formatErrorMessage } from "../infra/errors.js"; import { resolveUserPath } from "../utils.js"; +import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; import { createGeminiEmbeddingProvider, type GeminiEmbeddingClient, @@ -18,15 +19,6 @@ import { createOpenAiEmbeddingProvider, type OpenAiEmbeddingClient } from "./emb import { createVoyageEmbeddingProvider, type VoyageEmbeddingClient } from "./embeddings-voyage.js"; import { importNodeLlamaCpp } from "./node-llama.js"; -function sanitizeAndNormalizeEmbedding(vec: number[]): number[] { - const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0)); - const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0)); - if (magnitude < 1e-10) { - return sanitized; - } - return sanitized.map((value) => value / magnitude); -} - export type { GeminiEmbeddingClient } from "./embeddings-gemini.js"; export type { MistralEmbeddingClient } from "./embeddings-mistral.js"; export type { OpenAiEmbeddingClient } from "./embeddings-openai.js";