diff --git a/CHANGELOG.md b/CHANGELOG.md index d911a1c7992..fe066f4fd48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai ### Changes - Providers: add Cerebras as a bundled plugin with onboarding, static model catalog, docs, and manifest-owned endpoint metadata. Thanks @codex. +- Memory/OpenAI-compatible: add optional `memorySearch.inputType`, `queryInputType`, and `documentInputType` config for asymmetric embedding endpoints, including direct query embeddings and provider batch indexing. Carries forward #63313 and #60727. Thanks @HOYALIM and @prospect1314521. - Ollama/memory: add model-specific retrieval query prefixes for `nomic-embed-text`, `qwen3-embedding`, and `mxbai-embed-large` memory-search queries while leaving document batches unchanged. Carries forward #45013. Thanks @laolin5564. - Plugins/providers: move pre-runtime model-id normalization, provider endpoint host metadata, and OpenAI-compatible request-family hints into plugin manifests so core no longer carries bundled-provider routing tables. Thanks @codex. - Plugins/install: allow `OPENCLAW_PLUGIN_STAGE_DIR` to contain layered runtime-dependency roots, resolving read-only preinstalled deps before installing missing deps into the final writable root. Fixes #72396. Thanks @liorb-mountapps. diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 9e00f0b7038..c0b20b52936 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -454c34daa3f5f66a97f6a701968756a77a110fe611e013b0245fe6a9ef274997 config-baseline.json -56edd542252c0ec8b3005dcddcf083a568d5e7700f7675c509c2963e36a4597c config-baseline.core.json +6bd751bdfd55d644e8a83c575c4db8f67efae00df772c71553b7e7673c5ccc13 config-baseline.json +2bf9795740688c55b8ac50ecbfe954c3d3b0dce0a17b08e89a9385d56e197bee config-baseline.core.json 07963db49502132f26db396c56b36e018b110e6c55a68b3cb012d3ec96f43901 config-baseline.channel.json -f14d1d609ce93893f3bbd6c533251d30328f4deed5cf06da7cb2c9208147dc7a config-baseline.plugin.json +ed65cefbef96f034ce2b73069d9d5bacc341a43489ff9b20a34d40956b877f79 config-baseline.plugin.json diff --git a/docs/concepts/memory-search.md b/docs/concepts/memory-search.md index d0f0f8b6de9..30a403802f7 100644 --- a/docs/concepts/memory-search.md +++ b/docs/concepts/memory-search.md @@ -32,6 +32,11 @@ explicitly: For local embeddings with no API key, install the optional `node-llama-cpp` runtime package next to OpenClaw and use `provider: "local"`. +Some OpenAI-compatible embedding endpoints require asymmetric labels such as +`input_type: "query"` for searches and `input_type: "document"` or `"passage"` +for indexed chunks. Configure those with `memorySearch.queryInputType` and +`memorySearch.documentInputType`; see the [Memory configuration reference](/reference/memory-config#provider-specific-config). + ## Supported providers | Provider | ID | Needs API key | Notes | diff --git a/docs/providers/openai.md b/docs/providers/openai.md index 8c2662c7026..7bff1308889 100644 --- a/docs/providers/openai.md +++ b/docs/providers/openai.md @@ -86,6 +86,30 @@ through PI, `openclaw doctor` warns and leaves the route unchanged. | Realtime voice | Voice Call `realtime.provider: "openai"` / Control UI Talk | Yes | | Embeddings | memory embedding provider | Yes | +## Memory embeddings + +OpenClaw can use OpenAI, or an OpenAI-compatible embedding endpoint, for +`memory_search` indexing and query embeddings: + +```json5 +{ + agents: { + defaults: { + memorySearch: { + provider: "openai", + model: "text-embedding-3-small", + }, + }, + }, +} +``` + +For OpenAI-compatible endpoints that require asymmetric embedding labels, set +`queryInputType` and `documentInputType` under `memorySearch`. OpenClaw forwards +those as provider-specific `input_type` request fields: query embeddings use +`queryInputType`; indexed memory chunks and batch indexing use +`documentInputType`. See the [Memory configuration reference](/reference/memory-config#provider-specific-config) for the full example. + ## Getting started Choose your preferred auth method and follow the setup steps. diff --git a/docs/reference/memory-config.md b/docs/reference/memory-config.md index 65d96f9fd96..88ee5f5c09b 100644 --- a/docs/reference/memory-config.md +++ b/docs/reference/memory-config.md @@ -149,6 +149,37 @@ For custom OpenAI-compatible endpoints or overriding provider defaults: Changing model or `outputDimensionality` triggers an automatic full reindex. + + + OpenAI-compatible embedding endpoints can opt into provider-specific `input_type` request fields. This is useful for asymmetric embedding models that require different labels for query and document embeddings. + + | Key | Type | Default | Description | + | ------------------- | -------- | ------- | ------------------------------------------------------- | + | `inputType` | `string` | unset | Shared `input_type` for query and document embeddings | + | `queryInputType` | `string` | unset | Query-time `input_type`; overrides `inputType` | + | `documentInputType` | `string` | unset | Index/document `input_type`; overrides `inputType` | + + ```json5 + { + agents: { + defaults: { + memorySearch: { + provider: "openai", + remote: { + baseUrl: "https://embeddings.example/v1", + apiKey: "env:EMBEDDINGS_API_KEY", + }, + model: "asymmetric-embedder", + queryInputType: "query", + documentInputType: "passage", + }, + }, + }, + } + ``` + + Changing these values affects embedding cache identity for provider batch indexing and should be followed by a memory reindex when the upstream model treats the labels differently. + Bedrock uses the AWS SDK default credential chain — no API keys needed. If OpenClaw runs on EC2 with a Bedrock-enabled instance role, just set the provider and model: diff --git a/extensions/memory-core/src/memory/manager-provider-state.ts b/extensions/memory-core/src/memory/manager-provider-state.ts index 730a240c828..1a690b352e2 100644 --- a/extensions/memory-core/src/memory/manager-provider-state.ts +++ b/extensions/memory-core/src/memory/manager-provider-state.ts @@ -23,6 +23,9 @@ export function resolveMemoryPrimaryProviderRequest(params: { provider: string; model: string; remote: ResolvedMemorySearchConfig["remote"]; + inputType: ResolvedMemorySearchConfig["inputType"]; + queryInputType: ResolvedMemorySearchConfig["queryInputType"]; + documentInputType: ResolvedMemorySearchConfig["documentInputType"]; outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"]; fallback: ResolvedMemorySearchConfig["fallback"]; local: ResolvedMemorySearchConfig["local"]; @@ -31,6 +34,9 @@ export function resolveMemoryPrimaryProviderRequest(params: { provider: params.settings.provider, model: params.settings.model, remote: params.settings.remote, + inputType: params.settings.inputType, + queryInputType: params.settings.queryInputType, + documentInputType: params.settings.documentInputType, outputDimensionality: params.settings.outputDimensionality, fallback: params.settings.fallback, local: params.settings.local, @@ -75,6 +81,9 @@ export function resolveMemoryFallbackProviderRequest(params: { provider: string; model: string; remote: ResolvedMemorySearchConfig["remote"]; + inputType: ResolvedMemorySearchConfig["inputType"]; + queryInputType: ResolvedMemorySearchConfig["queryInputType"]; + documentInputType: ResolvedMemorySearchConfig["documentInputType"]; outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"]; fallback: "none"; local: ResolvedMemorySearchConfig["local"]; @@ -92,6 +101,9 @@ export function resolveMemoryFallbackProviderRequest(params: { provider: fallback, model: resolveEmbeddingProviderFallbackModel(fallback, params.settings.model, params.cfg), remote: params.settings.remote, + inputType: params.settings.inputType, + queryInputType: params.settings.queryInputType, + documentInputType: params.settings.documentInputType, outputDimensionality: params.settings.outputDimensionality, fallback: "none", local: params.settings.local, diff --git a/extensions/memory-core/src/memory/manager.mistral-provider.test.ts b/extensions/memory-core/src/memory/manager.mistral-provider.test.ts index b2c97396fb8..17140893e7b 100644 --- a/extensions/memory-core/src/memory/manager.mistral-provider.test.ts +++ b/extensions/memory-core/src/memory/manager.mistral-provider.test.ts @@ -136,6 +136,21 @@ describe("memory manager mistral provider wiring", () => { expect(request.outputDimensionality).toBe(1536); }); + it("includes memory input_type fields in the primary provider request", () => { + const request = resolveMemoryPrimaryProviderRequest({ + settings: { + ...createSettings({ provider: "openai" }), + inputType: "passage", + queryInputType: "query", + documentInputType: "document", + } as ResolvedMemorySearchConfig, + }); + + expect(request.inputType).toBe("passage"); + expect(request.queryInputType).toBe("query"); + expect(request.documentInputType).toBe("document"); + }); + it("uses default lmstudio model when activating lmstudio fallback", async () => { const request = resolveMemoryFallbackProviderRequest({ cfg: {} as OpenClawConfig, diff --git a/extensions/openai/embedding-provider.test.ts b/extensions/openai/embedding-provider.test.ts new file mode 100644 index 00000000000..2d5f364b186 --- /dev/null +++ b/extensions/openai/embedding-provider.test.ts @@ -0,0 +1,88 @@ +import type { MemoryEmbeddingProviderCreateOptions } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + fetchRemoteEmbeddingVectors: vi.fn(async () => [[1, 0]]), + resolveRemoteEmbeddingClient: vi.fn(async () => ({ + baseUrl: "https://embeddings.example/v1", + headers: { Authorization: "Bearer test" }, + model: "text-embedding-3-small", + })), +})); + +vi.mock("openclaw/plugin-sdk/memory-core-host-engine-embeddings", () => ({ + fetchRemoteEmbeddingVectors: mocks.fetchRemoteEmbeddingVectors, + resolveRemoteEmbeddingClient: mocks.resolveRemoteEmbeddingClient, +})); + +import { createOpenAiEmbeddingProvider } from "./embedding-provider.js"; + +function createOptions( + overrides: Partial = {}, +): MemoryEmbeddingProviderCreateOptions { + return { + config: {} as MemoryEmbeddingProviderCreateOptions["config"], + provider: "openai", + model: "text-embedding-3-small", + fallback: "none", + ...overrides, + }; +} + +describe("OpenAI embedding provider", () => { + beforeEach(() => { + mocks.fetchRemoteEmbeddingVectors.mockClear(); + mocks.resolveRemoteEmbeddingClient.mockClear(); + }); + + it("sends queryInputType on query embeddings", async () => { + const { provider } = await createOpenAiEmbeddingProvider( + createOptions({ inputType: "passage", queryInputType: "query" }), + ); + + await provider.embedQuery("hello"); + + expect(mocks.fetchRemoteEmbeddingVectors).toHaveBeenCalledWith( + expect.objectContaining({ + body: { + model: "text-embedding-3-small", + input: ["hello"], + input_type: "query", + }, + }), + ); + }); + + it("sends documentInputType on document batch embeddings", async () => { + const { provider } = await createOpenAiEmbeddingProvider( + createOptions({ inputType: "query", documentInputType: "document" }), + ); + + await provider.embedBatch(["doc one", "doc two"]); + + expect(mocks.fetchRemoteEmbeddingVectors).toHaveBeenCalledWith( + expect.objectContaining({ + body: { + model: "text-embedding-3-small", + input: ["doc one", "doc two"], + input_type: "document", + }, + }), + ); + }); + + it("omits input_type unless configured", async () => { + const { provider } = await createOpenAiEmbeddingProvider(createOptions()); + + await provider.embedBatch(["doc"]); + + expect(mocks.fetchRemoteEmbeddingVectors).toHaveBeenCalledWith( + expect.objectContaining({ + body: { + model: "text-embedding-3-small", + input: ["doc"], + }, + }), + ); + }); +}); diff --git a/extensions/openai/embedding-provider.ts b/extensions/openai/embedding-provider.ts index a536a93b0fe..0df74f9da3e 100644 --- a/extensions/openai/embedding-provider.ts +++ b/extensions/openai/embedding-provider.ts @@ -1,5 +1,5 @@ import { - createRemoteEmbeddingProvider, + fetchRemoteEmbeddingVectors, resolveRemoteEmbeddingClient, type MemoryEmbeddingProvider, type MemoryEmbeddingProviderCreateOptions, @@ -13,6 +13,9 @@ export type OpenAiEmbeddingClient = { ssrfPolicy?: SsrFPolicy; fetchImpl?: typeof fetch; model: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; }; const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; @@ -35,14 +38,46 @@ export async function createOpenAiEmbeddingProvider( options: MemoryEmbeddingProviderCreateOptions, ): Promise<{ provider: MemoryEmbeddingProvider; client: OpenAiEmbeddingClient }> { const client = await resolveOpenAiEmbeddingClient(options); + const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`; + + const resolveInputType = (kind: "query" | "document"): string | undefined => { + const explicit = kind === "query" ? client.queryInputType : client.documentInputType; + const value = explicit ?? client.inputType; + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; + }; + + const embed = async (input: string[], kind: "query" | "document"): Promise => { + if (input.length === 0) { + return []; + } + const inputType = resolveInputType(kind); + return await fetchRemoteEmbeddingVectors({ + url, + headers: client.headers, + ssrfPolicy: client.ssrfPolicy, + fetchImpl: client.fetchImpl, + body: { + model: client.model, + input, + ...(inputType ? { input_type: inputType } : {}), + }, + errorPrefix: "openai embeddings failed", + }); + }; return { - provider: createRemoteEmbeddingProvider({ + provider: { id: "openai", - client, - errorPrefix: "openai embeddings failed", - maxInputTokens: OPENAI_MAX_INPUT_TOKENS[client.model], - }), + model: client.model, + ...(typeof OPENAI_MAX_INPUT_TOKENS[client.model] === "number" + ? { maxInputTokens: OPENAI_MAX_INPUT_TOKENS[client.model] } + : {}), + embedQuery: async (text) => { + const [vec] = await embed([text], "query"); + return vec ?? []; + }, + embedBatch: async (texts) => await embed(texts, "document"), + }, client, }; } @@ -50,10 +85,16 @@ export async function createOpenAiEmbeddingProvider( export async function resolveOpenAiEmbeddingClient( options: MemoryEmbeddingProviderCreateOptions, ): Promise { - return await resolveRemoteEmbeddingClient({ + const client = await resolveRemoteEmbeddingClient({ provider: "openai", options, defaultBaseUrl: DEFAULT_OPENAI_BASE_URL, normalizeModel: normalizeOpenAiModel, }); + return { + ...client, + inputType: options.inputType, + queryInputType: options.queryInputType, + documentInputType: options.documentInputType, + }; } diff --git a/extensions/openai/memory-embedding-adapter.test.ts b/extensions/openai/memory-embedding-adapter.test.ts new file mode 100644 index 00000000000..2cdc26d4742 --- /dev/null +++ b/extensions/openai/memory-embedding-adapter.test.ts @@ -0,0 +1,76 @@ +import type { MemoryEmbeddingProvider } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + createOpenAiEmbeddingProvider: vi.fn(), + runOpenAiEmbeddingBatches: vi.fn(async () => new Map([["0", [1, 0]]])), +})); + +vi.mock("./embedding-provider.js", () => ({ + DEFAULT_OPENAI_EMBEDDING_MODEL: "text-embedding-3-small", + createOpenAiEmbeddingProvider: mocks.createOpenAiEmbeddingProvider, +})); + +vi.mock("./embedding-batch.js", () => ({ + OPENAI_BATCH_ENDPOINT: "/v1/embeddings", + runOpenAiEmbeddingBatches: mocks.runOpenAiEmbeddingBatches, +})); + +import { openAiMemoryEmbeddingProviderAdapter } from "./memory-embedding-adapter.js"; + +const provider: MemoryEmbeddingProvider = { + id: "openai", + model: "text-embedding-3-small", + embedQuery: async () => [1, 0], + embedBatch: async (texts) => texts.map(() => [1, 0]), +}; + +describe("OpenAI memory embedding adapter", () => { + beforeEach(() => { + mocks.createOpenAiEmbeddingProvider.mockReset(); + mocks.runOpenAiEmbeddingBatches.mockClear(); + mocks.createOpenAiEmbeddingProvider.mockResolvedValue({ + provider, + client: { + baseUrl: "https://embeddings.example/v1", + headers: {}, + model: "text-embedding-3-small", + inputType: "passage", + documentInputType: "document", + }, + }); + }); + + it("sends document input_type in OpenAI batch embedding requests", async () => { + const result = await openAiMemoryEmbeddingProviderAdapter.create({ + config: {} as never, + provider: "openai", + model: "text-embedding-3-small", + fallback: "none", + }); + + await result.runtime?.batchEmbed?.({ + agentId: "main", + chunks: [{ text: "doc one" }], + wait: true, + concurrency: 1, + pollIntervalMs: 1000, + timeoutMs: 60_000, + debug: () => {}, + }); + + expect(mocks.runOpenAiEmbeddingBatches).toHaveBeenCalledWith( + expect.objectContaining({ + requests: [ + expect.objectContaining({ + body: { + model: "text-embedding-3-small", + input: "doc one", + input_type: "document", + }, + }), + ], + }), + ); + }); +}); diff --git a/extensions/openai/memory-embedding-adapter.ts b/extensions/openai/memory-embedding-adapter.ts index 16a255c4af9..b592b6c005b 100644 --- a/extensions/openai/memory-embedding-adapter.ts +++ b/extensions/openai/memory-embedding-adapter.ts @@ -32,9 +32,11 @@ export const openAiMemoryEmbeddingProviderAdapter: MemoryEmbeddingProviderAdapte provider: "openai", baseUrl: client.baseUrl, model: client.model, + documentInputType: client.documentInputType ?? client.inputType, headers: sanitizeEmbeddingCacheHeaders(client.headers, ["authorization"]), }, batchEmbed: async (batch) => { + const inputType = client.documentInputType ?? client.inputType; const byCustomId = await runOpenAiEmbeddingBatches({ openAi: client, agentId: batch.agentId, @@ -45,6 +47,7 @@ export const openAiMemoryEmbeddingProviderAdapter: MemoryEmbeddingProviderAdapte body: { model: client.model, input: chunk.text, + ...(inputType ? { input_type: inputType } : {}), }, })), wait: batch.wait, diff --git a/src/agents/memory-search.test.ts b/src/agents/memory-search.test.ts index 9ce85d81dc9..992724337f5 100644 --- a/src/agents/memory-search.test.ts +++ b/src/agents/memory-search.test.ts @@ -480,6 +480,33 @@ describe("memory search config", () => { expect(resolved?.model).toBe("nomic-embed-text"); }); + it("merges memory search input_type overrides", () => { + const cfg = asConfig({ + agents: { + defaults: { + memorySearch: { + provider: "openai", + inputType: "passage", + queryInputType: "query", + }, + }, + list: [ + { + id: "main", + default: true, + memorySearch: { + documentInputType: "document", + }, + }, + ], + }, + }); + const resolved = resolveMemorySearchConfig(cfg, "main"); + expect(resolved?.inputType).toBe("passage"); + expect(resolved?.queryInputType).toBe("query"); + expect(resolved?.documentInputType).toBe("document"); + }); + it("defaults session delta thresholds", () => { const cfg = asConfig({ agents: { diff --git a/src/agents/memory-search.ts b/src/agents/memory-search.ts index f5fbbd7c9b1..aaf430db5fc 100644 --- a/src/agents/memory-search.ts +++ b/src/agents/memory-search.ts @@ -35,6 +35,9 @@ export type ResolvedMemorySearchConfig = { }; fallback: string; model: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; outputDimensionality?: number; local: { modelPath?: string; @@ -193,6 +196,11 @@ function mergeConfig( : undefined; const modelDefault = provider === "auto" ? undefined : primaryAdapter?.defaultModel; const model = overrides?.model ?? defaults?.model ?? modelDefault ?? ""; + const inputType = overrides?.inputType?.trim() || defaults?.inputType?.trim() || undefined; + const queryInputType = + overrides?.queryInputType?.trim() || defaults?.queryInputType?.trim() || undefined; + const documentInputType = + overrides?.documentInputType?.trim() || defaults?.documentInputType?.trim() || undefined; const outputDimensionality = overrides?.outputDimensionality ?? defaults?.outputDimensionality; const local = { modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath, @@ -306,6 +314,9 @@ function mergeConfig( }, fallback, model, + inputType, + queryInputType, + documentInputType, outputDimensionality, local, store, diff --git a/src/config/schema.base.generated.ts b/src/config/schema.base.generated.ts index 666d7c8c479..e63da5864dc 100644 --- a/src/config/schema.base.generated.ts +++ b/src/config/schema.base.generated.ts @@ -4415,6 +4415,27 @@ export const GENERATED_BASE_CONFIG_SCHEMA: BaseConfigSchemaResponse = { description: "Embedding model override used by the selected memory provider when a non-default model is required. Set this only when you need explicit recall quality/cost tuning beyond provider defaults.", }, + inputType: { + type: "string", + minLength: 1, + title: "Memory Search Input Type", + description: + "Optional provider-specific `input_type` value forwarded to compatible embedding requests when the same label should apply to both query and document embeddings. For asymmetric providers, prefer queryInputType and documentInputType.", + }, + queryInputType: { + type: "string", + minLength: 1, + title: "Memory Search Query Input Type", + description: + "Optional provider-specific `input_type` value for query-time memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a query label.", + }, + documentInputType: { + type: "string", + minLength: 1, + title: "Memory Search Document Input Type", + description: + "Optional provider-specific `input_type` value for document and indexing memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a passage or document label.", + }, outputDimensionality: { type: "integer", exclusiveMinimum: 0, @@ -6363,6 +6384,18 @@ export const GENERATED_BASE_CONFIG_SCHEMA: BaseConfigSchemaResponse = { model: { type: "string", }, + inputType: { + type: "string", + minLength: 1, + }, + queryInputType: { + type: "string", + minLength: 1, + }, + documentInputType: { + type: "string", + minLength: 1, + }, outputDimensionality: { type: "integer", exclusiveMinimum: 0, @@ -26056,6 +26089,21 @@ export const GENERATED_BASE_CONFIG_SCHEMA: BaseConfigSchemaResponse = { help: "Embedding model override used by the selected memory provider when a non-default model is required. Set this only when you need explicit recall quality/cost tuning beyond provider defaults.", tags: ["models"], }, + "agents.defaults.memorySearch.inputType": { + label: "Memory Search Input Type", + help: "Optional provider-specific `input_type` value forwarded to compatible embedding requests when the same label should apply to both query and document embeddings. For asymmetric providers, prefer queryInputType and documentInputType.", + tags: ["advanced"], + }, + "agents.defaults.memorySearch.queryInputType": { + label: "Memory Search Query Input Type", + help: "Optional provider-specific `input_type` value for query-time memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a query label.", + tags: ["advanced"], + }, + "agents.defaults.memorySearch.documentInputType": { + label: "Memory Search Document Input Type", + help: "Optional provider-specific `input_type` value for document and indexing memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a passage or document label.", + tags: ["advanced"], + }, "agents.defaults.memorySearch.outputDimensionality": { label: "Memory Search Output Dimensionality", help: "Provider-specific output vector size override for memory embeddings. Gemini embedding-2 supports 768, 1536, or 3072; Bedrock families such as Titan V2, Cohere V4, and Nova expose their own allowed sizes. Expect a full reindex when you change it because stored vector dimensions must stay consistent.", diff --git a/src/config/schema.help.quality.test.ts b/src/config/schema.help.quality.test.ts index 03521d6fb0f..34c3354435b 100644 --- a/src/config/schema.help.quality.test.ts +++ b/src/config/schema.help.quality.test.ts @@ -93,6 +93,9 @@ const TARGET_KEYS = [ "agents.defaults.memorySearch.remote.batch.timeoutMinutes", "agents.defaults.memorySearch.local.modelPath", "agents.defaults.memorySearch.store.path", + "agents.defaults.memorySearch.inputType", + "agents.defaults.memorySearch.queryInputType", + "agents.defaults.memorySearch.documentInputType", "agents.defaults.memorySearch.outputDimensionality", "agents.defaults.memorySearch.store.vector.enabled", "agents.defaults.memorySearch.store.vector.extensionPath", diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 8cfc9f0dc14..eefcd8b54cb 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -1000,6 +1000,12 @@ export const FIELD_HELP: Record = { 'Selects the embedding backend used to build/query memory vectors: "openai", "gemini", "voyage", "mistral", "bedrock", "lmstudio", "ollama", or "local". Keep your most reliable provider here and configure fallback for resilience.', "agents.defaults.memorySearch.model": "Embedding model override used by the selected memory provider when a non-default model is required. Set this only when you need explicit recall quality/cost tuning beyond provider defaults.", + "agents.defaults.memorySearch.inputType": + "Optional provider-specific `input_type` value forwarded to compatible embedding requests when the same label should apply to both query and document embeddings. For asymmetric providers, prefer queryInputType and documentInputType.", + "agents.defaults.memorySearch.queryInputType": + "Optional provider-specific `input_type` value for query-time memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a query label.", + "agents.defaults.memorySearch.documentInputType": + "Optional provider-specific `input_type` value for document and indexing memory embeddings. Use this with OpenAI-compatible asymmetric embedding endpoints that require a passage or document label.", "agents.defaults.memorySearch.outputDimensionality": "Provider-specific output vector size override for memory embeddings. Gemini embedding-2 supports 768, 1536, or 3072; Bedrock families such as Titan V2, Cohere V4, and Nova expose their own allowed sizes. Expect a full reindex when you change it because stored vector dimensions must stay consistent.", "agents.defaults.memorySearch.remote.baseUrl": diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index f641e4da4c4..5927605ceb5 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -423,6 +423,9 @@ export const FIELD_LABELS: Record = { "agents.defaults.memorySearch.remote.batch.pollIntervalMs": "Remote Batch Poll Interval (ms)", "agents.defaults.memorySearch.remote.batch.timeoutMinutes": "Remote Batch Timeout (min)", "agents.defaults.memorySearch.model": "Memory Search Model", + "agents.defaults.memorySearch.inputType": "Memory Search Input Type", + "agents.defaults.memorySearch.queryInputType": "Memory Search Query Input Type", + "agents.defaults.memorySearch.documentInputType": "Memory Search Document Input Type", "agents.defaults.memorySearch.outputDimensionality": "Memory Search Output Dimensionality", "agents.defaults.memorySearch.fallback": "Memory Search Fallback", "agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path", diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index 84bb85e3bc6..0a54f94ce37 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -382,6 +382,12 @@ export type MemorySearchConfig = { fallback?: string; /** Embedding model id (remote) or alias (local). */ model?: string; + /** Optional provider-specific embedding input_type for query and document requests. */ + inputType?: string; + /** Optional provider-specific embedding input_type for query-time memory search. */ + queryInputType?: string; + /** Optional provider-specific embedding input_type for document/index embeddings. */ + documentInputType?: string; /** * Gemini embedding-2 models only: output vector dimensions. * Supported values today are 768, 1536, and 3072. diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index bde067c14ca..6a14022a2a0 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -680,6 +680,9 @@ export const MemorySearchSchema = z .optional(), fallback: z.string().optional(), model: z.string().optional(), + inputType: z.string().min(1).optional(), + queryInputType: z.string().min(1).optional(), + documentInputType: z.string().min(1).optional(), outputDimensionality: z.number().int().positive().optional(), local: z .object({ diff --git a/src/memory-host-sdk/host/embeddings.types.ts b/src/memory-host-sdk/host/embeddings.types.ts index 218e3c91508..ce906c3a066 100644 --- a/src/memory-host-sdk/host/embeddings.types.ts +++ b/src/memory-host-sdk/host/embeddings.types.ts @@ -34,6 +34,9 @@ export type EmbeddingProviderOptions = { headers?: Record; }; model: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; fallback?: EmbeddingProviderFallback; local?: { modelPath?: string; diff --git a/src/plugins/memory-embedding-providers.ts b/src/plugins/memory-embedding-providers.ts index a9620b2fff8..993f9db18b8 100644 --- a/src/plugins/memory-embedding-providers.ts +++ b/src/plugins/memory-embedding-providers.ts @@ -45,6 +45,9 @@ export type MemoryEmbeddingProviderCreateOptions = { headers?: Record; }; model: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; local?: { modelPath?: string; modelCacheDir?: string; diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 0df545c102e..26597215af2 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -913,6 +913,9 @@ export type ProviderCreateEmbeddingProviderContext = { headers?: Record; }; providerApiKey?: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; outputDimensionality?: number; taskType?: string; };