From 9860db5cea721bfda38bff4b6a3bcad7226f9136 Mon Sep 17 00:00:00 2001
From: scoootscooob <167050519+scoootscooob@users.noreply.github.com>
Date: Sat, 4 Apr 2026 18:24:20 -0700
Subject: [PATCH] fix(memory): allow Gemini multimodal fallback before registry
 hydration (#61085)

* fix(memory): allow Gemini multimodal fallback

* docs(memory): clarify multimodal fallback
---
 packages/memory-host-sdk/src/multimodal.ts |  1 +
 src/agents/memory-search.test.ts           | 95 ++++++++++++++--------
 src/agents/memory-search.ts                | 20 ++++-
 3 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/packages/memory-host-sdk/src/multimodal.ts b/packages/memory-host-sdk/src/multimodal.ts
index eb11867ac3a..5c62de35490 100644
--- a/packages/memory-host-sdk/src/multimodal.ts
+++ b/packages/memory-host-sdk/src/multimodal.ts
@@ -1,5 +1,6 @@
 export {
   isMemoryMultimodalEnabled,
   normalizeMemoryMultimodalSettings,
+  supportsMemoryMultimodalEmbeddings,
   type MemoryMultimodalSettings,
 } from "./host/multimodal.js";
diff --git a/src/agents/memory-search.test.ts b/src/agents/memory-search.test.ts
index 17f7b99fe3d..7b14d294d31 100644
--- a/src/agents/memory-search.test.ts
+++ b/src/agents/memory-search.test.ts
@@ -8,21 +8,20 @@ import { resolveMemorySearchConfig } from "./memory-search.js";
 
 const asConfig = (cfg: OpenClawConfig): OpenClawConfig => cfg;
 
-describe("memory search config", () => {
-  beforeEach(() => {
-    clearMemoryEmbeddingProviders();
-    registerMemoryEmbeddingProvider({
-      id: "openai",
-      defaultModel: "text-embedding-3-small",
-      transport: "remote",
-      create: async () => ({ provider: null }),
-    });
-    registerMemoryEmbeddingProvider({
-      id: "local",
-      defaultModel: "local-default",
-      transport: "local",
-      create: async () => ({ provider: null }),
-    });
+function registerBaseMemoryEmbeddingProviders(options?: { includeGemini?: boolean }): void {
+  registerMemoryEmbeddingProvider({
+    id: "openai",
+    defaultModel: "text-embedding-3-small",
+    transport: "remote",
+    create: async () => ({ provider: null }),
+  });
+  registerMemoryEmbeddingProvider({
+    id: "local",
+    defaultModel: "local-default",
+    transport: "local",
+    create: async () => ({ provider: null }),
+  });
+  if (options?.includeGemini !== false) {
     registerMemoryEmbeddingProvider({
       id: "gemini",
       defaultModel: "gemini-embedding-001",
@@ -34,24 +33,31 @@ describe("memory search config", () => {
           .replace(/^(gemini|google)\//, "") === "gemini-embedding-2-preview",
       create: async () => ({ provider: null }),
     });
-    registerMemoryEmbeddingProvider({
-      id: "voyage",
-      defaultModel: "voyage-4-large",
-      transport: "remote",
-      create: async () => ({ provider: null }),
-    });
-    registerMemoryEmbeddingProvider({
-      id: "mistral",
-      defaultModel: "mistral-embed",
-      transport: "remote",
-      create: async () => ({ provider: null }),
-    });
-    registerMemoryEmbeddingProvider({
-      id: "ollama",
-      defaultModel: "nomic-embed-text",
-      transport: "remote",
-      create: async () => ({ provider: null }),
-    });
+  }
+  registerMemoryEmbeddingProvider({
+    id: "voyage",
+    defaultModel: "voyage-4-large",
+    transport: "remote",
+    create: async () => ({ provider: null }),
+  });
+  registerMemoryEmbeddingProvider({
+    id: "mistral",
+    defaultModel: "mistral-embed",
+    transport: "remote",
+    create: async () => ({ provider: null }),
+  });
+  registerMemoryEmbeddingProvider({
+    id: "ollama",
+    defaultModel: "nomic-embed-text",
+    transport: "remote",
+    create: async () => ({ provider: null }),
+  });
+}
+
+describe("memory search config", () => {
+  beforeEach(() => {
+    clearMemoryEmbeddingProviders();
+    registerBaseMemoryEmbeddingProviders();
   });
 
   afterEach(() => {
@@ -313,6 +319,29 @@ describe("memory search config", () => {
     );
   });
 
+  it("accepts Gemini multimodal memory even when the runtime registry has not registered Gemini yet", () => {
+    clearMemoryEmbeddingProviders();
+    registerBaseMemoryEmbeddingProviders({ includeGemini: false });
+    const cfg = asConfig({
+      agents: {
+        defaults: {
+          memorySearch: {
+            provider: "gemini",
+            model: "gemini-embedding-2-preview",
+            multimodal: { enabled: true, modalities: ["image"] },
+          },
+        },
+      },
+    });
+    const resolved = resolveMemorySearchConfig(cfg, "main");
+    expect(resolved?.provider).toBe("gemini");
+    expect(resolved?.multimodal).toEqual({
+      enabled: true,
+      modalities: ["image"],
+      maxFileBytes: 10 * 1024 * 1024,
+    });
+  });
+
   it("rejects multimodal memory when fallback is configured", () => {
     const cfg = asConfig({
       agents: {
diff --git a/src/agents/memory-search.ts b/src/agents/memory-search.ts
index 88d48b00ae6..ec0e8c10178 100644
--- a/src/agents/memory-search.ts
+++ b/src/agents/memory-search.ts
@@ -6,6 +6,7 @@ import type { SecretInput } from "../config/types.secrets.js";
 import {
   isMemoryMultimodalEnabled,
   normalizeMemoryMultimodalSettings,
+  supportsMemoryMultimodalEmbeddings,
   type MemoryMultimodalSettings,
 } from "../memory-host-sdk/multimodal.js";
 import { getMemoryEmbeddingProvider } from "../plugins/memory-embedding-providers.js";
@@ -379,11 +380,24 @@ export function resolveMemorySearchConfig(
   const multimodalActive = isMemoryMultimodalEnabled(resolved.multimodal);
   const multimodalProvider =
     resolved.provider === "auto" ? undefined : getMemoryEmbeddingProvider(resolved.provider);
+  const builtinMultimodalSupport =
+    resolved.provider === "auto"
+      ? false
+      : supportsMemoryMultimodalEmbeddings({
+          provider: resolved.provider,
+          model: resolved.model,
+        });
   if (
     multimodalActive &&
-    !multimodalProvider?.supportsMultimodalEmbeddings?.({
-      model: resolved.model,
-    })
+    !(
+      // Fall back to the built-in helper when the provider is not registered yet
+      // or when a registered adapter does not implement multimodal capability checks.
+      (
+        multimodalProvider?.supportsMultimodalEmbeddings?.({
+          model: resolved.model,
+        }) ?? builtinMultimodalSupport
+      )
+    )
   ) {
     throw new Error(
       "agents.*.memorySearch.multimodal requires a provider adapter that supports multimodal embeddings for the configured model.",