Merge remote-tracking branch 'origin/main' into cs/codex-native-web-search-spec

# Conflicts:
#	src/agents/pi-embedded-runner/extra-params.ts
#	src/agents/pi-embedded-runner/openai-stream-wrappers.ts

extensions/google/gemini-cli-provider.ts

@@ -1,10 +1,10 @@
-import { fetchGeminiUsage } from "../../src/infra/provider-usage.fetch.js";
-import { buildOauthProviderAuthResult } from "../../src/plugin-sdk/provider-auth-result.js";
+import { buildOauthProviderAuthResult } from "openclaw/plugin-sdk/core";
 import type {
   OpenClawPluginApi,
   ProviderAuthContext,
   ProviderFetchUsageSnapshotContext,
-} from "../../src/plugins/types.js";
+} from "openclaw/plugin-sdk/core";
+import { fetchGeminiUsage } from "openclaw/plugin-sdk/provider-usage";
 import { loginGeminiCliOAuth } from "./oauth.js";
 import { isModernGoogleModel, resolveGoogle31ForwardCompatModel } from "./provider-models.js";
 

extensions/google/index.ts

@@ -1,16 +1,16 @@
+import { emptyPluginConfigSchema, type OpenClawPluginApi } from "openclaw/plugin-sdk/core";
+import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth";
+import {
+  GOOGLE_GEMINI_DEFAULT_MODEL,
+  applyGoogleGeminiModelDefault,
+} from "openclaw/plugin-sdk/provider-models";
 import {
   createPluginBackedWebSearchProvider,
   getScopedCredentialValue,
   setScopedCredentialValue,
-} from "../../src/agents/tools/web-search-plugin-factory.js";
-import {
-  GOOGLE_GEMINI_DEFAULT_MODEL,
-  applyGoogleGeminiModelDefault,
-} from "../../src/commands/google-gemini-model-default.js";
-import { emptyPluginConfigSchema } from "../../src/plugins/config-schema.js";
-import { createProviderApiKeyAuthMethod } from "../../src/plugins/provider-api-key-auth.js";
-import type { OpenClawPluginApi } from "../../src/plugins/types.js";
+} from "openclaw/plugin-sdk/provider-web-search";
 import { registerGoogleGeminiCliProvider } from "./gemini-cli-provider.js";
+import { googleMediaUnderstandingProvider } from "./media-understanding-provider.js";
 import { isModernGoogleModel, resolveGoogle31ForwardCompatModel } from "./provider-models.js";
 
 const googlePlugin = {
@@ -51,6 +51,7 @@ const googlePlugin = {
       isModernModelRef: ({ modelId }) => isModernGoogleModel(modelId),
     });
     registerGoogleGeminiCliProvider(api);
+    api.registerMediaUnderstandingProvider(googleMediaUnderstandingProvider);
     api.registerWebSearchProvider(
       createPluginBackedWebSearchProvider({
         id: "gemini",

extensions/google/media-understanding-provider.ts

@@ -0,0 +1,149 @@
+import { normalizeGoogleModelId, parseGeminiAuth } from "openclaw/plugin-sdk/google";
+import {
+  assertOkOrThrowHttpError,
+  describeImageWithModel,
+  describeImagesWithModel,
+  normalizeBaseUrl,
+  postJsonRequest,
+  type AudioTranscriptionRequest,
+  type AudioTranscriptionResult,
+  type MediaUnderstandingProvider,
+  type VideoDescriptionRequest,
+  type VideoDescriptionResult,
+} from "openclaw/plugin-sdk/media-understanding";
+
+export const DEFAULT_GOOGLE_AUDIO_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+export const DEFAULT_GOOGLE_VIDEO_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
+const DEFAULT_GOOGLE_AUDIO_MODEL = "gemini-3-flash-preview";
+const DEFAULT_GOOGLE_VIDEO_MODEL = "gemini-3-flash-preview";
+const DEFAULT_GOOGLE_AUDIO_PROMPT = "Transcribe the audio.";
+const DEFAULT_GOOGLE_VIDEO_PROMPT = "Describe the video.";
+
+async function generateGeminiInlineDataText(params: {
+  buffer: Buffer;
+  mime?: string;
+  apiKey: string;
+  baseUrl?: string;
+  headers?: Record<string, string>;
+  model?: string;
+  prompt?: string;
+  timeoutMs: number;
+  fetchFn?: typeof fetch;
+  defaultBaseUrl: string;
+  defaultModel: string;
+  defaultPrompt: string;
+  defaultMime: string;
+  httpErrorLabel: string;
+  missingTextError: string;
+}): Promise<{ text: string; model: string }> {
+  const fetchFn = params.fetchFn ?? fetch;
+  const baseUrl = normalizeBaseUrl(params.baseUrl, params.defaultBaseUrl);
+  const allowPrivate = Boolean(params.baseUrl?.trim());
+  const model = (() => {
+    const trimmed = params.model?.trim();
+    if (!trimmed) {
+      return params.defaultModel;
+    }
+    return normalizeGoogleModelId(trimmed);
+  })();
+  const url = `${baseUrl}/models/${model}:generateContent`;
+
+  const authHeaders = parseGeminiAuth(params.apiKey);
+  const headers = new Headers(params.headers);
+  for (const [key, value] of Object.entries(authHeaders.headers)) {
+    if (!headers.has(key)) {
+      headers.set(key, value);
+    }
+  }
+
+  const prompt = (() => {
+    const trimmed = params.prompt?.trim();
+    return trimmed || params.defaultPrompt;
+  })();
+
+  const body = {
+    contents: [
+      {
+        role: "user",
+        parts: [
+          { text: prompt },
+          {
+            inline_data: {
+              mime_type: params.mime ?? params.defaultMime,
+              data: params.buffer.toString("base64"),
+            },
+          },
+        ],
+      },
+    ],
+  };
+
+  const { response: res, release } = await postJsonRequest({
+    url,
+    headers,
+    body,
+    timeoutMs: params.timeoutMs,
+    fetchFn,
+    allowPrivateNetwork: allowPrivate,
+  });
+
+  try {
+    await assertOkOrThrowHttpError(res, params.httpErrorLabel);
+
+    const payload = (await res.json()) as {
+      candidates?: Array<{
+        content?: { parts?: Array<{ text?: string }> };
+      }>;
+    };
+    const parts = payload.candidates?.[0]?.content?.parts ?? [];
+    const text = parts
+      .map((part) => part?.text?.trim())
+      .filter(Boolean)
+      .join("\n");
+    if (!text) {
+      throw new Error(params.missingTextError);
+    }
+    return { text, model };
+  } finally {
+    await release();
+  }
+}
+
+export async function transcribeGeminiAudio(
+  params: AudioTranscriptionRequest,
+): Promise<AudioTranscriptionResult> {
+  const { text, model } = await generateGeminiInlineDataText({
+    ...params,
+    defaultBaseUrl: DEFAULT_GOOGLE_AUDIO_BASE_URL,
+    defaultModel: DEFAULT_GOOGLE_AUDIO_MODEL,
+    defaultPrompt: DEFAULT_GOOGLE_AUDIO_PROMPT,
+    defaultMime: "audio/wav",
+    httpErrorLabel: "Audio transcription failed",
+    missingTextError: "Audio transcription response missing text",
+  });
+  return { text, model };
+}
+
+export async function describeGeminiVideo(
+  params: VideoDescriptionRequest,
+): Promise<VideoDescriptionResult> {
+  const { text, model } = await generateGeminiInlineDataText({
+    ...params,
+    defaultBaseUrl: DEFAULT_GOOGLE_VIDEO_BASE_URL,
+    defaultModel: DEFAULT_GOOGLE_VIDEO_MODEL,
+    defaultPrompt: DEFAULT_GOOGLE_VIDEO_PROMPT,
+    defaultMime: "video/mp4",
+    httpErrorLabel: "Video description failed",
+    missingTextError: "Video description response missing text",
+  });
+  return { text, model };
+}
+
+export const googleMediaUnderstandingProvider: MediaUnderstandingProvider = {
+  id: "google",
+  capabilities: ["image", "audio", "video"],
+  describeImage: describeImageWithModel,
+  describeImages: describeImagesWithModel,
+  transcribeAudio: transcribeGeminiAudio,
+  describeVideo: describeGeminiVideo,
+};

extensions/google/oauth.flow.ts

@@ -1,6 +1,6 @@
 import { createHash, randomBytes } from "node:crypto";
 import { createServer } from "node:http";
-import { isWSL2Sync } from "../../src/infra/wsl.js";
+import { isWSL2Sync } from "openclaw/plugin-sdk/infra-runtime";
 import { resolveOAuthClientConfig } from "./oauth.credentials.js";
 import { AUTH_URL, REDIRECT_URI, SCOPES } from "./oauth.shared.js";
 

extensions/google/oauth.http.ts

@@ -1,4 +1,4 @@
-import { fetchWithSsrFGuard } from "../../src/infra/net/fetch-guard.js";
+import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/infra-runtime";
 import { DEFAULT_FETCH_TIMEOUT_MS } from "./oauth.shared.js";
 
 export async function fetchWithTimeout(

extensions/google/provider-models.ts

@@ -1,39 +1,14 @@
-import { normalizeModelCompat } from "../../src/agents/model-compat.js";
 import type {
   ProviderResolveDynamicModelContext,
   ProviderRuntimeModel,
-} from "../../src/plugins/types.js";
+} from "openclaw/plugin-sdk/core";
+import { cloneFirstTemplateModel } from "../../src/plugins/provider-model-helpers.js";
 
 const GEMINI_3_1_PRO_PREFIX = "gemini-3.1-pro";
 const GEMINI_3_1_FLASH_PREFIX = "gemini-3.1-flash";
 const GEMINI_3_1_PRO_TEMPLATE_IDS = ["gemini-3-pro-preview"] as const;
 const GEMINI_3_1_FLASH_TEMPLATE_IDS = ["gemini-3-flash-preview"] as const;
 
-function cloneFirstTemplateModel(params: {
-  providerId: string;
-  modelId: string;
-  templateIds: readonly string[];
-  ctx: ProviderResolveDynamicModelContext;
-}): ProviderRuntimeModel | undefined {
-  const trimmedModelId = params.modelId.trim();
-  for (const templateId of [...new Set(params.templateIds)].filter(Boolean)) {
-    const template = params.ctx.modelRegistry.find(
-      params.providerId,
-      templateId,
-    ) as ProviderRuntimeModel | null;
-    if (!template) {
-      continue;
-    }
-    return normalizeModelCompat({
-      ...template,
-      id: trimmedModelId,
-      name: trimmedModelId,
-      reasoning: true,
-    } as ProviderRuntimeModel);
-  }
-  return undefined;
-}
-
 export function resolveGoogle31ForwardCompatModel(params: {
   providerId: string;
   ctx: ProviderResolveDynamicModelContext;
@@ -55,6 +30,7 @@ export function resolveGoogle31ForwardCompatModel(params: {
     modelId: trimmed,
     templateIds,
     ctx: params.ctx,
+    patch: { reasoning: true },
   });
 }