From e7b652798fdde69b3ff112a5dea2b9b56cd3eaa3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 06:32:45 +0100 Subject: [PATCH] fix(xai): remove synthetic batch STT model (#103419) --- docs/providers/xai.md | 9 ++- extensions/xai/doctor-contract-api.test.ts | 64 +++++++++++++++++++ extensions/xai/doctor-contract-api.ts | 50 +++++++++++++++ extensions/xai/index.test.ts | 2 +- extensions/xai/openclaw.plugin.json | 3 - extensions/xai/openclaw.plugin.test.ts | 5 ++ extensions/xai/stt.test.ts | 16 ++--- extensions/xai/stt.ts | 6 -- extensions/xai/xai.live.test.ts | 4 +- .../runner.auto-audio.test.ts | 49 ++++++++++++++ src/media-understanding/runner.ts | 16 +++-- src/plugins/doctor-contract-registry.test.ts | 21 ++++++ src/plugins/doctor-contract-registry.ts | 31 +++++++++ 13 files changed, 243 insertions(+), 33 deletions(-) diff --git a/docs/providers/xai.md b/docs/providers/xai.md index 40f5dc04813b..c82464321c93 100644 --- a/docs/providers/xai.md +++ b/docs/providers/xai.md @@ -351,9 +351,10 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 The bundled `xai` plugin registers batch speech-to-text through OpenClaw's media-understanding transcription surface. - - Default model: `grok-stt` - Endpoint: xAI REST `/v1/stt` - Input path: multipart audio file upload + - Model selection: xAI chooses the transcription model internally; the + endpoint has no model selector - Used wherever inbound audio transcription reads `tools.media.audio`, including Discord voice-channel segments and channel audio attachments @@ -368,7 +369,6 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 { type: "provider", provider: "xai", - model: "grok-stt", }, ], }, @@ -379,9 +379,8 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 Language can be supplied through the shared audio media config or per-call transcription request. Prompt hints are accepted by the shared OpenClaw - surface, but the xAI REST STT integration forwards only file, model, and - language because those map cleanly to the current public xAI - endpoint. + surface, but the xAI REST STT integration forwards only file and language + because those map to the current public xAI endpoint. diff --git a/extensions/xai/doctor-contract-api.test.ts b/extensions/xai/doctor-contract-api.test.ts index d4aeae7923a4..29c28964ba5d 100644 --- a/extensions/xai/doctor-contract-api.test.ts +++ b/extensions/xai/doctor-contract-api.test.ts @@ -109,6 +109,70 @@ describe("xAI doctor contract", () => { expect(normalizeCompatibilityConfig({ cfg: config })).toEqual({ config, changes: [] }); }); + + it("removes only the obsolete xAI STT model selector from media entries", () => { + const config = { + tools: { + media: { + models: [ + { + provider: " xAI ", + model: " GROK-STT ", + language: "en", + timeoutSeconds: 30, + }, + { type: "provider", provider: "xai", model: "custom-stt" }, + { type: "provider", provider: "openai", model: "grok-stt" }, + { type: "cli", provider: "xai", model: "grok-stt", command: "whisper" }, + { provider: "x-ai", model: "grok-stt" }, + { provider: "xai" }, + ], + audio: { + models: [ + { + type: "provider", + provider: "xai", + model: "grok-stt", + profile: "speech", + timeoutSeconds: 45, + }, + ], + }, + }, + }, + } as unknown as OpenClawConfig; + + expect( + legacyConfigRules.filter((rule) => rule.match(readPathForTest(config, rule.path))), + ).toHaveLength(2); + + const result = normalizeCompatibilityConfig({ cfg: config }); + + expect(result.changes).toHaveLength(2); + expect(result.config).not.toBe(config); + expect(result.config.tools?.media?.models).toEqual([ + { provider: " xAI ", language: "en", timeoutSeconds: 30 }, + { type: "provider", provider: "xai", model: "custom-stt" }, + { type: "provider", provider: "openai", model: "grok-stt" }, + { type: "cli", provider: "xai", model: "grok-stt", command: "whisper" }, + { provider: "x-ai", model: "grok-stt" }, + { provider: "xai" }, + ]); + expect(result.config.tools?.media?.audio?.models).toEqual([ + { + type: "provider", + provider: "xai", + profile: "speech", + timeoutSeconds: 45, + }, + ]); + expect(config.tools?.media?.models?.[0]).toHaveProperty("model", " GROK-STT "); + expect(config.tools?.media?.audio?.models?.[0]).toHaveProperty("model", "grok-stt"); + expect(normalizeCompatibilityConfig({ cfg: result.config })).toEqual({ + config: result.config, + changes: [], + }); + }); }); function readPathForTest(root: unknown, path: readonly (string | number)[]): unknown { diff --git a/extensions/xai/doctor-contract-api.ts b/extensions/xai/doctor-contract-api.ts index 002d0a13eb4b..b57f8adb7f78 100644 --- a/extensions/xai/doctor-contract-api.ts +++ b/extensions/xai/doctor-contract-api.ts @@ -54,6 +54,10 @@ const PLUGIN_MODEL_MIGRATIONS: PluginModelMigration[] = [ ["plugins", "entries", "xai", "config", "xSearch"], ].map((path) => ({ path, retiredModels: RETIRED_CODE_MODELS, targetModel: "grok-build-0.1" })), ]; +const XAI_STT_MODEL_LIST_PATHS = [ + ["tools", "media", "models"], + ["tools", "media", "audio", "models"], +] as const; function asRecord(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) @@ -81,12 +85,34 @@ function hasLegacyBuiltinCatalogRows(value: unknown): boolean { return Array.isArray(value) && value.some((model) => isLegacyXaiBuiltinModel(model)); } +function isLegacyXaiSttEntry(value: unknown): boolean { + const entry = asRecord(value); + if (!entry || (entry.type !== undefined && entry.type !== "provider")) { + return false; + } + return ( + typeof entry.provider === "string" && + entry.provider.trim().toLowerCase() === "xai" && + typeof entry.model === "string" && + entry.model.trim().toLowerCase() === "grok-stt" + ); +} + +function hasLegacyXaiSttEntries(value: unknown): boolean { + return Array.isArray(value) && value.some(isLegacyXaiSttEntry); +} + export const legacyConfigRules: LegacyConfigRule[] = [ ...PLUGIN_MODEL_MIGRATIONS.map((migration) => ({ path: migration.path, message: `${migration.path.join(".")}.model uses a retired xAI model; run "openclaw doctor --fix" to use ${migration.targetModel}.`, match: (value: unknown) => isRetiredToolModel(value, migration.retiredModels), })), + ...XAI_STT_MODEL_LIST_PATHS.map((path) => ({ + path: [...path], + message: `${path.join(".")} contains the obsolete xAI grok-stt model selector; run "openclaw doctor --fix" to remove it.`, + match: hasLegacyXaiSttEntries, + })), { path: ["models", "providers", "xai", "models"], message: @@ -121,6 +147,30 @@ export function normalizeCompatibilityConfig({ cfg }: { cfg: OpenClawConfig }): ); } + for (const path of XAI_STT_MODEL_LIST_PATHS) { + if (!hasLegacyXaiSttEntries(readPath(next, path))) { + continue; + } + if (next === cfg) { + next = structuredClone(cfg); + } + const entries = readPath(next, path); + if (!Array.isArray(entries)) { + continue; + } + let removed = 0; + for (const entry of entries) { + if (!isLegacyXaiSttEntry(entry)) { + continue; + } + delete asRecord(entry)?.model; + removed += 1; + } + changes.push( + `Removed the obsolete xAI grok-stt model selector from ${removed} ${path.join(".")} entr${removed === 1 ? "y" : "ies"}.`, + ); + } + const modelsPath = ["models", "providers", "xai", "models"]; const configuredModels = readPath(next, modelsPath); if (hasLegacyBuiltinCatalogRows(configuredModels)) { diff --git a/extensions/xai/index.test.ts b/extensions/xai/index.test.ts index 2853f8b3e0dc..2d0af820b9aa 100644 --- a/extensions/xai/index.test.ts +++ b/extensions/xai/index.test.ts @@ -488,7 +488,7 @@ describe("xai provider plugin", () => { const mediaProvider = requireEntry(mediaProviders, "xai"); expect(mediaProvider.capabilities).toEqual(["audio"]); - expect(mediaProvider.defaultModels).toEqual({ audio: "grok-stt" }); + expect(mediaProvider.defaultModels).toBeUndefined(); const realtimeProvider = requireEntry(realtimeTranscriptionProviders, "xai"); expect(realtimeProvider.label).toBe("xAI Realtime Transcription"); expect(realtimeProvider.aliases).toContain("xai-realtime"); diff --git a/extensions/xai/openclaw.plugin.json b/extensions/xai/openclaw.plugin.json index 164f1735dfaf..b0d7b48bf7ba 100644 --- a/extensions/xai/openclaw.plugin.json +++ b/extensions/xai/openclaw.plugin.json @@ -195,9 +195,6 @@ "mediaUnderstandingProviderMetadata": { "xai": { "capabilities": ["audio"], - "defaultModels": { - "audio": "grok-stt" - }, "autoPriority": { "audio": 25 } diff --git a/extensions/xai/openclaw.plugin.test.ts b/extensions/xai/openclaw.plugin.test.ts index 877c947c61be..6c2a2cd697b7 100644 --- a/extensions/xai/openclaw.plugin.test.ts +++ b/extensions/xai/openclaw.plugin.test.ts @@ -11,6 +11,7 @@ const manifest = JSON.parse( modelCatalog?: { suppressions?: Array<{ provider?: string; model?: string }>; }; + mediaUnderstandingProviderMetadata?: Record }>; }; const XAI_MULTI_AGENT_MODELS = [ @@ -68,4 +69,8 @@ describe("xAI plugin manifest", () => { expect(suppressionRefs).toContain(`xai/${model}`); } }); + + it("does not advertise a batch STT model selector", () => { + expect(manifest.mediaUnderstandingProviderMetadata?.xai?.defaultModels).toBeUndefined(); + }); }); diff --git a/extensions/xai/stt.test.ts b/extensions/xai/stt.test.ts index 33245527e792..42921b089fb3 100644 --- a/extensions/xai/stt.test.ts +++ b/extensions/xai/stt.test.ts @@ -1,10 +1,6 @@ // Xai tests cover stt plugin behavior. import { describe, expect, it, vi } from "vitest"; -import { - buildXaiMediaUnderstandingProvider, - transcribeXaiAudio, - XAI_DEFAULT_STT_MODEL, -} from "./stt.js"; +import { buildXaiMediaUnderstandingProvider, transcribeXaiAudio } from "./stt.js"; const { postTranscriptionRequestMock } = vi.hoisted(() => ({ postTranscriptionRequestMock: vi.fn( @@ -59,13 +55,13 @@ describe("xai stt", () => { mime: "audio/wav", apiKey: "xai-key", baseUrl: "https://api.x.ai/v1/", - model: XAI_DEFAULT_STT_MODEL, + model: "grok-4.3", language: "en", prompt: "ignored provider hint", timeoutMs: 10_000, }); - expect(result).toEqual({ text: "hello from audio", model: XAI_DEFAULT_STT_MODEL }); + expect(result).toEqual({ text: "hello from audio" }); const call = requireLastPostTranscriptionCall(); expect(call.url).toBe("https://api.x.ai/v1/stt"); expect(call.timeoutMs).toBe(10_000); @@ -73,7 +69,7 @@ describe("xai stt", () => { expect(call.headers.get("authorization")).toBe("Bearer xai-key"); expect(call.body).toBeInstanceOf(FormData); const form = call.body as FormData; - expect(form.get("model")).toBe(XAI_DEFAULT_STT_MODEL); + expect(form.get("model")).toBeNull(); expect(form.get("language")).toBe("en"); expect(form.get("prompt")).toBeNull(); expect(form.get("file")).toBeInstanceOf(Blob); @@ -83,7 +79,7 @@ describe("xai stt", () => { const provider = buildXaiMediaUnderstandingProvider(); expect(provider.id).toBe("xai"); expect(provider.capabilities).toEqual(["audio"]); - expect(provider.defaultModels).toEqual({ audio: XAI_DEFAULT_STT_MODEL }); + expect(provider.defaultModels).toBeUndefined(); expect(provider.autoPriority).toEqual({ audio: 25 }); }); @@ -98,7 +94,7 @@ describe("xai stt", () => { mime: "audio/wav", apiKey: "core-resolved-bearer", baseUrl: "https://api.x.ai/v1/", - model: XAI_DEFAULT_STT_MODEL, + model: "grok-4.3", timeoutMs: 10_000, }); const call = requireLastPostTranscriptionCall(); diff --git a/extensions/xai/stt.ts b/extensions/xai/stt.ts index 6dd8043ff821..040c253e6b2a 100644 --- a/extensions/xai/stt.ts +++ b/extensions/xai/stt.ts @@ -15,8 +15,6 @@ import { import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { XAI_BASE_URL } from "./model-definitions.js"; -export const XAI_DEFAULT_STT_MODEL = "grok-stt"; - type XaiSttResponse = { text?: string; }; @@ -44,14 +42,12 @@ export async function transcribeXaiAudio( transport: "media-understanding", }); - const model = normalizeOptionalString(params.model); const language = normalizeOptionalString(params.language); const form = buildAudioTranscriptionFormData({ buffer: params.buffer, fileName: params.fileName, mime: params.mime, fields: { - model, language, }, }); @@ -72,7 +68,6 @@ export async function transcribeXaiAudio( const payload = await readProviderJsonResponse(response, "xai.stt"); return { text: requireTranscriptionText(payload.text, "xAI transcription response missing text"), - ...(model ? { model } : {}), }; } finally { await release(); @@ -86,7 +81,6 @@ export function buildXaiMediaUnderstandingProvider(): MediaUnderstandingProvider return { id: "xai", capabilities: ["audio"], - defaultModels: { audio: XAI_DEFAULT_STT_MODEL }, autoPriority: { audio: 25 }, transcribeAudio: transcribeXaiAudio, }; diff --git a/extensions/xai/xai.live.test.ts b/extensions/xai/xai.live.test.ts index 27a9ada26eb0..2a1e9a4a2df6 100644 --- a/extensions/xai/xai.live.test.ts +++ b/extensions/xai/xai.live.test.ts @@ -19,7 +19,6 @@ import { isBillingErrorMessage } from "openclaw/plugin-sdk/test-env"; import { describe, expect, it } from "vitest"; import { createCodeExecutionTool } from "./code-execution.js"; import plugin from "./index.js"; -import { XAI_DEFAULT_STT_MODEL } from "./stt.js"; const XAI_API_KEY = process.env.XAI_API_KEY ?? ""; const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_XAI_IMAGE_MODEL?.trim() || "grok-imagine-image"; @@ -306,12 +305,11 @@ describeLive("xai plugin live", () => { mime: "audio/mpeg", apiKey: XAI_API_KEY, baseUrl: "https://api.x.ai/v1", - model: XAI_DEFAULT_STT_MODEL, timeoutMs: 90_000, }); const normalized = transcript?.text.toLowerCase() ?? ""; - expect(transcript?.model).toBe(XAI_DEFAULT_STT_MODEL); + expect(transcript?.model).toBeUndefined(); expectOpenClawLiveTranscriptMarker(normalized); expect(normalized).toContain("speech"); expect(normalized).toContain("text"); diff --git a/src/media-understanding/runner.auto-audio.test.ts b/src/media-understanding/runner.auto-audio.test.ts index 452fb66962cf..a3bec2579ff6 100644 --- a/src/media-understanding/runner.auto-audio.test.ts +++ b/src/media-understanding/runner.auto-audio.test.ts @@ -290,6 +290,55 @@ describe("runCapability auto audio entries", () => { expect(seenModel).toBe("gpt-4o-transcribe"); }); + it("does not leak the active xAI chat model into model-less batch STT", async () => { + let runResult: Awaited> | undefined; + let seenModel: string | undefined; + + await withAudioFixture("openclaw-auto-audio-xai", async ({ ctx, media, cache }) => { + const providerRegistry = createProviderRegistry({ + xai: { + id: "xai", + capabilities: ["audio"], + transcribeAudio: async (req) => { + seenModel = req.model; + return { text: "xai audio" }; + }, + }, + }); + const cfg = { + models: { + providers: { + xai: { + apiKey: "xai-test-key", // pragma: allowlist secret + models: [], + }, + }, + }, + } as unknown as OpenClawConfig; + + runResult = await runCapability({ + capability: "audio", + cfg, + ctx, + attachments: cache, + media, + providerRegistry, + activeModel: { provider: "xai", model: "grok-4.3" }, + }); + }); + + if (!runResult) { + throw new Error("expected xAI audio result"); + } + expect(requireCapabilityOutput(runResult, 0)).toEqual({ + kind: "audio.transcription", + attachmentIndex: 0, + provider: "xai", + text: "xai audio", + }); + expect(seenModel).toBeUndefined(); + }); + it("prefers provider keys over auto-detected local whisper", async () => { const binDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auto-audio-bin-")); try { diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts index a72bf56419aa..ba885315d089 100644 --- a/src/media-understanding/runner.ts +++ b/src/media-understanding/runner.ts @@ -623,6 +623,8 @@ async function resolveKeyEntry(params: { ) { return null; } + // The supplied model can belong to the active chat route. Audio providers + // use their own default or stay model-less; explicit media entries bypass this auto path. const resolvedModel = capability === "image" ? await resolveAutoImageModelId({ @@ -632,14 +634,18 @@ async function resolveKeyEntry(params: { explicitModel: model, workspaceDir, }) - : capability === "video" - ? (model ?? + : capability === "audio" + ? resolveDefaultMediaModelFromRegistry({ + providerId, + capability: "audio", + providerRegistry, + }) + : (model ?? resolveDefaultMediaModelFromRegistry({ providerId, capability: "video", providerRegistry, - })) - : model; + })); if (capability === "image" && !resolvedModel) { return null; } @@ -922,7 +928,7 @@ async function resolveActiveModelEntry(params: { providerRegistry: params.providerRegistry, }); } - if ((params.capability === "image" || params.capability === "audio") && !model) { + if (params.capability === "image" && !model) { return null; } return { diff --git a/src/plugins/doctor-contract-registry.test.ts b/src/plugins/doctor-contract-registry.test.ts index 7e2e740f52bd..19ee248ad173 100644 --- a/src/plugins/doctor-contract-registry.test.ts +++ b/src/plugins/doctor-contract-registry.test.ts @@ -391,6 +391,27 @@ describe("doctor-contract-registry module loader", () => { ).toEqual(["ollama-cloud"]); }); + it("collects provider ids from media model entries", () => { + const raw = { + tools: { + media: { + models: [{ provider: " xAI " }, { provider: " " }], + audio: { models: [{ provider: "XAI", model: "grok-stt" }] }, + image: { models: [{ provider: "openai", model: "gpt-5.5" }] }, + video: { models: [{ provider: "gemini", model: "veo" }] }, + }, + }, + }; + + expect(collectRelevantDoctorPluginIds(raw)).toEqual(["gemini", "openai", "xai"]); + expect( + collectRelevantDoctorPluginIdsForTouchedPaths({ + raw, + touchedPaths: [["tools", "media", "audio", "models", "0", "model"]], + }), + ).toEqual(["gemini", "openai", "xai"]); + }); + it("loads a plugin doctor contract when scoped by a contributed provider id", () => { const pluginRoot = makeTempDir(); fs.writeFileSync(path.join(pluginRoot, "doctor-contract-api.ts"), "export {};\n", "utf-8"); diff --git a/src/plugins/doctor-contract-registry.ts b/src/plugins/doctor-contract-registry.ts index bc13c7903104..a045c78fac96 100644 --- a/src/plugins/doctor-contract-registry.ts +++ b/src/plugins/doctor-contract-registry.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id"; import { asNullableRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; import type { LegacyConfigRule } from "../config/legacy.shared.js"; @@ -235,6 +236,30 @@ function hasLegacyElevenLabsTalkFields(raw: unknown): boolean { ); } +function collectMediaProviderIds(root: Record, ids: Set): void { + const media = asNullableRecord(asNullableRecord(root.tools)?.media); + if (!media) { + return; + } + const modelLists = [ + media.models, + asNullableRecord(media.audio)?.models, + asNullableRecord(media.image)?.models, + asNullableRecord(media.video)?.models, + ]; + for (const models of modelLists) { + if (!Array.isArray(models)) { + continue; + } + for (const model of models) { + const provider = asNullableRecord(model)?.provider; + if (typeof provider === "string" && provider.trim()) { + ids.add(normalizeProviderId(provider)); + } + } + } +} + export function collectRelevantDoctorPluginIds(raw: unknown): string[] { const ids = new Set(); const root = asNullableRecord(raw); @@ -265,6 +290,8 @@ export function collectRelevantDoctorPluginIds(raw: unknown): string[] { } } + collectMediaProviderIds(root, ids); + if (hasLegacyElevenLabsTalkFields(root)) { ids.add("elevenlabs"); } @@ -307,6 +334,10 @@ export function collectRelevantDoctorPluginIdsForTouchedPaths(params: { ids.add(third); continue; } + if (first === "tools" && second === "media") { + collectMediaProviderIds(root, ids); + continue; + } if (first === "talk" && hasLegacyElevenLabsTalkFields(root)) { ids.add("elevenlabs"); }