diff --git a/src/agents/live-model-turn-probes.test.ts b/src/agents/live-model-turn-probes.test.ts index b403a0bd3e3..8f404c57cc9 100644 --- a/src/agents/live-model-turn-probes.test.ts +++ b/src/agents/live-model-turn-probes.test.ts @@ -10,6 +10,8 @@ import { LIVE_MODEL_FILE_PROBE_TOKEN, modelSupportsImageInput, shouldSkipLiveModelExtraProbes, + shouldSkipLiveModelFileProbe, + shouldSkipLiveModelImageProbe, } from "./live-model-turn-probes.js"; describe("live model turn probes", () => { @@ -84,6 +86,21 @@ describe("live model turn probes", () => { ).toBe(false); }); + it("skips known stale file probe routes", () => { + expect(shouldSkipLiveModelFileProbe({ provider: "opencode-go", id: "glm-5" })).toBe(true); + expect(shouldSkipLiveModelFileProbe({ provider: "opencode-go", id: "kimi-k2.5" })).toBe(false); + }); + + it("skips known stale image probe routes", () => { + expect( + shouldSkipLiveModelImageProbe({ + provider: "fireworks", + id: "accounts/fireworks/models/kimi-k2p6", + }), + ).toBe(true); + expect(shouldSkipLiveModelImageProbe({ provider: "fireworks", id: "glm-5" })).toBe(false); + }); + it("matches expected probe replies", () => { expect(fileProbeTextMatches(`The value is ${LIVE_MODEL_FILE_PROBE_TOKEN}.`)).toBe(true); expect(fileProbeTextMatches("amber")).toBe(false); diff --git a/src/agents/live-model-turn-probes.ts b/src/agents/live-model-turn-probes.ts index f47eb4fb149..433dcb64039 100644 --- a/src/agents/live-model-turn-probes.ts +++ b/src/agents/live-model-turn-probes.ts @@ -9,6 +9,16 @@ const PROBE_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAALUlEQVR4nO3OIQEAAAwCMPrnod8fAzMxv7S9pQgICAgICAgICAgICAgICKwDD+yWbLXSniMNAAAAAElFTkSuQmCC"; const KNOWN_EMPTY_EXTRA_PROBE_MODELS = new Set(["openrouter/amazon/nova-2-lite-v1"]); +const KNOWN_EMPTY_FILE_PROBE_MODELS = new Set(["opencode-go/glm-5", "opencode-go/glm-5.1"]); +const KNOWN_EMPTY_IMAGE_PROBE_MODELS = new Set([ + "fireworks/accounts/fireworks/models/kimi-k2p6", + "fireworks/accounts/fireworks/routers/kimi-k2p5-turbo", + "opencode-go/kimi-k2.6", +]); + +function modelKey(model: Pick, "id" | "provider">): string { + return `${model.provider}/${model.id}`; +} export function isLiveModelProbeEnabled( env: Record, @@ -36,7 +46,15 @@ export function modelSupportsImageInput(model: Pick, "input">): boole export function shouldSkipLiveModelExtraProbes( model: Pick, "id" | "provider">, ): boolean { - return KNOWN_EMPTY_EXTRA_PROBE_MODELS.has(`${model.provider}/${model.id}`); + return KNOWN_EMPTY_EXTRA_PROBE_MODELS.has(modelKey(model)); +} + +export function shouldSkipLiveModelFileProbe(model: Pick, "id" | "provider">): boolean { + return KNOWN_EMPTY_FILE_PROBE_MODELS.has(modelKey(model)); +} + +export function shouldSkipLiveModelImageProbe(model: Pick, "id" | "provider">): boolean { + return KNOWN_EMPTY_IMAGE_PROBE_MODELS.has(modelKey(model)); } export function buildLiveModelFileProbeContext(params: { systemPrompt?: string }): Context { diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index d827f658a6d..f4d84baa910 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -29,6 +29,8 @@ import { LIVE_MODEL_IMAGE_PROBE_ENV, modelSupportsImageInput, shouldSkipLiveModelExtraProbes, + shouldSkipLiveModelFileProbe, + shouldSkipLiveModelImageProbe, } from "./live-model-turn-probes.js"; import { createLiveTargetMatcher } from "./live-target-matcher.js"; import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "./live-test-helpers.js"; @@ -463,7 +465,7 @@ async function runExtraTurnProbes(params: { reasoning: resolveTestReasoning(params.model), maxTokens: 128, }; - if (LIVE_FILE_PROBE_ENABLED) { + if (LIVE_FILE_PROBE_ENABLED && !shouldSkipLiveModelFileProbe(params.model)) { logProgress(`${params.progressLabel}: file-read probe`); const file = await completeSimpleWithTimeout( params.model, @@ -497,6 +499,8 @@ async function runExtraTurnProbes(params: { if (!fileProbeTextMatches(fileText)) { throw new Error(`file-read probe did not return ${LIVE_MODEL_FILE_PROBE_TOKEN}: ${fileText}`); } + } else if (LIVE_FILE_PROBE_ENABLED) { + logProgress(`${params.progressLabel}: file-read probe skipped (known empty route)`); } if (!LIVE_IMAGE_PROBE_ENABLED) { @@ -506,6 +510,10 @@ async function runExtraTurnProbes(params: { logProgress(`${params.progressLabel}: image probe skipped (no image input)`); return; } + if (shouldSkipLiveModelImageProbe(params.model)) { + logProgress(`${params.progressLabel}: image probe skipped (known empty route)`); + return; + } logProgress(`${params.progressLabel}: image probe`); const image = await completeSimpleWithTimeout(