openclaw/src/plugin-sdk/provider-catalog-shared.test.ts

// Provider catalog shared tests cover catalog hashing, normalization, and model visibility.
import type { ModelCatalogProvider } from "@openclaw/model-catalog-core/model-catalog-types";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
  applyProviderNativeStreamingUsageCompat,
  buildManifestModelProviderConfig,
  clearLiveCatalogCacheForTests,
  getCachedLiveCatalogValue,
  readConfiguredProviderCatalogEntries,
  supportsNativeStreamingUsageCompat,
} from "./provider-catalog-shared.js";
import type { ModelDefinitionConfig } from "./provider-model-shared.js";

function buildModel(id: string, supportsUsageInStreaming?: boolean): ModelDefinitionConfig {
  return {
    id,
    name: id,
    reasoning: false,
    input: ["text"],
    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
    contextWindow: 1024,
    maxTokens: 1024,
    ...(supportsUsageInStreaming === undefined ? {} : { compat: { supportsUsageInStreaming } }),
  };
}

describe("provider-catalog-shared live catalog cache", () => {
  beforeEach(() => {
    clearLiveCatalogCacheForTests();
  });

  it("reuses in-flight and fresh live catalog loads for matching keys", async () => {
    let now = 1_000;
    const load = vi.fn(async () => ({ models: ["a"] }));

    const first = getCachedLiveCatalogValue({
      keyParts: ["provider", "models", "secret-token"],
      load,
      ttlMs: 100,
      now: () => now,
    });
    const second = getCachedLiveCatalogValue({
      keyParts: ["provider", "models", "secret-token"],
      load,
      ttlMs: 100,
      now: () => now,
    });

    await expect(first).resolves.toEqual({ models: ["a"] });
    await expect(second).resolves.toEqual({ models: ["a"] });
    expect(load).toHaveBeenCalledTimes(1);

    now = 1_050;
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models", "secret-token"],
        load,
        ttlMs: 100,
        now: () => now,
      }),
    ).resolves.toEqual({ models: ["a"] });
    expect(load).toHaveBeenCalledTimes(1);

    now = 1_101;
    await getCachedLiveCatalogValue({
      keyParts: ["provider", "models", "secret-token"],
      load,
      ttlMs: 100,
      now: () => now,
    });
    expect(load).toHaveBeenCalledTimes(2);
  });

  it("does not cache failed live catalog loads", async () => {
    const load = vi
      .fn<() => Promise<string>>()
      .mockRejectedValueOnce(new Error("boom"))
      .mockResolvedValueOnce("ok");

    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models"],
        load,
      }),
    ).rejects.toThrow("boom");
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models"],
        load,
      }),
    ).resolves.toBe("ok");
    expect(load).toHaveBeenCalledTimes(2);
  });

  it("does not retain resolved live catalog values rejected by the cache predicate", async () => {
    const load = vi
      .fn<() => Promise<string>>()
      .mockResolvedValueOnce("empty")
      .mockResolvedValueOnce("usable");

    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models"],
        load,
        shouldCache: (value) => value !== "empty",
      }),
    ).resolves.toBe("empty");
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models"],
        load,
        shouldCache: (value) => value !== "empty",
      }),
    ).resolves.toBe("usable");
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models"],
        load,
        shouldCache: (value) => value !== "empty",
      }),
    ).resolves.toBe("usable");

    expect(load).toHaveBeenCalledTimes(2);
  });

  it("evicts the oldest live catalog cache entry when the cache is full", async () => {
    const load = vi.fn(async (id: number) => `value-${id}`);

    for (let i = 0; i < 100; i += 1) {
      await expect(
        getCachedLiveCatalogValue({
          keyParts: ["provider", "models", i],
          load: () => load(i),
          ttlMs: 60_000,
        }),
      ).resolves.toBe(`value-${i}`);
    }
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models", 100],
        load: () => load(100),
        ttlMs: 60_000,
      }),
    ).resolves.toBe("value-100");
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models", 0],
        load: () => load(0),
        ttlMs: 60_000,
      }),
    ).resolves.toBe("value-0");

    expect(load).toHaveBeenCalledTimes(102);
  });

  it("does not cache live catalog loads when the expiry would exceed Date range", async () => {
    const load = vi
      .fn<() => Promise<string>>()
      .mockResolvedValueOnce("first")
      .mockResolvedValueOnce("second");

    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models", "overflow"],
        load,
        ttlMs: 1,
        now: () => 8_640_000_000_000_000,
      }),
    ).resolves.toBe("first");
    await expect(
      getCachedLiveCatalogValue({
        keyParts: ["provider", "models", "overflow"],
        load,
        ttlMs: 1,
        now: () => 8_640_000_000_000_000,
      }),
    ).resolves.toBe("second");
    expect(load).toHaveBeenCalledTimes(2);
  });
});

describe("provider-catalog-shared native streaming usage compat", () => {
  it("detects native streaming usage compat from the endpoint capabilities", () => {
    expect(
      supportsNativeStreamingUsageCompat({
        providerId: "custom-qwen",
        baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
      }),
    ).toBe(true);
    expect(
      supportsNativeStreamingUsageCompat({
        providerId: "custom-kimi",
        baseUrl: "https://api.moonshot.ai/v1",
      }),
    ).toBe(true);
    expect(
      supportsNativeStreamingUsageCompat({
        providerId: "custom-proxy",
        baseUrl: "https://proxy.example.com/v1",
      }),
    ).toBe(false);
  });

  it("opts models into streaming usage for native endpoints while preserving explicit overrides", () => {
    const provider = applyProviderNativeStreamingUsageCompat({
      providerId: "custom-qwen",
      providerConfig: {
        api: "openai-completions",
        baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
        models: [buildModel("qwen-plus"), buildModel("qwen-max", false)],
      },
    });

    expect(provider.models?.[0]?.compat?.supportsUsageInStreaming).toBe(true);
    expect(provider.models?.[1]?.compat?.supportsUsageInStreaming).toBe(false);
  });
});

describe("provider-catalog-shared configured catalog entries", () => {
  it("normalizes bare retired Gemini ids for Google-owned configured providers", () => {
    expect(
      readConfiguredProviderCatalogEntries({
        providerId: "google",
        config: {
          models: {
            providers: {
              google: {
                baseUrl: "https://generativelanguage.googleapis.com/v1beta",
                models: [
                  {
                    id: "gemini-3-pro-preview",
                    name: "Gemini 3 Pro Preview",
                    input: ["text", "image"],
                    reasoning: true,
                    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
                    contextWindow: 1048576,
                    maxTokens: 65536,
                  },
                ],
              },
            },
          },
        },
      }),
    ).toEqual([
      {
        provider: "google",
        id: "gemini-3.1-pro-preview",
        name: "Gemini 3 Pro Preview",
        input: ["text", "image"],
        reasoning: true,
        contextWindow: 1048576,
      },
    ]);
  });

  it("preserves configured audio and video input modalities while normalizing nested Gemini ids", () => {
    expect(
      readConfiguredProviderCatalogEntries({
        providerId: "kilocode",
        config: {
          models: {
            providers: {
              kilocode: {
                baseUrl: "https://api.kilo.ai/api/gateway/",
                api: "openai-completions",
                models: [
                  {
                    id: "google/gemini-3-pro-preview",
                    name: "Gemini 3 Pro Preview",
                    input: ["text", "image", "video", "audio"],
                    reasoning: true,
                    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
                    contextWindow: 1048576,
                    maxTokens: 65536,
                  },
                ],
              },
            },
          },
        },
      }),
    ).toEqual([
      {
        provider: "kilocode",
        id: "google/gemini-3.1-pro-preview",
        name: "Gemini 3 Pro Preview",
        input: ["text", "image", "video", "audio"],
        reasoning: true,
        contextWindow: 1048576,
      },
    ]);
  });
});

describe("provider-catalog-shared manifest provider configs", () => {
  it("converts manifest model catalog rows into provider config rows", () => {
    const catalog: ModelCatalogProvider = {
      baseUrl: "https://api.example.test/v1",
      api: "openai-completions",
      headers: { "x-provider": "example" },
      models: [
        {
          id: "example-model",
          name: "Example Model",
          input: ["text", "image"],
          reasoning: true,
          contextWindow: 128_000,
          contextTokens: 64_000,
          maxTokens: 8192,
          thinkingLevelMap: { off: null, minimal: "low", max: "max" },
          mediaInput: {
            image: { maxSidePx: 2048, preferredSidePx: 1024, tokenMode: "detail" },
          },
          cost: {
            input: 1,
            output: 2,
            cacheRead: 0.25,
            cacheWrite: 0.5,
            tieredPricing: [
              {
                input: 0.5,
                output: 1,
                cacheRead: 0.1,
                cacheWrite: 0.2,
                range: [0, 1_000_000],
              },
            ],
          },
          compat: { supportsUsageInStreaming: true },
        },
      ],
    };

    expect(buildManifestModelProviderConfig({ providerId: "example", catalog })).toEqual({
      baseUrl: "https://api.example.test/v1",
      api: "openai-completions",
      headers: { "x-provider": "example" },
      models: [
        {
          id: "example-model",
          name: "Example Model",
          reasoning: true,
          input: ["text", "image"],
          cost: {
            input: 1,
            output: 2,
            cacheRead: 0.25,
            cacheWrite: 0.5,
            tieredPricing: [
              {
                input: 0.5,
                output: 1,
                cacheRead: 0.1,
                cacheWrite: 0.2,
                range: [0, 1_000_000],
              },
            ],
          },
          contextWindow: 128_000,
          contextTokens: 64_000,
          maxTokens: 8192,
          thinkingLevelMap: { off: null, minimal: "low", max: "max" },
          mediaInput: {
            image: { maxSidePx: 2048, preferredSidePx: 1024, tokenMode: "detail" },
          },
          compat: { supportsUsageInStreaming: true },
        },
      ],
    });
  });

  it("normalizes retired nested Gemini ids before emitting manifest provider config", () => {
    const catalog: ModelCatalogProvider = {
      baseUrl: "https://api.kilo.ai/api/gateway/",
      api: "openai-completions",
      models: [
        {
          id: "google/gemini-3-pro-preview",
          name: "Gemini 3 Pro Preview",
          input: ["text", "image"],
          reasoning: true,
          contextWindow: 1_048_576,
          maxTokens: 65_536,
        },
      ],
    };

    expect(buildManifestModelProviderConfig({ providerId: "kilocode", catalog })).toEqual({
      baseUrl: "https://api.kilo.ai/api/gateway/",
      api: "openai-completions",
      models: [
        {
          id: "google/gemini-3.1-pro-preview",
          name: "Gemini 3 Pro Preview",
          reasoning: true,
          input: ["text", "image"],
          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
          contextWindow: 1_048_576,
          maxTokens: 65_536,
        },
      ],
    });
  });

  it("rejects incomplete manifest rows before building provider runtime config", () => {
    expect(() =>
      buildManifestModelProviderConfig({
        providerId: "example",
        catalog: {
          baseUrl: "https://api.example.test/v1",
          models: [
            {
              id: "missing-context",
              maxTokens: 8192,
            },
          ],
        },
      }),
    ).toThrow("missing contextWindow");
  });

  it("rejects catalog data that cannot become runtime provider config", () => {
    expect(() =>
      buildManifestModelProviderConfig({
        providerId: "example",
        catalog: {
          models: [
            {
              id: "missing-base-url",
              contextWindow: 1024,
              maxTokens: 1024,
            },
          ],
        },
      }),
    ).toThrow("providers.example.baseUrl");

    expect(() =>
      buildManifestModelProviderConfig({
        providerId: "example",
        catalog: {
          baseUrl: "https://api.example.test/v1",
          models: [
            {
              id: "document-model",
              input: ["document"],
              contextWindow: 1024,
              maxTokens: 1024,
            },
          ],
        },
      }),
    ).toThrow("unsupported runtime input document");
  });

  it("rejects manifest catalogs when normalization drops a model row", () => {
    expect(() =>
      buildManifestModelProviderConfig({
        providerId: "example",
        catalog: {
          baseUrl: "https://api.example.test/v1",
          models: [
            {
              id: "valid",
              contextWindow: 1024,
              maxTokens: 1024,
            },
            {
              id: "",
              contextWindow: 1024,
              maxTokens: 1024,
            },
          ],
        },
      }),
    ).toThrow("providers.example.models");
  });
});