openclaw/src/agents/chutes-models.test.ts

import { describe, expect, it, vi, beforeEach } from "vitest";
import {
  buildChutesModelDefinition,
  CHUTES_MODEL_CATALOG,
  discoverChutesModels,
  clearChutesModelCache,
} from "./chutes-models.js";

describe("chutes-models", () => {
  beforeEach(() => {
    clearChutesModelCache();
  });

  it("buildChutesModelDefinition returns config with required fields", () => {
    const entry = CHUTES_MODEL_CATALOG[0];
    const def = buildChutesModelDefinition(entry);
    expect(def.id).toBe(entry.id);
    expect(def.name).toBe(entry.name);
    expect(def.reasoning).toBe(entry.reasoning);
    expect(def.input).toEqual(entry.input);
    expect(def.cost).toEqual(entry.cost);
    expect(def.contextWindow).toBe(entry.contextWindow);
    expect(def.maxTokens).toBe(entry.maxTokens);
    expect(def.compat?.supportsUsageInStreaming).toBe(false);
  });

  it("discoverChutesModels returns static catalog when accessToken is empty", async () => {
    const models = await discoverChutesModels("");
    expect(models).toHaveLength(CHUTES_MODEL_CATALOG.length);
    expect(models.map((m) => m.id)).toEqual(CHUTES_MODEL_CATALOG.map((m) => m.id));
  });

  it("discoverChutesModels returns static catalog in test env by default", async () => {
    const models = await discoverChutesModels("test-token");
    expect(models).toHaveLength(CHUTES_MODEL_CATALOG.length);
    expect(models[0]?.id).toBe("Qwen/Qwen3-32B");
  });

  it("discoverChutesModels correctly maps API response when not in test env", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi.fn().mockResolvedValue({
      ok: true,
      json: async () => ({
        data: [
          { id: "zai-org/GLM-4.7-TEE" },
          {
            id: "new-provider/new-model-r1",
            supported_features: ["reasoning"],
            input_modalities: ["text", "image"],
            context_length: 200000,
            max_output_length: 16384,
            pricing: { prompt: 0.1, completion: 0.2 },
          },
          { id: "new-provider/simple-model" },
        ],
      }),
    });
    vi.stubGlobal("fetch", mockFetch);

    try {
      const models = await discoverChutesModels("test-token-real-fetch");
      expect(models.length).toBeGreaterThan(0);
      if (models.length === 3) {
        expect(models[0]?.id).toBe("zai-org/GLM-4.7-TEE");
        expect(models[1]?.reasoning).toBe(true);
        expect(models[1]?.compat?.supportsUsageInStreaming).toBe(false);
      }
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });

  it("discoverChutesModels retries without auth on 401", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi.fn().mockImplementation((url, init) => {
      if (init?.headers?.Authorization === "Bearer test-token-error") {
        // pragma: allowlist secret
        return Promise.resolve({
          ok: false,
          status: 401,
        });
      }
      return Promise.resolve({
        ok: true,
        json: async () => ({
          data: [
            {
              id: "Qwen/Qwen3-32B",
              name: "Qwen/Qwen3-32B",
              supported_features: ["reasoning"],
              input_modalities: ["text"],
              context_length: 40960,
              max_output_length: 40960,
              pricing: { prompt: 0.08, completion: 0.24 },
            },
            {
              id: "unsloth/Mistral-Nemo-Instruct-2407",
              name: "unsloth/Mistral-Nemo-Instruct-2407",
              input_modalities: ["text"],
              context_length: 131072,
              max_output_length: 131072,
              pricing: { prompt: 0.02, completion: 0.04 },
            },
            {
              id: "deepseek-ai/DeepSeek-V3-0324-TEE",
              name: "deepseek-ai/DeepSeek-V3-0324-TEE",
              supported_features: ["reasoning"],
              input_modalities: ["text"],
              context_length: 131072,
              max_output_length: 65536,
              pricing: { prompt: 0.28, completion: 0.42 },
            },
          ],
        }),
      });
    });
    vi.stubGlobal("fetch", mockFetch);

    try {
      const models = await discoverChutesModels("test-token-error");
      expect(models.length).toBeGreaterThan(0);
      expect(mockFetch).toHaveBeenCalled();
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });

  it("caches fallback static catalog for non-OK responses", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi.fn().mockResolvedValue({
      ok: false,
      status: 503,
    });
    vi.stubGlobal("fetch", mockFetch);

    try {
      const first = await discoverChutesModels("chutes-fallback-token");
      const second = await discoverChutesModels("chutes-fallback-token");
      expect(first.map((m) => m.id)).toEqual(CHUTES_MODEL_CATALOG.map((m) => m.id));
      expect(second.map((m) => m.id)).toEqual(CHUTES_MODEL_CATALOG.map((m) => m.id));
      expect(mockFetch).toHaveBeenCalledTimes(1);
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });

  it("scopes discovery cache by access token", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi
      .fn()
      .mockImplementation((_url, init?: { headers?: Record<string, string> }) => {
        const auth = init?.headers?.Authorization;
        if (auth === "Bearer chutes-token-a") {
          return Promise.resolve({
            ok: true,
            json: async () => ({
              data: [{ id: "private/model-a" }],
            }),
          });
        }
        if (auth === "Bearer chutes-token-b") {
          return Promise.resolve({
            ok: true,
            json: async () => ({
              data: [{ id: "private/model-b" }],
            }),
          });
        }
        return Promise.resolve({
          ok: true,
          json: async () => ({
            data: [{ id: "public/model" }],
          }),
        });
      });
    vi.stubGlobal("fetch", mockFetch);

    try {
      const modelsA = await discoverChutesModels("chutes-token-a");
      const modelsB = await discoverChutesModels("chutes-token-b");
      const modelsASecond = await discoverChutesModels("chutes-token-a");
      expect(modelsA[0]?.id).toBe("private/model-a");
      expect(modelsB[0]?.id).toBe("private/model-b");
      expect(modelsASecond[0]?.id).toBe("private/model-a");
      // One request per token, then cache hit for the repeated token-a call.
      expect(mockFetch).toHaveBeenCalledTimes(2);
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });

  it("evicts oldest token entries when cache reaches max size", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi
      .fn()
      .mockImplementation((_url, init?: { headers?: Record<string, string> }) => {
        const auth = init?.headers?.Authorization ?? "";
        return Promise.resolve({
          ok: true,
          json: async () => ({
            data: [{ id: auth ? `${auth}-model` : "public-model" }],
          }),
        });
      });
    vi.stubGlobal("fetch", mockFetch);

    try {
      for (let i = 0; i < 150; i += 1) {
        await discoverChutesModels(`cache-token-${i}`);
      }

      // The oldest key should have been evicted once we exceed the cap.
      await discoverChutesModels("cache-token-0");
      expect(mockFetch).toHaveBeenCalledTimes(151);
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });

  it("prunes expired token cache entries during subsequent discovery", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;
    vi.useFakeTimers();
    vi.setSystemTime(new Date("2026-03-01T00:00:00.000Z"));

    const mockFetch = vi
      .fn()
      .mockImplementation((_url, init?: { headers?: Record<string, string> }) => {
        const auth = init?.headers?.Authorization ?? "";
        return Promise.resolve({
          ok: true,
          json: async () => ({
            data: [{ id: auth ? `${auth}-model` : "public-model" }],
          }),
        });
      });
    vi.stubGlobal("fetch", mockFetch);

    try {
      await discoverChutesModels("token-a");
      vi.advanceTimersByTime(5 * 60 * 1000 + 1);
      await discoverChutesModels("token-b");
      await discoverChutesModels("token-a");
      expect(mockFetch).toHaveBeenCalledTimes(3);
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
      vi.useRealTimers();
    }
  });

  it("does not cache 401 fallback under the failed token key", async () => {
    const oldNodeEnv = process.env.NODE_ENV;
    const oldVitest = process.env.VITEST;
    delete process.env.NODE_ENV;
    delete process.env.VITEST;

    const mockFetch = vi
      .fn()
      .mockImplementation((_url, init?: { headers?: Record<string, string> }) => {
        if (init?.headers?.Authorization === "Bearer failed-token") {
          return Promise.resolve({
            ok: false,
            status: 401,
          });
        }
        return Promise.resolve({
          ok: true,
          json: async () => ({
            data: [{ id: "public/model" }],
          }),
        });
      });
    vi.stubGlobal("fetch", mockFetch);

    try {
      await discoverChutesModels("failed-token");
      await discoverChutesModels("failed-token");
      // Two calls each perform: authenticated attempt (401) + public fallback.
      expect(mockFetch).toHaveBeenCalledTimes(4);
    } finally {
      process.env.NODE_ENV = oldNodeEnv;
      process.env.VITEST = oldVitest;
      vi.unstubAllGlobals();
    }
  });
});