vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 05:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: use candidate seam for fallback ordering cases

Browse Source
This commit is contained in:
Peter Steinberger
2026-05-06 01:48:32 +01:00
parent 1ddc2650c6
commit cc3eb0b53e
1 changed files with 56 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
src/agents/model-fallback.test.ts
View File

@@ -907,7 +907,7 @@ describe("runWithModelFallback", () => {
});
});
it("falls back directly to configured primary when an override model fails", async () => {
it("puts configured primary next when an override model is requested", () => {
const cfg = makeCfg({
agents: {
defaults: {
@@ -919,31 +919,19 @@ describe("runWithModelFallback", () => {
},
});
const run = createOverrideFailureRun({
overrideProvider: "anthropic",
overrideModel: "claude-opus-4-5",
fallbackProvider: "openai",
fallbackModel: "gpt-4.1-mini",
firstError: Object.assign(new Error("unauthorized"), { status: 401 }),
});
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-opus-4-5",
run,
});
expect(result.result).toBe("ok");
expect(result.provider).toBe("openai");
expect(result.model).toBe("gpt-4.1-mini");
expect(run.mock.calls).toEqual([
["anthropic", "claude-opus-4-5"],
["openai", "gpt-4.1-mini"],
expect(
__testing.resolveFallbackCandidates({
cfg,
provider: "anthropic",
model: "claude-opus-4-5",
}),
).toEqual([
{ provider: "anthropic", model: "claude-opus-4-5" },
{ provider: "openai", model: "gpt-4.1-mini" },
]);
});
it("keeps configured fallback chain when current model is a configured fallback", async () => {
it("keeps configured fallback chain when current model is a configured fallback", () => {
const cfg = makeCfg({
agents: {
defaults: {
@@ -955,33 +943,20 @@ describe("runWithModelFallback", () => {
},
});
const run = vi.fn().mockImplementation(async (provider: string, model: string) => {
if (provider === "anthropic" && model === "claude-haiku-3-5") {
throw Object.assign(new Error("rate-limited"), { status: 429 });
}
if (provider === "openrouter" && model === "openrouter/deepseek-chat") {
return "ok";
}
throw new Error(`unexpected fallback candidate: ${provider}/${model}`);
});
const result = await runWithModelFallback({
cfg,
provider: "anthropic",
model: "claude-haiku-3-5",
run,
});
expect(result.result).toBe("ok");
expect(result.provider).toBe("openrouter");
expect(result.model).toBe("openrouter/deepseek-chat");
expect(run.mock.calls).toEqual([
["anthropic", "claude-haiku-3-5"],
["openrouter", "openrouter/deepseek-chat"],
expect(
__testing.resolveFallbackCandidates({
cfg,
provider: "anthropic",
model: "claude-haiku-3-5",
}),
).toEqual([
{ provider: "anthropic", model: "claude-haiku-3-5" },
{ provider: "openrouter", model: "openrouter/deepseek-chat" },
{ provider: "openai", model: "gpt-4.1-mini" },
]);
});
it("treats normalized default refs as primary and keeps configured fallback chain", async () => {
it("treats normalized default refs as primary and keeps configured fallback chain", () => {
const cfg = makeCfg({
agents: {
defaults: {
@@ -993,44 +968,18 @@ describe("runWithModelFallback", () => {
},
});
const run = vi
.fn()
.mockRejectedValueOnce(Object.assign(new Error("nope"), { status: 401 }))
.mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: " OpenAI ",
model: "gpt-4.1-mini",
run,
});
expect(result.result).toBe("ok");
expect(run.mock.calls).toEqual([
["openai", "gpt-4.1-mini"],
["anthropic", "claude-haiku-3-5"],
expect(
__testing.resolveFallbackCandidates({
cfg,
provider: " OpenAI ",
model: "gpt-4.1-mini",
}),
).toEqual([
{ provider: "openai", model: "gpt-4.1-mini" },
{ provider: "anthropic", model: "claude-haiku-3-5" },
]);
});
it("records 400 insufficient_quota payloads as billing during fallback", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(Object.assign(new Error(INSUFFICIENT_QUOTA_PAYLOAD), { status: 400 }))
.mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "openai",
model: "gpt-4.1-mini",
run,
});
expect(result.result).toBe("ok");
expect(result.attempts).toHaveLength(1);
expect(result.attempts[0]?.reason).toBe("billing");
});
it("falls back to configured primary for override credential validation errors", async () => {
const cfg = makeCfg();
const run = createOverrideFailureRun({
@@ -1055,6 +1004,25 @@ describe("runWithModelFallback", () => {
]);
});
it("records 400 insufficient_quota payloads as billing during fallback", async () => {
const cfg = makeCfg();
const run = vi
.fn()
.mockRejectedValueOnce(Object.assign(new Error(INSUFFICIENT_QUOTA_PAYLOAD), { status: 400 }))
.mockResolvedValueOnce("ok");
const result = await runWithModelFallback({
cfg,
provider: "openai",
model: "gpt-4.1-mini",
run,
});
expect(result.result).toBe("ok");
expect(result.attempts).toHaveLength(1);
expect(result.attempts[0]?.reason).toBe("billing");
});
it("falls back on model-not-found error shapes", async () => {
const cases: Array<{
name: string;
@@ -1257,7 +1225,7 @@ describe("runWithModelFallback", () => {
expect(result.attempts).toEqual([]);
});
it("does not append configured primary when fallbacksOverride is set", async () => {
it("does not append configured primary when fallbacksOverride is set", () => {
const cfg = makeCfg({
agents: {
defaults: {
@@ -1267,23 +1235,17 @@ describe("runWithModelFallback", () => {
},
},
});
const run = vi
.fn()
.mockImplementation(() => Promise.reject(Object.assign(new Error("nope"), { status: 401 })));
await expect(
runWithModelFallback({
expect(
__testing.resolveFallbackCandidates({
cfg,
provider: "anthropic",
model: "claude-opus-4-5",
fallbacksOverride: ["anthropic/claude-haiku-3-5"],
run,
}),
).rejects.toThrow("All models failed");
expect(run.mock.calls).toEqual([
["anthropic", "claude-opus-4-5"],
["anthropic", "claude-haiku-3-5"],
).toEqual([
{ provider: "anthropic", model: "claude-opus-4-5" },
{ provider: "anthropic", model: "claude-haiku-3-5" },
]);
});