mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-03 12:00:22 +00:00
fix(agents): comprehensive quota fallback fixes - session overrides + surgical cooldown logic (#23816)
Merged via /review-pr -> /prepare-pr -> /merge-pr.
Prepared head SHA: e6f2b4742b
Co-authored-by: ramezgaberiel <844893+ramezgaberiel@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
Ramez
@@ -143,10 +143,22 @@ async function expectSkippedUnavailableProvider(params: {
|
||||
}) {
|
||||
const provider = `${params.providerPrefix}-${crypto.randomUUID()}`;
|
||||
const cfg = makeProviderFallbackCfg(provider);
|
||||
const store = makeSingleProviderStore({
|
||||
const primaryStore = makeSingleProviderStore({
|
||||
provider,
|
||||
usageStat: params.usageStat,
|
||||
});
|
||||
// Include fallback provider profile so the fallback is attempted (not skipped as no-profile).
|
||||
const store: AuthProfileStore = {
|
||||
...primaryStore,
|
||||
profiles: {
|
||||
...primaryStore.profiles,
|
||||
"fallback:default": {
|
||||
type: "api_key",
|
||||
provider: "fallback",
|
||||
key: "test-key",
|
||||
},
|
||||
},
|
||||
};
|
||||
const run = createFallbackOnlyRun();
|
||||
|
||||
const result = await runWithStoredAuth({
|
||||
@@ -436,11 +448,11 @@ describe("runWithModelFallback", () => {
|
||||
run,
|
||||
});
|
||||
|
||||
// Override model failed with model_not_found → falls back to configured primary.
|
||||
// Override model failed with model_not_found → tries fallbacks first (same provider).
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run.mock.calls[1]?.[0]).toBe("openai");
|
||||
expect(run.mock.calls[1]?.[1]).toBe("gpt-4.1-mini");
|
||||
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
||||
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
||||
});
|
||||
|
||||
it("skips providers when all profiles are in cooldown", async () => {
|
||||
@@ -794,6 +806,296 @@ describe("runWithModelFallback", () => {
|
||||
expect(result.provider).toBe("openai");
|
||||
expect(result.model).toBe("gpt-4.1-mini");
|
||||
});
|
||||
|
||||
// Tests for Bug A fix: Model fallback with session overrides
|
||||
describe("fallback behavior with session model overrides", () => {
|
||||
it("allows fallbacks when session model differs from config within same provider", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-5", "google/gemini-2.5-flash"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("Rate limit exceeded")) // Session model fails
|
||||
.mockResolvedValueOnce("fallback success"); // First fallback succeeds
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-20250514", // Different from config primary
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("fallback success");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run).toHaveBeenNthCalledWith(1, "anthropic", "claude-sonnet-4-20250514");
|
||||
expect(run).toHaveBeenNthCalledWith(2, "anthropic", "claude-sonnet-4-5"); // Fallback tried
|
||||
});
|
||||
|
||||
it("allows fallbacks with model version differences within same provider", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("Weekly quota exceeded"))
|
||||
.mockResolvedValueOnce("groq success");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-5", // Version difference from config
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("groq success");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run).toHaveBeenNthCalledWith(2, "groq", "llama-3.3-70b-versatile");
|
||||
});
|
||||
|
||||
it("still skips fallbacks when using different provider than config", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: [], // Empty fallbacks to match working pattern
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('No credentials found for profile "openai:default".'))
|
||||
.mockResolvedValueOnce("config primary worked");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai", // Different provider
|
||||
model: "gpt-4.1-mini",
|
||||
run,
|
||||
});
|
||||
|
||||
// Cross-provider requests should skip configured fallbacks but still try configured primary
|
||||
expect(result.result).toBe("config primary worked");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run).toHaveBeenNthCalledWith(1, "openai", "gpt-4.1-mini"); // Original request
|
||||
expect(run).toHaveBeenNthCalledWith(2, "anthropic", "claude-opus-4-6"); // Config primary as final fallback
|
||||
});
|
||||
|
||||
it("uses fallbacks when session model exactly matches config primary", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("Quota exceeded"))
|
||||
.mockResolvedValueOnce("fallback worked");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-6", // Exact match
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("fallback worked");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run).toHaveBeenNthCalledWith(2, "groq", "llama-3.3-70b-versatile");
|
||||
});
|
||||
});
|
||||
|
||||
// Tests for Bug B fix: Rate limit vs auth/billing cooldown distinction
|
||||
describe("fallback behavior with provider cooldowns", () => {
|
||||
async function makeAuthStoreWithCooldown(
|
||||
provider: string,
|
||||
reason: "rate_limit" | "auth" | "billing",
|
||||
): Promise<{ store: AuthProfileStore; dir: string }> {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-test-"));
|
||||
const now = Date.now();
|
||||
const store: AuthProfileStore = {
|
||||
version: AUTH_STORE_VERSION,
|
||||
profiles: {
|
||||
[`${provider}:default`]: { type: "api_key", provider, key: "test-key" },
|
||||
},
|
||||
usageStats: {
|
||||
[`${provider}:default`]:
|
||||
reason === "rate_limit"
|
||||
? {
|
||||
// Real rate-limit cooldowns are tracked through cooldownUntil
|
||||
// and failureCounts, not disabledReason.
|
||||
cooldownUntil: now + 300000,
|
||||
failureCounts: { rate_limit: 1 },
|
||||
}
|
||||
: {
|
||||
// Auth/billing issues use disabledUntil
|
||||
disabledUntil: now + 300000,
|
||||
disabledReason: reason,
|
||||
},
|
||||
},
|
||||
};
|
||||
saveAuthProfileStore(store, tmpDir);
|
||||
return { store, dir: tmpDir };
|
||||
}
|
||||
|
||||
it("attempts same-provider fallbacks during rate limit cooldown", async () => {
|
||||
const { dir } = await makeAuthStoreWithCooldown("anthropic", "rate_limit");
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-5", "groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi.fn().mockResolvedValueOnce("sonnet success"); // Fallback succeeds
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-6",
|
||||
run,
|
||||
agentDir: dir,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("sonnet success");
|
||||
expect(run).toHaveBeenCalledTimes(1); // Primary skipped, fallback attempted
|
||||
expect(run).toHaveBeenNthCalledWith(1, "anthropic", "claude-sonnet-4-5");
|
||||
});
|
||||
|
||||
it("skips same-provider models on auth cooldown but still tries no-profile fallback providers", async () => {
|
||||
const { dir } = await makeAuthStoreWithCooldown("anthropic", "auth");
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-5", "groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi.fn().mockResolvedValueOnce("groq success");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-6",
|
||||
run,
|
||||
agentDir: dir,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("groq success");
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
expect(run).toHaveBeenNthCalledWith(1, "groq", "llama-3.3-70b-versatile");
|
||||
});
|
||||
|
||||
it("skips same-provider models on billing cooldown but still tries no-profile fallback providers", async () => {
|
||||
const { dir } = await makeAuthStoreWithCooldown("anthropic", "billing");
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-5", "groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi.fn().mockResolvedValueOnce("groq success");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-6",
|
||||
run,
|
||||
agentDir: dir,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("groq success");
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
expect(run).toHaveBeenNthCalledWith(1, "groq", "llama-3.3-70b-versatile");
|
||||
});
|
||||
|
||||
it("tries cross-provider fallbacks when same provider has rate limit", async () => {
|
||||
// Anthropic in rate limit cooldown, Groq available
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-test-"));
|
||||
const store: AuthProfileStore = {
|
||||
version: AUTH_STORE_VERSION,
|
||||
profiles: {
|
||||
"anthropic:default": { type: "api_key", provider: "anthropic", key: "test-key" },
|
||||
"groq:default": { type: "api_key", provider: "groq", key: "test-key" },
|
||||
},
|
||||
usageStats: {
|
||||
"anthropic:default": {
|
||||
// Rate-limit reason is inferred from failureCounts for cooldown windows.
|
||||
cooldownUntil: Date.now() + 300000,
|
||||
failureCounts: { rate_limit: 2 },
|
||||
},
|
||||
// Groq not in cooldown
|
||||
},
|
||||
};
|
||||
saveAuthProfileStore(store, tmpDir);
|
||||
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "anthropic/claude-opus-4-6",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-5", "groq/llama-3.3-70b-versatile"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("Still rate limited")) // Sonnet still fails
|
||||
.mockResolvedValueOnce("groq success"); // Groq works
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-6",
|
||||
run,
|
||||
agentDir: tmpDir,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("groq success");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(run).toHaveBeenNthCalledWith(1, "anthropic", "claude-sonnet-4-5"); // Rate limit allows attempt
|
||||
expect(run).toHaveBeenNthCalledWith(2, "groq", "llama-3.3-70b-versatile"); // Cross-provider works
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("runWithImageModelFallback", () => {
|
||||
|
||||
Reference in New Issue
Block a user