perf(agents): keep model fallback auth runtime cold

This commit is contained in:
Vincent Koc
2026-04-13 16:50:19 +01:00
parent 285bfb3f93
commit 95517edaeb
3 changed files with 43 additions and 26 deletions

View File

@@ -0,0 +1,7 @@
export { resolveAuthProfileOrder } from "./auth-profiles/order.js";
export { ensureAuthProfileStore, loadAuthProfileStoreForRuntime } from "./auth-profiles/store.js";
export {
getSoonestCooldownExpiry,
isProfileInCooldown,
resolveProfilesUnavailableReason,
} from "./auth-profiles/usage.js";

View File

@@ -7,6 +7,7 @@ import type { OpenClawConfig } from "../config/config.js";
import { resetLogger, setLoggerOverride } from "../logging/logger.js";
import { createWarnLogCapture } from "../logging/test-helpers/warn-log-capture.js";
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
import * as authProfileSourceCheckModule from "./auth-profiles/source-check.js";
import * as authProfileStoreModule from "./auth-profiles/store.js";
import { saveAuthProfileStore } from "./auth-profiles/store.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
@@ -200,7 +201,7 @@ const CONNECTION_ERROR_MESSAGE = "Connection error.";
describe("runWithModelFallback", () => {
it("skips auth store bootstrap when no auth profile sources exist", async () => {
const hasSourcesSpy = vi
.spyOn(authProfileStoreModule, "hasAnyAuthProfileStoreSource")
.spyOn(authProfileSourceCheckModule, "hasAnyAuthProfileStoreSource")
.mockReturnValue(false);
const ensureStoreSpy = vi.spyOn(authProfileStoreModule, "ensureAuthProfileStore");
const run = vi.fn().mockResolvedValueOnce("ok");

View File

@@ -7,17 +7,8 @@ import { formatErrorMessage } from "../infra/errors.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { normalizeOptionalString } from "../shared/string-coerce.js";
import { sanitizeForLog } from "../terminal/ansi.js";
import { resolveAuthProfileOrder } from "./auth-profiles/order.js";
import {
ensureAuthProfileStore,
hasAnyAuthProfileStoreSource,
loadAuthProfileStoreForRuntime,
} from "./auth-profiles/store.js";
import {
getSoonestCooldownExpiry,
isProfileInCooldown,
resolveProfilesUnavailableReason,
} from "./auth-profiles/usage.js";
import { hasAnyAuthProfileStoreSource } from "./auth-profiles/source-check.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js";
import {
FailoverError,
@@ -150,6 +141,15 @@ type ModelFallbackRunResult<T> = {
attempts: FallbackAttempt[];
};
type ModelFallbackAuthRuntime = typeof import("./model-fallback-auth.runtime.js");
let modelFallbackAuthRuntimePromise: Promise<ModelFallbackAuthRuntime> | undefined;
async function loadModelFallbackAuthRuntime() {
modelFallbackAuthRuntimePromise ??= import("./model-fallback-auth.runtime.js");
return await modelFallbackAuthRuntimePromise;
}
function buildFallbackSuccess<T>(params: {
result: T;
provider: string;
@@ -286,29 +286,30 @@ function throwFallbackFailureSummary(params: {
}
function resolveFallbackSoonestCooldownExpiry(params: {
authStore: ReturnType<typeof ensureAuthProfileStore> | null;
authRuntime: ModelFallbackAuthRuntime | null;
authStore: AuthProfileStore | null;
agentDir?: string;
cfg: OpenClawConfig | undefined;
candidates: ModelCandidate[];
}): number | null {
if (!params.authStore) {
if (!params.authRuntime || !params.authStore) {
return null;
}
// Refresh from persisted state because embedded attempts can update auth
// cooldowns through a separate store instance while the fallback loop runs.
const refreshedStore = loadAuthProfileStoreForRuntime(params.agentDir, {
const refreshedStore = params.authRuntime.loadAuthProfileStoreForRuntime(params.agentDir, {
readOnly: true,
allowKeychainPrompt: false,
});
let soonest: number | null = null;
for (const candidate of params.candidates) {
const ids = resolveAuthProfileOrder({
const ids = params.authRuntime.resolveAuthProfileOrder({
cfg: params.cfg,
store: refreshedStore,
provider: candidate.provider,
});
const candidateSoonest = getSoonestCooldownExpiry(refreshedStore, ids, {
const candidateSoonest = params.authRuntime.getSoonestCooldownExpiry(refreshedStore, ids, {
forModel: candidate.model,
});
if (
@@ -506,7 +507,8 @@ function shouldProbePrimaryDuringCooldown(params: {
hasFallbackCandidates: boolean;
now: number;
throttleKey: string;
authStore: ReturnType<typeof ensureAuthProfileStore>;
authRuntime: ModelFallbackAuthRuntime;
authStore: AuthProfileStore;
profileIds: string[];
model: string;
}): boolean {
@@ -518,7 +520,7 @@ function shouldProbePrimaryDuringCooldown(params: {
return false;
}
const soonest = getSoonestCooldownExpiry(params.authStore, params.profileIds, {
const soonest = params.authRuntime.getSoonestCooldownExpiry(params.authStore, params.profileIds, {
now: params.now,
forModel: params.model,
});
@@ -562,7 +564,8 @@ function resolveCooldownDecision(params: {
hasFallbackCandidates: boolean;
now: number;
probeThrottleKey: string;
authStore: ReturnType<typeof ensureAuthProfileStore>;
authRuntime: ModelFallbackAuthRuntime;
authStore: AuthProfileStore;
profileIds: string[];
}): CooldownDecision {
const shouldProbe = shouldProbePrimaryDuringCooldown({
@@ -570,13 +573,14 @@ function resolveCooldownDecision(params: {
hasFallbackCandidates: params.hasFallbackCandidates,
now: params.now,
throttleKey: params.probeThrottleKey,
authRuntime: params.authRuntime,
authStore: params.authStore,
profileIds: params.profileIds,
model: params.candidate.model,
});
const inferredReason =
resolveProfilesUnavailableReason({
params.authRuntime.resolveProfilesUnavailableReason({
store: params.authStore,
profileIds: params.profileIds,
now: params.now,
@@ -644,10 +648,13 @@ export async function runWithModelFallback<T>(params: {
model: params.model,
fallbacksOverride: params.fallbacksOverride,
});
const authStore =
const authRuntime =
params.cfg && hasAnyAuthProfileStoreSource(params.agentDir)
? ensureAuthProfileStore(params.agentDir, { allowKeychainPrompt: false })
? await loadModelFallbackAuthRuntime()
: null;
const authStore = authRuntime
? authRuntime.ensureAuthProfileStore(params.agentDir, { allowKeychainPrompt: false })
: null;
const attempts: FallbackAttempt[] = [];
let lastError: unknown;
const cooldownProbeUsedProviders = new Set<string>();
@@ -662,14 +669,14 @@ export async function runWithModelFallback<T>(params: {
let runOptions: ModelFallbackRunOptions | undefined;
let attemptedDuringCooldown = false;
let transientProbeProviderForAttempt: string | null = null;
if (authStore) {
const profileIds = resolveAuthProfileOrder({
if (authRuntime && authStore) {
const profileIds = authRuntime.resolveAuthProfileOrder({
cfg: params.cfg,
store: authStore,
provider: candidate.provider,
});
const isAnyProfileAvailable = profileIds.some(
(id) => !isProfileInCooldown(authStore, id, undefined, candidate.model),
(id) => !authRuntime.isProfileInCooldown(authStore, id, undefined, candidate.model),
);
if (profileIds.length > 0 && !isAnyProfileAvailable) {
@@ -683,6 +690,7 @@ export async function runWithModelFallback<T>(params: {
hasFallbackCandidates,
now,
probeThrottleKey,
authRuntime,
authStore,
profileIds,
});
@@ -898,6 +906,7 @@ export async function runWithModelFallback<T>(params: {
attempt.reason ? ` (${attempt.reason})` : ""
}`,
soonestCooldownExpiry: resolveFallbackSoonestCooldownExpiry({
authRuntime,
authStore,
agentDir: params.agentDir,
cfg: params.cfg,