diff --git a/CHANGELOG.md b/CHANGELOG.md index 747e86b9a94..dc1036c673f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ Docs: https://docs.openclaw.ai - Discord/PluralKit: canonicalize proxied webhook turns to the original Discord message id for inbound dedupe, while preserving the proxy message id for reply routing. Thanks @acgh213. - Discord: only inject thread starter context on the first turn of the effective thread session, so follow-up thread replies do not repeat the starter block. Fixes #41355; supersedes #44447 and #44449. Thanks @p3nchan. - Gateway/diagnostics: include a bounded redacted startup error message in stability bundles, so crash-loop reports identify the failing plugin or contract without exposing secrets. Refs #75797. Thanks @ymebosma. +- Gateway/pricing: defer optional model pricing catalog refresh until after sidecars and channels reach the ready path, so slow OpenRouter or LiteLLM pricing fetches cannot block Gateway readiness. Fixes #74128; supersedes #73486. Thanks @ctbritt and @alprclbi. - Gateway/pricing: abort in-flight model pricing catalog fetches when Gateway shutdown stops the refresh loop, and avoid post-stop cache writes or refresh timers. Fixes #72208. Thanks @rzcq. - Codex/app-server: make startup retry cleanup ownership-aware so concurrent Codex lanes cannot close another lane's freshly restarted shared app-server client. Thanks @vincentkoc. - Google Meet/Twilio: report missing dial-in details during setup and explain that Twilio cannot join Meet URLs without a phone dial plan. diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index a41eac28443..231775c65e4 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -74,10 +74,10 @@ The `models` root also owns global model-catalog behavior. - `models.mode`: provider catalog behavior (`merge` or `replace`). - `models.providers`: custom provider map keyed by provider id. -- `models.pricing.enabled`: controls the background pricing bootstrap. When - `false`, Gateway startup skips OpenRouter and LiteLLM pricing-catalog fetches; - configured `models.providers.*.models[].cost` values still work for local cost - estimates. +- `models.pricing.enabled`: controls the background pricing bootstrap that + starts after sidecars and channels reach the Gateway ready path. When `false`, + the Gateway skips OpenRouter and LiteLLM pricing-catalog fetches; configured + `models.providers.*.models[].cost` values still work for local cost estimates. ## MCP diff --git a/docs/reference/token-use.md b/docs/reference/token-use.md index 50661b7e29f..3adb5499843 100644 --- a/docs/reference/token-use.md +++ b/docs/reference/token-use.md @@ -120,12 +120,13 @@ These are **USD per 1M tokens** for `input`, `output`, `cacheRead`, and `cacheWrite`. If pricing is missing, OpenClaw shows tokens only. OAuth tokens never show dollar cost. -Gateway startup also performs an optional background pricing bootstrap for -configured model refs that do not already have local pricing. That bootstrap -fetches remote OpenRouter and LiteLLM pricing catalogs. Set -`models.pricing.enabled: false` to skip those startup catalog fetches on offline -or restricted networks; explicit `models.providers.*.models[].cost` entries -continue to drive local cost estimates. +After sidecars and channels reach the Gateway ready path, OpenClaw starts an +optional background pricing bootstrap for configured model refs that do not +already have local pricing. That bootstrap fetches remote OpenRouter and LiteLLM +pricing catalogs. Set `models.pricing.enabled: false` to skip those catalog +fetches on offline or restricted networks; explicit +`models.providers.*.models[].cost` entries continue to drive local cost +estimates. ## Cache TTL and pruning impact diff --git a/src/gateway/server-runtime-services.test.ts b/src/gateway/server-runtime-services.test.ts index 006921f28ea..ef3204381f1 100644 --- a/src/gateway/server-runtime-services.test.ts +++ b/src/gateway/server-runtime-services.test.ts @@ -5,11 +5,13 @@ const hoisted = vi.hoisted(() => { stop: vi.fn(), updateConfig: vi.fn(), }; + const stopModelPricingRefresh = vi.fn(); return { heartbeatRunner, startHeartbeatRunner: vi.fn(() => heartbeatRunner), startChannelHealthMonitor: vi.fn(() => ({ stop: vi.fn() })), - startGatewayModelPricingRefresh: vi.fn(() => vi.fn()), + stopModelPricingRefresh, + startGatewayModelPricingRefresh: vi.fn(() => stopModelPricingRefresh), loadModelPricingCacheModule: vi.fn(), isVitestRuntimeEnv: vi.fn(() => false), recoverPendingDeliveries: vi.fn(async () => undefined), @@ -61,6 +63,7 @@ describe("server-runtime-services", () => { hoisted.startHeartbeatRunner.mockClear(); hoisted.startChannelHealthMonitor.mockClear(); hoisted.startGatewayModelPricingRefresh.mockClear(); + hoisted.stopModelPricingRefresh.mockClear(); hoisted.loadModelPricingCacheModule.mockClear(); hoisted.isVitestRuntimeEnv.mockReset().mockReturnValue(false); hoisted.recoverPendingDeliveries.mockClear(); @@ -69,14 +72,13 @@ describe("server-runtime-services", () => { }); it("skips model pricing bootstrap import when pricing is disabled", async () => { - startGatewayRuntimeServices({ + activateGatewayScheduledServices({ minimalTestGateway: false, cfgAtStart: { models: { pricing: { enabled: false } } } as never, - channelManager: { - getRuntimeSnapshot: vi.fn(), - isHealthMonitorEnabled: vi.fn(), - isManuallyStopped: vi.fn(), - } as never, + deps: {} as never, + sessionDeliveryRecoveryMaxEnqueuedAt: 123, + cron: { start: vi.fn(async () => undefined) }, + logCron: { error: vi.fn() }, log: createLog(), }); @@ -86,7 +88,7 @@ describe("server-runtime-services", () => { expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled(); }); - it("keeps scheduled services inert during initial runtime setup", async () => { + it("keeps scheduled services and pricing refresh inert during initial runtime setup", async () => { const services = startGatewayRuntimeServices({ minimalTestGateway: false, cfgAtStart: {} as never, @@ -100,7 +102,8 @@ describe("server-runtime-services", () => { expect(hoisted.startChannelHealthMonitor).toHaveBeenCalledTimes(1); await vi.dynamicImportSettled(); - expect(hoisted.startGatewayModelPricingRefresh).toHaveBeenCalledWith({ config: {} }); + expect(hoisted.loadModelPricingCacheModule).not.toHaveBeenCalled(); + expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled(); expect(hoisted.startHeartbeatRunner).not.toHaveBeenCalled(); expect(hoisted.recoverPendingDeliveries).not.toHaveBeenCalled(); @@ -108,40 +111,45 @@ describe("server-runtime-services", () => { expect(hoisted.heartbeatRunner.stop).not.toHaveBeenCalled(); }); - it("passes startup plugin lookup metadata to the initial pricing refresh", async () => { + it("starts model pricing refresh after scheduled services activate", async () => { const pluginLookUpTable = { index: { plugins: [] }, manifestRegistry: { plugins: [], diagnostics: [] }, }; + const cron = { start: vi.fn(async () => undefined) }; + const log = createLog(); - startGatewayRuntimeServices({ + const services = activateGatewayScheduledServices({ minimalTestGateway: false, cfgAtStart: {} as never, - channelManager: { - getRuntimeSnapshot: vi.fn(), - isHealthMonitorEnabled: vi.fn(), - isManuallyStopped: vi.fn(), - } as never, - log: createLog(), + deps: {} as never, + sessionDeliveryRecoveryMaxEnqueuedAt: 123, + cron, + logCron: { error: vi.fn() }, + log, pluginLookUpTable: pluginLookUpTable as never, }); + expect(hoisted.startHeartbeatRunner).toHaveBeenCalledTimes(1); + expect(cron.start).toHaveBeenCalledTimes(1); await vi.dynamicImportSettled(); expect(hoisted.startGatewayModelPricingRefresh).toHaveBeenCalledWith({ config: {}, pluginLookUpTable, }); + services.stopModelPricingRefresh(); + expect(hoisted.stopModelPricingRefresh).toHaveBeenCalledTimes(1); }); - it("does not start model pricing refresh after early stop", async () => { - const services = startGatewayRuntimeServices({ + it("does not start model pricing refresh after scheduled services stop before import settles", async () => { + const cron = { start: vi.fn(async () => undefined) }; + const services = activateGatewayScheduledServices({ minimalTestGateway: false, cfgAtStart: {} as never, - channelManager: { - getRuntimeSnapshot: vi.fn(), - isHealthMonitorEnabled: vi.fn(), - isManuallyStopped: vi.fn(), - } as never, + deps: {} as never, + sessionDeliveryRecoveryMaxEnqueuedAt: 123, + cron, + logCron: { error: vi.fn() }, log: createLog(), }); @@ -149,6 +157,7 @@ describe("server-runtime-services", () => { await vi.dynamicImportSettled(); expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled(); + expect(hoisted.stopModelPricingRefresh).not.toHaveBeenCalled(); }); it("activates heartbeat, cron, and delivery recovery after sidecars are ready", async () => { diff --git a/src/gateway/server-runtime-services.ts b/src/gateway/server-runtime-services.ts index 84d943dd80f..96eb22be89f 100644 --- a/src/gateway/server-runtime-services.ts +++ b/src/gateway/server-runtime-services.ts @@ -125,7 +125,6 @@ export function startGatewayRuntimeServices(params: { cfgAtStart: OpenClawConfig; channelManager: GatewayChannelManager; log: GatewayRuntimeServiceLogger; - pluginLookUpTable?: PluginMetadataRegistryView; }): { heartbeatRunner: HeartbeatRunner; channelHealthMonitor: ChannelHealthMonitor | null; @@ -139,14 +138,7 @@ export function startGatewayRuntimeServices(params: { return { heartbeatRunner: createNoopHeartbeatRunner(), channelHealthMonitor, - stopModelPricingRefresh: - !params.minimalTestGateway && !isVitestRuntimeEnv() - ? startGatewayModelPricingRefreshOnDemand({ - config: params.cfgAtStart, - ...(params.pluginLookUpTable ? { pluginLookUpTable: params.pluginLookUpTable } : {}), - log: params.log, - }) - : () => {}, + stopModelPricingRefresh: () => {}, }; } @@ -158,9 +150,10 @@ export function activateGatewayScheduledServices(params: { cron: { start: () => Promise }; logCron: { error: (message: string) => void }; log: GatewayRuntimeServiceLogger; -}): { heartbeatRunner: HeartbeatRunner } { + pluginLookUpTable?: PluginMetadataRegistryView; +}): { heartbeatRunner: HeartbeatRunner; stopModelPricingRefresh: () => void } { if (params.minimalTestGateway) { - return { heartbeatRunner: createNoopHeartbeatRunner() }; + return { heartbeatRunner: createNoopHeartbeatRunner(), stopModelPricingRefresh: () => {} }; } const heartbeatRunner = startHeartbeatRunner({ cfg: params.cfgAtStart }); startGatewayCronWithLogging({ @@ -176,5 +169,12 @@ export function activateGatewayScheduledServices(params: { log: params.log, maxEnqueuedAt: params.sessionDeliveryRecoveryMaxEnqueuedAt, }); - return { heartbeatRunner }; + const stopModelPricingRefresh = !isVitestRuntimeEnv() + ? startGatewayModelPricingRefreshOnDemand({ + config: params.cfgAtStart, + ...(params.pluginLookUpTable ? { pluginLookUpTable: params.pluginLookUpTable } : {}), + log: params.log, + }) + : () => {}; + return { heartbeatRunner, stopModelPricingRefresh }; } diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 6b0f4193f5d..ed6273b7a04 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -827,7 +827,9 @@ export async function startGatewayServer( }); deps.cron = runtimeState.cronState.cron; + let closePreludeStarted = false; const runClosePrelude = async () => { + closePreludeStarted = true; clearCurrentPluginMetadataSnapshot(); const { runGatewayClosePrelude } = await loadGatewayCloseModule(); await runGatewayClosePrelude({ @@ -973,7 +975,6 @@ export async function startGatewayServer( cfgAtStart, channelManager, log, - pluginLookUpTable, }), ); @@ -1166,6 +1167,31 @@ export async function startGatewayServer( await startListening(); startupTrace.mark("http.bound"); const sessionDeliveryRecoveryMaxEnqueuedAt = Date.now(); + let postAttachRuntimeReturned = false; + let scheduledServicesActivated = false; + const activateScheduledServicesWhenReady = () => { + if ( + closePreludeStarted || + !postAttachRuntimeReturned || + !startupSidecarsReady || + scheduledServicesActivated + ) { + return; + } + const activated = activateGatewayScheduledServices({ + minimalTestGateway, + cfgAtStart, + deps, + sessionDeliveryRecoveryMaxEnqueuedAt, + cron: runtimeState.cronState.cron, + logCron, + log, + pluginLookUpTable, + }); + scheduledServicesActivated = true; + runtimeState.heartbeatRunner = activated.heartbeatRunner; + runtimeState.stopModelPricingRefresh = activated.stopModelPricingRefresh; + }; ({ stopGatewayUpdateCheck: runtimeState.stopGatewayUpdateCheck, tailscaleCleanup: runtimeState.tailscaleCleanup, @@ -1221,23 +1247,15 @@ export async function startGatewayServer( }, onSidecarsReady: () => { startupSidecarsReady = true; + activateScheduledServicesWhenReady(); }, startupTrace, deferSidecars: opts.deferStartupSidecars === true, }), )); startupTrace.mark("ready"); - - const activated = activateGatewayScheduledServices({ - minimalTestGateway, - cfgAtStart, - deps, - sessionDeliveryRecoveryMaxEnqueuedAt, - cron: runtimeState.cronState.cron, - logCron, - log, - }); - runtimeState.heartbeatRunner = activated.heartbeatRunner; + postAttachRuntimeReturned = true; + activateScheduledServicesWhenReady(); const { startManagedGatewayConfigReloader } = await import("./server-reload-handlers.js"); runtimeState.configReloader = startManagedGatewayConfigReloader({