diff --git a/scripts/bench-gateway-startup.ts b/scripts/bench-gateway-startup.ts index 1019988d8df..4b76a896538 100644 --- a/scripts/bench-gateway-startup.ts +++ b/scripts/bench-gateway-startup.ts @@ -323,7 +323,7 @@ async function waitForProbe(params: { function requestStatus(port: number, pathname: string): Promise { return new Promise((resolve, reject) => { const req = request( - { host: "127.0.0.1", method: "GET", path: pathname, port, timeout: 1000 }, + { host: "127.0.0.1", method: "GET", path: pathname, port, timeout: 100 }, (res) => { res.resume(); res.on("end", () => resolve(res.statusCode ?? 0)); @@ -511,20 +511,22 @@ async function runGatewaySample(options: { child.stdout.on("data", onChunk); child.stderr.on("data", onChunk); - const healthz = await waitForProbe({ - deadlineAt, - isDone: () => childExited, - path: "/healthz", - port, - startAt, - }); - const readyz = await waitForProbe({ - deadlineAt, - isDone: () => childExited, - path: "/readyz", - port, - startAt, - }); + const [healthz, readyz] = await Promise.all([ + waitForProbe({ + deadlineAt, + isDone: () => childExited, + path: "/healthz", + port, + startAt, + }), + waitForProbe({ + deadlineAt, + isDone: () => childExited, + path: "/readyz", + port, + startAt, + }), + ]); const exit = await stopChild(child); await childExitPromise.catch(() => null); rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 }); diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index acf91ce9861..7341fa66534 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -22,6 +22,7 @@ import type { GatewayWsLogStyle } from "../../gateway/ws-logging.js"; import { setGatewayWsLogStyle } from "../../gateway/ws-logging.js"; import { setVerbose } from "../../globals.js"; import { resolveControlUiRootSync } from "../../infra/control-ui-assets.js"; +import { isTruthyEnvValue } from "../../infra/env.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { GatewayLockError } from "../../infra/gateway-lock.js"; import { formatPortDiagnostics, inspectPortUsage } from "../../infra/ports.js"; @@ -98,6 +99,8 @@ const GATEWAY_RUN_BOOLEAN_KEYS = [ const SUPERVISED_GATEWAY_LOCK_RETRY_MS = 5000; +type Awaitable = T | Promise; + /** * EX_CONFIG (78) from sysexits.h — used for configuration errors so systemd * (via RestartPreventExitStatus=78) stops restarting instead of entering a @@ -113,6 +116,36 @@ const GATEWAY_AUTH_MODES: readonly GatewayAuthMode[] = [ ]; const GATEWAY_TAILSCALE_MODES: readonly GatewayTailscaleMode[] = ["off", "serve", "funnel"]; +function createGatewayCliStartupTrace() { + const enabled = isTruthyEnvValue(process.env.OPENCLAW_GATEWAY_STARTUP_TRACE); + const started = performance.now(); + let last = started; + const emit = (name: string, durationMs: number, totalMs: number) => { + if (enabled) { + gatewayLog.info( + `startup trace: ${name} ${durationMs.toFixed(1)}ms total=${totalMs.toFixed(1)}ms`, + ); + } + }; + return { + mark(name: string) { + const now = performance.now(); + emit(name, now - last, now - started); + last = now; + }, + async measure(name: string, run: () => Awaitable): Promise { + const before = performance.now(); + try { + return await run(); + } finally { + const now = performance.now(); + emit(name, now - before, now - started); + last = now; + } + }, + }; +} + function warnInlinePasswordFlag() { defaultRuntime.error( "Warning: --password can be exposed via process listings. Prefer --password-file or OPENCLAW_GATEWAY_PASSWORD.", @@ -284,22 +317,28 @@ async function runGatewayCommand(opts: GatewayRunOpts) { process.env.OPENCLAW_RAW_STREAM_PATH = rawStreamPath; } + const startupTrace = createGatewayCliStartupTrace(); + // The heaviest part of gateway startup is loading the server module tree // (channels, plugins, HTTP stack, etc.). Show a spinner so the user sees // progress instead of a silent 15-20 s pause (especially on Windows/NTFS). - const { startGatewayServer } = await withProgress( - { label: "Loading gateway modules…", indeterminate: true }, - async () => import("../../gateway/server.js"), + const { startGatewayServer } = await startupTrace.measure("cli.server-import", () => + withProgress( + { label: "Loading gateway modules…", indeterminate: true }, + async () => import("../../gateway/server.js"), + ), ); setConsoleTimestampPrefix(true); if (devMode) { - await ensureDevGatewayConfig({ reset: Boolean(opts.reset) }); + await startupTrace.measure("cli.dev-config", () => + ensureDevGatewayConfig({ reset: Boolean(opts.reset) }), + ); } gatewayLog.info("loading configuration…"); - const cfg = loadConfig(); + const cfg = await startupTrace.measure("cli.config-load", () => loadConfig()); maybeLogPendingControlUiBuild(cfg); const portOverride = parsePort(opts.port); if (opts.port !== undefined && portOverride === null) { @@ -422,7 +461,9 @@ async function runGatewayCommand(opts: GatewayRunOpts) { const tokenRaw = toOptionString(opts.token); gatewayLog.info("resolving authentication…"); - const snapshot = await readConfigFileSnapshot().catch(() => null); + const snapshot = await startupTrace.measure("cli.config-snapshot", () => + readConfigFileSnapshot().catch(() => null), + ); const configExists = snapshot?.exists ?? fs.existsSync(CONFIG_PATH); const configAuditPath = path.join(resolveStateDir(process.env), "logs", "config-audit.jsonl"); const effectiveCfg = snapshot?.valid ? snapshot.config : cfg; @@ -449,12 +490,14 @@ async function runGatewayCommand(opts: GatewayRunOpts) { ...(passwordRaw ? { password: passwordRaw } : {}), } : undefined; - const resolvedAuth = resolveGatewayAuth({ - authConfig: cfg.gateway?.auth, - authOverride, - env: process.env, - tailscaleMode: tailscaleMode ?? cfg.gateway?.tailscale?.mode ?? "off", - }); + const resolvedAuth = await startupTrace.measure("cli.auth-resolve", () => + resolveGatewayAuth({ + authConfig: cfg.gateway?.auth, + authOverride, + env: process.env, + tailscaleMode: tailscaleMode ?? cfg.gateway?.tailscale?.mode ?? "off", + }), + ); const resolvedAuthMode = resolvedAuth.mode; const tokenValue = resolvedAuth.token; const passwordValue = resolvedAuth.password; @@ -537,6 +580,7 @@ async function runGatewayCommand(opts: GatewayRunOpts) { : undefined; gatewayLog.info("starting..."); + startupTrace.mark("cli.gateway-loop"); const startLoop = async () => await runGatewayLoop({ runtime: defaultRuntime, diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index eb5f18f5128..23c9cbf0f44 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -148,6 +148,21 @@ describe("startGatewayPostAttachRuntime", () => { expect(hoisted.logGatewayStartup).toHaveBeenCalledWith( expect.objectContaining({ loadedPluginIds: ["beta", "alpha"] }), ); + expect(hoisted.startGatewayMemoryBackend).not.toHaveBeenCalled(); + }); + + it("starts the qmd memory backend only when configured", async () => { + await startGatewayPostAttachRuntime({ + ...createPostAttachParams(), + gatewayPluginConfigAtStart: { + hooks: { internal: { enabled: false } }, + memory: { backend: "qmd" }, + } as never, + }); + + await vi.waitFor(() => { + expect(hoisted.startGatewayMemoryBackend).toHaveBeenCalledTimes(1); + }); }); it("keeps startup-gated methods unavailable while sidecars are still resuming", async () => { diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 4800517eaf1..b06c782fb12 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -17,6 +17,29 @@ import type { startGatewayTailscaleExposure } from "./server-tailscale.js"; const SESSION_LOCK_STALE_MS = 30 * 60 * 1000; +type Awaitable = T | Promise; + +type GatewayStartupTrace = { + mark: (name: string) => void; + measure: (name: string, run: () => Awaitable) => Promise; +}; + +async function measureStartup( + startupTrace: GatewayStartupTrace | undefined, + name: string, + run: () => Awaitable, +): Promise { + return startupTrace ? startupTrace.measure(name, run) : await run(); +} + +function shouldCheckRestartSentinel(env: NodeJS.ProcessEnv = process.env): boolean { + return !env.VITEST && env.NODE_ENV !== "test"; +} + +function shouldStartGatewayMemoryBackend(cfg: OpenClawConfig): boolean { + return cfg.memory?.backend === "qmd"; +} + async function prewarmConfiguredPrimaryModel(params: { cfg: OpenClawConfig; log: { warn: (msg: string) => void }; @@ -88,114 +111,126 @@ export async function startGatewaySidecars(params: { error: (msg: string) => void; }; logChannels: { info: (msg: string) => void; error: (msg: string) => void }; + startupTrace?: GatewayStartupTrace; }) { - try { - const [{ resolveStateDir }, { resolveAgentSessionDirs }, { cleanStaleLockFiles }] = - await Promise.all([ - import("../config/paths.js"), - import("../agents/session-dirs.js"), - import("../agents/session-write-lock.js"), - ]); - const stateDir = resolveStateDir(process.env); - const sessionDirs = await resolveAgentSessionDirs(stateDir); - for (const sessionsDir of sessionDirs) { - await cleanStaleLockFiles({ - sessionsDir, - staleMs: SESSION_LOCK_STALE_MS, - removeStale: true, - log: { warn: (message) => params.log.warn(message) }, + await measureStartup(params.startupTrace, "sidecars.session-locks", async () => { + try { + const [{ resolveStateDir }, { resolveAgentSessionDirs }, { cleanStaleLockFiles }] = + await Promise.all([ + import("../config/paths.js"), + import("../agents/session-dirs.js"), + import("../agents/session-write-lock.js"), + ]); + const stateDir = resolveStateDir(process.env); + const sessionDirs = await resolveAgentSessionDirs(stateDir); + for (const sessionsDir of sessionDirs) { + await cleanStaleLockFiles({ + sessionsDir, + staleMs: SESSION_LOCK_STALE_MS, + removeStale: true, + log: { warn: (message) => params.log.warn(message) }, + }); + } + } catch (err) { + params.log.warn(`session lock cleanup failed on startup: ${String(err)}`); + } + }); + + await measureStartup(params.startupTrace, "sidecars.gmail-watch", async () => { + if (params.cfg.hooks?.enabled && params.cfg.hooks.gmail?.account) { + const { startGmailWatcherWithLogs } = await import("../hooks/gmail-watcher-lifecycle.js"); + await startGmailWatcherWithLogs({ + cfg: params.cfg, + log: params.logHooks, }); } - } catch (err) { - params.log.warn(`session lock cleanup failed on startup: ${String(err)}`); - } + }); - if (params.cfg.hooks?.enabled && params.cfg.hooks.gmail?.account) { - const { startGmailWatcherWithLogs } = await import("../hooks/gmail-watcher-lifecycle.js"); - await startGmailWatcherWithLogs({ - cfg: params.cfg, - log: params.logHooks, - }); - } - - if (params.cfg.hooks?.gmail?.model) { - const [ - { DEFAULT_MODEL, DEFAULT_PROVIDER }, - { loadModelCatalog }, - { getModelRefStatus, resolveConfiguredModelRef, resolveHooksGmailModel }, - ] = await Promise.all([ - import("../agents/defaults.js"), - import("../agents/model-catalog.js"), - import("../agents/model-selection.js"), - ]); - const hooksModelRef = resolveHooksGmailModel({ - cfg: params.cfg, - defaultProvider: DEFAULT_PROVIDER, - }); - if (hooksModelRef) { - const { provider: resolvedDefaultProvider, model: defaultModel } = resolveConfiguredModelRef({ + await measureStartup(params.startupTrace, "sidecars.gmail-model", async () => { + if (params.cfg.hooks?.gmail?.model) { + const [ + { DEFAULT_MODEL, DEFAULT_PROVIDER }, + { loadModelCatalog }, + { getModelRefStatus, resolveConfiguredModelRef, resolveHooksGmailModel }, + ] = await Promise.all([ + import("../agents/defaults.js"), + import("../agents/model-catalog.js"), + import("../agents/model-selection.js"), + ]); + const hooksModelRef = resolveHooksGmailModel({ cfg: params.cfg, defaultProvider: DEFAULT_PROVIDER, - defaultModel: DEFAULT_MODEL, }); - const catalog = await loadModelCatalog({ config: params.cfg }); - const status = getModelRefStatus({ - cfg: params.cfg, - catalog, - ref: hooksModelRef, - defaultProvider: resolvedDefaultProvider, - defaultModel, - }); - if (!status.allowed) { - params.logHooks.warn( - `hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`, - ); - } - if (!status.inCatalog) { - params.logHooks.warn( - `hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`, - ); + if (hooksModelRef) { + const { provider: resolvedDefaultProvider, model: defaultModel } = + resolveConfiguredModelRef({ + cfg: params.cfg, + defaultProvider: DEFAULT_PROVIDER, + defaultModel: DEFAULT_MODEL, + }); + const catalog = await loadModelCatalog({ config: params.cfg }); + const status = getModelRefStatus({ + cfg: params.cfg, + catalog, + ref: hooksModelRef, + defaultProvider: resolvedDefaultProvider, + defaultModel, + }); + if (!status.allowed) { + params.logHooks.warn( + `hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`, + ); + } + if (!status.inCatalog) { + params.logHooks.warn( + `hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`, + ); + } } } - } + }); const internalHooksConfigured = hasConfiguredInternalHooks(params.cfg); - try { - if (internalHooksConfigured) { - const [{ setInternalHooksEnabled }, { loadInternalHooks }] = await Promise.all([ - import("../hooks/internal-hooks.js"), - import("../hooks/loader.js"), - ]); - setInternalHooksEnabled(params.cfg.hooks?.internal?.enabled !== false); - const loadedCount = await loadInternalHooks(params.cfg, params.defaultWorkspaceDir); - if (loadedCount > 0) { - params.logHooks.info( - `loaded ${loadedCount} internal hook handler${loadedCount > 1 ? "s" : ""}`, - ); + await measureStartup(params.startupTrace, "sidecars.internal-hooks", async () => { + try { + if (internalHooksConfigured) { + const [{ setInternalHooksEnabled }, { loadInternalHooks }] = await Promise.all([ + import("../hooks/internal-hooks.js"), + import("../hooks/loader.js"), + ]); + setInternalHooksEnabled(params.cfg.hooks?.internal?.enabled !== false); + const loadedCount = await loadInternalHooks(params.cfg, params.defaultWorkspaceDir); + if (loadedCount > 0) { + params.logHooks.info( + `loaded ${loadedCount} internal hook handler${loadedCount > 1 ? "s" : ""}`, + ); + } } + } catch (err) { + params.logHooks.error(`failed to load hooks: ${String(err)}`); } - } catch (err) { - params.logHooks.error(`failed to load hooks: ${String(err)}`); - } + }); const skipChannels = isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS); - if (!skipChannels) { - try { - await prewarmConfiguredPrimaryModel({ - cfg: params.cfg, - log: params.log, - }); - await params.startChannels(); - } catch (err) { - params.logChannels.error(`channel startup failed: ${String(err)}`); + await measureStartup(params.startupTrace, "sidecars.channels", async () => { + if (!skipChannels) { + try { + await prewarmConfiguredPrimaryModel({ + cfg: params.cfg, + log: params.log, + }); + await params.startChannels(); + } catch (err) { + params.logChannels.error(`channel startup failed: ${String(err)}`); + } + } else { + params.logChannels.info( + "skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", + ); } - } else { - params.logChannels.info( - "skipping channel start (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)", - ); - } + }); if (internalHooksConfigured) { setTimeout(() => { @@ -213,16 +248,18 @@ export async function startGatewaySidecars(params: { } let pluginServices: PluginServicesHandle | null = null; - try { - const { startPluginServices } = await import("../plugins/services.js"); - pluginServices = await startPluginServices({ - registry: params.pluginRegistry, - config: params.cfg, - workspaceDir: params.defaultWorkspaceDir, - }); - } catch (err) { - params.log.warn(`plugin services failed to start: ${String(err)}`); - } + await measureStartup(params.startupTrace, "sidecars.plugin-services", async () => { + try { + const { startPluginServices } = await import("../plugins/services.js"); + pluginServices = await startPluginServices({ + registry: params.pluginRegistry, + config: params.cfg, + workspaceDir: params.defaultWorkspaceDir, + }); + } catch (err) { + params.log.warn(`plugin services failed to start: ${String(err)}`); + } + }); if (params.cfg.acp?.enabled) { const [{ getAcpSessionManager }, { ACP_SESSION_IDENTITY_RENDERER_VERSION }] = await Promise.all( @@ -243,30 +280,48 @@ export async function startGatewaySidecars(params: { }); } - void import("./server-startup-memory.js") - .then(({ startGatewayMemoryBackend }) => - startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }), - ) - .catch((err) => { - params.log.warn(`qmd memory startup initialization failed: ${String(err)}`); + await measureStartup(params.startupTrace, "sidecars.memory", async () => { + if (!shouldStartGatewayMemoryBackend(params.cfg)) { + return; + } + setImmediate(() => { + void import("./server-startup-memory.js") + .then(({ startGatewayMemoryBackend }) => + startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }), + ) + .catch((err) => { + params.log.warn(`qmd memory startup initialization failed: ${String(err)}`); + }); }); + }); - const { shouldWakeFromRestartSentinel, scheduleRestartSentinelWake } = - await import("./server-restart-sentinel.js"); - if (shouldWakeFromRestartSentinel()) { + await measureStartup(params.startupTrace, "sidecars.restart-sentinel", async () => { + if (!shouldCheckRestartSentinel()) { + return; + } + const { hasRestartSentinel } = await import("../infra/restart-sentinel.js"); + if (!(await hasRestartSentinel())) { + return; + } setTimeout(() => { - void scheduleRestartSentinelWake({ deps: params.deps }); + void import("./server-restart-sentinel.js") + .then(({ scheduleRestartSentinelWake }) => + scheduleRestartSentinelWake({ deps: params.deps }), + ) + .catch((err) => { + params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`); + }); }, 750); - } + }); - const { scheduleSubagentOrphanRecovery } = await import("../agents/subagent-registry.js"); - scheduleSubagentOrphanRecovery(); + await measureStartup(params.startupTrace, "sidecars.subagent-recovery", async () => { + const { scheduleSubagentOrphanRecovery } = await import("../agents/subagent-registry.js"); + scheduleSubagentOrphanRecovery(); + }); return { pluginServices }; } -type Awaitable = T | Promise; - type GatewayPostAttachRuntimeDeps = { getGlobalHookRunner: () => Awaitable>; logGatewayStartup: (params: Parameters[0]) => Awaitable; @@ -329,26 +384,29 @@ export async function startGatewayPostAttachRuntime( unavailableGatewayMethods: Set; onPluginServices?: (pluginServices: PluginServicesHandle | null) => void; onSidecarsReady?: () => void; + startupTrace?: GatewayStartupTrace; }, runtimeDeps: GatewayPostAttachRuntimeDeps = defaultGatewayPostAttachRuntimeDeps, ) { - await runtimeDeps.logGatewayStartup({ - cfg: params.cfgAtStart, - bindHost: params.bindHost, - bindHosts: params.bindHosts, - port: params.port, - tlsEnabled: params.tlsEnabled, - loadedPluginIds: params.pluginRegistry.plugins - .filter((plugin) => plugin.status === "loaded") - .map((plugin) => plugin.id), - log: params.log, - isNixMode: params.isNixMode, - startupStartedAt: params.startupStartedAt, - }); + await measureStartup(params.startupTrace, "post-attach.log", () => + runtimeDeps.logGatewayStartup({ + cfg: params.cfgAtStart, + bindHost: params.bindHost, + bindHosts: params.bindHosts, + port: params.port, + tlsEnabled: params.tlsEnabled, + loadedPluginIds: params.pluginRegistry.plugins + .filter((plugin) => plugin.status === "loaded") + .map((plugin) => plugin.id), + log: params.log, + isNixMode: params.isNixMode, + startupStartedAt: params.startupStartedAt, + }), + ); const stopGatewayUpdateCheckPromise = params.minimalTestGateway ? Promise.resolve(() => {}) - : Promise.resolve( + : measureStartup(params.startupTrace, "post-attach.update-check", () => runtimeDeps.scheduleGatewayUpdateCheck({ cfg: params.cfgAtStart, log: params.log, @@ -364,7 +422,7 @@ export async function startGatewayPostAttachRuntime( ? Promise.resolve(null) : params.tailscaleMode === "off" && !params.resetOnExit ? Promise.resolve(null) - : Promise.resolve( + : measureStartup(params.startupTrace, "post-attach.tailscale", () => runtimeDeps.startGatewayTailscaleExposure({ tailscaleMode: params.tailscaleMode, resetOnExit: params.resetOnExit, @@ -378,21 +436,25 @@ export async function startGatewayPostAttachRuntime( ? Promise.resolve({ pluginServices: null }) : new Promise((resolve) => setImmediate(resolve)).then(async () => { params.log.info("starting channels and sidecars..."); - const result = await runtimeDeps.startGatewaySidecars({ - cfg: params.gatewayPluginConfigAtStart, - pluginRegistry: params.pluginRegistry, - defaultWorkspaceDir: params.defaultWorkspaceDir, - deps: params.deps, - startChannels: params.startChannels, - log: params.log, - logHooks: params.logHooks, - logChannels: params.logChannels, - }); + const result = await measureStartup(params.startupTrace, "sidecars.total", () => + runtimeDeps.startGatewaySidecars({ + cfg: params.gatewayPluginConfigAtStart, + pluginRegistry: params.pluginRegistry, + defaultWorkspaceDir: params.defaultWorkspaceDir, + deps: params.deps, + startChannels: params.startChannels, + log: params.log, + logHooks: params.logHooks, + logChannels: params.logChannels, + startupTrace: params.startupTrace, + }), + ); for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) { params.unavailableGatewayMethods.delete(method); } params.onPluginServices?.(result.pluginServices); params.onSidecarsReady?.(); + params.startupTrace?.mark("sidecars.ready"); return result; }); diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 0bb31dcbb14..365fd6fed6c 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -830,6 +830,7 @@ export async function startGatewayServer( onSidecarsReady: () => { startupSidecarsReady = true; }, + startupTrace, }), )); startupTrace.mark("ready"); diff --git a/src/gateway/server/readiness.test.ts b/src/gateway/server/readiness.test.ts index f875eb8dfb7..80ef1f9d90f 100644 --- a/src/gateway/server/readiness.test.ts +++ b/src/gateway/server/readiness.test.ts @@ -102,6 +102,28 @@ describe("createReadinessChecker", () => { }); }); + it("does not cache startup-pending readiness", () => { + withReadinessClock(() => { + let startupPending = true; + const { manager, readiness } = createReadinessHarness({ + startedAgoMs: 5 * 60_000, + accounts: {}, + getStartupPending: () => startupPending, + cacheTtlMs: 1_000, + }); + expect(readiness()).toEqual({ + ready: false, + failing: ["startup-sidecars"], + uptimeMs: 300_000, + }); + expect(manager.getRuntimeSnapshot).not.toHaveBeenCalled(); + + startupPending = false; + expect(readiness()).toEqual({ ready: true, failing: [], uptimeMs: 300_000 }); + expect(manager.getRuntimeSnapshot).toHaveBeenCalledTimes(1); + }); + }); + it("ignores disabled and unconfigured channels", () => { withReadinessClock(() => { const { readiness } = createReadinessHarness({ diff --git a/src/gateway/server/readiness.ts b/src/gateway/server/readiness.ts index 312177a0f6f..f7eb873c2fb 100644 --- a/src/gateway/server/readiness.ts +++ b/src/gateway/server/readiness.ts @@ -46,15 +46,15 @@ export function createReadinessChecker(deps: { return (): ReadinessResult => { const now = Date.now(); const uptimeMs = now - startedAt; + if (deps.getStartupPending?.()) { + return { ready: false, failing: ["startup-sidecars"], uptimeMs }; + } if (cachedState && now - cachedAt < cacheTtlMs) { return { ...cachedState, uptimeMs }; } const snapshot = channelManager.getRuntimeSnapshot(); const failing: string[] = []; - if (deps.getStartupPending?.()) { - failing.push("startup-sidecars"); - } for (const [channelId, accounts] of Object.entries(snapshot.channelAccounts)) { if (!accounts) { diff --git a/src/infra/restart-sentinel.ts b/src/infra/restart-sentinel.ts index baf8168047d..60cd97a613c 100644 --- a/src/infra/restart-sentinel.ts +++ b/src/infra/restart-sentinel.ts @@ -96,6 +96,15 @@ export async function readRestartSentinel( } } +export async function hasRestartSentinel(env: NodeJS.ProcessEnv = process.env): Promise { + try { + await fs.access(resolveRestartSentinelPath(env)); + return true; + } catch { + return false; + } +} + export async function consumeRestartSentinel( env: NodeJS.ProcessEnv = process.env, ): Promise {