From 59faa023febc5ed0135826a583bd4efb9f81be29 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 21:34:18 +0100 Subject: [PATCH] fix(gateway): unblock sidecar startup --- CHANGELOG.md | 1 + extensions/discord/src/monitor/provider.ts | 36 ++++++++-- scripts/run-node.mjs | 20 +++++- src/cli/gateway-cli/run-loop.test.ts | 13 +++- src/cli/gateway-cli/run-loop.ts | 10 ++- .../server-startup-post-attach.test.ts | 31 +++++++- src/gateway/server-startup-post-attach.ts | 32 ++++++++- src/infra/restart.ts | 5 ++ src/infra/run-node.test.ts | 65 +++++++++++++++++ src/tasks/runtime-internal.ts | 1 + src/tasks/task-registry.test.ts | 70 +++++++++++++++++++ src/tasks/task-registry.ts | 24 +++++-- 12 files changed, 287 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d12a496ac0..8571d8d616e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Docs: https://docs.openclaw.ai - CLI/message: resolve targeted `openclaw message` channels to their owning plugin before loading the registry, and fall back to configured channel plugins when the channel must be inferred, so scripted sends avoid full bundled plugin registry scans without assuming channel ids match plugin ids. Fixes #73006. Thanks @jasonftl. - Plugins/startup: parse strict JSON plugin manifests with native JSON first and keep JSON5 as the compatibility fallback, reducing manifest registry CPU during Gateway boot and CLI startup. Fixes #73011. Thanks @jasonftl. - CLI/models: keep route-first `models status --json` stdout reserved for the JSON payload by routing auth-profile and startup diagnostics to stderr. Fixes #72962. Thanks @vishutdhar. +- Gateway/runtime: keep dirty-tree status calls from rebuilding live `dist`, clear stale task and restart state across in-process restarts, retry transient Discord lazy imports, and let channel startup continue after slow model warmup so browser, Discord, and voice-call sidecars come online. Thanks @vincentkoc. - Sessions: ignore future-dated session activity timestamps during reset freshness checks and cap future `updatedAt` values at the merge boundary so clock-skewed messages cannot keep stale sessions alive forever. Fixes #72989. Thanks @martingarramon. - Sessions: apply search, activity filters, and limits before gateway row enrichment so bounded session lists avoid scanning discarded transcripts. Carries forward #72978. Thanks @yeager. - Sessions: remove trajectory runtime and pointer sidecars when session maintenance prunes, caps, or disk-evicts their owning session, while preserving sidecars still referenced by live rows. Fixes #73000. Thanks @jared-rebel. diff --git a/extensions/discord/src/monitor/provider.ts b/extensions/discord/src/monitor/provider.ts index c350832750e..b360d62d5a7 100644 --- a/extensions/discord/src/monitor/provider.ts +++ b/extensions/discord/src/monitor/provider.ts @@ -145,21 +145,45 @@ async function loadDiscordVoiceRuntime(): Promise { if (loadDiscordVoiceRuntimeForTesting) { return await loadDiscordVoiceRuntimeForTesting(); } - discordVoiceRuntimePromise ??= import("../voice/manager.runtime.js"); - return await discordVoiceRuntimePromise; + const promise = discordVoiceRuntimePromise ?? import("../voice/manager.runtime.js"); + discordVoiceRuntimePromise = promise; + try { + return await promise; + } catch (error) { + if (discordVoiceRuntimePromise === promise) { + discordVoiceRuntimePromise = undefined; + } + throw error; + } } async function loadDiscordProviderSessionRuntime(): Promise { if (loadDiscordProviderSessionRuntimeForTesting) { return await loadDiscordProviderSessionRuntimeForTesting(); } - discordProviderSessionRuntimePromise ??= import("./provider-session.runtime.js"); - return await discordProviderSessionRuntimePromise; + const promise = discordProviderSessionRuntimePromise ?? import("./provider-session.runtime.js"); + discordProviderSessionRuntimePromise = promise; + try { + return await promise; + } catch (error) { + if (discordProviderSessionRuntimePromise === promise) { + discordProviderSessionRuntimePromise = undefined; + } + throw error; + } } async function loadPluginRuntime() { - pluginRuntimePromise ??= import("openclaw/plugin-sdk/plugin-runtime"); - return await pluginRuntimePromise; + const promise = pluginRuntimePromise ?? import("openclaw/plugin-sdk/plugin-runtime"); + pluginRuntimePromise = promise; + try { + return await promise; + } catch (error) { + if (pluginRuntimePromise === promise) { + pluginRuntimePromise = undefined; + } + throw error; + } } function normalizeBooleanForTesting(value: unknown): boolean | undefined { diff --git a/scripts/run-node.mjs b/scripts/run-node.mjs index f6e083415f9..167fcfc6f83 100644 --- a/scripts/run-node.mjs +++ b/scripts/run-node.mjs @@ -801,6 +801,15 @@ const writeBuildStamp = (deps) => { const shouldSkipCleanWatchRuntimeSync = (deps) => deps.env.OPENCLAW_WATCH_MODE === "1"; +const isGatewayClientCommand = (args) => + args[0] === "gateway" && (args[1] === "call" || args[1] === "status"); + +const shouldUseExistingDistForGatewayClient = (deps, buildRequirement) => + buildRequirement.reason === "dirty_watched_tree" && + isGatewayClientCommand(deps.args) && + deps.env.OPENCLAW_FORCE_BUILD !== "1" && + statMtime(deps.distEntry, deps.fs) != null; + export async function runNodeMain(params = {}) { const deps = { spawn: params.spawn ?? spawn, @@ -834,9 +843,16 @@ export async function runNodeMain(params = {}) { try { let exitCode = 1; - const buildRequirement = resolveBuildRequirement(deps); + let buildRequirement = resolveBuildRequirement(deps); + const useExistingGatewayClientDist = shouldUseExistingDistForGatewayClient( + deps, + buildRequirement, + ); + if (useExistingGatewayClientDist) { + buildRequirement = { shouldBuild: false, reason: "gateway_client_existing_dist" }; + } if (!buildRequirement.shouldBuild) { - if (!shouldSkipCleanWatchRuntimeSync(deps)) { + if (!useExistingGatewayClientDist && !shouldSkipCleanWatchRuntimeSync(deps)) { const runtimePostBuildRequirement = resolveRuntimePostBuildRequirement(deps); if (runtimePostBuildRequirement.shouldSync) { const synced = await withRunNodeBuildLock(deps, async () => { diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts index 17e2e5b545a..bb36f947fbd 100644 --- a/src/cli/gateway-cli/run-loop.test.ts +++ b/src/cli/gateway-cli/run-loop.test.ts @@ -10,6 +10,7 @@ const consumeGatewayRestartIntentSync = vi.fn(() => false); const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false); const markGatewaySigusr1RestartHandled = vi.fn(); const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined); +const resetGatewayRestartStateForInProcessRestart = vi.fn(); const scheduleGatewaySigusr1Restart = vi.fn((_opts?: { delayMs?: number; reason?: string }) => ({ ok: true, pid: process.pid, @@ -23,6 +24,7 @@ const getActiveTaskCount = vi.fn(() => 0); const markGatewayDraining = vi.fn(); const waitForActiveTasks = vi.fn(async (_timeoutMs?: number) => ({ drained: true })); const resetAllLanes = vi.fn(); +const reloadTaskRegistryFromStore = vi.fn(); const getActiveBundledRuntimeDepsInstallCount = vi.fn(() => 0); const waitForBundledRuntimeDepsInstallIdle = vi.fn(async (_timeoutMs?: number) => ({ drained: true, @@ -71,6 +73,7 @@ vi.mock("../../infra/restart.js", () => ({ isGatewaySigusr1RestartExternallyAllowed: () => isGatewaySigusr1RestartExternallyAllowed(), markGatewaySigusr1RestartHandled: () => markGatewaySigusr1RestartHandled(), peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(), + resetGatewayRestartStateForInProcessRestart: () => resetGatewayRestartStateForInProcessRestart(), scheduleGatewaySigusr1Restart: (opts?: { delayMs?: number; reason?: string }) => scheduleGatewaySigusr1Restart(opts), })); @@ -91,6 +94,10 @@ vi.mock("../../process/command-queue.js", () => ({ resetAllLanes: () => resetAllLanes(), })); +vi.mock("../../tasks/runtime-internal.js", () => ({ + reloadTaskRegistryFromStore: () => reloadTaskRegistryFromStore(), +})); + vi.mock("../../plugins/bundled-runtime-deps-activity.js", () => ({ getActiveBundledRuntimeDepsInstallCount: () => getActiveBundledRuntimeDepsInstallCount(), waitForBundledRuntimeDepsInstallIdle: (timeoutMs?: number) => @@ -306,7 +313,7 @@ describe("runGatewayLoop", () => { }); }); - it("restarts after SIGUSR1 even when drain times out, and resets lanes for the new iteration", async () => { + it("restarts after SIGUSR1 even when drain times out, and resets runtime state for the new iteration", async () => { vi.clearAllMocks(); loadConfig.mockReturnValue({ gateway: { @@ -395,6 +402,8 @@ describe("runGatewayLoop", () => { }); expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(1); expect(resetAllLanes).toHaveBeenCalledTimes(1); + expect(resetGatewayRestartStateForInProcessRestart).toHaveBeenCalledTimes(1); + expect(reloadTaskRegistryFromStore).toHaveBeenCalledTimes(1); sigusr1(); @@ -407,6 +416,8 @@ describe("runGatewayLoop", () => { expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(2); expect(markGatewayDraining).toHaveBeenCalledTimes(2); expect(resetAllLanes).toHaveBeenCalledTimes(2); + expect(resetGatewayRestartStateForInProcessRestart).toHaveBeenCalledTimes(2); + expect(reloadTaskRegistryFromStore).toHaveBeenCalledTimes(2); expect(acquireGatewayLock).toHaveBeenCalledTimes(3); sigterm(); diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index 83b2784b6cb..e768130f335 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -19,6 +19,7 @@ import { isGatewaySigusr1RestartExternallyAllowed, markGatewaySigusr1RestartHandled, peekGatewaySigusr1RestartReason, + resetGatewayRestartStateForInProcessRestart, scheduleGatewaySigusr1Restart, } from "../../infra/restart.js"; import { detectRespawnSupervisor } from "../../infra/supervisor-markers.js"; @@ -36,6 +37,7 @@ import { } from "../../process/command-queue.js"; import { createRestartIterationHook } from "../../process/restart-recovery.js"; import type { RuntimeEnv } from "../../runtime.js"; +import { reloadTaskRegistryFromStore } from "../../tasks/runtime-internal.js"; const gatewayLog = createSubsystemLogger("gateway"); const LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS = 1500; @@ -422,10 +424,12 @@ export async function runGatewayLoop(params: { // After an in-process restart (SIGUSR1), reset command-queue lane state. // Interrupted tasks from the previous lifecycle may have left `active` // counts elevated (their finally blocks never ran), permanently blocking - // new work from draining. This must happen here — at the restart - // coordinator level — rather than inside individual subsystem init - // functions, to avoid surprising cross-cutting side effects. + // new work from draining. The same boundary also discards stale restart + // deferral timers and reloads the task registry from durable state so + // cancelled/completed work is not kept alive by old in-memory maps. resetAllLanes(); + resetGatewayRestartStateForInProcessRestart(); + reloadTaskRegistryFromStore(); }); // Keep process alive; SIGUSR1 triggers an in-process restart (no supervisor required). diff --git a/src/gateway/server-startup-post-attach.test.ts b/src/gateway/server-startup-post-attach.test.ts index d3ebcf51fd9..8a796a2a09e 100644 --- a/src/gateway/server-startup-post-attach.test.ts +++ b/src/gateway/server-startup-post-attach.test.ts @@ -127,7 +127,7 @@ vi.mock("./server-tailscale.js", () => ({ startGatewayTailscaleExposure: hoisted.startGatewayTailscaleExposure, })); -const { startGatewayPostAttachRuntime, startGatewaySidecars } = +const { startGatewayPostAttachRuntime, startGatewaySidecars, __testing } = await import("./server-startup-post-attach.js"); const { STARTUP_UNAVAILABLE_GATEWAY_METHODS } = await import("./server-startup-unavailable-methods.js"); @@ -225,6 +225,35 @@ describe("startGatewayPostAttachRuntime", () => { expect(returned).toBe(true); }); + it("continues channel startup when primary model prewarm hangs", async () => { + vi.useFakeTimers(); + const log = { warn: vi.fn() }; + const prewarm = vi.fn(async () => { + await new Promise(() => undefined); + }); + + try { + const promise = __testing.prewarmConfiguredPrimaryModelWithTimeout( + { + cfg: {} as never, + log, + timeoutMs: 25, + }, + prewarm as never, + ); + + await vi.advanceTimersByTimeAsync(25); + await promise; + + expect(prewarm).toHaveBeenCalledTimes(1); + expect(log.warn).toHaveBeenCalledWith( + "startup model warmup timed out after 25ms; continuing channel startup", + ); + } finally { + vi.useRealTimers(); + } + }); + it("keeps startup-gated methods unavailable while sidecars are still resuming", async () => { let resumeSidecars!: () => void; const sidecarsReady = new Promise<{ pluginServices: null }>((resolve) => { diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 2da02d6565d..d9008f4e779 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -21,6 +21,7 @@ import type { startGatewayTailscaleExposure } from "./server-tailscale.js"; const SESSION_LOCK_STALE_MS = 30 * 60 * 1000; const ACP_BACKEND_READY_TIMEOUT_MS = 5_000; const ACP_BACKEND_READY_POLL_MS = 50; +const PRIMARY_MODEL_PREWARM_TIMEOUT_MS = 5_000; type Awaitable = T | Promise; @@ -162,6 +163,34 @@ async function prewarmConfiguredPrimaryModel(params: { } } +async function prewarmConfiguredPrimaryModelWithTimeout( + params: { + cfg: OpenClawConfig; + log: { warn: (msg: string) => void }; + timeoutMs?: number; + }, + prewarm: typeof prewarmConfiguredPrimaryModel = prewarmConfiguredPrimaryModel, +): Promise { + let settled = false; + const warmup = prewarm(params) + .catch((err) => { + params.log.warn(`startup model warmup failed: ${String(err)}`); + }) + .finally(() => { + settled = true; + }); + const timeout = sleep(params.timeoutMs ?? PRIMARY_MODEL_PREWARM_TIMEOUT_MS, undefined, { + ref: false, + }).then(() => { + if (!settled) { + params.log.warn( + `startup model warmup timed out after ${params.timeoutMs ?? PRIMARY_MODEL_PREWARM_TIMEOUT_MS}ms; continuing channel startup`, + ); + } + }); + await Promise.race([warmup, timeout]); +} + export async function startGatewaySidecars(params: { cfg: OpenClawConfig; pluginRegistry: ReturnType; @@ -289,7 +318,7 @@ export async function startGatewaySidecars(params: { await measureStartup(params.startupTrace, "sidecars.channels", async () => { if (!skipChannels) { try { - await prewarmConfiguredPrimaryModel({ + await prewarmConfiguredPrimaryModelWithTimeout({ cfg: params.cfg, log: params.log, }); @@ -608,4 +637,5 @@ export async function startGatewayPostAttachRuntime( export const __testing = { prewarmConfiguredPrimaryModel, + prewarmConfiguredPrimaryModelWithTimeout, }; diff --git a/src/infra/restart.ts b/src/infra/restart.ts index 3bfed4024b1..bea0bcaab1b 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -72,6 +72,11 @@ function clearActiveDeferralPolls(): void { activeDeferralPolls.clear(); } +export function resetGatewayRestartStateForInProcessRestart(): void { + clearActiveDeferralPolls(); + clearPendingScheduledRestart(); +} + export type RestartAuditInfo = { actor?: string; deviceId?: string; diff --git a/src/infra/run-node.test.ts b/src/infra/run-node.test.ts index bc53a2afbec..05d3a5e5601 100644 --- a/src/infra/run-node.test.ts +++ b/src/infra/run-node.test.ts @@ -107,6 +107,10 @@ function statusCommandSpawn() { return [process.execPath, "openclaw.mjs", "status"]; } +function gatewayCallStatusCommandSpawn() { + return [process.execPath, "openclaw.mjs", "gateway", "call", "status", "--json"]; +} + function resolvePath(tmp: string, relativePath: string) { return path.join(tmp, relativePath); } @@ -230,6 +234,32 @@ async function runStatusCommand(params: { }); } +async function runGatewayCallStatusCommand(params: { + tmp: string; + spawn: (cmd: string, args: string[]) => ReturnType; + spawnSync?: (cmd: string, args: string[]) => { status: number; stdout: string }; + env?: Record; + runRuntimePostBuild?: (params?: { + cwd?: string; + env?: Record; + }) => void | Promise; +}) { + return await runNodeMain({ + cwd: params.tmp, + args: ["gateway", "call", "status", "--json"], + env: { + ...process.env, + OPENCLAW_RUNNER_LOG: "0", + ...params.env, + }, + spawn: params.spawn, + ...(params.spawnSync ? { spawnSync: params.spawnSync } : {}), + ...(params.runRuntimePostBuild ? { runRuntimePostBuild: params.runRuntimePostBuild } : {}), + execPath: process.execPath, + platform: process.platform, + }); +} + async function runQaCommand(params: { tmp: string; spawn: (cmd: string, args: string[]) => ReturnType; @@ -1057,6 +1087,41 @@ describe("run-node script", () => { }); }); + it("does not rebuild for gateway client calls against an existing dirty dist", async () => { + await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { + await setupTrackedProject(tmp, { + files: { + [ROOT_SRC]: "export const value = 1;\n", + [RUNTIME_POSTBUILD_STAMP]: '{"head":"abc123"}\n', + }, + buildPaths: [ + ROOT_SRC, + ROOT_TSCONFIG, + ROOT_PACKAGE, + DIST_ENTRY, + BUILD_STAMP, + RUNTIME_POSTBUILD_STAMP, + ], + }); + + const runRuntimePostBuild = vi.fn(); + const { spawnCalls, spawn, spawnSync } = createSpawnRecorder({ + gitHead: "abc123\n", + gitStatus: ` M ${ROOT_SRC}\n`, + }); + const exitCode = await runGatewayCallStatusCommand({ + tmp, + spawn, + spawnSync, + runRuntimePostBuild, + }); + + expect(exitCode).toBe(0); + expect(spawnCalls).toEqual([gatewayCallStatusCommandSpawn()]); + expect(runRuntimePostBuild).not.toHaveBeenCalled(); + }); + }); + it("reports a clean tree explicitly when dist is current", async () => { await withTempDir({ prefix: "openclaw-run-node-" }, async (tmp) => { await setupTrackedProject(tmp, { diff --git a/src/tasks/runtime-internal.ts b/src/tasks/runtime-internal.ts index 02027db6c8f..78354bff3b2 100644 --- a/src/tasks/runtime-internal.ts +++ b/src/tasks/runtime-internal.ts @@ -23,6 +23,7 @@ export { markTaskTerminalByRunId, maybeDeliverTaskTerminalUpdate, recordTaskProgressByRunId, + reloadTaskRegistryFromStore, resetTaskRegistryDeliveryRuntimeForTests, resolveTaskForLookupToken, resetTaskRegistryForTests, diff --git a/src/tasks/task-registry.test.ts b/src/tasks/task-registry.test.ts index 24ab1abed6a..f01503662c2 100644 --- a/src/tasks/task-registry.test.ts +++ b/src/tasks/task-registry.test.ts @@ -34,6 +34,7 @@ import { markTaskTerminalById, markTaskTerminalByRunId, recordTaskProgressByRunId, + reloadTaskRegistryFromStore, resetTaskRegistryControlRuntimeForTests, resetTaskRegistryDeliveryRuntimeForTests, resetTaskRegistryForTests, @@ -1834,6 +1835,75 @@ describe("task-registry", () => { }); }); + it("reloads from durable state instead of preserving stale in-memory tasks", async () => { + await withTaskRegistryTempDir(async (root) => { + process.env.OPENCLAW_STATE_DIR = root; + resetTaskRegistryForTests(); + const now = Date.now(); + let durableTasks = new Map>(); + configureTaskRegistryRuntime({ + store: { + loadSnapshot: () => ({ + tasks: durableTasks, + deliveryStates: new Map(), + }), + saveSnapshot: () => {}, + upsertTask: () => {}, + upsertTaskWithDeliveryState: () => {}, + }, + }); + + const staleTask = createTaskRecord({ + runtime: "cli", + ownerKey: "agent:main:main", + scopeKind: "session", + requesterSessionKey: "agent:main:main", + runId: "run-stale-memory", + task: "Stale in-memory task", + status: "running", + deliveryStatus: "pending", + notifyPolicy: "silent", + }); + setTaskTimingById({ + taskId: staleTask.taskId, + startedAt: now - 60_000, + lastEventAt: now - 60_000, + }); + expect(getTaskRegistrySummary().active).toBe(1); + + durableTasks = new Map([ + [ + "task-durable", + { + taskId: "task-durable", + runtime: "cli", + requesterSessionKey: "agent:main:main", + ownerKey: "agent:main:main", + scopeKind: "session", + runId: "run-durable", + task: "Durable terminal task", + status: "cancelled", + deliveryStatus: "not_applicable", + notifyPolicy: "silent", + createdAt: now - 30_000, + startedAt: now - 30_000, + endedAt: now - 10_000, + lastEventAt: now - 10_000, + }, + ], + ]); + + reloadTaskRegistryFromStore(); + + expect(findTaskByRunId("run-stale-memory")).toBeUndefined(); + expect(findTaskByRunId("run-durable")).toMatchObject({ + taskId: "task-durable", + status: "cancelled", + }); + expect(getTaskRegistrySummary().active).toBe(0); + }); + }); + it("summarizes inspectable task audit findings", async () => { await withTaskRegistryTempDir(async (root) => { process.env.OPENCLAW_STATE_DIR = root; diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index a571bf9827c..c036480a1e7 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -319,6 +319,16 @@ function persistTaskDeliveryStateDelete(taskId: string) { }); } +function clearTaskRegistryMemory(): void { + tasks.clear(); + taskDeliveryStates.clear(); + taskIdsByRunId.clear(); + taskIdsByOwnerKey.clear(); + taskIdsByParentFlowId.clear(); + taskIdsByRelatedSessionKey.clear(); + tasksWithPendingDelivery.clear(); +} + function ensureDeliveryStatus(params: { ownerKey: string; scopeKind: TaskScopeKind; @@ -946,6 +956,12 @@ export function ensureTaskRegistryReady() { ensureListener(); } +export function reloadTaskRegistryFromStore(): void { + clearTaskRegistryMemory(); + restoreAttempted = false; + restoreTaskRegistryOnce(); +} + function updateTask(taskId: string, patch: Partial): TaskRecord | null { const current = tasks.get(taskId); if (!current) { @@ -2057,13 +2073,7 @@ export function deleteTaskRecordById(taskId: string): boolean { } export function resetTaskRegistryForTests(opts?: { persist?: boolean }) { - tasks.clear(); - taskDeliveryStates.clear(); - taskIdsByRunId.clear(); - taskIdsByOwnerKey.clear(); - taskIdsByParentFlowId.clear(); - taskIdsByRelatedSessionKey.clear(); - tasksWithPendingDelivery.clear(); + clearTaskRegistryMemory(); restoreAttempted = false; resetTaskRegistryRuntimeForTests(); if (listenerStop) {