/** * Session write-lock implementation. * * Uses lock files with owner metadata, stale detection, signal cleanup, and watchdog checks to serialize writes. */ import "../infra/fs-safe-defaults.js"; import type fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; import { createFileLockManager } from "../infra/file-lock-manager.js"; import { readGatewayProcessArgsSync as readProcessArgsSync } from "../infra/gateway-processes.js"; import { getProcessStartTime, isPidAlive } from "../shared/pid-alive.js"; import { SessionWriteLockStaleError, SessionWriteLockTimeoutError, } from "./session-write-lock-error.js"; type LockFilePayload = { pid?: number; createdAt?: string; /** Process start time in clock ticks (from /proc/pid/stat field 22). */ starttime?: number; maxHoldMs?: number; }; function isValidLockNumber(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } export type SessionLockInspection = { lockPath: string; pid: number | null; pidAlive: boolean; createdAt: string | null; ageMs: number | null; stale: boolean; staleReasons: string[]; removable: boolean; removed: boolean; }; export type SessionLockOwnerProcessArgsReader = (pid: number) => string[] | null; const CLEANUP_SIGNALS = ["SIGINT", "SIGTERM", "SIGQUIT", "SIGABRT"] as const; type CleanupSignal = (typeof CLEANUP_SIGNALS)[number]; const CLEANUP_STATE_KEY = Symbol.for("openclaw.sessionWriteLockCleanupState"); const WATCHDOG_STATE_KEY = Symbol.for("openclaw.sessionWriteLockWatchdogState"); const DEFAULT_SESSION_WRITE_LOCK_STALE_MS = 30 * 60 * 1000; const DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS = 5 * 60 * 1000; const DEFAULT_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS = 60_000; const ABORTABLE_SESSION_WRITE_LOCK_POLL_MS = 100; const DEFAULT_WATCHDOG_INTERVAL_MS = 60_000; const DEFAULT_TIMEOUT_GRACE_MS = 2 * 60 * 1000; const REPORT_ONLY_STALE_LOCK_REASONS = new Set(["too-old", "hold-exceeded"]); /** * Yield control to the event loop so other sessions can make progress * while lock contention callbacks run synchronous I/O. */ function yieldEventLoop(): Promise { return new Promise((resolve) => { setImmediate(resolve); }); } // A payload-less lock can be left behind during the window between open("wx") // and the owner metadata write if the owner is suspended (CPU pressure, // container freeze, I/O stall, GC pause). 30 s covers realistic system // pauses while staying well below DEFAULT_TIMEOUT_GRACE_MS (120 s). const ORPHAN_LOCK_PAYLOAD_GRACE_MS = 30_000; const SHORT_TIMEOUT_ORPHAN_LOCK_PAYLOAD_GRACE_MS = 5_000; type CleanupState = { registered: boolean; exitHandler?: () => void; cleanupHandlers: Map void>; }; type WatchdogState = { started: boolean; intervalMs: number; timer?: NodeJS.Timeout; }; type LockInspectionDetails = Pick< SessionLockInspection, "pid" | "pidAlive" | "createdAt" | "ageMs" | "stale" | "staleReasons" >; const SESSION_LOCKS = createFileLockManager("openclaw.session-write-lock"); let resolveProcessStartTimeForLock = getProcessStartTime; function isFileLockError(error: unknown, code: string): boolean { return (error as { code?: unknown } | null)?.code === code; } export type SessionWriteLockAcquireTimeoutConfig = { session?: { writeLock?: { acquireTimeoutMs?: number; staleMs?: number; maxHoldMs?: number; }; }; }; type SessionWriteLockMsKey = "acquireTimeoutMs" | "staleMs" | "maxHoldMs"; const SESSION_WRITE_LOCK_ENV: Record = { acquireTimeoutMs: "OPENCLAW_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS", staleMs: "OPENCLAW_SESSION_WRITE_LOCK_STALE_MS", maxHoldMs: "OPENCLAW_SESSION_WRITE_LOCK_MAX_HOLD_MS", }; function readPositiveMsEnv( env: NodeJS.ProcessEnv, key: string, opts: { allowInfinity?: boolean } = {}, ): number | undefined { const raw = env[key]?.trim(); if (!raw) { return undefined; } if (raw === "Infinity") { return opts.allowInfinity ? Number.POSITIVE_INFINITY : undefined; } if (!/^\d+$/.test(raw)) { return undefined; } const value = Number(raw); return parsePositiveMs(value, opts); } function parsePositiveMs( value: number | undefined, opts: { allowInfinity?: boolean } = {}, ): number | undefined { if (typeof value !== "number" || Number.isNaN(value) || value <= 0) { return undefined; } if (value === Number.POSITIVE_INFINITY) { return opts.allowInfinity ? value : undefined; } if (!Number.isFinite(value)) { return undefined; } if (!Number.isSafeInteger(value)) { return undefined; } return value; } function resolveSessionWriteLockMs(params: { config?: SessionWriteLockAcquireTimeoutConfig; env?: NodeJS.ProcessEnv; key: SessionWriteLockMsKey; fallback: number; allowInfinity?: boolean; }): number { const opts = { allowInfinity: params.allowInfinity }; return ( readPositiveMsEnv(params.env ?? process.env, SESSION_WRITE_LOCK_ENV[params.key], opts) ?? parsePositiveMs(params.config?.session?.writeLock?.[params.key], opts) ?? params.fallback ); } export function resolveSessionWriteLockAcquireTimeoutMs( config?: SessionWriteLockAcquireTimeoutConfig, env?: NodeJS.ProcessEnv, ): number { return resolveSessionWriteLockMs({ config, env, key: "acquireTimeoutMs", fallback: DEFAULT_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS, allowInfinity: true, }); } export function resolveSessionWriteLockStaleMs( config?: SessionWriteLockAcquireTimeoutConfig, env?: NodeJS.ProcessEnv, ): number { return resolveSessionWriteLockMs({ config, env, key: "staleMs", fallback: DEFAULT_SESSION_WRITE_LOCK_STALE_MS, }); } function resolveSessionWriteLockMaxHoldMs( config?: SessionWriteLockAcquireTimeoutConfig, params: { env?: NodeJS.ProcessEnv; fallback?: number } = {}, ): number { return resolveSessionWriteLockMs({ config, env: params.env, key: "maxHoldMs", fallback: params.fallback ?? DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS, }); } export function resolveSessionWriteLockOptions( config?: SessionWriteLockAcquireTimeoutConfig, params: { env?: NodeJS.ProcessEnv; maxHoldMsFallback?: number } = {}, ): { timeoutMs: number; staleMs: number; maxHoldMs: number } { return { timeoutMs: resolveSessionWriteLockAcquireTimeoutMs(config, params.env), staleMs: resolveSessionWriteLockStaleMs(config, params.env), maxHoldMs: resolveSessionWriteLockMaxHoldMs(config, { env: params.env, fallback: params.maxHoldMsFallback, }), }; } function resolveCleanupState(): CleanupState { const proc = process as NodeJS.Process & { [CLEANUP_STATE_KEY]?: CleanupState; }; if (!proc[CLEANUP_STATE_KEY]) { proc[CLEANUP_STATE_KEY] = { registered: false, exitHandler: undefined, cleanupHandlers: new Map void>(), }; } return proc[CLEANUP_STATE_KEY]; } function resolveWatchdogState(): WatchdogState { const proc = process as NodeJS.Process & { [WATCHDOG_STATE_KEY]?: WatchdogState; }; if (!proc[WATCHDOG_STATE_KEY]) { proc[WATCHDOG_STATE_KEY] = { started: false, intervalMs: DEFAULT_WATCHDOG_INTERVAL_MS, }; } return proc[WATCHDOG_STATE_KEY]; } function resolvePositiveMs( value: number | undefined, fallback: number, opts: { allowInfinity?: boolean } = {}, ): number { if (typeof value !== "number" || Number.isNaN(value) || value <= 0) { return fallback; } if (value === Number.POSITIVE_INFINITY) { return opts.allowInfinity ? value : fallback; } if (!Number.isFinite(value)) { return fallback; } return value; } export function resolveSessionLockMaxHoldFromTimeout(params: { timeoutMs: number; graceMs?: number; minMs?: number; }): number { const minMs = resolvePositiveMs(params.minMs, DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS); const timeoutMs = resolvePositiveMs(params.timeoutMs, minMs, { allowInfinity: true }); if (timeoutMs === Number.POSITIVE_INFINITY) { return MAX_TIMER_TIMEOUT_MS; } const graceMs = resolvePositiveMs(params.graceMs, DEFAULT_TIMEOUT_GRACE_MS); return Math.min(MAX_TIMER_TIMEOUT_MS, Math.max(minMs, timeoutMs + graceMs)); } /** * Synchronously release all held locks. * Used during process exit when async operations aren't reliable. */ function releaseAllLocksSync(): void { SESSION_LOCKS.reset(); stopWatchdogTimer(); } async function runLockWatchdogCheck(nowMs = Date.now()): Promise { let released = 0; for (const held of SESSION_LOCKS.heldEntries()) { const maxHoldMs = typeof held.metadata.maxHoldMs === "number" ? held.metadata.maxHoldMs : DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS; const heldForMs = nowMs - held.acquiredAt; if (heldForMs <= maxHoldMs) { continue; } process.stderr.write( `[session-write-lock] releasing lock held for ${heldForMs}ms (max=${maxHoldMs}ms): ${held.lockPath}\n`, ); const didRelease = await held.forceRelease(); if (didRelease) { released += 1; } } return released; } function stopWatchdogTimer(): void { const watchdogState = resolveWatchdogState(); if (watchdogState.timer) { clearInterval(watchdogState.timer); watchdogState.timer = undefined; } watchdogState.started = false; } function shouldStartBackgroundWatchdog(): boolean { return process.env.VITEST !== "true" || process.env.OPENCLAW_TEST_SESSION_LOCK_WATCHDOG === "1"; } function ensureWatchdogStarted(intervalMs: number): void { if (!shouldStartBackgroundWatchdog()) { return; } const watchdogState = resolveWatchdogState(); if (watchdogState.started) { return; } watchdogState.started = true; watchdogState.intervalMs = intervalMs; watchdogState.timer = setInterval(() => { void runLockWatchdogCheck().catch(() => { // Ignore watchdog errors - best effort cleanup only. }); }, intervalMs); watchdogState.timer.unref?.(); } function handleTerminationSignal(signal: CleanupSignal): void { releaseAllLocksSync(); const cleanupState = resolveCleanupState(); const shouldReraise = process.listenerCount(signal) === 1; if (shouldReraise) { const handler = cleanupState.cleanupHandlers.get(signal); if (handler) { process.off(signal, handler); cleanupState.cleanupHandlers.delete(signal); } try { process.kill(process.pid, signal); } catch { // Ignore errors during shutdown } } } function registerCleanupHandlers(): void { const cleanupState = resolveCleanupState(); cleanupState.registered = true; if (!cleanupState.exitHandler) { // Cleanup on normal exit and process.exit() calls cleanupState.exitHandler = () => { releaseAllLocksSync(); }; process.on("exit", cleanupState.exitHandler); } ensureWatchdogStarted(DEFAULT_WATCHDOG_INTERVAL_MS); // Handle termination signals for (const signal of CLEANUP_SIGNALS) { if (cleanupState.cleanupHandlers.has(signal)) { continue; } try { const handler = () => handleTerminationSignal(signal); cleanupState.cleanupHandlers.set(signal, handler); process.on(signal, handler); } catch { // Ignore unsupported signals on this platform. } } } function unregisterCleanupHandlers(): void { const cleanupState = resolveCleanupState(); if (cleanupState.exitHandler) { process.off("exit", cleanupState.exitHandler); cleanupState.exitHandler = undefined; } for (const [signal, handler] of cleanupState.cleanupHandlers) { process.off(signal, handler); } cleanupState.cleanupHandlers.clear(); cleanupState.registered = false; } function parseLockPayload(raw: string): LockFilePayload | null { const parsed = JSON.parse(raw) as Record; const payload: LockFilePayload = {}; if (isValidLockNumber(parsed.pid) && parsed.pid > 0) { payload.pid = parsed.pid; } if (typeof parsed.createdAt === "string") { payload.createdAt = parsed.createdAt; } if (isValidLockNumber(parsed.starttime)) { payload.starttime = parsed.starttime; } if (isValidLockNumber(parsed.maxHoldMs) && parsed.maxHoldMs > 0) { payload.maxHoldMs = parsed.maxHoldMs; } return payload; } async function readLockPayload(lockPath: string): Promise { try { const raw = await fs.readFile(lockPath, "utf8"); return parseLockPayload(raw); } catch { return null; } } async function readLockPayloadForDiagnostics( lockPath: string, ): Promise<{ payload: LockFilePayload | null; missing: boolean }> { try { const raw = await fs.readFile(lockPath, "utf8"); return { payload: parseLockPayload(raw), missing: false }; } catch (error) { const code = (error as { code?: string } | null)?.code; return { payload: null, missing: code === "ENOENT" }; } } async function resolveNormalizedSessionFile(sessionFile: string): Promise { const resolvedSessionFile = path.resolve(sessionFile); const sessionDir = path.dirname(resolvedSessionFile); try { const normalizedDir = await fs.realpath(sessionDir); return path.join(normalizedDir, path.basename(resolvedSessionFile)); } catch { return resolvedSessionFile; } } function normalizeOwnerProcessArg(arg: string): string { return arg.trim().replaceAll("\\", "/").toLowerCase(); } function isOpenClawSessionOwnerArgv(args: string[]): boolean { const normalized = args.map(normalizeOwnerProcessArg).filter(Boolean); if (normalized.length === 0) { return false; } const exe = (normalized[0] ?? "").replace(/\.(bat|cmd|exe)$/i, ""); if (exe === "openclaw" || exe.endsWith("/openclaw") || exe.endsWith("/openclaw-gateway")) { return true; } if ( normalized.some( (arg) => arg === "openclaw" || arg.endsWith("/openclaw") || arg === "openclaw.mjs" || arg.endsWith("/openclaw.mjs"), ) ) { return true; } const entryCandidates = [ "dist/index.js", "dist/entry.js", "scripts/run-node.mjs", "src/entry.ts", "src/index.ts", ]; const hasOpenClawCommandToken = normalized.some((arg) => arg === "gateway" || arg === "agent"); return normalized.some( (arg) => entryCandidates.some((entry) => arg.endsWith(entry)) && hasOpenClawCommandToken, ); } function readOwnerProcessArgs( reader: SessionLockOwnerProcessArgsReader, pid: number, ): string[] | null { try { const args = reader(pid); return Array.isArray(args) ? args : null; } catch { return null; } } function inspectLockPayload( payload: LockFilePayload | null, staleMs: number, nowMs: number, opts: { respectMaxHold?: boolean } = {}, ): LockInspectionDetails { const pid = isValidLockNumber(payload?.pid) && payload.pid > 0 ? payload.pid : null; const pidAlive = pid !== null ? isPidAlive(pid) : false; const createdAt = typeof payload?.createdAt === "string" ? payload.createdAt : null; const createdAtMs = createdAt ? Date.parse(createdAt) : Number.NaN; const ageMs = Number.isFinite(createdAtMs) ? Math.max(0, nowMs - createdAtMs) : null; // Detect PID recycling: if the PID is alive but its start time differs from // what was recorded in the lock file, the original process died and the OS // reassigned the same PID to a different process. const storedStarttime = isValidLockNumber(payload?.starttime) ? payload.starttime : null; const pidRecycled = pidAlive && pid !== null && storedStarttime !== null ? (() => { const currentStarttime = resolveProcessStartTimeForLock(pid); return currentStarttime !== null && currentStarttime !== storedStarttime; })() : false; const staleReasons: string[] = []; if (pid === null) { staleReasons.push("missing-pid"); } else if (!pidAlive) { staleReasons.push("dead-pid"); } else if (pidRecycled) { staleReasons.push("recycled-pid"); } if (ageMs === null) { staleReasons.push("invalid-createdAt"); } else if (ageMs > staleMs) { staleReasons.push("too-old"); } const holderMaxHoldMs = isValidLockNumber(payload?.maxHoldMs) && payload.maxHoldMs > 0 ? payload.maxHoldMs : undefined; if ( opts.respectMaxHold === true && typeof holderMaxHoldMs === "number" && ageMs !== null && ageMs > holderMaxHoldMs ) { staleReasons.push("hold-exceeded"); } return { pid, pidAlive, createdAt, ageMs, stale: staleReasons.length > 0, staleReasons, }; } function shouldTreatAsNonOpenClawOwner(params: { payload: LockFilePayload | null; inspected: LockInspectionDetails; heldByThisProcess: boolean; readOwnerProcessArgs: SessionLockOwnerProcessArgsReader; }): boolean { if (params.inspected.pid === null || !params.inspected.pidAlive) { return false; } if (params.inspected.staleReasons.includes("recycled-pid")) { return false; } if (params.inspected.pid === process.pid && params.heldByThisProcess) { return false; } if (!isValidLockNumber(params.payload?.pid) || params.payload.pid <= 0) { return false; } const args = readOwnerProcessArgs(params.readOwnerProcessArgs, params.payload.pid); if (!args || args.every((arg) => !arg.trim())) { return false; } return !isOpenClawSessionOwnerArgv(args); } function lockInspectionNeedsMtimeStaleFallback(details: LockInspectionDetails): boolean { return ( details.stale && details.staleReasons.every( (reason) => reason === "missing-pid" || reason === "invalid-createdAt", ) ); } async function shouldReportContendedLockStale(params: { lockPath: string; details: LockInspectionDetails; heldByThisProcess: boolean; staleMs: number; nowMs: number; orphanPayloadGraceMs: number; }): Promise { if (!params.details.stale) { return false; } if (params.heldByThisProcess) { return false; } if (lockInspectionNeedsMtimeStaleFallback(params.details)) { try { const stat = await fs.stat(params.lockPath); const ageMs = Math.max(0, params.nowMs - stat.mtimeMs); return ageMs > Math.min(params.staleMs, params.orphanPayloadGraceMs); } catch (error) { const code = (error as { code?: string } | null)?.code; return code !== "ENOENT"; } } return true; } async function shouldRemoveContendedLockFile( lockPath: string, details: LockInspectionDetails, staleMs: number, nowMs: number, orphanPayloadGraceMs = ORPHAN_LOCK_PAYLOAD_GRACE_MS, ): Promise { if (!details.stale) { return false; } if (details.staleReasons.every((reason) => REPORT_ONLY_STALE_LOCK_REASONS.has(reason))) { return false; } if (!lockInspectionNeedsMtimeStaleFallback(details)) { return true; } try { const stat = await fs.stat(lockPath); const ageMs = Math.max(0, nowMs - stat.mtimeMs); return ageMs > Math.min(staleMs, orphanPayloadGraceMs); } catch (error) { const code = (error as { code?: string } | null)?.code; return code !== "ENOENT"; } } function resolveOrphanLockPayloadGraceMs(timeoutMs: number): number { if (timeoutMs < ORPHAN_LOCK_PAYLOAD_GRACE_MS) { return SHORT_TIMEOUT_ORPHAN_LOCK_PAYLOAD_GRACE_MS; } return ORPHAN_LOCK_PAYLOAD_GRACE_MS; } function resolveRemainingAcquireTimeoutMs( timeoutMs: number, startedAtMs: number, nowMs: number, ): number { if (timeoutMs === Number.POSITIVE_INFINITY) { return Number.POSITIVE_INFINITY; } const elapsedMs = Math.max(0, nowMs - startedAtMs); return Math.max(0, timeoutMs - elapsedMs); } async function shouldRetryStaleAcquireFailure(params: { lockPath: string; lockMissingAtDiagnostics: boolean; inspected: LockInspectionDetails; heldByThisProcess: boolean; staleMs: number; nowMs: number; orphanPayloadGraceMs: number; }): Promise { if (params.lockMissingAtDiagnostics) { return true; } return !(await shouldReportContendedLockStale({ lockPath: params.lockPath, details: params.inspected, heldByThisProcess: params.heldByThisProcess, staleMs: params.staleMs, nowMs: params.nowMs, orphanPayloadGraceMs: params.orphanPayloadGraceMs, })); } async function shouldRemoveLockDuringCleanup( lockPath: string, details: LockInspectionDetails, staleMs: number, nowMs: number, ): Promise { if (!details.stale) { return false; } return await shouldRemoveContendedLockFile(lockPath, details, staleMs, nowMs); } function sessionLockHeldByThisProcess(normalizedSessionFile: string): boolean { return SESSION_LOCKS.heldEntries().some( (entry) => entry.normalizedTargetPath === normalizedSessionFile, ); } function shouldTreatAsOrphanSelfLock(params: { payload: LockFilePayload | null; heldByThisProcess: boolean; reclaimLockWithoutStarttime: boolean; }): boolean { const pid = isValidLockNumber(params.payload?.pid) ? params.payload.pid : null; if (pid !== process.pid) { return false; } if (params.heldByThisProcess) { return false; } const storedStarttime = isValidLockNumber(params.payload?.starttime) ? params.payload.starttime : null; if (storedStarttime === null) { return params.reclaimLockWithoutStarttime; } const currentStarttime = resolveProcessStartTimeForLock(process.pid); return currentStarttime !== null && currentStarttime === storedStarttime; } function describeLockOwnerForError(params: { payload: LockFilePayload | null; inspected: LockInspectionDetails; }): string { const parts: string[] = []; if (params.inspected.pid !== null) { parts.push(`pid=${params.inspected.pid}`); parts.push(`alive=${params.inspected.pidAlive ? "true" : "false"}`); } else if (typeof params.payload?.pid === "number") { parts.push(`pid=${params.payload.pid}`); } else { parts.push("owner=unknown"); } if (typeof params.inspected.ageMs === "number") { parts.push(`ageMs=${Math.floor(params.inspected.ageMs)}`); } return parts.join(" "); } function inspectLockPayloadForSession(params: { payload: LockFilePayload | null; staleMs: number; nowMs: number; heldByThisProcess: boolean; reclaimLockWithoutStarttime: boolean; readOwnerProcessArgs: SessionLockOwnerProcessArgsReader; respectMaxHold?: boolean; }): LockInspectionDetails { const inspected = inspectLockPayload(params.payload, params.staleMs, params.nowMs, { respectMaxHold: params.respectMaxHold, }); if ( shouldTreatAsOrphanSelfLock({ payload: params.payload, heldByThisProcess: params.heldByThisProcess, reclaimLockWithoutStarttime: params.reclaimLockWithoutStarttime, }) ) { return { ...inspected, stale: true, staleReasons: inspected.staleReasons.includes("orphan-self-pid") ? inspected.staleReasons : [...inspected.staleReasons, "orphan-self-pid"], }; } if ( shouldTreatAsNonOpenClawOwner({ payload: params.payload, inspected, heldByThisProcess: params.heldByThisProcess, readOwnerProcessArgs: params.readOwnerProcessArgs, }) ) { return { ...inspected, stale: true, staleReasons: [...inspected.staleReasons, "non-openclaw-owner"], }; } return inspected; } export async function cleanStaleLockFiles(params: { sessionsDir: string; config?: SessionWriteLockAcquireTimeoutConfig; env?: NodeJS.ProcessEnv; staleMs?: number; removeStale?: boolean; nowMs?: number; readOwnerProcessArgs?: SessionLockOwnerProcessArgsReader; log?: { warn?: (message: string) => void; info?: (message: string) => void; }; }): Promise<{ locks: SessionLockInspection[]; cleaned: SessionLockInspection[] }> { const sessionsDir = path.resolve(params.sessionsDir); const staleMs = resolvePositiveMs( params.staleMs, resolveSessionWriteLockStaleMs(params.config, params.env), ); const removeStale = params.removeStale !== false; const nowMs = params.nowMs ?? Date.now(); const baseOwnerProcessArgsReader = params.readOwnerProcessArgs ?? readProcessArgsSync; // Memoize per-invocation: many locks in the same sweep often share a pid (gateway, MCP), // and resolving owner argv is the most expensive per-lock syscall (PowerShell on Windows // is ~0.5–1s per pid) — pids do not recycle within a single sweep. (#86509) const ownerArgsByPid = new Map(); const ownerProcessArgsReader: SessionLockOwnerProcessArgsReader = (pid) => { const cached = ownerArgsByPid.get(pid); if (cached !== undefined) { return cached; } const args = baseOwnerProcessArgsReader(pid); ownerArgsByPid.set(pid, args); return args; }; let entries: fsSync.Dirent[]; try { entries = await fs.readdir(sessionsDir, { withFileTypes: true }); } catch (err) { const code = (err as { code?: string }).code; if (code === "ENOENT") { return { locks: [], cleaned: [] }; } throw err; } const locks: SessionLockInspection[] = []; const cleaned: SessionLockInspection[] = []; const lockEntries = entries .filter((entry) => entry.name.endsWith(".jsonl.lock")) .toSorted((a, b) => a.name.localeCompare(b.name)); for (const entry of lockEntries) { // Yield to the event loop between locks so concurrent timers/HTTP polling can run // while this sweep does per-lock sync syscalls (isPidAlive, /proc reads, PowerShell). (#86509) await new Promise((resolve) => { setImmediate(resolve); }); const lockPath = path.join(sessionsDir, entry.name); const payload = await readLockPayload(lockPath); const inspected = inspectLockPayloadForSession({ payload, staleMs, nowMs, heldByThisProcess: false, reclaimLockWithoutStarttime: false, readOwnerProcessArgs: ownerProcessArgsReader, }); const removable = await shouldRemoveLockDuringCleanup(lockPath, inspected, staleMs, nowMs); const lockInfo: SessionLockInspection = { lockPath, ...inspected, removable, removed: false, }; if (removeStale && removable) { await fs.rm(lockPath, { force: true }); lockInfo.removed = true; cleaned.push(lockInfo); params.log?.warn?.( `removed stale session lock: ${lockPath} (${lockInfo.staleReasons.join(", ") || "unknown"})`, ); } locks.push(lockInfo); } return { locks, cleaned }; } export async function acquireSessionWriteLock(params: { sessionFile: string; timeoutMs?: number; staleMs?: number; maxHoldMs?: number; allowReentrant?: boolean; signal?: AbortSignal; }): Promise<{ release: () => Promise; }> { const throwIfAborted = () => { if (!params.signal?.aborted) { return; } if (params.signal.reason instanceof Error) { throw params.signal.reason; } const error = new Error("request aborted", { cause: params.signal.reason }); error.name = "AbortError"; throw error; }; throwIfAborted(); registerCleanupHandlers(); const allowReentrant = params.allowReentrant ?? false; const defaultOptions = resolveSessionWriteLockOptions(); const timeoutMs = resolvePositiveMs(params.timeoutMs, defaultOptions.timeoutMs, { allowInfinity: true, }); const staleMs = resolvePositiveMs(params.staleMs, defaultOptions.staleMs); const maxHoldMs = resolvePositiveMs(params.maxHoldMs, defaultOptions.maxHoldMs); const orphanPayloadGraceMs = resolveOrphanLockPayloadGraceMs(timeoutMs); const sessionFile = path.resolve(params.sessionFile); const sessionDir = path.dirname(sessionFile); const normalizedSessionFile = await resolveNormalizedSessionFile(sessionFile); const lockPath = `${normalizedSessionFile}.lock`; await fs.mkdir(sessionDir, { recursive: true }); const startedAtMs = Date.now(); while (true) { throwIfAborted(); const remainingTimeoutMs = resolveRemainingAcquireTimeoutMs(timeoutMs, startedAtMs, Date.now()); if (remainingTimeoutMs <= 0) { const payload = await readLockPayload(lockPath); const nowMs = Date.now(); const heldByThisProcess = sessionLockHeldByThisProcess(normalizedSessionFile); const inspected = inspectLockPayloadForSession({ payload, staleMs, nowMs, heldByThisProcess, reclaimLockWithoutStarttime: true, readOwnerProcessArgs: readProcessArgsSync, respectMaxHold: !heldByThisProcess, }); const owner = describeLockOwnerForError({ payload, inspected }); throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath }); } try { const acquireAttemptTimeoutMs = params.signal ? Math.min(remainingTimeoutMs, ABORTABLE_SESSION_WRITE_LOCK_POLL_MS) : remainingTimeoutMs; const lock = await SESSION_LOCKS.acquire(sessionFile, { staleMs, timeoutMs: acquireAttemptTimeoutMs, retry: { minTimeout: 50, maxTimeout: 1000, factor: 1 }, staleRecovery: "remove-if-unchanged", allowReentrant, metadata: { maxHoldMs }, payload: () => { const createdAt = new Date().toISOString(); const starttime = resolveProcessStartTimeForLock(process.pid); const lockPayload: LockFilePayload = { pid: process.pid, createdAt, maxHoldMs }; if (starttime !== null) { lockPayload.starttime = starttime; } return lockPayload as Record; }, shouldReclaim: async ({ payload, nowMs, heldByThisProcess }) => { // Yield to the event loop before synchronous process inspection // to prevent lock contention retries from starving other sessions. await yieldEventLoop(); const inspected = inspectLockPayloadForSession({ payload: payload as LockFilePayload | null, staleMs, nowMs, heldByThisProcess, reclaimLockWithoutStarttime: true, readOwnerProcessArgs: readProcessArgsSync, respectMaxHold: !heldByThisProcess, }); return await shouldReportContendedLockStale({ lockPath, details: inspected, heldByThisProcess, staleMs, nowMs, orphanPayloadGraceMs, }); }, shouldRemoveStaleLock: async ({ lockPath: lockPathLocal, normalizedTargetPath, payload, }) => { await yieldEventLoop(); const nowMs = Date.now(); const heldByThisProcess = sessionLockHeldByThisProcess(normalizedTargetPath); const inspected = inspectLockPayloadForSession({ payload: payload as LockFilePayload | null, staleMs, nowMs, heldByThisProcess, reclaimLockWithoutStarttime: true, readOwnerProcessArgs: readProcessArgsSync, respectMaxHold: !heldByThisProcess, }); return await shouldRemoveContendedLockFile( lockPathLocal, inspected, staleMs, nowMs, orphanPayloadGraceMs, ); }, }); return { release: lock.release }; } catch (err) { throwIfAborted(); if (!isFileLockError(err, "file_lock_timeout") && !isFileLockError(err, "file_lock_stale")) { throw err; } if ( params.signal && isFileLockError(err, "file_lock_timeout") && resolveRemainingAcquireTimeoutMs(timeoutMs, startedAtMs, Date.now()) > 0 ) { continue; } const errorLockPath = (err as { lockPath?: string }).lockPath ?? lockPath; const { payload, missing: lockMissingAtDiagnostics } = await readLockPayloadForDiagnostics(errorLockPath); const nowMs = Date.now(); const heldByThisProcess = sessionLockHeldByThisProcess(normalizedSessionFile); const inspected = inspectLockPayloadForSession({ payload, staleMs, nowMs, heldByThisProcess, reclaimLockWithoutStarttime: true, readOwnerProcessArgs: readProcessArgsSync, respectMaxHold: !heldByThisProcess, }); const owner = describeLockOwnerForError({ payload, inspected }); if (isFileLockError(err, "file_lock_stale")) { if ( resolveRemainingAcquireTimeoutMs(timeoutMs, startedAtMs, Date.now()) > 0 && (await shouldRetryStaleAcquireFailure({ lockPath: errorLockPath, lockMissingAtDiagnostics, inspected, heldByThisProcess, staleMs, nowMs, orphanPayloadGraceMs, })) ) { continue; } throw new SessionWriteLockStaleError({ owner, lockPath: errorLockPath, staleReasons: inspected.staleReasons, }); } throw new SessionWriteLockTimeoutError({ timeoutMs, owner, lockPath: errorLockPath }); } } } export const testing = { cleanupSignals: [...CLEANUP_SIGNALS], handleTerminationSignal, inspectLockPayloadForTest: inspectLockPayload, releaseAllLocksSync, runLockWatchdogCheck, resolveRemainingAcquireTimeoutMs, setProcessStartTimeResolverForTest(resolver: ((pid: number) => number | null) | null): void { resolveProcessStartTimeForLock = resolver ?? getProcessStartTime; }, }; export async function drainSessionWriteLockStateForTest(): Promise { await SESSION_LOCKS.drain(); stopWatchdogTimer(); unregisterCleanupHandlers(); } export function resetSessionWriteLockStateForTest(): void { releaseAllLocksSync(); stopWatchdogTimer(); unregisterCleanupHandlers(); resolveProcessStartTimeForLock = getProcessStartTime; } export { testing as __testing };