mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-14 05:56:11 +00:00
fix: suspension and restart wait for background execs (#104172)
* fix: block restart and suspension on background execs * test: cover background exec suspension inspector
This commit is contained in:
committed by
GitHub
parent
b8502e1c0a
commit
02cd9bf648
@@ -81,6 +81,8 @@ Actions:
|
||||
Notes:
|
||||
|
||||
- Only backgrounded sessions are listed/persisted — in memory only, not on disk. Sessions are lost on process restart.
|
||||
- A live background session blocks cooperative host suspension and safe Gateway restart until the process owner confirms its actual exit.
|
||||
- `process remove` can hide a running session immediately after requesting termination; suspension and restart remain blocked until exit confirmation.
|
||||
- Session logs are only saved to chat history if you run `process poll`/`log` and the tool result is recorded.
|
||||
- `process` is scoped per agent; it only sees sessions started by that agent.
|
||||
- Use `poll`/`log` for status, logs, or completion confirmation when automatic completion wake is unavailable.
|
||||
|
||||
@@ -134,8 +134,10 @@ transports, or control the hosting platform. The host must fence its ingress
|
||||
before preparation and remains responsible for wake, snapshot/freeze, and
|
||||
stop. `activeCount` is the aggregate tracked-work count, while `blockers`
|
||||
contains the non-zero category counts and bounded task details. This is not a
|
||||
general process-quiescence barrier. Channel health, maintenance, cache refresh,
|
||||
established plugin WebSocket sessions, and plugin-owned background work can
|
||||
general process-quiescence barrier. A `background-exec` blocker is aggregate
|
||||
only: command text, process IDs, output, and session or scope identifiers never
|
||||
cross the protocol. Channel health, maintenance, cache refresh, established
|
||||
plugin WebSocket sessions, and unregistered plugin-owned background work can
|
||||
remain active.
|
||||
The hosting platform must freeze or snapshot the full process tree and its
|
||||
filesystem consistently; unregistered work cannot be proven idle by this first
|
||||
|
||||
@@ -44,6 +44,36 @@ describe("gateway suspension protocol", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps background exec blockers count-only", () => {
|
||||
const result = {
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
retryAfterMs: 20_000,
|
||||
activeCount: 1,
|
||||
blockers: [
|
||||
{
|
||||
kind: "background-exec",
|
||||
count: 1,
|
||||
message: "1 active background exec session(s)",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(validateGatewaySuspendPrepareResult(result)).toBe(true);
|
||||
expect(
|
||||
validateGatewaySuspendPrepareResult({
|
||||
...result,
|
||||
blockers: [{ ...result.blockers[0], sessionIds: ["private-session-id"] }],
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
validateGatewaySuspendPrepareResult({
|
||||
...result,
|
||||
blockers: [{ ...result.blockers[0], command: "private command" }],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("validates status and resume results", () => {
|
||||
expect(validateGatewaySuspendStatusResult({ status: "running" })).toBe(true);
|
||||
expect(validateGatewaySuspendStatusResult({ status: "ready", expiresAtMs: 123 })).toBe(true);
|
||||
|
||||
@@ -27,6 +27,7 @@ export const GatewaySuspendBlockerSchema = Type.Object(
|
||||
Type.Literal("queue"),
|
||||
Type.Literal("reply"),
|
||||
Type.Literal("embedded-run"),
|
||||
Type.Literal("background-exec"),
|
||||
Type.Literal("cron-run"),
|
||||
Type.Literal("task"),
|
||||
Type.Literal("root-request"),
|
||||
|
||||
@@ -9,8 +9,12 @@ import type { ProcessSession } from "./bash-process-registry.js";
|
||||
import {
|
||||
addSession,
|
||||
appendOutput,
|
||||
createSessionSlug,
|
||||
deleteSession,
|
||||
drainSession,
|
||||
getActiveBackgroundExecSessionCount,
|
||||
listFinishedSessions,
|
||||
listRunningSessions,
|
||||
markBackgrounded,
|
||||
markExited,
|
||||
resetProcessRegistryForTests,
|
||||
@@ -19,6 +23,14 @@ import {
|
||||
} from "./bash-process-registry.js";
|
||||
import { createProcessSessionFixture } from "./bash-process-registry.test-helpers.js";
|
||||
|
||||
const randomMocks = vi.hoisted(() => ({
|
||||
generateSecureInt: vi.fn(() => 0),
|
||||
}));
|
||||
|
||||
vi.mock("../infra/secure-random.js", () => ({
|
||||
generateSecureInt: randomMocks.generateSecureInt,
|
||||
}));
|
||||
|
||||
describe("bash process registry", () => {
|
||||
function createRegistrySession(params: {
|
||||
id?: string;
|
||||
@@ -37,6 +49,8 @@ describe("bash process registry", () => {
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
randomMocks.generateSecureInt.mockReset();
|
||||
randomMocks.generateSecureInt.mockReturnValue(0);
|
||||
resetProcessRegistryForTests();
|
||||
});
|
||||
|
||||
@@ -173,6 +187,79 @@ describe("bash process registry", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("tracks only live backgrounded sessions", () => {
|
||||
const session = createRegistrySession({
|
||||
maxOutputChars: 100,
|
||||
pendingMaxOutputChars: 30_000,
|
||||
backgrounded: false,
|
||||
});
|
||||
|
||||
addSession(session);
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(0);
|
||||
|
||||
markBackgrounded(session);
|
||||
markBackgrounded(session);
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(1);
|
||||
|
||||
markExited(session, 0, null, "completed");
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(0);
|
||||
|
||||
markBackgrounded(session);
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps a hidden background session active until its process exits", () => {
|
||||
const session = createRegistrySession({
|
||||
id: "hidden-until-exit",
|
||||
maxOutputChars: 100,
|
||||
pendingMaxOutputChars: 30_000,
|
||||
backgrounded: false,
|
||||
});
|
||||
|
||||
addSession(session);
|
||||
markBackgrounded(session);
|
||||
deleteSession(session.id);
|
||||
|
||||
expect(listRunningSessions()).toHaveLength(0);
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(1);
|
||||
|
||||
markExited(session, null, "SIGTERM", "killed");
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps a hidden active session id reserved until exit", () => {
|
||||
const session = createRegistrySession({
|
||||
id: "amber-atlas",
|
||||
maxOutputChars: 100,
|
||||
pendingMaxOutputChars: 30_000,
|
||||
backgrounded: false,
|
||||
});
|
||||
|
||||
addSession(session);
|
||||
markBackgrounded(session);
|
||||
deleteSession(session.id);
|
||||
expect(createSessionSlug()).toBe("amber-atlas-2");
|
||||
|
||||
session.backgrounded = false;
|
||||
markExited(session, 0, null, "completed");
|
||||
expect(createSessionSlug()).toBe("amber-atlas");
|
||||
});
|
||||
|
||||
it("clears background activity in the test reset", () => {
|
||||
const session = createRegistrySession({
|
||||
maxOutputChars: 100,
|
||||
pendingMaxOutputChars: 30_000,
|
||||
backgrounded: false,
|
||||
});
|
||||
|
||||
addSession(session);
|
||||
markBackgrounded(session);
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(1);
|
||||
|
||||
resetProcessRegistryForTests();
|
||||
expect(getActiveBackgroundExecSessionCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("clamps a zero retention TTL to one minute", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -109,11 +109,14 @@ interface FinishedSession {
|
||||
|
||||
const runningSessions = new Map<string, ProcessSession>();
|
||||
const finishedSessions = new Map<string, FinishedSession>();
|
||||
const activeBackgroundExecSessionIds = new Set<string>();
|
||||
|
||||
let sweeper: NodeJS.Timeout | null = null;
|
||||
|
||||
function isSessionIdTaken(id: string) {
|
||||
return runningSessions.has(id) || finishedSessions.has(id);
|
||||
return (
|
||||
runningSessions.has(id) || finishedSessions.has(id) || activeBackgroundExecSessionIds.has(id)
|
||||
);
|
||||
}
|
||||
|
||||
/** Creates a unique short session id that avoids running and retained sessions. */
|
||||
@@ -137,7 +140,7 @@ export function getFinishedSession(id: string) {
|
||||
return finishedSessions.get(id);
|
||||
}
|
||||
|
||||
/** Removes a session from both running and finished registries. */
|
||||
/** Removes visible session records without changing live-process activity. */
|
||||
export function deleteSession(id: string) {
|
||||
runningSessions.delete(id);
|
||||
finishedSessions.delete(id);
|
||||
@@ -194,6 +197,9 @@ export function markExited(
|
||||
exitReason?: TerminationReason,
|
||||
noOutputTimedOut?: boolean,
|
||||
) {
|
||||
// Visibility can be cleared before process termination. Keep suspension
|
||||
// blocked until the process owner reports the actual terminal transition.
|
||||
activeBackgroundExecSessionIds.delete(session.id);
|
||||
session.exited = true;
|
||||
session.exitCode = exitCode;
|
||||
session.exitSignal = exitSignal;
|
||||
@@ -206,6 +212,14 @@ export function markExited(
|
||||
/** Marks a running session as reconnectable after the exec call returns. */
|
||||
export function markBackgrounded(session: ProcessSession) {
|
||||
session.backgrounded = true;
|
||||
if (!session.exited) {
|
||||
activeBackgroundExecSessionIds.add(session.id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the number of live background exec sessions without exposing process details. */
|
||||
export function getActiveBackgroundExecSessionCount(): number {
|
||||
return activeBackgroundExecSessionIds.size;
|
||||
}
|
||||
|
||||
function moveToFinished(session: ProcessSession, status: ProcessStatus) {
|
||||
@@ -334,6 +348,7 @@ export function listFinishedSessions() {
|
||||
export function resetProcessRegistryForTests() {
|
||||
runningSessions.clear();
|
||||
finishedSessions.clear();
|
||||
activeBackgroundExecSessionIds.clear();
|
||||
stopSweeper();
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi, type Mock } from "vitest";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi, type Mock } from "vitest";
|
||||
import {
|
||||
onInternalDiagnosticEvent,
|
||||
resetDiagnosticEventsForTest,
|
||||
@@ -29,8 +29,18 @@ import {
|
||||
} from "../infra/exec-authorization-plan.js";
|
||||
import { buildAuthorizedShellCommandFromPlan } from "../infra/exec-authorization-render.js";
|
||||
import { resolvePolicyTargetCandidatePath } from "../infra/exec-command-resolution.js";
|
||||
import { createSafeGatewayRestartPreflight } from "../infra/restart-coordinator.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import type { ExecApprovalFollowupTarget } from "./bash-tools.exec-host-shared.js";
|
||||
import type { ExecApprovalFollowupFactory } from "./bash-tools.exec-types.js";
|
||||
import type {
|
||||
ExecApprovalFollowupFactory,
|
||||
ExecApprovalFollowupOutcome,
|
||||
} from "./bash-tools.exec-types.js";
|
||||
|
||||
type StrictInlineEvalBoundary =
|
||||
typeof import("./bash-tools.exec-host-shared.js").enforceStrictInlineEvalApprovalBoundary;
|
||||
@@ -155,6 +165,7 @@ const resolveExecHostApprovalContextMock = vi.hoisted(() =>
|
||||
),
|
||||
);
|
||||
const runExecProcessMock = vi.hoisted(() => vi.fn());
|
||||
const markBackgroundedMock = vi.hoisted(() => vi.fn());
|
||||
const sendExecApprovalFollowupResultMock = vi.hoisted(() =>
|
||||
vi.fn<SendExecApprovalFollowupResult>(async () => undefined),
|
||||
);
|
||||
@@ -222,7 +233,8 @@ vi.mock("./bash-tools.exec-runtime.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("./bash-process-registry.js", () => ({
|
||||
markBackgrounded: vi.fn(),
|
||||
getActiveBackgroundExecSessionCount: vi.fn(() => 0),
|
||||
markBackgrounded: markBackgroundedMock,
|
||||
tail: vi.fn((value) => value),
|
||||
}));
|
||||
|
||||
@@ -291,6 +303,7 @@ describe("processGatewayAllowlist", () => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
resetDiagnosticEventsForTest();
|
||||
buildExecApprovalPendingToolResultMock.mockReset();
|
||||
buildExecApprovalFollowupTargetMock.mockReset();
|
||||
@@ -341,6 +354,7 @@ describe("processGatewayAllowlist", () => {
|
||||
askFallback: "deny",
|
||||
});
|
||||
runExecProcessMock.mockReset();
|
||||
markBackgroundedMock.mockReset();
|
||||
sendExecApprovalFollowupResultMock.mockReset();
|
||||
enforceStrictInlineEvalApprovalBoundaryMock.mockReset();
|
||||
enforceStrictInlineEvalApprovalBoundaryMock.mockImplementation((value) => ({
|
||||
@@ -367,6 +381,10 @@ describe("processGatewayAllowlist", () => {
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
function runGatewayAllowlist(
|
||||
overrides: Partial<GatewayAllowlistParams> & Pick<GatewayAllowlistParams, "command">,
|
||||
) {
|
||||
@@ -2010,6 +2028,136 @@ EOF`,
|
||||
},
|
||||
);
|
||||
|
||||
it("waits outside admission, then atomically hands an approved process to the registry", async () => {
|
||||
let resolveApproval: (decision: ExecApprovalDecision) => void = () => {};
|
||||
const approval = new Promise<ExecApprovalDecision>((resolve) => {
|
||||
resolveApproval = resolve;
|
||||
});
|
||||
let resolveOutcome: (outcome: ExecApprovalFollowupOutcome) => void = () => {};
|
||||
const outcome = new Promise<ExecApprovalFollowupOutcome>((resolve) => {
|
||||
resolveOutcome = resolve;
|
||||
});
|
||||
let allowSpawn: () => void = () => {};
|
||||
const spawnAllowed = new Promise<void>((resolve) => {
|
||||
allowSpawn = resolve;
|
||||
});
|
||||
let announceSpawn: () => void = () => {};
|
||||
const spawnStarted = new Promise<void>((resolve) => {
|
||||
announceSpawn = resolve;
|
||||
});
|
||||
resolveApprovalDecisionOrUndefinedMock.mockReturnValue(approval);
|
||||
createExecApprovalDecisionStateMock.mockReturnValue({
|
||||
baseDecision: { timedOut: false },
|
||||
approvedByAsk: true,
|
||||
deniedReason: null,
|
||||
});
|
||||
commitExecAuthorizationMock.mockImplementation(async () => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
});
|
||||
runExecProcessMock.mockImplementation(async () => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
announceSpawn();
|
||||
await spawnAllowed;
|
||||
return { session: { id: "sess-atomic" }, promise: outcome };
|
||||
});
|
||||
markBackgroundedMock.mockImplementation(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
});
|
||||
buildExecApprovalFollowupTargetMock.mockImplementation((value) => value);
|
||||
|
||||
const result = await runGatewayAllowlist({
|
||||
command: "find . -maxdepth 1",
|
||||
turnSourceChannel: "feishu",
|
||||
});
|
||||
expect(result.pendingResult?.details.status).toBe("approval-pending");
|
||||
await vi.waitFor(() => {
|
||||
expect(resolveApprovalDecisionOrUndefinedMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
resolveApproval("allow-once");
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(commitExecAuthorizationMock).not.toHaveBeenCalled();
|
||||
expect(runExecProcessMock).not.toHaveBeenCalled();
|
||||
expect(markBackgroundedMock).not.toHaveBeenCalled();
|
||||
|
||||
suspension?.release();
|
||||
await spawnStarted;
|
||||
expect(
|
||||
createSafeGatewayRestartPreflight({
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
}),
|
||||
).toMatchObject({
|
||||
safe: false,
|
||||
counts: { rootRequests: 1, totalActive: 1 },
|
||||
blockers: [{ kind: "root-request", count: 1 }],
|
||||
});
|
||||
allowSpawn();
|
||||
await vi.waitFor(() => {
|
||||
expect(markBackgroundedMock).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
expect(commitExecAuthorizationMock).toHaveBeenCalledOnce();
|
||||
expect(runExecProcessMock).toHaveBeenCalledOnce();
|
||||
|
||||
resolveOutcome({
|
||||
status: "completed",
|
||||
exitCode: 0,
|
||||
timedOut: false,
|
||||
aggregated: "done",
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(sendExecApprovalFollowupResultMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
it("denies a detached approved process when restart drain wins admission", async () => {
|
||||
let resolveApproval: (decision: ExecApprovalDecision) => void = () => {};
|
||||
resolveApprovalDecisionOrUndefinedMock.mockReturnValue(
|
||||
new Promise<ExecApprovalDecision>((resolve) => {
|
||||
resolveApproval = resolve;
|
||||
}),
|
||||
);
|
||||
createExecApprovalDecisionStateMock.mockReturnValue({
|
||||
baseDecision: { timedOut: false },
|
||||
approvedByAsk: true,
|
||||
deniedReason: null,
|
||||
});
|
||||
buildExecApprovalFollowupTargetMock.mockImplementation((value) => value);
|
||||
|
||||
const result = await runGatewayAllowlist({
|
||||
command: "find . -maxdepth 1",
|
||||
turnSourceChannel: "feishu",
|
||||
});
|
||||
expect(result.pendingResult?.details.status).toBe("approval-pending");
|
||||
await vi.waitFor(() => {
|
||||
expect(resolveApprovalDecisionOrUndefinedMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
markGatewayRestartDraining();
|
||||
resolveApproval("allow-once");
|
||||
await vi.waitFor(() => {
|
||||
expect(sendExecApprovalFollowupResultMock).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"Exec denied (gateway id=req-1, gateway-draining): find . -maxdepth 1",
|
||||
);
|
||||
});
|
||||
expect(sendExecApprovalFollowupResultMock).toHaveBeenCalledOnce();
|
||||
expect(commitExecAuthorizationMock).not.toHaveBeenCalled();
|
||||
expect(runExecProcessMock).not.toHaveBeenCalled();
|
||||
expect(markBackgroundedMock).not.toHaveBeenCalled();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps the fire-and-forget path for channels without native approval clients", async () => {
|
||||
resolveApprovalDecisionOrUndefinedMock.mockResolvedValue("allow-once");
|
||||
createExecApprovalDecisionStateMock.mockReturnValue({
|
||||
|
||||
@@ -39,6 +39,10 @@ import {
|
||||
type ExecAutoReviewInput,
|
||||
} from "../infra/exec-auto-review.js";
|
||||
import type { SafeBinProfile } from "../infra/exec-safe-bin-policy.js";
|
||||
import {
|
||||
GatewayDrainingError,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { isNativeApprovalChannel, normalizeMessageChannel } from "../utils/message-channel.js";
|
||||
import { markBackgrounded, tail } from "./bash-process-registry.js";
|
||||
import {
|
||||
@@ -1120,40 +1124,75 @@ export async function processGatewayAllowlist(
|
||||
return;
|
||||
}
|
||||
|
||||
let admitted:
|
||||
| { status: "started"; run: Awaited<ReturnType<typeof runExecProcess>> }
|
||||
| { status: "approval-state-write-failed" }
|
||||
| { status: "spawn-failed" };
|
||||
try {
|
||||
await commitExecutionAuthorization({
|
||||
source: approvalDecision.authorizationSource,
|
||||
resolvedPath: resolvedPath ?? undefined,
|
||||
...(approvalDecision.allowAlwaysDecision
|
||||
? { allowAlwaysDecision: approvalDecision.allowAlwaysDecision }
|
||||
: {}),
|
||||
admitted = await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
try {
|
||||
await commitExecutionAuthorization({
|
||||
source: approvalDecision.authorizationSource,
|
||||
resolvedPath: resolvedPath ?? undefined,
|
||||
...(approvalDecision.allowAlwaysDecision
|
||||
? { allowAlwaysDecision: approvalDecision.allowAlwaysDecision }
|
||||
: {}),
|
||||
});
|
||||
} catch {
|
||||
return { status: "approval-state-write-failed" as const };
|
||||
}
|
||||
|
||||
let run: Awaited<ReturnType<typeof runExecProcess>>;
|
||||
try {
|
||||
run = await runExecProcess({
|
||||
command: params.command,
|
||||
execCommand: approvalDecision.execCommandOverride,
|
||||
workdir: params.workdir,
|
||||
env: params.env,
|
||||
pathPrepend: params.pathPrepend,
|
||||
sandbox: undefined,
|
||||
containerWorkdir: null,
|
||||
usePty: params.pty,
|
||||
warnings: params.warnings,
|
||||
maxOutput: params.maxOutput,
|
||||
pendingMaxOutput: params.pendingMaxOutput,
|
||||
notifyOnExit: false,
|
||||
notifyOnExitEmptySuccess: false,
|
||||
scopeKey: params.scopeKey,
|
||||
sessionKey: params.notifySessionKey ?? params.sessionKey,
|
||||
timeoutSec: effectiveTimeout,
|
||||
});
|
||||
} catch {
|
||||
return { status: "spawn-failed" as const };
|
||||
}
|
||||
|
||||
// Keep the admitted root until the registry owns the live process.
|
||||
// Suspension must observe one side of this handoff at every instant.
|
||||
markBackgrounded(run.session);
|
||||
return { status: "started" as const, run };
|
||||
});
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof GatewayDrainingError ||
|
||||
(error instanceof Error && error.message === "gateway is draining for restart")
|
||||
) {
|
||||
await sendExecApprovalFollowupResult(
|
||||
followupTarget,
|
||||
`Exec denied (gateway id=${approvalId}, gateway-draining): ${params.command}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Detached approval work must always settle through a follow-up. Treat
|
||||
// any unexpected admission failure as a spawn failure, never an
|
||||
// unhandled rejection from this fire-and-forget chain.
|
||||
admitted = { status: "spawn-failed" };
|
||||
}
|
||||
|
||||
if (admitted.status === "approval-state-write-failed") {
|
||||
await denyApprovalStateWriteFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
let run: Awaited<ReturnType<typeof runExecProcess>> | null;
|
||||
try {
|
||||
run = await runExecProcess({
|
||||
command: params.command,
|
||||
execCommand: approvalDecision.execCommandOverride,
|
||||
workdir: params.workdir,
|
||||
env: params.env,
|
||||
pathPrepend: params.pathPrepend,
|
||||
sandbox: undefined,
|
||||
containerWorkdir: null,
|
||||
usePty: params.pty,
|
||||
warnings: params.warnings,
|
||||
maxOutput: params.maxOutput,
|
||||
pendingMaxOutput: params.pendingMaxOutput,
|
||||
notifyOnExit: false,
|
||||
notifyOnExitEmptySuccess: false,
|
||||
scopeKey: params.scopeKey,
|
||||
sessionKey: params.notifySessionKey ?? params.sessionKey,
|
||||
timeoutSec: effectiveTimeout,
|
||||
});
|
||||
} catch {
|
||||
if (admitted.status === "spawn-failed") {
|
||||
await sendExecApprovalFollowupResult(
|
||||
followupTarget,
|
||||
`Exec denied (gateway id=${approvalId}, spawn-failed): ${params.command}`,
|
||||
@@ -1161,7 +1200,7 @@ export async function processGatewayAllowlist(
|
||||
return;
|
||||
}
|
||||
|
||||
markBackgrounded(run.session);
|
||||
const { run } = admitted;
|
||||
|
||||
const outcome = await run.promise;
|
||||
const dynamicFollowupText = await resolveGatewayExecApprovalFollowupText({
|
||||
|
||||
@@ -2,6 +2,13 @@
|
||||
* Gateway config reload handler tests.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
addSession,
|
||||
markBackgrounded,
|
||||
markExited,
|
||||
resetProcessRegistryForTests,
|
||||
} from "../agents/bash-process-registry.js";
|
||||
import { createProcessSessionFixture } from "../agents/bash-process-registry.test-helpers.js";
|
||||
import type { ConfigWriteNotification } from "../config/config.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
@@ -16,6 +23,7 @@ import {
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayIndependentRootWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js";
|
||||
@@ -234,12 +242,14 @@ function createReloadHandlersForTest(
|
||||
// shared vitest worker imports those helpers before this file runs, the leaked
|
||||
// env routes reloads into the skip branch and channel restarts never fire.
|
||||
beforeEach(() => {
|
||||
resetProcessRegistryForTests();
|
||||
delete process.env.OPENCLAW_SKIP_CHANNELS;
|
||||
delete process.env.OPENCLAW_SKIP_PROVIDERS;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
resetProcessRegistryForTests();
|
||||
hoisted.startGmailWatcherWithLogs.mockClear();
|
||||
hoisted.stopGmailWatcher.mockClear();
|
||||
hoisted.activeTaskCount.value = 0;
|
||||
@@ -500,6 +510,111 @@ describe("gateway restart deferral preflight", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("defers config restart until a background exec actually exits", async () => {
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
const logReload = { info: vi.fn(), warn: vi.fn() };
|
||||
const { requestGatewayRestart } = createReloadHandlersForTest(logReload);
|
||||
const session = createProcessSessionFixture({
|
||||
id: "background-restart-blocker",
|
||||
command: "private command",
|
||||
pid: 12345,
|
||||
});
|
||||
addSession(session);
|
||||
markBackgrounded(session);
|
||||
const signalSpy = vi.fn();
|
||||
process.once("SIGUSR1", signalSpy);
|
||||
vi.useFakeTimers();
|
||||
|
||||
try {
|
||||
expect(
|
||||
requestGatewayRestart(
|
||||
{
|
||||
changedPaths: ["gateway.port"],
|
||||
restartGateway: true,
|
||||
restartReasons: ["gateway.port"],
|
||||
hotReasons: [],
|
||||
reloadHooks: false,
|
||||
restartGmailWatcher: false,
|
||||
restartCron: false,
|
||||
restartHeartbeat: false,
|
||||
restartHealthMonitor: false,
|
||||
reloadPlugins: false,
|
||||
restartChannels: new Set(),
|
||||
disposeMcpRuntimes: false,
|
||||
noopPaths: [],
|
||||
},
|
||||
{},
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
expect(signalSpy).not.toHaveBeenCalled();
|
||||
expect(logReload.warn).toHaveBeenCalledWith(
|
||||
"config change requires gateway restart (gateway.port) — deferring until 1 background exec session(s) complete",
|
||||
);
|
||||
|
||||
markExited(session, 0, null, "completed");
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
|
||||
expect(signalSpy).toHaveBeenCalledOnce();
|
||||
expect(logReload.info).toHaveBeenCalledWith(
|
||||
"all operations and replies completed; restarting gateway now",
|
||||
);
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", signalSpy);
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("defers config restart across an admitted process handoff", async () => {
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
const logReload = { info: vi.fn(), warn: vi.fn() };
|
||||
const { requestGatewayRestart } = createReloadHandlersForTest(logReload);
|
||||
const handoff = tryBeginGatewayIndependentRootWorkAdmission();
|
||||
const signalSpy = vi.fn();
|
||||
process.once("SIGUSR1", signalSpy);
|
||||
vi.useFakeTimers();
|
||||
|
||||
try {
|
||||
expect(
|
||||
requestGatewayRestart(
|
||||
{
|
||||
changedPaths: ["gateway.port"],
|
||||
restartGateway: true,
|
||||
restartReasons: ["gateway.port"],
|
||||
hotReasons: [],
|
||||
reloadHooks: false,
|
||||
restartGmailWatcher: false,
|
||||
restartCron: false,
|
||||
restartHeartbeat: false,
|
||||
restartHealthMonitor: false,
|
||||
reloadPlugins: false,
|
||||
restartChannels: new Set(),
|
||||
disposeMcpRuntimes: false,
|
||||
noopPaths: [],
|
||||
},
|
||||
{},
|
||||
),
|
||||
).toBe(true);
|
||||
expect(signalSpy).not.toHaveBeenCalled();
|
||||
expect(logReload.warn).toHaveBeenCalledWith(
|
||||
"config change requires gateway restart (gateway.port) — deferring until 1 gateway request(s) complete",
|
||||
);
|
||||
|
||||
handoff?.release();
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
|
||||
expect(signalSpy).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
handoff?.release();
|
||||
process.removeListener("SIGUSR1", signalSpy);
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("defers channel hot reload until active embedded work drains", async () => {
|
||||
const previousSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS;
|
||||
const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Gateway hot-reload handlers.
|
||||
// Applies config reload plans to hooks, cron, heartbeat, plugins, channels, and restarts.
|
||||
import { disposeAllSessionMcpRuntimes } from "../agents/agent-bundle-mcp-tools.js";
|
||||
import { getActiveBackgroundExecSessionCount } from "../agents/bash-process-registry.js";
|
||||
import { refreshContextWindowCache } from "../agents/context.js";
|
||||
import {
|
||||
getActiveEmbeddedRunCount,
|
||||
@@ -28,7 +29,10 @@ import {
|
||||
setGatewaySigusr1RestartPolicy,
|
||||
} from "../infra/restart.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
clearSecretsRuntimeSnapshot,
|
||||
getActiveSecretsRuntimeSnapshot,
|
||||
@@ -234,13 +238,23 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
const queueSize = getTotalQueueSize();
|
||||
const pendingReplies = getTotalPendingReplies();
|
||||
const embeddedRuns = getActiveEmbeddedRunCount();
|
||||
const backgroundExecSessions = getActiveBackgroundExecSessionCount();
|
||||
const rootRequests = getActiveGatewayRootWorkCount({ excludeCurrent: true });
|
||||
const activeTasks = getInspectableActiveTaskRestartBlockers().length;
|
||||
return {
|
||||
queueSize,
|
||||
pendingReplies,
|
||||
embeddedRuns,
|
||||
backgroundExecSessions,
|
||||
rootRequests,
|
||||
activeTasks,
|
||||
totalActive: queueSize + pendingReplies + embeddedRuns + activeTasks,
|
||||
totalActive:
|
||||
queueSize +
|
||||
pendingReplies +
|
||||
embeddedRuns +
|
||||
backgroundExecSessions +
|
||||
rootRequests +
|
||||
activeTasks,
|
||||
};
|
||||
};
|
||||
const formatActiveDetails = (counts: ReturnType<typeof getActiveCounts>) => {
|
||||
@@ -254,6 +268,12 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
if (counts.embeddedRuns > 0) {
|
||||
details.push(`${counts.embeddedRuns} embedded run(s)`);
|
||||
}
|
||||
if (counts.backgroundExecSessions > 0) {
|
||||
details.push(`${counts.backgroundExecSessions} background exec session(s)`);
|
||||
}
|
||||
if (counts.rootRequests > 0) {
|
||||
details.push(`${counts.rootRequests} gateway request(s)`);
|
||||
}
|
||||
if (counts.activeTasks > 0) {
|
||||
details.push(`${counts.activeTasks} background task run(s)`);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
// and WebSocket surfaces, config reload hooks, and graceful restart/shutdown.
|
||||
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
|
||||
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
|
||||
import { getActiveBackgroundExecSessionCount } from "../agents/bash-process-registry.js";
|
||||
import {
|
||||
getActiveEmbeddedRunCount,
|
||||
resolveActiveEmbeddedRunSessionId,
|
||||
@@ -57,6 +58,7 @@ import {
|
||||
pinActivePluginSessionExtensionRegistry,
|
||||
} from "../plugins/runtime.js";
|
||||
import { getTotalQueueSize, isGatewayDraining } from "../process/command-queue.js";
|
||||
import { getActiveGatewayRootWorkCount } from "../process/gateway-work-admission.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import {
|
||||
clearSecretsRuntimeSnapshot,
|
||||
@@ -651,6 +653,8 @@ export async function startGatewayServer(
|
||||
getTotalPendingReplies() +
|
||||
getActiveEmbeddedRunCount() +
|
||||
getActiveCronJobCount() +
|
||||
getActiveBackgroundExecSessionCount() +
|
||||
getActiveGatewayRootWorkCount() +
|
||||
getActiveTaskCount(),
|
||||
);
|
||||
// Unconditional startup migration: seed gateway.controlUi.allowedOrigins for existing
|
||||
|
||||
@@ -57,6 +57,7 @@ function createRootOnlyInspectors(): GatewayActiveWorkInspectors {
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Collects process activity shared by restart and host-suspension decisions.
|
||||
import { getActiveBackgroundExecSessionCount } from "../agents/bash-process-registry.js";
|
||||
import { getActiveEmbeddedRunCount } from "../agents/embedded-agent-runner/run-state.js";
|
||||
import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js";
|
||||
import { getActiveCronJobCount } from "../cron/active-jobs.js";
|
||||
@@ -19,6 +20,7 @@ export type GatewayActiveWorkCounts = {
|
||||
queueSize: number;
|
||||
pendingReplies: number;
|
||||
embeddedRuns: number;
|
||||
backgroundExecSessions: number;
|
||||
cronRuns: number;
|
||||
activeTasks: number;
|
||||
rootRequests: number;
|
||||
@@ -37,6 +39,7 @@ export type GatewayActiveWorkBlocker = {
|
||||
| "queue"
|
||||
| "reply"
|
||||
| "embedded-run"
|
||||
| "background-exec"
|
||||
| "cron-run"
|
||||
| "task"
|
||||
| "root-request"
|
||||
@@ -61,6 +64,7 @@ export type GatewayActiveWorkInspectors = {
|
||||
getQueueSize: () => number;
|
||||
getPendingReplies: () => number;
|
||||
getEmbeddedRuns: () => number;
|
||||
getBackgroundExecSessions: () => number;
|
||||
getCronRuns: () => number;
|
||||
getActiveTasks: () => number;
|
||||
getTaskBlockers: () => ActiveTaskRestartBlocker[];
|
||||
@@ -77,6 +81,7 @@ const defaultInspectors: GatewayActiveWorkInspectors = {
|
||||
getQueueSize: getTotalQueueSize,
|
||||
getPendingReplies: getTotalPendingReplies,
|
||||
getEmbeddedRuns: getActiveEmbeddedRunCount,
|
||||
getBackgroundExecSessions: getActiveBackgroundExecSessionCount,
|
||||
getCronRuns: () => Math.max(getActiveCronJobCount(), getSuspensionVisibleCronTaskRunCount()),
|
||||
getActiveTasks: () => getInspectableActiveTaskRestartBlockers().length,
|
||||
getTaskBlockers: getInspectableActiveTaskRestartBlockers,
|
||||
@@ -101,6 +106,7 @@ export function createGatewayActiveWorkSnapshot(
|
||||
queueSize: normalizeCount(resolved.getQueueSize()),
|
||||
pendingReplies: normalizeCount(resolved.getPendingReplies()),
|
||||
embeddedRuns: normalizeCount(resolved.getEmbeddedRuns()),
|
||||
backgroundExecSessions: normalizeCount(resolved.getBackgroundExecSessions()),
|
||||
cronRuns: normalizeCount(resolved.getCronRuns()),
|
||||
activeTasks: normalizeCount(resolved.getActiveTasks()),
|
||||
rootRequests: normalizeCount(resolved.getRootRequests()),
|
||||
@@ -130,6 +136,11 @@ export function createGatewayActiveWorkSnapshot(
|
||||
`${counts.pendingReplies} pending reply delivery operation(s)`,
|
||||
);
|
||||
add(counts.embeddedRuns, "embedded-run", `${counts.embeddedRuns} active embedded run(s)`);
|
||||
add(
|
||||
counts.backgroundExecSessions,
|
||||
"background-exec",
|
||||
`${counts.backgroundExecSessions} active background exec session(s)`,
|
||||
);
|
||||
add(counts.cronRuns, "cron-run", `${counts.cronRuns} active cron run(s)`);
|
||||
add(counts.rootRequests, "root-request", `${counts.rootRequests} active gateway request(s)`);
|
||||
add(
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
// Covers atomic refuse-only suspension preparation, renewal, and release.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
addSession,
|
||||
deleteSession,
|
||||
getActiveBackgroundExecSessionCount,
|
||||
markBackgrounded,
|
||||
markExited,
|
||||
resetProcessRegistryForTests,
|
||||
} from "../agents/bash-process-registry.js";
|
||||
import { createProcessSessionFixture } from "../agents/bash-process-registry.test-helpers.js";
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
markGatewayRestartDraining,
|
||||
@@ -7,6 +16,7 @@ import {
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import type { GatewayActiveWorkInspectors } from "./gateway-active-work.js";
|
||||
import {
|
||||
GATEWAY_SUSPEND_RETRY_AFTER_MS,
|
||||
GATEWAY_SUSPEND_TTL_MS,
|
||||
getGatewaySuspendStatus,
|
||||
prepareGatewaySuspend,
|
||||
@@ -21,6 +31,7 @@ function inspectors(
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
@@ -36,11 +47,13 @@ function inspectors(
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetProcessRegistryForTests();
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetProcessRegistryForTests();
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
@@ -85,6 +98,50 @@ describe("gateway suspend coordinator", () => {
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
});
|
||||
|
||||
it("stays busy after a background session is hidden until its process exits", () => {
|
||||
const session = createProcessSessionFixture({
|
||||
id: "private-background-session",
|
||||
command: "private command",
|
||||
});
|
||||
addSession(session);
|
||||
markBackgrounded(session);
|
||||
deleteSession(session.id);
|
||||
|
||||
const inspect = inspectors({
|
||||
getBackgroundExecSessions: getActiveBackgroundExecSessionCount,
|
||||
});
|
||||
const busy = prepareGatewaySuspend({
|
||||
requestId: "request-background-exec",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
inspect,
|
||||
});
|
||||
|
||||
expect(busy).toEqual({
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
retryAfterMs: GATEWAY_SUSPEND_RETRY_AFTER_MS,
|
||||
activeCount: 1,
|
||||
blockers: [
|
||||
{
|
||||
kind: "background-exec",
|
||||
count: 1,
|
||||
message: "1 active background exec session(s)",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
markExited(session, 0, null, "completed");
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-background-exec",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
inspect,
|
||||
}),
|
||||
).toMatchObject({ status: "ready", activeCount: 0, blockers: [] });
|
||||
});
|
||||
|
||||
it("keeps admission closed until a failed busy rollback resumes scheduling", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
// Covers safe gateway restart preflight and requests.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
createSafeGatewayRestartPreflight,
|
||||
requestSafeGatewayRestart,
|
||||
@@ -11,6 +15,14 @@ vi.mock("./restart.js", () => ({
|
||||
scheduleGatewaySigusr1Restart: (opts: unknown) => scheduleGatewaySigusr1Restart(opts),
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
describe("safe gateway restart coordinator", () => {
|
||||
it("reports safe when no restart blockers are active", () => {
|
||||
const preflight = createSafeGatewayRestartPreflight({
|
||||
@@ -18,6 +30,8 @@ describe("safe gateway restart coordinator", () => {
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getRootRequests: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
});
|
||||
@@ -29,6 +43,8 @@ describe("safe gateway restart coordinator", () => {
|
||||
pendingReplies: 0,
|
||||
embeddedRuns: 0,
|
||||
cronRuns: 0,
|
||||
backgroundExecSessions: 0,
|
||||
rootRequests: 0,
|
||||
activeTasks: 0,
|
||||
totalActive: 0,
|
||||
},
|
||||
@@ -43,6 +59,8 @@ describe("safe gateway restart coordinator", () => {
|
||||
getPendingReplies: () => 1,
|
||||
getEmbeddedRuns: () => 1,
|
||||
getCronRuns: () => 1,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getRootRequests: () => 1,
|
||||
getActiveTasks: () => 1,
|
||||
getTaskBlockers: () => [
|
||||
{
|
||||
@@ -57,18 +75,79 @@ describe("safe gateway restart coordinator", () => {
|
||||
});
|
||||
|
||||
expect(preflight.safe).toBe(false);
|
||||
expect(preflight.counts.totalActive).toBe(6);
|
||||
expect(preflight.counts.totalActive).toBe(7);
|
||||
expect(preflight.blockers.map((blocker) => blocker.kind)).toEqual([
|
||||
"queue",
|
||||
"reply",
|
||||
"embedded-run",
|
||||
"cron-run",
|
||||
"root-request",
|
||||
"task",
|
||||
]);
|
||||
expect(preflight.summary).toContain("restart deferred");
|
||||
expect(preflight.summary).toContain("taskId=task-1");
|
||||
});
|
||||
|
||||
it("defers restart for aggregate background exec sessions", () => {
|
||||
const preflight = createSafeGatewayRestartPreflight({
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 2,
|
||||
getRootRequests: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
});
|
||||
|
||||
expect(preflight.safe).toBe(false);
|
||||
expect(preflight.counts).toMatchObject({
|
||||
backgroundExecSessions: 2,
|
||||
totalActive: 2,
|
||||
});
|
||||
expect(preflight.blockers).toEqual([
|
||||
{
|
||||
kind: "background-exec",
|
||||
count: 2,
|
||||
message: "2 active background exec session(s)",
|
||||
},
|
||||
]);
|
||||
expect(preflight.summary).toBe("restart deferred: 2 active background exec session(s)");
|
||||
});
|
||||
|
||||
it("counts an admitted spawn handoff while excluding the preflight request", async () => {
|
||||
const handoff = tryBeginGatewayIndependentRootWorkAdmission();
|
||||
const request = tryBeginGatewayIndependentRootWorkAdmission();
|
||||
expect(handoff).not.toBeNull();
|
||||
expect(request).not.toBeNull();
|
||||
|
||||
try {
|
||||
await request?.run(async () => {
|
||||
const preflight = createSafeGatewayRestartPreflight({
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
});
|
||||
|
||||
expect(preflight.counts).toMatchObject({ rootRequests: 1, totalActive: 1 });
|
||||
expect(preflight.blockers).toEqual([
|
||||
{
|
||||
kind: "root-request",
|
||||
count: 1,
|
||||
message: "1 active gateway request(s)",
|
||||
},
|
||||
]);
|
||||
});
|
||||
} finally {
|
||||
request?.release();
|
||||
handoff?.release();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps truncated task titles on complete UTF-16 code points", () => {
|
||||
const preflight = createSafeGatewayRestartPreflight({
|
||||
getQueueSize: () => 0,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { getActiveGatewayRootWorkCount } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
createGatewayActiveWorkSnapshot,
|
||||
type GatewayActiveWorkBlocker,
|
||||
@@ -12,11 +13,20 @@ export type SafeGatewayRestartCounts = {
|
||||
pendingReplies: number;
|
||||
embeddedRuns: number;
|
||||
cronRuns: number;
|
||||
backgroundExecSessions: number;
|
||||
rootRequests: number;
|
||||
activeTasks: number;
|
||||
totalActive: number;
|
||||
};
|
||||
export type SafeGatewayRestartBlocker = Omit<GatewayActiveWorkBlocker, "kind"> & {
|
||||
kind: "queue" | "reply" | "embedded-run" | "cron-run" | "task";
|
||||
kind:
|
||||
| "queue"
|
||||
| "reply"
|
||||
| "embedded-run"
|
||||
| "cron-run"
|
||||
| "background-exec"
|
||||
| "root-request"
|
||||
| "task";
|
||||
};
|
||||
|
||||
type SafeRestartInspectors = Pick<
|
||||
@@ -25,6 +35,8 @@ type SafeRestartInspectors = Pick<
|
||||
| "getPendingReplies"
|
||||
| "getEmbeddedRuns"
|
||||
| "getCronRuns"
|
||||
| "getBackgroundExecSessions"
|
||||
| "getRootRequests"
|
||||
| "getActiveTasks"
|
||||
| "getTaskBlockers"
|
||||
>;
|
||||
@@ -48,7 +60,10 @@ export function createSafeGatewayRestartPreflight(
|
||||
): SafeGatewayRestartPreflight {
|
||||
const snapshot = createGatewayActiveWorkSnapshot({
|
||||
...inspectors,
|
||||
getRootRequests: () => 0,
|
||||
// Restart RPC preflight itself owns a root. Count every other admitted
|
||||
// handoff so signal emission cannot split spawn from durable ownership.
|
||||
getRootRequests:
|
||||
inspectors.getRootRequests ?? (() => getActiveGatewayRootWorkCount({ excludeCurrent: true })),
|
||||
getSessionAdmissions: () => 0,
|
||||
getSessionMutations: () => 0,
|
||||
getChatRuns: () => 0,
|
||||
@@ -61,12 +76,16 @@ export function createSafeGatewayRestartPreflight(
|
||||
pendingReplies: snapshot.counts.pendingReplies,
|
||||
embeddedRuns: snapshot.counts.embeddedRuns,
|
||||
cronRuns: snapshot.counts.cronRuns,
|
||||
backgroundExecSessions: snapshot.counts.backgroundExecSessions,
|
||||
rootRequests: snapshot.counts.rootRequests,
|
||||
activeTasks: snapshot.counts.activeTasks,
|
||||
totalActive:
|
||||
snapshot.counts.queueSize +
|
||||
snapshot.counts.pendingReplies +
|
||||
snapshot.counts.embeddedRuns +
|
||||
snapshot.counts.cronRuns +
|
||||
snapshot.counts.backgroundExecSessions +
|
||||
snapshot.counts.rootRequests +
|
||||
snapshot.counts.activeTasks,
|
||||
};
|
||||
const blockers = snapshot.blockers as SafeGatewayRestartBlocker[];
|
||||
@@ -83,7 +102,7 @@ export function createSafeGatewayRestartPreflight(
|
||||
};
|
||||
}
|
||||
|
||||
/** Schedule a gateway restart after collecting queue/reply/task blockers. */
|
||||
/** Schedule a gateway restart after collecting tracked active-work blockers. */
|
||||
export function requestSafeGatewayRestart(
|
||||
opts: {
|
||||
reason?: string;
|
||||
|
||||
@@ -24,6 +24,7 @@ function inspectors(): GatewayActiveWorkInspectors {
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getBackgroundExecSessions: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
|
||||
Reference in New Issue
Block a user