mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-03 06:43:18 +00:00
* fix(gateway): bound busy channel health by real run age The channel health policy treats a channel as healthy-busy even while disconnected, bounded only by a 25 minute stale ceiling measured from lastRunActivityAt. The run-state heartbeat refreshes lastRunActivityAt every 60 seconds for as long as any run is active, so a run that hangs forever (for example a send blocking on a dead socket after the transport already reported connected:false) keeps that timestamp fresh and the stuck ceiling is never reached. The account is then reported healthy forever by the health monitor, readiness probe, and health CLI, and no restart ever fires. createRunStateMachine now tracks each in-flight run's start time keyed by an opaque run handle and publishes the oldest still-active run's start as activeRunStartedAt. The health policy busy override keys its ceiling off the real run age, so a run stuck longer than the threshold reports stuck and the monitor can restart it. Because the reported start is the oldest active run and advances to the next-oldest as runs complete, a channel churning through many short overlapping runs (activeRuns above 1 across concurrent queue keys) stays healthy; only a genuinely hung run breaches the ceiling. Short and active runs stay healthy and the existing lastRunActivityAt fallback is preserved for snapshots without a start time. * fix(channels): retain run-state callback compatibility Keep the released zero-argument onRunEnd callback source-compatible while allowing internal queue callers to pass a run handle for exact concurrent-run accounting. The compatibility path closes the oldest active run, preserving existing lifecycle behavior for consumers that do not use handles. * fix(channels): keep anonymous runs out of age tracking The zero-argument lifecycle callbacks cannot identify which concurrent run completed, so they must not update the identity-sensitive run start used by channel health. Keep their busy count separately and reserve exact start tracking for the shared queue's handle-aware lifecycle path. * fix(channels): keep tracked runs internal Keep the public run-state lifecycle callbacks unchanged. The channel queue now owns opaque run identity and augments its status updates with the oldest active queue run, so implementation details do not expand the SDK surface. * fix(channels): type queue run start status Keep activeRunStartedAt in the internal status patch type so the queue can publish its private tracked-run age through the existing status sink. * fix(channels): wrap isActive to satisfy unbound-method lint * fix(gateway): gate busy run-age ceiling on disconnected transport
200 lines
5.4 KiB
TypeScript
200 lines
5.4 KiB
TypeScript
/**
|
|
* Tests channel lifecycle queue ordering and failure handling.
|
|
*/
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import { createChannelRunQueue } from "./channel-lifecycle.core.js";
|
|
|
|
function createDeferred() {
|
|
let resolve: (() => void) | undefined;
|
|
const promise = new Promise<void>((innerResolve) => {
|
|
resolve = innerResolve;
|
|
});
|
|
return { promise, resolve };
|
|
}
|
|
|
|
async function flushAsyncWork() {
|
|
for (let i = 0; i < 20; i += 1) {
|
|
await Promise.resolve();
|
|
}
|
|
}
|
|
|
|
describe("createChannelRunQueue", () => {
|
|
it("serializes work per key while allowing unrelated keys to run", async () => {
|
|
const first = createDeferred();
|
|
const second = createDeferred();
|
|
const third = createDeferred();
|
|
const order: string[] = [];
|
|
const queue = createChannelRunQueue({});
|
|
|
|
queue.enqueue("same", async () => {
|
|
order.push("start:first");
|
|
await first.promise;
|
|
order.push("end:first");
|
|
});
|
|
queue.enqueue("same", async () => {
|
|
order.push("start:second");
|
|
await second.promise;
|
|
order.push("end:second");
|
|
});
|
|
queue.enqueue("other", async () => {
|
|
order.push("start:third");
|
|
await third.promise;
|
|
order.push("end:third");
|
|
});
|
|
|
|
await flushAsyncWork();
|
|
expect(order).toEqual(["start:first", "start:third"]);
|
|
|
|
third.resolve?.();
|
|
await third.promise;
|
|
await flushAsyncWork();
|
|
expect(order).toEqual(["start:first", "start:third", "end:third"]);
|
|
|
|
first.resolve?.();
|
|
await first.promise;
|
|
await flushAsyncWork();
|
|
expect(order).toEqual(["start:first", "start:third", "end:third", "end:first", "start:second"]);
|
|
|
|
second.resolve?.();
|
|
await second.promise;
|
|
});
|
|
|
|
it("updates run status and routes async errors", async () => {
|
|
const taskError = new Error("boom");
|
|
const setStatus = vi.fn();
|
|
const onError = vi.fn();
|
|
const queue = createChannelRunQueue({ setStatus, onError });
|
|
|
|
queue.enqueue("key", async () => {
|
|
throw taskError;
|
|
});
|
|
|
|
await flushAsyncWork();
|
|
|
|
expect(setStatus).toHaveBeenCalledTimes(3);
|
|
const [initialStatus, busyStatus, finalStatus] = setStatus.mock.calls.map(([status]) => status);
|
|
expect(initialStatus).toEqual({ activeRuns: 0, busy: false, activeRunStartedAt: null });
|
|
expect(busyStatus?.activeRuns).toBe(1);
|
|
expect(busyStatus?.busy).toBe(true);
|
|
expect(typeof busyStatus?.lastRunActivityAt).toBe("number");
|
|
expect(typeof busyStatus?.activeRunStartedAt).toBe("number");
|
|
expect(finalStatus?.activeRuns).toBe(0);
|
|
expect(finalStatus?.busy).toBe(false);
|
|
expect(typeof finalStatus?.lastRunActivityAt).toBe("number");
|
|
expect(finalStatus?.activeRunStartedAt).toBeNull();
|
|
expect(onError).toHaveBeenCalledWith(taskError);
|
|
});
|
|
|
|
it("keeps the oldest run start while a newer concurrent task completes", async () => {
|
|
vi.useFakeTimers();
|
|
try {
|
|
const first = createDeferred();
|
|
const second = createDeferred();
|
|
const setStatus = vi.fn();
|
|
const queue = createChannelRunQueue({ setStatus });
|
|
|
|
vi.setSystemTime(1_000);
|
|
queue.enqueue("first", async () => {
|
|
await first.promise;
|
|
});
|
|
await flushAsyncWork();
|
|
|
|
vi.setSystemTime(2_000);
|
|
queue.enqueue("second", async () => {
|
|
await second.promise;
|
|
});
|
|
await flushAsyncWork();
|
|
|
|
second.resolve?.();
|
|
await second.promise;
|
|
await flushAsyncWork();
|
|
|
|
expect(setStatus.mock.calls.at(-1)?.[0]).toMatchObject({
|
|
activeRuns: 1,
|
|
busy: true,
|
|
activeRunStartedAt: 1_000,
|
|
});
|
|
|
|
queue.deactivate();
|
|
first.resolve?.();
|
|
await first.promise;
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("advances to the next-oldest run start when the oldest run ends", async () => {
|
|
vi.useFakeTimers();
|
|
try {
|
|
const first = createDeferred();
|
|
const second = createDeferred();
|
|
const setStatus = vi.fn();
|
|
const queue = createChannelRunQueue({ setStatus });
|
|
|
|
vi.setSystemTime(1_000);
|
|
queue.enqueue("first", async () => {
|
|
await first.promise;
|
|
});
|
|
await flushAsyncWork();
|
|
|
|
vi.setSystemTime(2_000);
|
|
queue.enqueue("second", async () => {
|
|
await second.promise;
|
|
});
|
|
await flushAsyncWork();
|
|
|
|
first.resolve?.();
|
|
await first.promise;
|
|
await flushAsyncWork();
|
|
|
|
expect(setStatus.mock.calls.at(-1)?.[0]).toMatchObject({
|
|
activeRuns: 1,
|
|
busy: true,
|
|
activeRunStartedAt: 2_000,
|
|
});
|
|
|
|
queue.deactivate();
|
|
second.resolve?.();
|
|
await second.promise;
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("contains reporting hook errors", async () => {
|
|
const taskError = new Error("boom");
|
|
const onError = vi.fn(() => {
|
|
throw new Error("report failed");
|
|
});
|
|
const queue = createChannelRunQueue({
|
|
onError,
|
|
});
|
|
|
|
queue.enqueue("key", async () => {
|
|
throw taskError;
|
|
});
|
|
|
|
await flushAsyncWork();
|
|
expect(onError).toHaveBeenCalledWith(taskError);
|
|
});
|
|
|
|
it("skips queued work after deactivation", async () => {
|
|
const first = createDeferred();
|
|
const task = vi.fn();
|
|
const queue = createChannelRunQueue({});
|
|
|
|
queue.enqueue("key", async () => {
|
|
await first.promise;
|
|
});
|
|
queue.enqueue("key", task);
|
|
await flushAsyncWork();
|
|
|
|
queue.deactivate();
|
|
first.resolve?.();
|
|
await first.promise;
|
|
await flushAsyncWork();
|
|
|
|
expect(task).not.toHaveBeenCalled();
|
|
});
|
|
});
|