mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-27 07:57:50 +00:00
fix(codex): keep run lane timeout progress-aware
This commit is contained in:
@@ -391,6 +391,72 @@ describe("command queue", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("task timeout renews from progress timestamps", async () => {
|
||||
const lane = `timeout-progress-lane-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
setCommandLaneConcurrency(lane, 1);
|
||||
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let progressAtMs = Date.now();
|
||||
const blocker = createDeferred();
|
||||
const first = enqueueCommandInLane(
|
||||
lane,
|
||||
async () => {
|
||||
await blocker.promise;
|
||||
return "first";
|
||||
},
|
||||
{
|
||||
taskTimeoutMs: 25,
|
||||
taskTimeoutProgressAtMs: () => progressAtMs,
|
||||
},
|
||||
);
|
||||
let secondRan = false;
|
||||
const second = enqueueCommandInLane(lane, async () => {
|
||||
secondRan = true;
|
||||
return "second";
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
progressAtMs = Date.now();
|
||||
await vi.advanceTimersByTimeAsync(20);
|
||||
expect(secondRan).toBe(false);
|
||||
|
||||
blocker.resolve();
|
||||
await expect(first).resolves.toBe("first");
|
||||
await expect(second).resolves.toBe("second");
|
||||
expect(secondRan).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("task timeout falls back when progress timestamp callback throws", async () => {
|
||||
const lane = `timeout-progress-throw-lane-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
setCommandLaneConcurrency(lane, 1);
|
||||
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const first = enqueueCommandInLane(lane, async () => new Promise<never>(() => {}), {
|
||||
taskTimeoutMs: 25,
|
||||
taskTimeoutProgressAtMs: () => {
|
||||
throw new Error("progress failed");
|
||||
},
|
||||
});
|
||||
const firstRejected = expect(first).rejects.toBeInstanceOf(CommandLaneTaskTimeoutError);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
await firstRejected;
|
||||
|
||||
expect(
|
||||
diagnosticMocks.diag.warn.mock.calls.some(([message]) =>
|
||||
String(message).includes("lane task timeout progress callback failed"),
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps work queued while a lane has zero concurrency and drains after resume", async () => {
|
||||
const lane = `suspended-lane-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
setCommandLaneConcurrency(lane, 0);
|
||||
|
||||
@@ -63,6 +63,7 @@ type QueueEntry = {
|
||||
enqueuedAt: number;
|
||||
warnAfterMs: number;
|
||||
taskTimeoutMs?: number;
|
||||
taskTimeoutProgressAtMs?: () => number | undefined;
|
||||
onWait?: (waitMs: number, queuedAhead: number) => void;
|
||||
};
|
||||
|
||||
@@ -210,14 +211,33 @@ async function runQueueEntryTask(lane: string, entry: QueueEntry): Promise<unkno
|
||||
return await taskPromise;
|
||||
}
|
||||
|
||||
const startedAtMs = Date.now();
|
||||
const readLastProgressAtMs = () => {
|
||||
let value: number | undefined;
|
||||
try {
|
||||
value = entry.taskTimeoutProgressAtMs?.();
|
||||
} catch (err) {
|
||||
diag.warn(`lane task timeout progress callback failed: lane=${lane} error="${String(err)}"`);
|
||||
}
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0
|
||||
? Math.max(startedAtMs, Math.floor(value))
|
||||
: startedAtMs;
|
||||
};
|
||||
let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
|
||||
let timedOut = false;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutHandle = setTimeout(() => {
|
||||
timedOut = true;
|
||||
reject(new CommandLaneTaskTimeoutError(lane, taskTimeoutMs));
|
||||
}, taskTimeoutMs);
|
||||
timeoutHandle.unref?.();
|
||||
const armTimeout = () => {
|
||||
const elapsedMs = Math.max(0, Date.now() - readLastProgressAtMs());
|
||||
const remainingMs = taskTimeoutMs - elapsedMs;
|
||||
if (remainingMs <= 0) {
|
||||
timedOut = true;
|
||||
reject(new CommandLaneTaskTimeoutError(lane, taskTimeoutMs));
|
||||
return;
|
||||
}
|
||||
timeoutHandle = setTimeout(armTimeout, remainingMs);
|
||||
timeoutHandle.unref?.();
|
||||
};
|
||||
armTimeout();
|
||||
});
|
||||
|
||||
try {
|
||||
@@ -349,6 +369,7 @@ export function enqueueCommandInLane<T>(
|
||||
enqueuedAt: Date.now(),
|
||||
warnAfterMs,
|
||||
taskTimeoutMs: normalizeTaskTimeoutMs(opts?.taskTimeoutMs),
|
||||
taskTimeoutProgressAtMs: opts?.taskTimeoutProgressAtMs,
|
||||
onWait: opts?.onWait,
|
||||
});
|
||||
logLaneEnqueue(cleaned, getLaneDepth(state));
|
||||
|
||||
@@ -2,6 +2,7 @@ export type CommandQueueEnqueueOptions = {
|
||||
warnAfterMs?: number;
|
||||
onWait?: (waitMs: number, queuedAhead: number) => void;
|
||||
taskTimeoutMs?: number;
|
||||
taskTimeoutProgressAtMs?: () => number | undefined;
|
||||
};
|
||||
|
||||
export type CommandQueueEnqueueFn = <T>(
|
||||
|
||||
Reference in New Issue
Block a user