fix: scope codex attempt watchdog to turn progress

This commit is contained in:
Eva (agent)
2026-05-13 05:12:33 +07:00
committed by Peter Steinberger
parent 722161271e
commit d7d597cfd8
2 changed files with 77 additions and 15 deletions

View File

@@ -1644,6 +1644,46 @@ describe("runCodexAppServerAttempt", () => {
expect(harness.request.mock.calls.some(([method]) => method === "turn/interrupt")).toBe(false);
});
it("does not count non-turn app-server requests as turn attempt progress", async () => {
const harness = createStartedThreadHarness();
const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined);
const params = createParams(
path.join(tempDir, "session.jsonl"),
path.join(tempDir, "workspace"),
);
params.timeoutMs = 100;
const run = runCodexAppServerAttempt(params, {
turnCompletionIdleTimeoutMs: 500,
turnAssistantCompletionIdleTimeoutMs: 500,
turnTerminalIdleTimeoutMs: 500,
});
await harness.waitForMethod("turn/start");
await new Promise((resolve) => setTimeout(resolve, 60));
await harness.handleServerRequest({
id: "request-account-refresh",
method: "account/nonTurnRefresh",
params: {},
});
const result = await run;
expect(result.aborted).toBe(true);
expect(result.timedOut).toBe(true);
expect(result.promptError).toBe(
"codex app-server turn idle timed out waiting for turn/completed",
);
const warnCall = warn.mock.calls.find(
([message]) => message === "codex app-server turn idle timed out waiting for progress",
);
const warnData = warnCall?.[1] as
| { lastActivityReason?: string; timeoutMs?: number }
| undefined;
expect(warnData?.timeoutMs).toBe(100);
expect(warnData?.lastActivityReason).toBe("turn:start");
expect(harness.request.mock.calls.some(([method]) => method === "turn/interrupt")).toBe(true);
});
it("does not count account rate-limit updates as turn completion activity", async () => {
let notify: (notification: CodexServerNotification) => Promise<void> = async () => undefined;
let handleRequest:

View File

@@ -1068,6 +1068,9 @@ export async function runCodexAppServerAttempt(
let turnCompletionLastActivityAt = Date.now();
let turnCompletionLastActivityReason = "startup";
let turnCompletionLastActivityDetails: Record<string, unknown> | undefined;
let turnAttemptLastProgressAt = Date.now();
let turnAttemptLastProgressReason = "startup";
let turnAttemptLastProgressDetails: Record<string, unknown> | undefined;
let activeAppServerTurnRequests = 0;
const activeOpenClawDynamicToolCallIds = new Set<string>();
const activeTurnItemIds = new Set<string>();
@@ -1154,7 +1157,7 @@ export async function runCodexAppServerAttempt(
) {
return;
}
const idleMs = Math.max(0, Date.now() - turnCompletionLastActivityAt);
const idleMs = Math.max(0, Date.now() - turnAttemptLastProgressAt);
if (idleMs < turnAttemptIdleTimeoutMs) {
scheduleTurnAttemptIdleWatch();
return;
@@ -1169,16 +1172,16 @@ export async function runCodexAppServerAttempt(
turnId,
idleMs,
timeoutMs: turnAttemptIdleTimeoutMs,
lastActivityReason: turnCompletionLastActivityReason,
...turnCompletionLastActivityDetails,
lastActivityReason: turnAttemptLastProgressReason,
...turnAttemptLastProgressDetails,
});
embeddedAgentLog.warn("codex app-server turn idle timed out waiting for progress", {
threadId: thread.threadId,
turnId,
idleMs,
timeoutMs: turnAttemptIdleTimeoutMs,
lastActivityReason: turnCompletionLastActivityReason,
...turnCompletionLastActivityDetails,
lastActivityReason: turnAttemptLastProgressReason,
...turnAttemptLastProgressDetails,
});
runAbortController.abort("turn_progress_idle_timeout");
};
@@ -1296,7 +1299,7 @@ export async function runCodexAppServerAttempt(
) {
return;
}
const elapsedMs = Math.max(0, Date.now() - turnCompletionLastActivityAt);
const elapsedMs = Math.max(0, Date.now() - turnAttemptLastProgressAt);
const delayMs = Math.max(1, turnAttemptIdleTimeoutMs - elapsedMs);
turnAttemptIdleTimer = setTimeout(fireTurnAttemptIdleTimeout, delayMs);
turnAttemptIdleTimer.unref?.();
@@ -1318,13 +1321,24 @@ export async function runCodexAppServerAttempt(
turnTerminalIdleTimer.unref?.();
}
function scheduleTurnProgressWatches() {
scheduleTurnAttemptIdleWatch();
scheduleTurnCompletionIdleWatch();
scheduleTurnTerminalIdleWatch();
}
const touchTurnCompletionActivity = (
reason: string,
options?: { arm?: boolean; details?: Record<string, unknown> },
options?: { arm?: boolean; details?: Record<string, unknown>; attemptProgress?: boolean },
) => {
turnCompletionLastActivityAt = Date.now();
turnCompletionLastActivityReason = reason;
turnCompletionLastActivityDetails = options?.details;
if (options?.attemptProgress) {
turnAttemptLastProgressAt = turnCompletionLastActivityAt;
turnAttemptLastProgressReason = reason;
turnAttemptLastProgressDetails = options.details;
}
emitTrustedDiagnosticEvent({
type: "run.progress",
runId: params.runId,
@@ -1336,9 +1350,7 @@ export async function runCodexAppServerAttempt(
turnCompletionIdleWatchArmed = true;
turnCompletionIdleWatchPinnedByTerminalError = false;
}
scheduleTurnAttemptIdleWatch();
scheduleTurnCompletionIdleWatch();
scheduleTurnTerminalIdleWatch();
scheduleTurnProgressWatches();
};
const disarmTurnCompletionIdleWatch = () => {
@@ -1444,6 +1456,7 @@ export async function runCodexAppServerAttempt(
if (isCurrentTurnNotification) {
touchTurnCompletionActivity(`notification:${notification.method}`, {
details: describeNotificationActivity(notification),
attemptProgress: true,
});
reportCodexExecutionNotification(notification);
}
@@ -1569,8 +1582,8 @@ export async function runCodexAppServerAttempt(
activeAppServerTurnRequests += 1;
clearTurnCompletionIdleTimer();
disarmTurnAssistantCompletionIdleWatch();
touchTurnCompletionActivity(`request:${request.method}`);
let armCompletionWatchOnResponse = false;
let requestCountsAsTurnActivity = false;
try {
if (request.method === "account/chatgptAuthTokens/refresh") {
return refreshCodexAppServerAuthTokens({
@@ -1584,6 +1597,7 @@ export async function runCodexAppServerAttempt(
}
if (request.method === "mcpServer/elicitation/request") {
armCompletionWatchOnResponse = true;
requestCountsAsTurnActivity = true;
return handleCodexAppServerElicitationRequest({
requestParams: request.params,
paramsForRun: params,
@@ -1595,6 +1609,7 @@ export async function runCodexAppServerAttempt(
}
if (request.method === "item/tool/requestUserInput") {
armCompletionWatchOnResponse = true;
requestCountsAsTurnActivity = true;
return userInputBridge?.handleRequest({
id: request.id,
params: request.params,
@@ -1603,6 +1618,7 @@ export async function runCodexAppServerAttempt(
if (request.method !== "item/tool/call") {
if (isCodexAppServerApprovalRequest(request.method)) {
armCompletionWatchOnResponse = true;
requestCountsAsTurnActivity = true;
return handleApprovalRequest({
method: request.method,
params: request.params,
@@ -1619,6 +1635,7 @@ export async function runCodexAppServerAttempt(
return undefined;
}
armCompletionWatchOnResponse = true;
requestCountsAsTurnActivity = true;
turnCrossedToolHandoff = true;
activeOpenClawDynamicToolCallIds.add(call.callId);
trajectoryRecorder?.recordEvent("tool.call", {
@@ -1698,9 +1715,14 @@ export async function runCodexAppServerAttempt(
return response as JsonValue;
} finally {
activeAppServerTurnRequests = Math.max(0, activeAppServerTurnRequests - 1);
touchTurnCompletionActivity(`request:${request.method}:response`, {
arm: armCompletionWatchOnResponse,
});
if (requestCountsAsTurnActivity) {
touchTurnCompletionActivity(`request:${request.method}:response`, {
arm: armCompletionWatchOnResponse,
attemptProgress: true,
});
} else {
scheduleTurnProgressWatches();
}
}
});
@@ -1999,7 +2021,7 @@ export async function runCodexAppServerAttempt(
setActiveEmbeddedRun(params.sessionId, handle, params.sessionKey);
turnAttemptIdleWatchArmed = true;
turnTerminalIdleWatchArmed = true;
touchTurnCompletionActivity("turn:start", { arm: true });
touchTurnCompletionActivity("turn:start", { arm: true, attemptProgress: true });
const abortListener = () => {
const shouldRetireClient = timedOut;