mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-24 19:33:02 +00:00
fix: scope codex attempt watchdog to turn progress
This commit is contained in:
committed by
Peter Steinberger
parent
722161271e
commit
d7d597cfd8
@@ -1644,6 +1644,46 @@ describe("runCodexAppServerAttempt", () => {
|
||||
expect(harness.request.mock.calls.some(([method]) => method === "turn/interrupt")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not count non-turn app-server requests as turn attempt progress", async () => {
|
||||
const harness = createStartedThreadHarness();
|
||||
const warn = vi.spyOn(embeddedAgentLog, "warn").mockImplementation(() => undefined);
|
||||
const params = createParams(
|
||||
path.join(tempDir, "session.jsonl"),
|
||||
path.join(tempDir, "workspace"),
|
||||
);
|
||||
params.timeoutMs = 100;
|
||||
|
||||
const run = runCodexAppServerAttempt(params, {
|
||||
turnCompletionIdleTimeoutMs: 500,
|
||||
turnAssistantCompletionIdleTimeoutMs: 500,
|
||||
turnTerminalIdleTimeoutMs: 500,
|
||||
});
|
||||
await harness.waitForMethod("turn/start");
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
await harness.handleServerRequest({
|
||||
id: "request-account-refresh",
|
||||
method: "account/nonTurnRefresh",
|
||||
params: {},
|
||||
});
|
||||
|
||||
const result = await run;
|
||||
expect(result.aborted).toBe(true);
|
||||
expect(result.timedOut).toBe(true);
|
||||
expect(result.promptError).toBe(
|
||||
"codex app-server turn idle timed out waiting for turn/completed",
|
||||
);
|
||||
const warnCall = warn.mock.calls.find(
|
||||
([message]) => message === "codex app-server turn idle timed out waiting for progress",
|
||||
);
|
||||
const warnData = warnCall?.[1] as
|
||||
| { lastActivityReason?: string; timeoutMs?: number }
|
||||
| undefined;
|
||||
expect(warnData?.timeoutMs).toBe(100);
|
||||
expect(warnData?.lastActivityReason).toBe("turn:start");
|
||||
expect(harness.request.mock.calls.some(([method]) => method === "turn/interrupt")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not count account rate-limit updates as turn completion activity", async () => {
|
||||
let notify: (notification: CodexServerNotification) => Promise<void> = async () => undefined;
|
||||
let handleRequest:
|
||||
|
||||
@@ -1068,6 +1068,9 @@ export async function runCodexAppServerAttempt(
|
||||
let turnCompletionLastActivityAt = Date.now();
|
||||
let turnCompletionLastActivityReason = "startup";
|
||||
let turnCompletionLastActivityDetails: Record<string, unknown> | undefined;
|
||||
let turnAttemptLastProgressAt = Date.now();
|
||||
let turnAttemptLastProgressReason = "startup";
|
||||
let turnAttemptLastProgressDetails: Record<string, unknown> | undefined;
|
||||
let activeAppServerTurnRequests = 0;
|
||||
const activeOpenClawDynamicToolCallIds = new Set<string>();
|
||||
const activeTurnItemIds = new Set<string>();
|
||||
@@ -1154,7 +1157,7 @@ export async function runCodexAppServerAttempt(
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const idleMs = Math.max(0, Date.now() - turnCompletionLastActivityAt);
|
||||
const idleMs = Math.max(0, Date.now() - turnAttemptLastProgressAt);
|
||||
if (idleMs < turnAttemptIdleTimeoutMs) {
|
||||
scheduleTurnAttemptIdleWatch();
|
||||
return;
|
||||
@@ -1169,16 +1172,16 @@ export async function runCodexAppServerAttempt(
|
||||
turnId,
|
||||
idleMs,
|
||||
timeoutMs: turnAttemptIdleTimeoutMs,
|
||||
lastActivityReason: turnCompletionLastActivityReason,
|
||||
...turnCompletionLastActivityDetails,
|
||||
lastActivityReason: turnAttemptLastProgressReason,
|
||||
...turnAttemptLastProgressDetails,
|
||||
});
|
||||
embeddedAgentLog.warn("codex app-server turn idle timed out waiting for progress", {
|
||||
threadId: thread.threadId,
|
||||
turnId,
|
||||
idleMs,
|
||||
timeoutMs: turnAttemptIdleTimeoutMs,
|
||||
lastActivityReason: turnCompletionLastActivityReason,
|
||||
...turnCompletionLastActivityDetails,
|
||||
lastActivityReason: turnAttemptLastProgressReason,
|
||||
...turnAttemptLastProgressDetails,
|
||||
});
|
||||
runAbortController.abort("turn_progress_idle_timeout");
|
||||
};
|
||||
@@ -1296,7 +1299,7 @@ export async function runCodexAppServerAttempt(
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const elapsedMs = Math.max(0, Date.now() - turnCompletionLastActivityAt);
|
||||
const elapsedMs = Math.max(0, Date.now() - turnAttemptLastProgressAt);
|
||||
const delayMs = Math.max(1, turnAttemptIdleTimeoutMs - elapsedMs);
|
||||
turnAttemptIdleTimer = setTimeout(fireTurnAttemptIdleTimeout, delayMs);
|
||||
turnAttemptIdleTimer.unref?.();
|
||||
@@ -1318,13 +1321,24 @@ export async function runCodexAppServerAttempt(
|
||||
turnTerminalIdleTimer.unref?.();
|
||||
}
|
||||
|
||||
function scheduleTurnProgressWatches() {
|
||||
scheduleTurnAttemptIdleWatch();
|
||||
scheduleTurnCompletionIdleWatch();
|
||||
scheduleTurnTerminalIdleWatch();
|
||||
}
|
||||
|
||||
const touchTurnCompletionActivity = (
|
||||
reason: string,
|
||||
options?: { arm?: boolean; details?: Record<string, unknown> },
|
||||
options?: { arm?: boolean; details?: Record<string, unknown>; attemptProgress?: boolean },
|
||||
) => {
|
||||
turnCompletionLastActivityAt = Date.now();
|
||||
turnCompletionLastActivityReason = reason;
|
||||
turnCompletionLastActivityDetails = options?.details;
|
||||
if (options?.attemptProgress) {
|
||||
turnAttemptLastProgressAt = turnCompletionLastActivityAt;
|
||||
turnAttemptLastProgressReason = reason;
|
||||
turnAttemptLastProgressDetails = options.details;
|
||||
}
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.progress",
|
||||
runId: params.runId,
|
||||
@@ -1336,9 +1350,7 @@ export async function runCodexAppServerAttempt(
|
||||
turnCompletionIdleWatchArmed = true;
|
||||
turnCompletionIdleWatchPinnedByTerminalError = false;
|
||||
}
|
||||
scheduleTurnAttemptIdleWatch();
|
||||
scheduleTurnCompletionIdleWatch();
|
||||
scheduleTurnTerminalIdleWatch();
|
||||
scheduleTurnProgressWatches();
|
||||
};
|
||||
|
||||
const disarmTurnCompletionIdleWatch = () => {
|
||||
@@ -1444,6 +1456,7 @@ export async function runCodexAppServerAttempt(
|
||||
if (isCurrentTurnNotification) {
|
||||
touchTurnCompletionActivity(`notification:${notification.method}`, {
|
||||
details: describeNotificationActivity(notification),
|
||||
attemptProgress: true,
|
||||
});
|
||||
reportCodexExecutionNotification(notification);
|
||||
}
|
||||
@@ -1569,8 +1582,8 @@ export async function runCodexAppServerAttempt(
|
||||
activeAppServerTurnRequests += 1;
|
||||
clearTurnCompletionIdleTimer();
|
||||
disarmTurnAssistantCompletionIdleWatch();
|
||||
touchTurnCompletionActivity(`request:${request.method}`);
|
||||
let armCompletionWatchOnResponse = false;
|
||||
let requestCountsAsTurnActivity = false;
|
||||
try {
|
||||
if (request.method === "account/chatgptAuthTokens/refresh") {
|
||||
return refreshCodexAppServerAuthTokens({
|
||||
@@ -1584,6 +1597,7 @@ export async function runCodexAppServerAttempt(
|
||||
}
|
||||
if (request.method === "mcpServer/elicitation/request") {
|
||||
armCompletionWatchOnResponse = true;
|
||||
requestCountsAsTurnActivity = true;
|
||||
return handleCodexAppServerElicitationRequest({
|
||||
requestParams: request.params,
|
||||
paramsForRun: params,
|
||||
@@ -1595,6 +1609,7 @@ export async function runCodexAppServerAttempt(
|
||||
}
|
||||
if (request.method === "item/tool/requestUserInput") {
|
||||
armCompletionWatchOnResponse = true;
|
||||
requestCountsAsTurnActivity = true;
|
||||
return userInputBridge?.handleRequest({
|
||||
id: request.id,
|
||||
params: request.params,
|
||||
@@ -1603,6 +1618,7 @@ export async function runCodexAppServerAttempt(
|
||||
if (request.method !== "item/tool/call") {
|
||||
if (isCodexAppServerApprovalRequest(request.method)) {
|
||||
armCompletionWatchOnResponse = true;
|
||||
requestCountsAsTurnActivity = true;
|
||||
return handleApprovalRequest({
|
||||
method: request.method,
|
||||
params: request.params,
|
||||
@@ -1619,6 +1635,7 @@ export async function runCodexAppServerAttempt(
|
||||
return undefined;
|
||||
}
|
||||
armCompletionWatchOnResponse = true;
|
||||
requestCountsAsTurnActivity = true;
|
||||
turnCrossedToolHandoff = true;
|
||||
activeOpenClawDynamicToolCallIds.add(call.callId);
|
||||
trajectoryRecorder?.recordEvent("tool.call", {
|
||||
@@ -1698,9 +1715,14 @@ export async function runCodexAppServerAttempt(
|
||||
return response as JsonValue;
|
||||
} finally {
|
||||
activeAppServerTurnRequests = Math.max(0, activeAppServerTurnRequests - 1);
|
||||
touchTurnCompletionActivity(`request:${request.method}:response`, {
|
||||
arm: armCompletionWatchOnResponse,
|
||||
});
|
||||
if (requestCountsAsTurnActivity) {
|
||||
touchTurnCompletionActivity(`request:${request.method}:response`, {
|
||||
arm: armCompletionWatchOnResponse,
|
||||
attemptProgress: true,
|
||||
});
|
||||
} else {
|
||||
scheduleTurnProgressWatches();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1999,7 +2021,7 @@ export async function runCodexAppServerAttempt(
|
||||
setActiveEmbeddedRun(params.sessionId, handle, params.sessionKey);
|
||||
turnAttemptIdleWatchArmed = true;
|
||||
turnTerminalIdleWatchArmed = true;
|
||||
touchTurnCompletionActivity("turn:start", { arm: true });
|
||||
touchTurnCompletionActivity("turn:start", { arm: true, attemptProgress: true });
|
||||
|
||||
const abortListener = () => {
|
||||
const shouldRetireClient = timedOut;
|
||||
|
||||
Reference in New Issue
Block a user