From fee89ff4cb0b47fdcac274c08755c9554b4737bf Mon Sep 17 00:00:00 2001 From: Peter Lee Date: Sun, 12 Jul 2026 18:33:24 -0500 Subject: [PATCH] fix(provider-runtime): align transient network error codes across retry paths (#101496) * fix(provider-runtime): align transient network retries Centralize the shared connection error policy, preserve the bounded provider-only ENOTFOUND retry, and keep gateway DNS/config failures and provider creates fail-fast.\n\nCo-authored-by: Peter Lee * test(provider-runtime): cover root retry error codes Exercise direct Node error.code and nested cause.code shapes through the public retry wrapper. Co-authored-by: Peter Lee --------- Co-authored-by: Peter Steinberger --- CHANGELOG.md | 1 + src/agents/run-wait.test.ts | 24 ++++++++ src/agents/run-wait.ts | 7 ++- src/infra/retryable-network-errors.ts | 7 +++ src/provider-runtime/operation-retry.test.ts | 61 ++++++++++++++++++++ src/provider-runtime/operation-retry.ts | 18 ++++-- 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 src/infra/retryable-network-errors.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f02e8969073..f59ae59c2db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- **Provider network retries:** align provider read/poll/download and agent-wait recovery for transient connection errors, retry bounded provider `ENOTFOUND` failures while leaving gateway `ENOTFOUND` and non-idempotent create operations fail-fast. (#101496) Thanks @xialonglee. - **Session retry classification:** stop permanent provider errors whose identifiers or payload details merely contain 429/5xx digit sequences from re-sending full context, and share bounded rate-limit-window parsing across retry paths. (#105258) Thanks @destire-mio. - **LINE directive templates:** suppress confirms and buttons with blank required fields or unlabeled actions while preserving valid titleless buttons and surrounding reply text. (#105520) Thanks @edenfunf. - **SQLite maintenance schema validation:** reject current-version global and agent databases with missing or drifted canonical tables, constraints, indexes, triggers, or table options before compaction, while accepting supported additive-migration layouts. diff --git a/src/agents/run-wait.test.ts b/src/agents/run-wait.test.ts index 9fce135bd367..a56df9875e4e 100644 --- a/src/agents/run-wait.test.ts +++ b/src/agents/run-wait.test.ts @@ -1069,3 +1069,27 @@ describe("waitForAgentRunsToDrain", () => { } }); }); + +describe("isRecoverableAgentWaitError", () => { + it.each([ + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "EPIPE", + "EHOSTUNREACH", + "ENETUNREACH", + "EAI_AGAIN", + ])("recovers from %s connection failures", (code) => { + expect(isRecoverableAgentWaitError(`connect ${code} 127.0.0.1:443`)).toBe(true); + }); + + it.each([ + undefined, + "", + "gateway timeout", + "ENOENT: no such file", + "getaddrinfo ENOTFOUND gateway.example.com", + ])("does not recover from %s", (error) => { + expect(isRecoverableAgentWaitError(error)).toBe(false); + }); +}); diff --git a/src/agents/run-wait.ts b/src/agents/run-wait.ts index 7779b791c90f..21b319b03f7d 100644 --- a/src/agents/run-wait.ts +++ b/src/agents/run-wait.ts @@ -14,6 +14,7 @@ import { } from "@openclaw/normalization-core/number-coercion"; import { callGateway } from "../gateway/call.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { hasRetryableConnectionErrorCode } from "../infra/retryable-network-errors.js"; import { normalizeBlockedLivenessWaitStatus } from "../shared/agent-liveness.js"; import { isOpenClawInternalSourceReplyMirrorAssistantMessage, @@ -139,7 +140,6 @@ const RECOVERABLE_AGENT_WAIT_ERROR_PATTERNS: readonly RegExp[] = [ /gateway not connected/i, /no active .* listener/i, /socket hang up/i, - /\b(ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|EHOSTUNREACH|ENETUNREACH)\b/i, ]; /** Return true for transient gateway/transport failures that callers may retry. */ @@ -151,7 +151,10 @@ export function isRecoverableAgentWaitError(error: string | undefined): boolean if (message.includes("gateway timeout")) { return false; } - return RECOVERABLE_AGENT_WAIT_ERROR_PATTERNS.some((pattern) => pattern.test(message)); + return ( + hasRetryableConnectionErrorCode(message) || + RECOVERABLE_AGENT_WAIT_ERROR_PATTERNS.some((pattern) => pattern.test(message)) + ); } function normalizePendingRunIds(runIds: Iterable): string[] { diff --git a/src/infra/retryable-network-errors.ts b/src/infra/retryable-network-errors.ts new file mode 100644 index 000000000000..b5f0c069f7e1 --- /dev/null +++ b/src/infra/retryable-network-errors.ts @@ -0,0 +1,7 @@ +// Keep connection retry policy aligned across provider reads and gateway waits. +const RETRYABLE_CONNECTION_ERROR_CODE_RE = + /\b(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN)\b/i; + +export function hasRetryableConnectionErrorCode(message: string): boolean { + return RETRYABLE_CONNECTION_ERROR_CODE_RE.test(message); +} diff --git a/src/provider-runtime/operation-retry.test.ts b/src/provider-runtime/operation-retry.test.ts index 4aa65df314b6..edce5cc0deb3 100644 --- a/src/provider-runtime/operation-retry.test.ts +++ b/src/provider-runtime/operation-retry.test.ts @@ -42,4 +42,65 @@ describe("executeProviderOperationWithRetry", () => { expect(operation).toHaveBeenCalledTimes(1); }); + + it.each([ + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "EPIPE", + "EHOSTUNREACH", + "ENETUNREACH", + "EAI_AGAIN", + "ENOTFOUND", + ])("retries %s network failures from structured errors", async (code) => { + const cause = Object.assign(new Error("connect failed"), { code }); + const error = + code === "EPIPE" + ? Object.assign(new Error("socket closed"), { code }) + : new Error("fetch failed", { cause }); + const operation = vi + .fn<() => Promise>() + .mockRejectedValueOnce(error) + .mockResolvedValue("ok"); + + await expect( + executeProviderOperationWithRetry({ + provider: "test", + stage: "read", + operation, + retry: { attempts: 2, baseDelayMs: 0, maxDelayMs: 0 }, + }), + ).resolves.toBe("ok"); + expect(operation).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["HTTP 400", Object.assign(new Error("Bad Request"), { status: 400 })], + ["ENOENT", new Error("ENOENT: no such file or directory")], + ])("does not retry %s failures", async (_label, error) => { + const operation = vi.fn(async () => { + throw error; + }); + + await expect( + executeProviderOperationWithRetry({ + provider: "test", + stage: "read", + operation, + retry: { attempts: 2, baseDelayMs: 0, maxDelayMs: 0 }, + }), + ).rejects.toThrow(); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it("does not retry create operations by default", async () => { + const operation = vi.fn(async () => { + throw Object.assign(new Error("write EPIPE"), { code: "EPIPE" }); + }); + + await expect( + executeProviderOperationWithRetry({ provider: "test", stage: "create", operation }), + ).rejects.toThrow("EPIPE"); + expect(operation).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/provider-runtime/operation-retry.ts b/src/provider-runtime/operation-retry.ts index 1d70cc97ff80..ce00e7ae9521 100644 --- a/src/provider-runtime/operation-retry.ts +++ b/src/provider-runtime/operation-retry.ts @@ -1,6 +1,7 @@ // Provider operation retry helpers run retryable provider operations with backoff. import { sleepWithAbort } from "../infra/backoff.js"; import { formatErrorMessage } from "../infra/errors.js"; +import { hasRetryableConnectionErrorCode } from "../infra/retryable-network-errors.js"; export type ProviderOperationRetryStage = "read" | "poll" | "download" | "create"; @@ -103,13 +104,20 @@ function readErrorCause(error: unknown): unknown { return (error as { cause?: unknown }).cause; } +// Provider reads get one bounded retry for negative DNS responses. Gateway +// waits exclude ENOTFOUND because their configured gateway address needs repair. +const PROVIDER_RETRYABLE_DNS_ERROR_CODE_RE = /\bENOTFOUND\b/i; + +function hasProviderRetryableNetworkCode(value: string): boolean { + return hasRetryableConnectionErrorCode(value) || PROVIDER_RETRYABLE_DNS_ERROR_CODE_RE.test(value); +} + function hasTransientNetworkSignal(error: unknown, message: string): boolean { - const transientCodes = /\b(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN)\b/i; - if (transientCodes.test(message)) { + if (hasProviderRetryableNetworkCode(message)) { return true; } const code = readErrorCode(error); - if (code && transientCodes.test(code)) { + if (code && hasProviderRetryableNetworkCode(code)) { return true; } const cause = readErrorCause(error); @@ -117,11 +125,11 @@ function hasTransientNetworkSignal(error: unknown, message: string): boolean { return false; } const causeCode = readErrorCode(cause); - if (causeCode && transientCodes.test(causeCode)) { + if (causeCode && hasProviderRetryableNetworkCode(causeCode)) { return true; } const causeMessage = formatErrorMessage(cause); - return transientCodes.test(causeMessage); + return hasProviderRetryableNetworkCode(causeMessage); } function hasTimeoutSignal(error: unknown, message: string): boolean {