vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-29 18:12:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: handle Cloudflare 521 and transient 5xx errors gracefully

Browse Source 
- Add isCloudflareOrHtmlErrorPage() to detect HTML error pages (521-530)
- Add isTransientHttpError() for retryable 5xx codes (500,502,503,521-524,529)
- Sanitize HTML error pages to clean user message instead of raw HTML
- Add single retry with 2.5s backoff for transient HTTP errors in agent runner
- Never send raw HTML/Cloudflare error pages to users
This commit is contained in:
Rodrigo Uroz
2026-02-10 13:01:59 -03:00
committed by Tak Hoffman
parent c28cbac512
commit 1e388a503a
7 changed files with 258 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/agents/pi-embedded-helpers.formatrawassistanterrorforui.test.ts
View File

@@ -22,4 +22,16 @@ describe("formatRawAssistantErrorForUi", () => {
"HTTP 500: Internal Server Error",
);
});
it("sanitizes HTML error pages into a clean unavailable message", () => {
const htmlError = `521 <!DOCTYPE html>
<html lang="en-US">
<head><title>Web server is down | example.com | Cloudflare</title></head>
<body>Ray ID: abc123</body>
</html>`;
expect(formatRawAssistantErrorForUi(htmlError)).toBe(
"The AI service is temporarily unavailable (HTTP 521). Please try again in a moment.",
);
});
});

24
src/agents/pi-embedded-helpers.iscloudflareorhtmlerrorpage.test.ts Normal file
View File

@@ -0,0 +1,24 @@
import { describe, expect, it } from "vitest";
import { isCloudflareOrHtmlErrorPage } from "./pi-embedded-helpers.js";
describe("isCloudflareOrHtmlErrorPage", () => {
it("detects Cloudflare 521 HTML pages", () => {
const htmlError = `521 <!DOCTYPE html>
<html lang="en-US">
<head><title>Web server is down | example.com | Cloudflare</title></head>
<body><h1>Web server is down</h1></body>
</html>`;
expect(isCloudflareOrHtmlErrorPage(htmlError)).toBe(true);
});
it("detects generic 5xx HTML pages", () => {
const htmlError = `503 <html><head><title>Service Unavailable</title></head><body>down</body></html>`;
expect(isCloudflareOrHtmlErrorPage(htmlError)).toBe(true);
});
it("does not flag non-HTML status lines", () => {
expect(isCloudflareOrHtmlErrorPage("500 Internal Server Error")).toBe(false);
expect(isCloudflareOrHtmlErrorPage("429 Too Many Requests")).toBe(false);
});
});

18
src/agents/pi-embedded-helpers.istransienthttperror.test.ts Normal file
View File

@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { isTransientHttpError } from "./pi-embedded-helpers.js";
describe("isTransientHttpError", () => {
it("returns true for retryable 5xx status codes", () => {
expect(isTransientHttpError("500 Internal Server Error")).toBe(true);
expect(isTransientHttpError("502 Bad Gateway")).toBe(true);
expect(isTransientHttpError("503 Service Unavailable")).toBe(true);
expect(isTransientHttpError("521 <!DOCTYPE html><html></html>")).toBe(true);
expect(isTransientHttpError("529 Overloaded")).toBe(true);
});
it("returns false for non-retryable or non-http text", () => {
expect(isTransientHttpError("504 Gateway Timeout")).toBe(false);
expect(isTransientHttpError("429 Too Many Requests")).toBe(false);
expect(isTransientHttpError("network timeout")).toBe(false);
});
});

2
src/agents/pi-embedded-helpers.ts
View File

@@ -17,6 +17,7 @@ export {
parseApiErrorInfo,
sanitizeUserFacingText,
isBillingErrorMessage,
isCloudflareOrHtmlErrorPage,
isCloudCodeAssistFormatError,
isCompactionFailureError,
isContextOverflowError,
@@ -29,6 +30,7 @@ export {
isRawApiErrorPayload,
isRateLimitAssistantError,
isRateLimitErrorMessage,
isTransientHttpError,
isTimeoutErrorMessage,
parseImageDimensionError,
parseImageSizeError,

54
src/agents/pi-embedded-helpers/errors.ts
View File

@@ -78,6 +78,10 @@ const ERROR_PREFIX_RE =
const CONTEXT_OVERFLOW_ERROR_HEAD_RE =
/^(?:context overflow:|request_too_large\b|request size exceeds\b|request exceeds the maximum size\b|context length exceeded\b|maximum context length\b|prompt is too long\b|exceeds model context window\b)/i;
const HTTP_STATUS_PREFIX_RE = /^(?:http\s*)?(\d{3})\s+(.+)$/i;
const HTTP_STATUS_CODE_PREFIX_RE = /^(?:http\s*)?(\d{3})(?:\s+([\s\S]+))?$/i;
const HTML_ERROR_PREFIX_RE = /^\s*(?:<!doctype\s+html\b|<html\b)/i;
const CLOUDFLARE_HTML_ERROR_CODES = new Set([521, 522, 523, 524, 525, 526, 530]);
const TRANSIENT_HTTP_ERROR_CODES = new Set([500, 502, 503, 521, 522, 523, 524, 529]);
const HTTP_ERROR_HINTS = [
"error",
"bad request",
@@ -96,6 +100,48 @@ const HTTP_ERROR_HINTS = [
"permission",
];
function extractLeadingHttpStatus(raw: string): { code: number; rest: string } | null {
const match = raw.match(HTTP_STATUS_CODE_PREFIX_RE);
if (!match) {
return null;
}
const code = Number(match[1]);
if (!Number.isFinite(code)) {
return null;
}
return { code, rest: (match[2] ?? "").trim() };
}
export function isCloudflareOrHtmlErrorPage(raw: string): boolean {
const trimmed = raw.trim();
if (!trimmed) {
return false;
}
const status = extractLeadingHttpStatus(trimmed);
if (!status || status.code < 500) {
return false;
}
if (CLOUDFLARE_HTML_ERROR_CODES.has(status.code)) {
return true;
}
return status.code < 600 && HTML_ERROR_PREFIX_RE.test(status.rest);
}
export function isTransientHttpError(raw: string): boolean {
const trimmed = raw.trim();
if (!trimmed) {
return false;
}
const status = extractLeadingHttpStatus(trimmed);
if (!status) {
return false;
}
return TRANSIENT_HTTP_ERROR_CODES.has(status.code);
}
function stripFinalTagsFromText(text: string): string {
if (!text) {
return text;
@@ -133,6 +179,9 @@ function collapseConsecutiveDuplicateBlocks(text: string): string {
}
function isLikelyHttpErrorText(raw: string): boolean {
if (isCloudflareOrHtmlErrorPage(raw)) {
return true;
}
const match = raw.match(HTTP_STATUS_PREFIX_RE);
if (!match) {
return false;
@@ -311,6 +360,11 @@ export function formatRawAssistantErrorForUi(raw?: string): string {
return "LLM request failed with an unknown error.";
}
const leadingStatus = extractLeadingHttpStatus(trimmed);
if (leadingStatus && isCloudflareOrHtmlErrorPage(trimmed)) {
return `The AI service is temporarily unavailable (HTTP ${leadingStatus.code}). Please try again in a moment.`;
}
const httpMatch = trimmed.match(HTTP_STATUS_PREFIX_RE);
if (httpMatch) {
const rest = httpMatch[2].trim();

20
src/auto-reply/reply/agent-runner-execution.ts
View File

@@ -14,6 +14,7 @@ import {
isCompactionFailureError,
isContextOverflowError,
isLikelyContextOverflowError,
isTransientHttpError,
sanitizeUserFacingText,
} from "../../agents/pi-embedded-helpers.js";
import { runEmbeddedPiAgent } from "../../agents/pi-embedded.js";
@@ -79,6 +80,7 @@ export async function runAgentTurnWithFallback(params: {
storePath?: string;
resolvedVerboseLevel: VerboseLevel;
}): Promise<AgentRunLoopResult> {
const TRANSIENT_HTTP_RETRY_DELAY_MS = 2_500;
let didLogHeartbeatStrip = false;
let autoCompactionCompleted = false;
// Track payloads sent directly (not via pipeline) during tool flush to avoid duplicates.
@@ -97,6 +99,7 @@ export async function runAgentTurnWithFallback(params: {
let fallbackProvider = params.followupRun.run.provider;
let fallbackModel = params.followupRun.run.model;
let didResetAfterCompactionFailure = false;
let didRetryTransientHttpError = false;
while (true) {
try {
@@ -506,6 +509,7 @@ export async function runAgentTurnWithFallback(params: {
const isCompactionFailure = isCompactionFailureError(message);
const isSessionCorruption = /function call turn comes immediately after/i.test(message);
const isRoleOrderingError = /incorrect role information|roles must alternate/i.test(message);
const isTransientHttp = isTransientHttpError(message);
if (
isCompactionFailure &&
@@ -577,8 +581,22 @@ export async function runAgentTurnWithFallback(params: {
};
}
if (isTransientHttp && !didRetryTransientHttpError) {
didRetryTransientHttpError = true;
defaultRuntime.error(
`Transient HTTP provider error before reply (${message}). Retrying once in ${TRANSIENT_HTTP_RETRY_DELAY_MS}ms.`,
);
await new Promise<void>((resolve) => {
setTimeout(resolve, TRANSIENT_HTTP_RETRY_DELAY_MS);
});
continue;
}
defaultRuntime.error(`Embedded agent failed before reply: ${message}`);
const trimmedMessage = message.replace(/\.\s*$/, "");
const safeMessage = isTransientHttp
? sanitizeUserFacingText(message, { errorContext: true })
: message;
const trimmedMessage = safeMessage.replace(/\.\s*$/, "");
const fallbackText = isContextOverflow
? "⚠️ Context overflow — prompt too large for this model. Try a shorter message or a larger-context model."
: isRoleOrderingError

129
src/auto-reply/reply/agent-runner.transient-http-retry.test.ts Normal file
View File

@@ -0,0 +1,129 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { TemplateContext } from "../templating.js";
import type { FollowupRun, QueueSettings } from "./queue.js";
import { createMockTypingController } from "./test-helpers.js";
const runEmbeddedPiAgentMock = vi.fn();
vi.mock("../../agents/model-fallback.js", () => ({
runWithModelFallback: async ({
provider,
model,
run,
}: {
provider: string;
model: string;
run: (provider: string, model: string) => Promise<unknown>;
}) => ({
result: await run(provider, model),
provider,
model,
}),
}));
vi.mock("../../agents/pi-embedded.js", () => ({
queueEmbeddedPiMessage: vi.fn().mockReturnValue(false),
runEmbeddedPiAgent: (params: unknown) => runEmbeddedPiAgentMock(params),
}));
vi.mock("./queue.js", async () => {
const actual = await vi.importActual<typeof import("./queue.js")>("./queue.js");
return {
...actual,
enqueueFollowupRun: vi.fn(),
scheduleFollowupDrain: vi.fn(),
};
});
import { runReplyAgent } from "./agent-runner.js";
describe("runReplyAgent transient HTTP retry", () => {
beforeEach(() => {
runEmbeddedPiAgentMock.mockReset();
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("retries once after transient 521 HTML failure and then succeeds", async () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
runEmbeddedPiAgentMock
.mockRejectedValueOnce(
new Error(
`521 <!DOCTYPE html><html lang="en-US"><head><title>Web server is down</title></head><body>Cloudflare</body></html>`,
),
)
.mockResolvedValueOnce({
payloads: [{ text: "Recovered response" }],
meta: {},
});
const typing = createMockTypingController();
const sessionCtx = {
Provider: "telegram",
MessageSid: "msg",
} as unknown as TemplateContext;
const resolvedQueue = { mode: "interrupt" } as unknown as QueueSettings;
const followupRun = {
prompt: "hello",
summaryLine: "hello",
enqueuedAt: Date.now(),
run: {
sessionId: "session",
sessionKey: "main",
messageProvider: "telegram",
sessionFile: "/tmp/session.jsonl",
workspaceDir: "/tmp",
config: {},
skillsSnapshot: {},
provider: "anthropic",
model: "claude",
thinkLevel: "low",
verboseLevel: "off",
elevatedLevel: "off",
bashElevated: {
enabled: false,
allowed: false,
defaultLevel: "off",
},
timeoutMs: 1_000,
blockReplyBreak: "message_end",
},
} as unknown as FollowupRun;
const runPromise = runReplyAgent({
commandBody: "hello",
followupRun,
queueKey: "main",
resolvedQueue,
shouldSteer: false,
shouldFollowup: false,
isActive: false,
isStreaming: false,
typing,
sessionCtx,
defaultModel: "anthropic/claude-opus-4-5",
resolvedVerboseLevel: "off",
isNewSession: false,
blockStreamingEnabled: false,
resolvedBlockStreamingBreak: "message_end",
shouldInjectGroupIntro: false,
typingMode: "instant",
});
await vi.advanceTimersByTimeAsync(2_500);
const result = await runPromise;
expect(runEmbeddedPiAgentMock).toHaveBeenCalledTimes(2);
expect(errorSpy).toHaveBeenCalledWith(
expect.stringContaining("Transient HTTP provider error before reply"),
);
const payload = Array.isArray(result) ? result[0] : result;
expect(payload?.text).toContain("Recovered response");
errorSpy.mockRestore();
});
});