diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b037a1ba0e..a11180f4256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ Docs: https://docs.openclaw.ai - Codex/failover: classify `deactivated_workspace` as a permanent auth failure so configured fallback models can advance when a Codex workspace is deactivated. (#55893) Thanks @litang9. - Exec: keep configured `tools.exec.pathPrepend` entries ahead of user shell startup PATH changes on POSIX gateway runs. (#81403) Thanks @medns. - Gateway/sessions: allow shared-secret bearer callers to read and stream session history without an explicit scope header. (#81815) Thanks @medns. +- Agents/embedded runner: classify HTML auth provider responses as `auth_html` and return a re-authentication hint instead of the CDN-blocked copy that `upstream_html` returns. Cloudflare Access login pages, nginx basic-auth challenges, and gateway login walls all produce HTML auth bodies that were previously misdiagnosed as transient CDN blocks. (#79900) Thanks @martingarramon. ## 2026.5.20 diff --git a/src/agents/pi-embedded-error-observation.test.ts b/src/agents/pi-embedded-error-observation.test.ts index 5ec20e4f8c0..c5372883adc 100644 --- a/src/agents/pi-embedded-error-observation.test.ts +++ b/src/agents/pi-embedded-error-observation.test.ts @@ -4,6 +4,7 @@ import { buildApiErrorObservationFields, buildTextObservationFields, sanitizeForConsole, + shouldSuppressRawErrorConsoleSuffix, } from "./pi-embedded-error-observation.js"; const OBSERVATION_BEARER_TOKEN = "sk-redact-test-token"; @@ -182,6 +183,14 @@ describe("buildApiErrorObservationFields", () => { expect(observed.httpCode).toBe("401"); expect(observed.providerRuntimeFailureKind).toBe("unclassified"); }); + + it("centralizes raw console suffix suppression for auth failures", () => { + expect(shouldSuppressRawErrorConsoleSuffix("auth_html")).toBe(true); + expect(shouldSuppressRawErrorConsoleSuffix("auth_scope")).toBe(true); + expect(shouldSuppressRawErrorConsoleSuffix("auth_refresh")).toBe(true); + expect(shouldSuppressRawErrorConsoleSuffix("timeout")).toBe(false); + expect(shouldSuppressRawErrorConsoleSuffix(undefined)).toBe(false); + }); }); describe("sanitizeForConsole", () => { diff --git a/src/agents/pi-embedded-error-observation.ts b/src/agents/pi-embedded-error-observation.ts index c857ac66063..6275c107353 100644 --- a/src/agents/pi-embedded-error-observation.ts +++ b/src/agents/pi-embedded-error-observation.ts @@ -22,6 +22,11 @@ const OBSERVATION_EXTRA_REDACT_PATTERNS = [ String.raw`"(?:api[-_]?key|api_key)"\s*:\s*"([^"]+)"`, String.raw`(?:\bCookie\b\s*[:=]\s*[^;=\s]+=|;\s*[^;=\s]+=)([^;\s\r\n]+)`, ]; +const RAW_ERROR_CONSOLE_SUPPRESSED_FAILURE_KINDS = new Set([ + "auth_html", + "auth_refresh", + "auth_scope", +]); function resolveConfiguredRedactPatterns(): string[] { const configured = readLoggingConfig()?.redactPatterns; @@ -76,6 +81,14 @@ function redactObservationText(text: string | undefined): string | undefined { }); } +export function shouldSuppressRawErrorConsoleSuffix( + providerRuntimeFailureKind?: ProviderRuntimeFailureKind, +): boolean { + return providerRuntimeFailureKind + ? RAW_ERROR_CONSOLE_SUPPRESSED_FAILURE_KINDS.has(providerRuntimeFailureKind) + : false; +} + function buildObservationFingerprint(params: { raw: string; requestId?: string; diff --git a/src/agents/pi-embedded-helpers.formatassistanterrortext.test.ts b/src/agents/pi-embedded-helpers.formatassistanterrortext.test.ts index 6e63f4e8849..d0981b714fa 100644 --- a/src/agents/pi-embedded-helpers.formatassistanterrortext.test.ts +++ b/src/agents/pi-embedded-helpers.formatassistanterrortext.test.ts @@ -342,10 +342,17 @@ describe("formatAssistantErrorText", () => { ); }); + it("returns re-authentication copy for HTML provider 401 auth failures", () => { + const msg = makeAssistantError("401 Unauthorized"); + expect(formatAssistantErrorText(msg)).toBe( + "Authentication failed at the provider. Re-authenticate and verify your provider credentials and account access.", + ); + }); + it("returns an HTML-403 auth message for HTML provider auth failures", () => { const msg = makeAssistantError("403 Access denied"); expect(formatAssistantErrorText(msg)).toBe( - "Authentication failed with an HTML 403 response from the provider. Re-authenticate and verify your provider account access.", + "Authentication failed at the provider. Re-authenticate and verify your provider credentials and account access.", ); }); diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index 23b6afa594f..7e6ab6ba51d 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -1510,7 +1510,15 @@ describe("classifyProviderRuntimeFailureKind", () => { classifyProviderRuntimeFailureKind( "403 Access denied", ), - ).toBe("auth_html_403"); + ).toBe("auth_html"); + }); + + it("classifies HTML 401 auth failures", () => { + expect( + classifyProviderRuntimeFailureKind( + "401 Unauthorized", + ), + ).toBe("auth_html"); }); it("classifies proxy, dns, timeout, schema, sandbox, and replay failures", () => { diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index 97873af38d1..fbe78602813 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -259,7 +259,7 @@ export type ProviderRuntimeFailureKind = | "refresh_contention" | "callback_timeout" | "callback_validation" - | "auth_html_403" + | "auth_html" | "upstream_html" | "proxy" | "rate_limit" @@ -976,7 +976,7 @@ export function classifyProviderRuntimeFailureKind( return "proxy"; } if (message && isHtmlErrorResponse(message, status)) { - return status === 403 ? "auth_html_403" : "upstream_html"; + return status === 401 || status === 403 ? "auth_html" : "upstream_html"; } const failoverClassification = classifyFailoverSignal({ ...normalizedSignal, @@ -1090,10 +1090,10 @@ export function formatAssistantErrorText( ); } - if (providerRuntimeFailureKind === "auth_html_403") { + if (providerRuntimeFailureKind === "auth_html") { return ( - "Authentication failed with an HTML 403 response from the provider. " + - "Re-authenticate and verify your provider account access." + "Authentication failed at the provider. " + + "Re-authenticate and verify your provider credentials and account access." ); } diff --git a/src/agents/pi-embedded-helpers/provider-error-patterns.test.ts b/src/agents/pi-embedded-helpers/provider-error-patterns.test.ts index 8024ce4dd3a..5c58f44b64d 100644 --- a/src/agents/pi-embedded-helpers/provider-error-patterns.test.ts +++ b/src/agents/pi-embedded-helpers/provider-error-patterns.test.ts @@ -210,9 +210,9 @@ describe("Cloudflare / CDN HTML error page classification (#67517)", () => { ); }); - it("classifies 403 HTML runtime failures as auth_html_403", () => { + it("classifies 403 HTML runtime failures as auth_html", () => { expect(classifyProviderRuntimeFailureKind({ status: 403, message: html403 })).toBe( - "auth_html_403", + "auth_html", ); }); diff --git a/src/agents/pi-embedded-runner/run/failover-observation.test.ts b/src/agents/pi-embedded-runner/run/failover-observation.test.ts index 0a72503b1db..b411e3f22c4 100644 --- a/src/agents/pi-embedded-runner/run/failover-observation.test.ts +++ b/src/agents/pi-embedded-runner/run/failover-observation.test.ts @@ -36,6 +36,8 @@ function firstWarnDetails(warnSpy: { mock: { calls: unknown[][] } }): { consoleMessage?: string; model?: string; provider?: string; + providerRuntimeFailureKind?: string; + rawErrorPreview?: string; sourceModel?: string; sourceProvider?: string; } { @@ -43,6 +45,8 @@ function firstWarnDetails(warnSpy: { mock: { calls: unknown[][] } }): { consoleMessage?: string; model?: string; provider?: string; + providerRuntimeFailureKind?: string; + rawErrorPreview?: string; sourceModel?: string; sourceProvider?: string; }; @@ -133,4 +137,33 @@ describe("createFailoverDecisionLogger", () => { expect(firstWarnDetails(warnSpy).consoleMessage).toContain("from=openai/gpt-5.4"); expect(firstWarnDetails(warnSpy).consoleMessage).not.toContain("to=openai/gpt-5.4"); }); + + it("omits raw HTML auth bodies from consoleMessage for HTML 401 auth failures", () => { + const warnSpy = vi.spyOn(log, "warn").mockImplementation(() => {}); + const logDecision = createFailoverDecisionLogger({ + stage: "assistant", + runId: "run:auth-html", + rawError: "401 Unauthorized", + failoverReason: "auth", + profileFailureReason: "auth", + provider: "openai-codex", + model: "gpt-5.4", + sourceProvider: "openai-codex", + sourceModel: "gpt-5.4", + profileId: "openai-codex:p1", + fallbackConfigured: true, + timedOut: false, + aborted: false, + }); + + logDecision("rotate_profile"); + + const observation = firstWarnDetails(warnSpy); + expect(observation.providerRuntimeFailureKind).toBe("auth_html"); + expect(observation.rawErrorPreview).toBe( + "401 Unauthorized", + ); + expect(observation.consoleMessage).not.toContain("rawError="); + expect(observation.consoleMessage).not.toContain(""); + }); }); diff --git a/src/agents/pi-embedded-runner/run/failover-observation.ts b/src/agents/pi-embedded-runner/run/failover-observation.ts index 8ec5ca0eb14..0de896a47ce 100644 --- a/src/agents/pi-embedded-runner/run/failover-observation.ts +++ b/src/agents/pi-embedded-runner/run/failover-observation.ts @@ -3,6 +3,7 @@ import type { AuthProfileFailureReason } from "../../auth-profiles.js"; import { buildApiErrorObservationFields, sanitizeForConsole, + shouldSuppressRawErrorConsoleSuffix, } from "../../pi-embedded-error-observation.js"; import type { FailoverReason } from "../../pi-embedded-helpers.js"; import { log } from "../logger.js"; @@ -58,12 +59,9 @@ export function createFailoverDecisionLogger( return (decision, extra) => { const observedError = buildApiErrorObservationFields(normalizedBase.rawError); const safeRawErrorPreview = sanitizeForConsole(observedError.rawErrorPreview); - const shouldSuppressRawErrorConsoleSuffix = - observedError.providerRuntimeFailureKind === "auth_html_403" || - observedError.providerRuntimeFailureKind === "auth_scope" || - observedError.providerRuntimeFailureKind === "auth_refresh"; const rawErrorConsoleSuffix = - safeRawErrorPreview && !shouldSuppressRawErrorConsoleSuffix + safeRawErrorPreview && + !shouldSuppressRawErrorConsoleSuffix(observedError.providerRuntimeFailureKind) ? ` rawError=${safeRawErrorPreview}` : ""; log.warn("embedded run failover decision", { diff --git a/src/agents/pi-embedded-subscribe.handlers.lifecycle.test.ts b/src/agents/pi-embedded-subscribe.handlers.lifecycle.test.ts index 28eaf15ad42..4ef00ea5cbe 100644 --- a/src/agents/pi-embedded-subscribe.handlers.lifecycle.test.ts +++ b/src/agents/pi-embedded-subscribe.handlers.lifecycle.test.ts @@ -206,28 +206,44 @@ describe("handleAgentEnd", () => { expect(meta.httpCode).toBe("401"); }); - it("omits raw HTML auth bodies from consoleMessage for HTML 403 auth failures", async () => { - const ctx = createContext({ - role: "assistant", - stopReason: "error", - provider: "openai-codex", - model: "gpt-5.4", + it.each([ + { errorMessage: "403 Access denied", - content: [{ type: "text", text: "" }], - }); + expectedError: + "Authentication failed at the provider. Re-authenticate and verify your provider credentials and account access.", + expectedKind: "auth_html", + expectedPreview: "403 Access denied", + }, + { + errorMessage: "401 Unauthorized", + expectedError: + "Authentication failed at the provider. Re-authenticate and verify your provider credentials and account access.", + expectedKind: "auth_html", + expectedPreview: "401 Unauthorized", + }, + ])( + "omits raw HTML auth bodies from consoleMessage for $expectedKind failures", + async ({ errorMessage, expectedError, expectedKind, expectedPreview }) => { + const ctx = createContext({ + role: "assistant", + stopReason: "error", + provider: "openai-codex", + model: "gpt-5.4", + errorMessage, + content: [{ type: "text", text: "" }], + }); - await handleAgentEnd(ctx); + await handleAgentEnd(ctx); - const meta = firstWarnMeta(ctx); - expect(meta.providerRuntimeFailureKind).toBe("auth_html_403"); - expect(meta.rawErrorPreview).toBe("403 Access denied"); - expect(meta.error).toBe( - "Authentication failed with an HTML 403 response from the provider. Re-authenticate and verify your provider account access.", - ); - const consoleMsg = typeof meta.consoleMessage === "string" ? meta.consoleMessage : ""; - expect(consoleMsg).not.toContain("rawError="); - expect(consoleMsg).not.toContain(""); - }); + const meta = firstWarnMeta(ctx); + expect(meta.providerRuntimeFailureKind).toBe(expectedKind); + expect(meta.rawErrorPreview).toBe(expectedPreview); + expect(meta.error).toBe(expectedError); + const consoleMsg = typeof meta.consoleMessage === "string" ? meta.consoleMessage : ""; + expect(consoleMsg).not.toContain("rawError="); + expect(consoleMsg).not.toContain(""); + }, + ); it("keeps non-error run-end logging on debug only", async () => { const ctx = createContext(undefined); diff --git a/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts b/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts index 66a9aa643e0..1eba7b0f9a4 100644 --- a/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts +++ b/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts @@ -4,6 +4,7 @@ import { buildApiErrorObservationFields, buildTextObservationFields, sanitizeForConsole, + shouldSuppressRawErrorConsoleSuffix, } from "./pi-embedded-error-observation.js"; import { classifyFailoverReason, formatAssistantErrorText } from "./pi-embedded-helpers.js"; import { hasCommittedMessagingToolDeliveryEvidence } from "./pi-embedded-runner/delivery-evidence.js"; @@ -92,12 +93,9 @@ export function handleAgentEnd(ctx: EmbeddedPiSubscribeContext): void | Promise< const safeModel = sanitizeForConsole(lastAssistant.model) ?? "unknown"; const safeProvider = sanitizeForConsole(lastAssistant.provider) ?? "unknown"; const safeRawErrorPreview = sanitizeForConsole(observedError.rawErrorPreview); - const shouldSuppressRawErrorConsoleSuffix = - observedError.providerRuntimeFailureKind === "auth_html_403" || - observedError.providerRuntimeFailureKind === "auth_scope" || - observedError.providerRuntimeFailureKind === "auth_refresh"; const rawErrorConsoleSuffix = - safeRawErrorPreview && !shouldSuppressRawErrorConsoleSuffix + safeRawErrorPreview && + !shouldSuppressRawErrorConsoleSuffix(observedError.providerRuntimeFailureKind) ? ` rawError=${safeRawErrorPreview}` : ""; ctx.log.warn("embedded run agent end", {