openai-codex: classify auth and runtime failures

This commit is contained in:
Eva
2026-04-10 18:55:28 +07:00
committed by Peter Steinberger
parent 776c8e037e
commit 8166d592d9
10 changed files with 407 additions and 10 deletions

View File

@@ -23,6 +23,7 @@ describe("buildApiErrorObservationFields", () => {
rawErrorPreview: expect.stringContaining('"request_id":"sha256:'),
rawErrorHash: expect.stringMatching(/^sha256:/),
rawErrorFingerprint: expect.stringMatching(/^sha256:/),
providerRuntimeFailureKind: "timeout",
providerErrorType: "overloaded_error",
providerErrorMessagePreview: "Overloaded",
requestIdHash: expect.stringMatching(/^sha256:/),
@@ -69,6 +70,7 @@ describe("buildApiErrorObservationFields", () => {
textPreview: expect.stringContaining('"request_id":"sha256:'),
textHash: expect.stringMatching(/^sha256:/),
textFingerprint: expect.stringMatching(/^sha256:/),
providerRuntimeFailureKind: "timeout",
providerErrorType: "overloaded_error",
providerErrorMessagePreview: "Overloaded",
requestIdHash: expect.stringMatching(/^sha256:/),
@@ -156,6 +158,7 @@ describe("buildApiErrorObservationFields", () => {
textHash: undefined,
textFingerprint: undefined,
httpCode: undefined,
providerRuntimeFailureKind: undefined,
providerErrorType: undefined,
providerErrorMessagePreview: undefined,
requestIdHash: undefined,
@@ -176,6 +179,17 @@ describe("buildApiErrorObservationFields", () => {
expect(observed.rawErrorPreview).not.toContain("custom-secret-abc123");
expect(observed.rawErrorPreview).toContain("custom");
});
it("records runtime failure kind for missing-scope auth payloads", () => {
const observed = buildApiErrorObservationFields(
'401 {"type":"error","error":{"type":"permission_error","message":"Missing scopes: api.responses.write"}}',
);
expect(observed).toMatchObject({
httpCode: "401",
providerRuntimeFailureKind: "auth_scope",
});
});
});
describe("sanitizeForConsole", () => {

View File

@@ -3,7 +3,11 @@ import { redactIdentifier } from "../logging/redact-identifier.js";
import { getDefaultRedactPatterns, redactSensitiveText } from "../logging/redact.js";
import { normalizeOptionalString } from "../shared/string-coerce.js";
import { sanitizeForConsole } from "./console-sanitize.js";
import { getApiErrorPayloadFingerprint, parseApiErrorInfo } from "./pi-embedded-helpers.js";
import {
classifyProviderRuntimeFailureKind,
getApiErrorPayloadFingerprint,
parseApiErrorInfo,
} from "./pi-embedded-helpers.js";
import { stableStringify } from "./stable-stringify.js";
export { sanitizeForConsole } from "./console-sanitize.js";
@@ -105,6 +109,7 @@ export function buildApiErrorObservationFields(rawError?: string): {
rawErrorHash?: string;
rawErrorFingerprint?: string;
httpCode?: string;
providerRuntimeFailureKind?: string;
providerErrorType?: string;
providerErrorMessagePreview?: string;
requestIdHash?: string;
@@ -138,6 +143,10 @@ export function buildApiErrorObservationFields(rawError?: string): {
? redactIdentifier(rawFingerprint, { len: 12 })
: undefined,
httpCode: parsed?.httpCode,
providerRuntimeFailureKind: classifyProviderRuntimeFailureKind({
status: parsed?.httpCode ? Number(parsed.httpCode) : undefined,
message: trimmed,
}),
providerErrorType: parsed?.type,
providerErrorMessagePreview: truncateForObservation(
redactedProviderMessage,
@@ -155,6 +164,7 @@ export function buildTextObservationFields(text?: string): {
textHash?: string;
textFingerprint?: string;
httpCode?: string;
providerRuntimeFailureKind?: string;
providerErrorType?: string;
providerErrorMessagePreview?: string;
requestIdHash?: string;
@@ -165,6 +175,7 @@ export function buildTextObservationFields(text?: string): {
textHash: observed.rawErrorHash,
textFingerprint: observed.rawErrorFingerprint,
httpCode: observed.httpCode,
providerRuntimeFailureKind: observed.providerRuntimeFailureKind,
providerErrorType: observed.providerErrorType,
providerErrorMessagePreview: observed.providerErrorMessagePreview,
requestIdHash: observed.requestIdHash,

View File

@@ -215,6 +215,38 @@ describe("formatAssistantErrorText", () => {
);
});
it("returns an explicit re-authentication message for OAuth refresh failures", () => {
const msg = makeAssistantError(
"OAuth token refresh failed for openai-codex: invalid_grant. Please try again or re-authenticate.",
);
expect(formatAssistantErrorText(msg)).toBe(
"Authentication refresh failed. Re-authenticate this provider and try again.",
);
});
it("returns a missing-scope message for OpenAI Codex scope failures", () => {
const msg = makeAssistantError(
'401 {"type":"error","error":{"type":"permission_error","message":"Missing scopes: api.responses.write model.request"}}',
);
expect(formatAssistantErrorText(msg)).toBe(
"Authentication is missing the required OpenAI Codex scopes. Re-run OpenAI/Codex login and try again.",
);
});
it("returns an HTML-403 auth message for HTML provider auth failures", () => {
const msg = makeAssistantError("403 <!DOCTYPE html><html><body>Access denied</body></html>");
expect(formatAssistantErrorText(msg)).toBe(
"Authentication failed with an HTML 403 response from the provider. Re-authenticate and verify your provider account access.",
);
});
it("returns a proxy-specific message for proxy misroutes", () => {
const msg = makeAssistantError("407 Proxy Authentication Required");
expect(formatAssistantErrorText(msg)).toBe(
"LLM request failed: proxy or tunnel configuration blocked the provider request.",
);
});
it("sanitizes invalid streaming event order errors", () => {
const msg = makeAssistantError(
'Unexpected event order, got message_start before receiving "message_stop"',

View File

@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
classifyProviderRuntimeFailureKind,
classifyFailoverReason,
classifyFailoverReasonFromHttpStatus,
extractObservedOverflowTokenCount,
@@ -1101,3 +1102,46 @@ describe("classifyFailoverReason", () => {
).toBe("auth_permanent");
});
});
describe("classifyProviderRuntimeFailureKind", () => {
it("classifies missing scope failures", () => {
expect(
classifyProviderRuntimeFailureKind(
'401 {"type":"error","error":{"type":"permission_error","message":"Missing scopes: api.responses.write"}}',
),
).toBe("auth_scope");
});
it("classifies OAuth refresh failures", () => {
expect(
classifyProviderRuntimeFailureKind(
"OAuth token refresh failed for openai-codex: invalid_grant. Please try again or re-authenticate.",
),
).toBe("auth_refresh");
});
it("classifies HTML 403 auth failures", () => {
expect(
classifyProviderRuntimeFailureKind(
"403 <!DOCTYPE html><html><body>Access denied</body></html>",
),
).toBe("auth_html_403");
});
it("classifies proxy, dns, timeout, schema, sandbox, and replay failures", () => {
expect(classifyProviderRuntimeFailureKind("407 Proxy Authentication Required")).toBe("proxy");
expect(
classifyProviderRuntimeFailureKind("dial tcp: lookup api.example.com: no such host"),
).toBe("dns");
expect(classifyProviderRuntimeFailureKind("socket hang up")).toBe("timeout");
expect(
classifyProviderRuntimeFailureKind("INVALID_REQUEST_ERROR: string should match pattern"),
).toBe("schema");
expect(classifyProviderRuntimeFailureKind("exec denied (allowlist-miss):")).toBe(
"sandbox_blocked",
);
expect(classifyProviderRuntimeFailureKind("tool_use.input: Field required")).toBe(
"replay_invalid",
);
});
});

View File

@@ -11,6 +11,7 @@ export {
} from "./pi-embedded-helpers/bootstrap.js";
export {
BILLING_ERROR_USER_MESSAGE,
classifyProviderRuntimeFailureKind,
formatBillingErrorMessage,
classifyFailoverReason,
classifyFailoverReasonFromHttpStatus,

View File

@@ -18,6 +18,7 @@ export {
isCloudflareOrHtmlErrorPage,
parseApiErrorInfo,
} from "../../shared/assistant-error-format.js";
import { classifyOAuthRefreshFailure } from "../auth-profiles/oauth-refresh-failure.js";
import { formatExecDeniedUserMessage } from "../exec-approval-result.js";
import { stripInternalRuntimeContext } from "../internal-runtime-context.js";
import { isModelNotFoundErrorMessage } from "../live-model-errors.js";
@@ -407,6 +408,19 @@ export type FailoverClassification =
kind: "context_overflow";
};
export type ProviderRuntimeFailureKind =
| "auth_scope"
| "auth_refresh"
| "auth_html_403"
| "proxy"
| "rate_limit"
| "dns"
| "timeout"
| "schema"
| "sandbox_blocked"
| "replay_invalid"
| "unknown";
const BILLING_402_HINTS = [
"insufficient credits",
"insufficient quota",
@@ -450,6 +464,102 @@ const TIMEOUT_ERROR_CODES = new Set([
"EPIPE",
"EAI_AGAIN",
]);
const AUTH_SCOPE_HINT_RE =
/\b(?:missing|required|requires|insufficient)\s+(?:the\s+following\s+)?scopes?\b|\bmissing\s+scope\b|\binsufficient\s+permissions?\b/i;
const AUTH_SCOPE_NAME_RE = /\b(?:api\.responses\.write|model\.request)\b/i;
const HTML_BODY_RE = /^\s*(?:<!doctype\s+html\b|<html\b)/i;
const HTML_CLOSE_RE = /<\/html>/i;
const PROXY_ERROR_RE =
/\bproxy\b|\bproxyconnect\b|\bhttps?_proxy\b|\b407\b|\bproxy authentication required\b|\btunnel connection failed\b|\bconnect tunnel\b|\bsocks proxy\b/i;
const DNS_ERROR_RE = /\benotfound\b|\beai_again\b|\bgetaddrinfo\b|\bno such host\b|\bdns\b/i;
const INTERRUPTED_NETWORK_ERROR_RE =
/\beconnrefused\b|\beconnreset\b|\beconnaborted\b|\benetreset\b|\behostunreach\b|\behostdown\b|\benetunreach\b|\bepipe\b|\bsocket hang up\b|\bconnection refused\b|\bconnection reset\b|\bconnection aborted\b|\bnetwork is unreachable\b|\bhost is unreachable\b|\bfetch failed\b|\bconnection error\b|\bnetwork request failed\b/i;
const REPLAY_INVALID_RE =
/\bprevious_response_id\b.*\b(?:invalid|unknown|not found|does not exist|expired|mismatch)\b|\btool_(?:use|call)\.(?:input|arguments)\b.*\b(?:missing|required)\b|\bincorrect role information\b|\broles must alternate\b/i;
const SANDBOX_BLOCKED_RE =
/\bapproval is required\b|\bapproval timed out\b|\bapproval was denied\b|\bblocked by sandbox\b|\bsandbox\b.*\b(?:blocked|denied|forbidden|disabled|not allowed)\b/i;
function inferSignalStatus(signal: FailoverSignal): number | undefined {
if (typeof signal.status === "number" && Number.isFinite(signal.status)) {
return signal.status;
}
return extractLeadingHttpStatus(signal.message?.trim() ?? "")?.code;
}
function isHtmlErrorResponse(raw: string, status?: number): boolean {
const trimmed = raw.trim();
if (!trimmed) {
return false;
}
const inferred =
typeof status === "number" && Number.isFinite(status)
? status
: extractLeadingHttpStatus(trimmed)?.code;
if (typeof inferred !== "number" || inferred < 400) {
return false;
}
const rest = extractLeadingHttpStatus(trimmed)?.rest ?? trimmed;
return HTML_BODY_RE.test(rest) && HTML_CLOSE_RE.test(rest);
}
function isAuthScopeErrorMessage(raw: string, status?: number): boolean {
if (!raw) {
return false;
}
const inferred =
typeof status === "number" && Number.isFinite(status)
? status
: extractLeadingHttpStatus(raw.trim())?.code;
if (inferred !== 401 && inferred !== 403) {
return false;
}
return AUTH_SCOPE_HINT_RE.test(raw) || AUTH_SCOPE_NAME_RE.test(raw);
}
function isProxyErrorMessage(raw: string, status?: number): boolean {
if (!raw) {
return false;
}
if (status === 407) {
return true;
}
return PROXY_ERROR_RE.test(raw);
}
function isDnsTransportErrorMessage(raw: string): boolean {
return DNS_ERROR_RE.test(raw);
}
function isReplayInvalidErrorMessage(raw: string): boolean {
return REPLAY_INVALID_RE.test(raw);
}
function isSandboxBlockedErrorMessage(raw: string): boolean {
return Boolean(formatExecDeniedUserMessage(raw)) || SANDBOX_BLOCKED_RE.test(raw);
}
function isSchemaErrorMessage(raw: string): boolean {
if (!raw || isReplayInvalidErrorMessage(raw) || isContextOverflowError(raw)) {
return false;
}
return classifyFailoverReason(raw) === "format" || matchesFormatErrorPattern(raw);
}
function isTimeoutTransportErrorMessage(raw: string, status?: number): boolean {
if (!raw) {
return false;
}
if (isTimeoutErrorMessage(raw) || INTERRUPTED_NETWORK_ERROR_RE.test(raw)) {
return true;
}
if (
typeof status === "number" &&
[408, 499, 500, 502, 503, 504, 521, 522, 523, 524, 529].includes(status)
) {
return true;
}
return false;
}
function includesAnyHint(text: string, hints: readonly string[]): boolean {
return hints.some((hint) => text.includes(hint));
@@ -774,10 +884,7 @@ function classifyFailoverClassificationFromMessage(
}
export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassification | null {
const inferredStatus =
typeof signal.status === "number" && Number.isFinite(signal.status)
? signal.status
: extractLeadingHttpStatus(signal.message?.trim() ?? "")?.code;
const inferredStatus = inferSignalStatus(signal);
const messageClassification = signal.message
? classifyFailoverClassificationFromMessage(signal.message, signal.provider)
: null;
@@ -796,6 +903,60 @@ export function classifyFailoverSignal(signal: FailoverSignal): FailoverClassifi
return messageClassification;
}
export function classifyProviderRuntimeFailureKind(
signal: FailoverSignal | string,
): ProviderRuntimeFailureKind {
const normalizedSignal = typeof signal === "string" ? { message: signal } : signal;
const message = normalizedSignal.message?.trim() ?? "";
const status = inferSignalStatus(normalizedSignal);
if (!message && typeof status !== "number") {
return "unknown";
}
if (message && classifyOAuthRefreshFailure(message)) {
return "auth_refresh";
}
if (message && isAuthScopeErrorMessage(message, status)) {
return "auth_scope";
}
if (message && status === 403 && isHtmlErrorResponse(message, status)) {
return "auth_html_403";
}
if (message && isProxyErrorMessage(message, status)) {
return "proxy";
}
const failoverClassification = classifyFailoverSignal({
...normalizedSignal,
status,
message: message || undefined,
});
if (failoverClassification?.kind === "reason" && failoverClassification.reason === "rate_limit") {
return "rate_limit";
}
if (message && isDnsTransportErrorMessage(message)) {
return "dns";
}
if (message && isSandboxBlockedErrorMessage(message)) {
return "sandbox_blocked";
}
if (message && isReplayInvalidErrorMessage(message)) {
return "replay_invalid";
}
if (message && isSchemaErrorMessage(message)) {
return "schema";
}
if (
failoverClassification?.kind === "reason" &&
(failoverClassification.reason === "timeout" || failoverClassification.reason === "overloaded")
) {
return "timeout";
}
if (message && isTimeoutTransportErrorMessage(message, status)) {
return "timeout";
}
return "unknown";
}
function coerceText(value: unknown): string {
if (typeof value === "string") {
return value;
@@ -947,6 +1108,12 @@ export function formatAssistantErrorText(
return "LLM request failed with an unknown error.";
}
const providerRuntimeFailureKind = classifyProviderRuntimeFailureKind({
status: extractLeadingHttpStatus(raw)?.code,
message: raw,
provider: opts?.provider ?? msg.provider,
});
const unknownTool =
raw.match(/unknown tool[:\s]+["']?([a-z0-9_-]+)["']?/i) ??
raw.match(/tool\s+["']?([a-z0-9_-]+)["']?\s+(?:not found|is not available)/i);
@@ -966,6 +1133,28 @@ export function formatAssistantErrorText(
return diskSpaceCopy;
}
if (providerRuntimeFailureKind === "auth_refresh") {
return "Authentication refresh failed. Re-authenticate this provider and try again.";
}
if (providerRuntimeFailureKind === "auth_scope") {
return (
"Authentication is missing the required OpenAI Codex scopes. " +
"Re-run OpenAI/Codex login and try again."
);
}
if (providerRuntimeFailureKind === "auth_html_403") {
return (
"Authentication failed with an HTML 403 response from the provider. " +
"Re-authenticate and verify your provider account access."
);
}
if (providerRuntimeFailureKind === "proxy") {
return "LLM request failed: proxy or tunnel configuration blocked the provider request.";
}
if (isContextOverflowError(raw)) {
return (
"Context overflow: prompt too large for the model. " +
@@ -1027,6 +1216,17 @@ export function formatAssistantErrorText(
return formatBillingErrorMessage(opts?.provider, opts?.model ?? msg.model);
}
if (providerRuntimeFailureKind === "schema") {
return "LLM request failed: provider rejected the request schema or tool payload.";
}
if (providerRuntimeFailureKind === "replay_invalid") {
return (
"Session history or replay state is invalid. " +
"Use /new to start a fresh session and try again."
);
}
if (isLikelyHttpErrorText(raw) || isRawApiErrorPayload(raw)) {
return formatRawAssistantErrorForUi(raw);
}

View File

@@ -68,6 +68,7 @@ describe("handleAgentEnd", () => {
event: "embedded_run_agent_end",
runId: "run-1",
error: "LLM request failed: connection refused by the provider endpoint.",
providerRuntimeFailureKind: "timeout",
rawErrorPreview: "connection refused",
consoleMessage:
"embedded run agent end: runId=run-1 isError=true model=unknown provider=unknown error=LLM request failed: connection refused by the provider endpoint. rawError=connection refused",
@@ -101,6 +102,7 @@ describe("handleAgentEnd", () => {
runId: "run-1",
error: "The AI service is temporarily overloaded. Please try again in a moment.",
failoverReason: "overloaded",
providerRuntimeFailureKind: "timeout",
providerErrorType: "overloaded_error",
consoleMessage:
'embedded run agent end: runId=run-1 isError=true model=claude-test provider=anthropic error=The AI service is temporarily overloaded. Please try again in a moment. rawError={"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}',
@@ -160,6 +162,26 @@ describe("handleAgentEnd", () => {
});
});
it("logs runtime failure kind for missing-scope auth errors", async () => {
const ctx = createContext({
role: "assistant",
stopReason: "error",
provider: "openai-codex",
model: "gpt-5.4",
errorMessage:
'401 {"type":"error","error":{"type":"permission_error","message":"Missing scopes: api.responses.write"}}',
content: [{ type: "text", text: "" }],
});
await handleAgentEnd(ctx);
expect(vi.mocked(ctx.log.warn).mock.calls[0]?.[1]).toMatchObject({
failoverReason: "auth",
providerRuntimeFailureKind: "auth_scope",
httpCode: "401",
});
});
it("keeps non-error run-end logging on debug only", async () => {
const ctx = createContext(undefined);

View File

@@ -5,7 +5,11 @@ import {
buildTextObservationFields,
sanitizeForConsole,
} from "./pi-embedded-error-observation.js";
import { classifyFailoverReason, formatAssistantErrorText } from "./pi-embedded-helpers.js";
import {
classifyFailoverReason,
classifyProviderRuntimeFailureKind,
formatAssistantErrorText,
} from "./pi-embedded-helpers.js";
import {
consumePendingToolMediaReply,
hasAssistantVisibleReply,
@@ -51,6 +55,10 @@ export function handleAgentEnd(ctx: EmbeddedPiSubscribeContext): void | Promise<
const failoverReason = classifyFailoverReason(rawError ?? "", {
provider: lastAssistant.provider,
});
const providerRuntimeFailureKind = classifyProviderRuntimeFailureKind({
message: rawError ?? "",
provider: lastAssistant.provider,
});
const errorText = (friendlyError || lastAssistant.errorMessage || "LLM request failed.").trim();
const observedError = buildApiErrorObservationFields(rawError);
const safeErrorText =
@@ -68,6 +76,7 @@ export function handleAgentEnd(ctx: EmbeddedPiSubscribeContext): void | Promise<
isError: true,
error: safeErrorText,
failoverReason,
providerRuntimeFailureKind,
model: lastAssistant.model,
provider: lastAssistant.provider,
...observedError,

View File

@@ -88,7 +88,7 @@ describe("loginOpenAICodexOAuth", () => {
expect(runtime.error).not.toHaveBeenCalled();
});
it("passes through Pi-provided OAuth authorize URL without mutation", async () => {
it("adds required Codex OAuth scopes to Pi-provided authorize URLs", async () => {
const creds = {
provider: "openai-codex" as const,
access: "access-token",
@@ -109,10 +109,35 @@ describe("loginOpenAICodexOAuth", () => {
const { runtime } = await runCodexOAuth({ isRemote: false, openUrl });
expect(openUrl).toHaveBeenCalledWith(
"https://auth.openai.com/oauth/authorize?scope=openid+profile+email+offline_access&state=abc",
"https://auth.openai.com/oauth/authorize?scope=openid+profile+email+offline_access+model.request+api.responses.write&state=abc",
);
expect(runtime.log).toHaveBeenCalledWith(
"Open: https://auth.openai.com/oauth/authorize?scope=openid+profile+email+offline_access&state=abc",
"Open: https://auth.openai.com/oauth/authorize?scope=openid+profile+email+offline_access+model.request+api.responses.write&state=abc",
);
});
it("adds a scope parameter when the upstream authorize url omitted it", async () => {
const creds = {
provider: "openai-codex" as const,
access: "access-token",
refresh: "refresh-token",
expires: Date.now() + 60_000,
email: "user@example.com",
};
mocks.loginOpenAICodex.mockImplementation(
async (opts: { onAuth: (event: { url: string }) => Promise<void> }) => {
await opts.onAuth({
url: "https://auth.openai.com/oauth/authorize?state=abc",
});
return creds;
},
);
const openUrl = vi.fn(async () => {});
await runCodexOAuth({ isRemote: false, openUrl });
expect(openUrl).toHaveBeenCalledWith(
"https://auth.openai.com/oauth/authorize?state=abc&scope=openid+profile+email+offline_access+model.request+api.responses.write",
);
});

View File

@@ -10,6 +10,41 @@ import {
const manualInputPromptMessage = "Paste the authorization code (or full redirect URL):";
const openAICodexOAuthOriginator = "openclaw";
const OPENAI_CODEX_OAUTH_REQUIRED_SCOPES = [
"openid",
"profile",
"email",
"offline_access",
"model.request",
"api.responses.write",
] as const;
function normalizeOpenAICodexAuthorizeUrl(rawUrl: string): string {
const trimmed = rawUrl.trim();
if (!trimmed) {
return rawUrl;
}
try {
const url = new URL(trimmed);
if (!/openai\.com$/i.test(url.hostname) || !/\/oauth\/authorize$/i.test(url.pathname)) {
return rawUrl;
}
const existing = new Set(
(url.searchParams.get("scope") ?? "")
.split(/\s+/)
.map((scope) => scope.trim())
.filter(Boolean),
);
for (const scope of OPENAI_CODEX_OAUTH_REQUIRED_SCOPES) {
existing.add(scope);
}
url.searchParams.set("scope", Array.from(existing).join(" "));
return url.toString();
} catch {
return rawUrl;
}
}
export async function loginOpenAICodexOAuth(params: {
prompter: WizardPrompter;
@@ -60,7 +95,11 @@ export async function loginOpenAICodexOAuth(params: {
});
const creds = await loginOpenAICodex({
onAuth: baseOnAuth,
onAuth: async (event) =>
await baseOnAuth({
...event,
url: normalizeOpenAICodexAuthorizeUrl(event.url),
}),
onPrompt,
originator: openAICodexOAuthOriginator,
onManualCodeInput: isRemote