mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-31 11:51:22 +00:00
d81593c6e2
Merged via squash.
Prepared head SHA: 619bc4c1fa
Co-authored-by: joeykrug <5925937+joeykrug@users.noreply.github.com>
Co-authored-by: jalehman <550978+jalehman@users.noreply.github.com>
Reviewed-by: @jalehman
1860 lines
78 KiB
TypeScript
1860 lines
78 KiB
TypeScript
import { randomBytes } from "node:crypto";
|
|
import fs from "node:fs/promises";
|
|
import type { ThinkLevel } from "../../auto-reply/thinking.js";
|
|
import {
|
|
ensureContextEnginesInitialized,
|
|
resolveContextEngine,
|
|
} from "../../context-engine/index.js";
|
|
import { type BackoffPolicy, computeBackoff, sleepWithAbort } from "../../infra/backoff.js";
|
|
import { generateSecureToken } from "../../infra/secure-random.js";
|
|
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
|
import { prepareProviderRuntimeAuth } from "../../plugins/provider-runtime.js";
|
|
import type { PluginHookBeforeAgentStartResult } from "../../plugins/types.js";
|
|
import { enqueueCommandInLane } from "../../process/command-queue.js";
|
|
import { isMarkdownCapableMessageChannel } from "../../utils/message-channel.js";
|
|
import { resolveOpenClawAgentDir } from "../agent-paths.js";
|
|
import { hasConfiguredModelFallbacks } from "../agent-scope.js";
|
|
import {
|
|
type AuthProfileFailureReason,
|
|
isProfileInCooldown,
|
|
markAuthProfileFailure,
|
|
markAuthProfileGood,
|
|
markAuthProfileUsed,
|
|
resolveProfilesUnavailableReason,
|
|
} from "../auth-profiles.js";
|
|
import {
|
|
CONTEXT_WINDOW_HARD_MIN_TOKENS,
|
|
CONTEXT_WINDOW_WARN_BELOW_TOKENS,
|
|
evaluateContextWindowGuard,
|
|
resolveContextWindowInfo,
|
|
} from "../context-window-guard.js";
|
|
import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js";
|
|
import {
|
|
coerceToFailoverError,
|
|
describeFailoverError,
|
|
FailoverError,
|
|
resolveFailoverStatus,
|
|
} from "../failover-error.js";
|
|
import { shouldAllowCooldownProbeForReason } from "../failover-policy.js";
|
|
import {
|
|
applyLocalNoAuthHeaderOverride,
|
|
ensureAuthProfileStore,
|
|
getApiKeyForModel,
|
|
type ResolvedProviderAuth,
|
|
resolveAuthProfileOrder,
|
|
} from "../model-auth.js";
|
|
import { normalizeProviderId } from "../model-selection.js";
|
|
import { ensureOpenClawModelsJson } from "../models-config.js";
|
|
import {
|
|
classifyFailoverReason,
|
|
extractObservedOverflowTokenCount,
|
|
type FailoverReason,
|
|
formatAssistantErrorText,
|
|
formatBillingErrorMessage,
|
|
isAuthAssistantError,
|
|
isBillingAssistantError,
|
|
isCompactionFailureError,
|
|
isFailoverAssistantError,
|
|
isFailoverErrorMessage,
|
|
isLikelyContextOverflowError,
|
|
isRateLimitAssistantError,
|
|
isTimeoutErrorMessage,
|
|
parseImageDimensionError,
|
|
parseImageSizeError,
|
|
pickFallbackThinkingLevel,
|
|
} from "../pi-embedded-helpers.js";
|
|
import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js";
|
|
import { isLikelyMutatingToolName } from "../tool-mutation.js";
|
|
import { derivePromptTokens, normalizeUsage, type UsageLike } from "../usage.js";
|
|
import { redactRunIdentifier, resolveRunWorkspaceDir } from "../workspace-run.js";
|
|
import { runPostCompactionSideEffects } from "./compact.js";
|
|
import { buildEmbeddedCompactionRuntimeContext } from "./compaction-runtime-context.js";
|
|
import { runContextEngineMaintenance } from "./context-engine-maintenance.js";
|
|
import { resolveGlobalLane, resolveSessionLane } from "./lanes.js";
|
|
import { log } from "./logger.js";
|
|
import { resolveModelAsync } from "./model.js";
|
|
import { runEmbeddedAttempt } from "./run/attempt.js";
|
|
import { createFailoverDecisionLogger } from "./run/failover-observation.js";
|
|
import type { RunEmbeddedPiAgentParams } from "./run/params.js";
|
|
import { buildEmbeddedRunPayloads } from "./run/payloads.js";
|
|
import {
|
|
sessionLikelyHasOversizedToolResults,
|
|
truncateOversizedToolResultsInSession,
|
|
} from "./tool-result-truncation.js";
|
|
import type { EmbeddedPiAgentMeta, EmbeddedPiRunResult } from "./types.js";
|
|
import {
|
|
createUsageAccumulator,
|
|
mergeUsageIntoAccumulator,
|
|
resolveLastCallUsage,
|
|
toNormalizedUsage,
|
|
type UsageAccumulator,
|
|
} from "./usage-accumulator.js";
|
|
import { describeUnknownError } from "./utils.js";
|
|
|
|
type ApiKeyInfo = ResolvedProviderAuth;
|
|
|
|
type RuntimeAuthState = {
|
|
sourceApiKey: string;
|
|
authMode: string;
|
|
profileId?: string;
|
|
expiresAt?: number;
|
|
refreshTimer?: ReturnType<typeof setTimeout>;
|
|
refreshInFlight?: Promise<void>;
|
|
};
|
|
|
|
const RUNTIME_AUTH_REFRESH_MARGIN_MS = 5 * 60 * 1000;
|
|
const RUNTIME_AUTH_REFRESH_RETRY_MS = 60 * 1000;
|
|
const RUNTIME_AUTH_REFRESH_MIN_DELAY_MS = 5 * 1000;
|
|
// Keep overload pacing noticeable enough to avoid tight retry bursts, but short
|
|
// enough that fallback still feels responsive within a single turn.
|
|
const OVERLOAD_FAILOVER_BACKOFF_POLICY: BackoffPolicy = {
|
|
initialMs: 250,
|
|
maxMs: 1_500,
|
|
factor: 2,
|
|
jitter: 0.2,
|
|
};
|
|
|
|
// Avoid Anthropic's refusal test token poisoning session transcripts.
|
|
const ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL = "ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL";
|
|
const ANTHROPIC_MAGIC_STRING_REPLACEMENT = "ANTHROPIC MAGIC STRING TRIGGER REFUSAL (redacted)";
|
|
|
|
function scrubAnthropicRefusalMagic(prompt: string): string {
|
|
if (!prompt.includes(ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL)) {
|
|
return prompt;
|
|
}
|
|
return prompt.replaceAll(
|
|
ANTHROPIC_MAGIC_STRING_TRIGGER_REFUSAL,
|
|
ANTHROPIC_MAGIC_STRING_REPLACEMENT,
|
|
);
|
|
}
|
|
function createCompactionDiagId(): string {
|
|
return `ovf-${Date.now().toString(36)}-${generateSecureToken(4)}`;
|
|
}
|
|
|
|
// Defensive guard for the outer run loop across all retry branches.
|
|
const BASE_RUN_RETRY_ITERATIONS = 24;
|
|
const RUN_RETRY_ITERATIONS_PER_PROFILE = 8;
|
|
const MIN_RUN_RETRY_ITERATIONS = 32;
|
|
const MAX_RUN_RETRY_ITERATIONS = 160;
|
|
|
|
function resolveMaxRunRetryIterations(profileCandidateCount: number): number {
|
|
const scaled =
|
|
BASE_RUN_RETRY_ITERATIONS +
|
|
Math.max(1, profileCandidateCount) * RUN_RETRY_ITERATIONS_PER_PROFILE;
|
|
return Math.min(MAX_RUN_RETRY_ITERATIONS, Math.max(MIN_RUN_RETRY_ITERATIONS, scaled));
|
|
}
|
|
|
|
function resolveActiveErrorContext(params: {
|
|
lastAssistant: { provider?: string; model?: string } | undefined;
|
|
provider: string;
|
|
model: string;
|
|
}): { provider: string; model: string } {
|
|
return {
|
|
provider: params.lastAssistant?.provider ?? params.provider,
|
|
model: params.lastAssistant?.model ?? params.model,
|
|
};
|
|
}
|
|
|
|
function buildUsageAgentMetaFields(params: {
|
|
usageAccumulator: UsageAccumulator;
|
|
lastAssistantUsage?: UsageLike | null;
|
|
lastRunPromptUsage: ReturnType<typeof normalizeUsage> | undefined;
|
|
/** API-reported total from the most recent call, mirroring the success path correction. */
|
|
lastTurnTotal?: number;
|
|
}): Pick<EmbeddedPiAgentMeta, "usage" | "lastCallUsage" | "promptTokens"> {
|
|
const usage = toNormalizedUsage(params.usageAccumulator);
|
|
// Keep `usage.total` aligned with the API-reported latest-call total when
|
|
// available; accumulated totals are for billing, not context display.
|
|
if (usage && params.lastTurnTotal && params.lastTurnTotal > 0) {
|
|
usage.total = params.lastTurnTotal;
|
|
}
|
|
const lastCallUsage = resolveLastCallUsage(params.lastAssistantUsage, params.usageAccumulator);
|
|
const promptTokens = derivePromptTokens(params.lastRunPromptUsage);
|
|
return {
|
|
usage,
|
|
lastCallUsage,
|
|
promptTokens,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Build agentMeta for error return paths, preserving accumulated usage so that
|
|
* session totalTokens reflects the actual context size rather than going stale.
|
|
* Without this, error returns omit usage and the session keeps whatever
|
|
* totalTokens was set by the previous successful run.
|
|
*/
|
|
function buildErrorAgentMeta(params: {
|
|
sessionId: string;
|
|
provider: string;
|
|
model: string;
|
|
usageAccumulator: UsageAccumulator;
|
|
lastRunPromptUsage: ReturnType<typeof normalizeUsage> | undefined;
|
|
lastAssistant?: { usage?: unknown } | null;
|
|
/** API-reported total from the most recent call, mirroring the success path correction. */
|
|
lastTurnTotal?: number;
|
|
}): EmbeddedPiAgentMeta {
|
|
const usageMeta = buildUsageAgentMetaFields({
|
|
usageAccumulator: params.usageAccumulator,
|
|
lastAssistantUsage: params.lastAssistant?.usage as UsageLike | undefined,
|
|
lastRunPromptUsage: params.lastRunPromptUsage,
|
|
lastTurnTotal: params.lastTurnTotal,
|
|
});
|
|
return {
|
|
sessionId: params.sessionId,
|
|
provider: params.provider,
|
|
model: params.model,
|
|
// Only include usage fields when we have actual data from prior API calls.
|
|
...(usageMeta.usage ? { usage: usageMeta.usage } : {}),
|
|
...(usageMeta.lastCallUsage ? { lastCallUsage: usageMeta.lastCallUsage } : {}),
|
|
...(usageMeta.promptTokens ? { promptTokens: usageMeta.promptTokens } : {}),
|
|
};
|
|
}
|
|
|
|
export async function runEmbeddedPiAgent(
|
|
params: RunEmbeddedPiAgentParams,
|
|
): Promise<EmbeddedPiRunResult> {
|
|
const sessionLane = resolveSessionLane(params.sessionKey?.trim() || params.sessionId);
|
|
const globalLane = resolveGlobalLane(params.lane);
|
|
const enqueueGlobal =
|
|
params.enqueue ?? ((task, opts) => enqueueCommandInLane(globalLane, task, opts));
|
|
const enqueueSession =
|
|
params.enqueue ?? ((task, opts) => enqueueCommandInLane(sessionLane, task, opts));
|
|
const channelHint = params.messageChannel ?? params.messageProvider;
|
|
const resolvedToolResultFormat =
|
|
params.toolResultFormat ??
|
|
(channelHint
|
|
? isMarkdownCapableMessageChannel(channelHint)
|
|
? "markdown"
|
|
: "plain"
|
|
: "markdown");
|
|
const isProbeSession = params.sessionId?.startsWith("probe-") ?? false;
|
|
|
|
return enqueueSession(() =>
|
|
enqueueGlobal(async () => {
|
|
const started = Date.now();
|
|
const workspaceResolution = resolveRunWorkspaceDir({
|
|
workspaceDir: params.workspaceDir,
|
|
sessionKey: params.sessionKey,
|
|
agentId: params.agentId,
|
|
config: params.config,
|
|
});
|
|
const resolvedWorkspace = workspaceResolution.workspaceDir;
|
|
const redactedSessionId = redactRunIdentifier(params.sessionId);
|
|
const redactedSessionKey = redactRunIdentifier(params.sessionKey);
|
|
const redactedWorkspace = redactRunIdentifier(resolvedWorkspace);
|
|
if (workspaceResolution.usedFallback) {
|
|
log.warn(
|
|
`[workspace-fallback] caller=runEmbeddedPiAgent reason=${workspaceResolution.fallbackReason} run=${params.runId} session=${redactedSessionId} sessionKey=${redactedSessionKey} agent=${workspaceResolution.agentId} workspace=${redactedWorkspace}`,
|
|
);
|
|
}
|
|
ensureRuntimePluginsLoaded({
|
|
config: params.config,
|
|
workspaceDir: resolvedWorkspace,
|
|
allowGatewaySubagentBinding: params.allowGatewaySubagentBinding,
|
|
});
|
|
|
|
let provider = (params.provider ?? DEFAULT_PROVIDER).trim() || DEFAULT_PROVIDER;
|
|
let modelId = (params.model ?? DEFAULT_MODEL).trim() || DEFAULT_MODEL;
|
|
const agentDir = params.agentDir ?? resolveOpenClawAgentDir();
|
|
const fallbackConfigured = hasConfiguredModelFallbacks({
|
|
cfg: params.config,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
});
|
|
await ensureOpenClawModelsJson(params.config, agentDir);
|
|
|
|
// Run before_model_resolve hooks early so plugins can override the
|
|
// provider/model before resolveModel().
|
|
//
|
|
// Legacy compatibility: before_agent_start is also checked for override
|
|
// fields if present. New hook takes precedence when both are set.
|
|
let modelResolveOverride: { providerOverride?: string; modelOverride?: string } | undefined;
|
|
let legacyBeforeAgentStartResult: PluginHookBeforeAgentStartResult | undefined;
|
|
const hookRunner = getGlobalHookRunner();
|
|
const hookCtx = {
|
|
agentId: workspaceResolution.agentId,
|
|
sessionKey: params.sessionKey,
|
|
sessionId: params.sessionId,
|
|
workspaceDir: resolvedWorkspace,
|
|
messageProvider: params.messageProvider ?? undefined,
|
|
trigger: params.trigger,
|
|
channelId: params.messageChannel ?? params.messageProvider ?? undefined,
|
|
};
|
|
if (hookRunner?.hasHooks("before_model_resolve")) {
|
|
try {
|
|
modelResolveOverride = await hookRunner.runBeforeModelResolve(
|
|
{ prompt: params.prompt },
|
|
hookCtx,
|
|
);
|
|
} catch (hookErr) {
|
|
log.warn(`before_model_resolve hook failed: ${String(hookErr)}`);
|
|
}
|
|
}
|
|
if (hookRunner?.hasHooks("before_agent_start")) {
|
|
try {
|
|
legacyBeforeAgentStartResult = await hookRunner.runBeforeAgentStart(
|
|
{ prompt: params.prompt },
|
|
hookCtx,
|
|
);
|
|
modelResolveOverride = {
|
|
providerOverride:
|
|
modelResolveOverride?.providerOverride ??
|
|
legacyBeforeAgentStartResult?.providerOverride,
|
|
modelOverride:
|
|
modelResolveOverride?.modelOverride ?? legacyBeforeAgentStartResult?.modelOverride,
|
|
};
|
|
} catch (hookErr) {
|
|
log.warn(
|
|
`before_agent_start hook (legacy model resolve path) failed: ${String(hookErr)}`,
|
|
);
|
|
}
|
|
}
|
|
if (modelResolveOverride?.providerOverride) {
|
|
provider = modelResolveOverride.providerOverride;
|
|
log.info(`[hooks] provider overridden to ${provider}`);
|
|
}
|
|
if (modelResolveOverride?.modelOverride) {
|
|
modelId = modelResolveOverride.modelOverride;
|
|
log.info(`[hooks] model overridden to ${modelId}`);
|
|
}
|
|
|
|
const { model, error, authStorage, modelRegistry } = await resolveModelAsync(
|
|
provider,
|
|
modelId,
|
|
agentDir,
|
|
params.config,
|
|
);
|
|
if (!model) {
|
|
throw new FailoverError(error ?? `Unknown model: ${provider}/${modelId}`, {
|
|
reason: "model_not_found",
|
|
provider,
|
|
model: modelId,
|
|
});
|
|
}
|
|
let runtimeModel = model;
|
|
|
|
const ctxInfo = resolveContextWindowInfo({
|
|
cfg: params.config,
|
|
provider,
|
|
modelId,
|
|
modelContextWindow: runtimeModel.contextWindow,
|
|
defaultTokens: DEFAULT_CONTEXT_TOKENS,
|
|
});
|
|
// Apply contextTokens cap to model so pi-coding-agent's auto-compaction
|
|
// threshold uses the effective limit, not the native context window.
|
|
let effectiveModel =
|
|
ctxInfo.tokens < (runtimeModel.contextWindow ?? Infinity)
|
|
? { ...runtimeModel, contextWindow: ctxInfo.tokens }
|
|
: runtimeModel;
|
|
const ctxGuard = evaluateContextWindowGuard({
|
|
info: ctxInfo,
|
|
warnBelowTokens: CONTEXT_WINDOW_WARN_BELOW_TOKENS,
|
|
hardMinTokens: CONTEXT_WINDOW_HARD_MIN_TOKENS,
|
|
});
|
|
if (ctxGuard.shouldWarn) {
|
|
log.warn(
|
|
`low context window: ${provider}/${modelId} ctx=${ctxGuard.tokens} (warn<${CONTEXT_WINDOW_WARN_BELOW_TOKENS}) source=${ctxGuard.source}`,
|
|
);
|
|
}
|
|
if (ctxGuard.shouldBlock) {
|
|
log.error(
|
|
`blocked model (context window too small): ${provider}/${modelId} ctx=${ctxGuard.tokens} (min=${CONTEXT_WINDOW_HARD_MIN_TOKENS}) source=${ctxGuard.source}`,
|
|
);
|
|
throw new FailoverError(
|
|
`Model context window too small (${ctxGuard.tokens} tokens). Minimum is ${CONTEXT_WINDOW_HARD_MIN_TOKENS}.`,
|
|
{ reason: "unknown", provider, model: modelId },
|
|
);
|
|
}
|
|
|
|
const authStore = ensureAuthProfileStore(agentDir, {
|
|
allowKeychainPrompt: false,
|
|
});
|
|
const preferredProfileId = params.authProfileId?.trim();
|
|
let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined;
|
|
if (lockedProfileId) {
|
|
const lockedProfile = authStore.profiles[lockedProfileId];
|
|
if (
|
|
!lockedProfile ||
|
|
normalizeProviderId(lockedProfile.provider) !== normalizeProviderId(provider)
|
|
) {
|
|
lockedProfileId = undefined;
|
|
}
|
|
}
|
|
const profileOrder = resolveAuthProfileOrder({
|
|
cfg: params.config,
|
|
store: authStore,
|
|
provider,
|
|
preferredProfile: preferredProfileId,
|
|
});
|
|
if (lockedProfileId && !profileOrder.includes(lockedProfileId)) {
|
|
throw new Error(`Auth profile "${lockedProfileId}" is not configured for ${provider}.`);
|
|
}
|
|
const profileCandidates = lockedProfileId
|
|
? [lockedProfileId]
|
|
: profileOrder.length > 0
|
|
? profileOrder
|
|
: [undefined];
|
|
let profileIndex = 0;
|
|
|
|
const initialThinkLevel = params.thinkLevel ?? "off";
|
|
let thinkLevel = initialThinkLevel;
|
|
const attemptedThinking = new Set<ThinkLevel>();
|
|
let apiKeyInfo: ApiKeyInfo | null = null;
|
|
let lastProfileId: string | undefined;
|
|
let runtimeAuthState: RuntimeAuthState | null = null;
|
|
let runtimeAuthRefreshCancelled = false;
|
|
const hasRefreshableRuntimeAuth = () => Boolean(runtimeAuthState?.sourceApiKey.trim());
|
|
|
|
const clearRuntimeAuthRefreshTimer = () => {
|
|
if (!runtimeAuthState?.refreshTimer) {
|
|
return;
|
|
}
|
|
clearTimeout(runtimeAuthState.refreshTimer);
|
|
runtimeAuthState.refreshTimer = undefined;
|
|
};
|
|
|
|
const stopRuntimeAuthRefreshTimer = () => {
|
|
if (!runtimeAuthState) {
|
|
return;
|
|
}
|
|
runtimeAuthRefreshCancelled = true;
|
|
clearRuntimeAuthRefreshTimer();
|
|
};
|
|
|
|
const refreshRuntimeAuth = async (reason: string): Promise<void> => {
|
|
if (!runtimeAuthState) {
|
|
return;
|
|
}
|
|
if (runtimeAuthState.refreshInFlight) {
|
|
await runtimeAuthState.refreshInFlight;
|
|
return;
|
|
}
|
|
runtimeAuthState.refreshInFlight = (async () => {
|
|
const sourceApiKey = runtimeAuthState?.sourceApiKey.trim() ?? "";
|
|
if (!sourceApiKey) {
|
|
throw new Error(`Runtime auth refresh requires a source credential.`);
|
|
}
|
|
log.debug(`Refreshing runtime auth for ${runtimeModel.provider} (${reason})...`);
|
|
const preparedAuth = await prepareProviderRuntimeAuth({
|
|
provider: runtimeModel.provider,
|
|
config: params.config,
|
|
workspaceDir: resolvedWorkspace,
|
|
env: process.env,
|
|
context: {
|
|
config: params.config,
|
|
agentDir,
|
|
workspaceDir: resolvedWorkspace,
|
|
env: process.env,
|
|
provider: runtimeModel.provider,
|
|
modelId,
|
|
model: runtimeModel,
|
|
apiKey: sourceApiKey,
|
|
authMode: runtimeAuthState?.authMode ?? "unknown",
|
|
profileId: runtimeAuthState?.profileId,
|
|
},
|
|
});
|
|
if (!preparedAuth?.apiKey) {
|
|
throw new Error(
|
|
`Provider "${runtimeModel.provider}" does not support runtime auth refresh.`,
|
|
);
|
|
}
|
|
authStorage.setRuntimeApiKey(runtimeModel.provider, preparedAuth.apiKey);
|
|
if (preparedAuth.baseUrl) {
|
|
runtimeModel = { ...runtimeModel, baseUrl: preparedAuth.baseUrl };
|
|
effectiveModel = {
|
|
...effectiveModel,
|
|
baseUrl: preparedAuth.baseUrl,
|
|
};
|
|
}
|
|
runtimeAuthState = {
|
|
...runtimeAuthState,
|
|
expiresAt: preparedAuth.expiresAt,
|
|
};
|
|
if (preparedAuth.expiresAt) {
|
|
const remaining = preparedAuth.expiresAt - Date.now();
|
|
log.debug(
|
|
`Runtime auth refreshed for ${runtimeModel.provider}; expires in ${Math.max(0, Math.floor(remaining / 1000))}s.`,
|
|
);
|
|
}
|
|
})()
|
|
.catch((err) => {
|
|
log.warn(
|
|
`Runtime auth refresh failed for ${runtimeModel.provider}: ${describeUnknownError(err)}`,
|
|
);
|
|
throw err;
|
|
})
|
|
.finally(() => {
|
|
if (runtimeAuthState) {
|
|
runtimeAuthState.refreshInFlight = undefined;
|
|
}
|
|
});
|
|
await runtimeAuthState.refreshInFlight;
|
|
};
|
|
|
|
const scheduleRuntimeAuthRefresh = (): void => {
|
|
if (!runtimeAuthState || runtimeAuthRefreshCancelled) {
|
|
return;
|
|
}
|
|
if (!hasRefreshableRuntimeAuth()) {
|
|
log.warn(
|
|
`Skipping runtime auth refresh scheduling for ${runtimeModel.provider}; source credential missing.`,
|
|
);
|
|
return;
|
|
}
|
|
if (!runtimeAuthState.expiresAt) {
|
|
return;
|
|
}
|
|
clearRuntimeAuthRefreshTimer();
|
|
const now = Date.now();
|
|
const refreshAt = runtimeAuthState.expiresAt - RUNTIME_AUTH_REFRESH_MARGIN_MS;
|
|
const delayMs = Math.max(RUNTIME_AUTH_REFRESH_MIN_DELAY_MS, refreshAt - now);
|
|
const timer = setTimeout(() => {
|
|
if (runtimeAuthRefreshCancelled) {
|
|
return;
|
|
}
|
|
refreshRuntimeAuth("scheduled")
|
|
.then(() => scheduleRuntimeAuthRefresh())
|
|
.catch(() => {
|
|
if (runtimeAuthRefreshCancelled) {
|
|
return;
|
|
}
|
|
const retryTimer = setTimeout(() => {
|
|
if (runtimeAuthRefreshCancelled) {
|
|
return;
|
|
}
|
|
refreshRuntimeAuth("scheduled-retry")
|
|
.then(() => scheduleRuntimeAuthRefresh())
|
|
.catch(() => undefined);
|
|
}, RUNTIME_AUTH_REFRESH_RETRY_MS);
|
|
const activeRuntimeAuthState = runtimeAuthState;
|
|
if (activeRuntimeAuthState) {
|
|
activeRuntimeAuthState.refreshTimer = retryTimer;
|
|
}
|
|
if (runtimeAuthRefreshCancelled && activeRuntimeAuthState) {
|
|
clearTimeout(retryTimer);
|
|
activeRuntimeAuthState.refreshTimer = undefined;
|
|
}
|
|
});
|
|
}, delayMs);
|
|
runtimeAuthState.refreshTimer = timer;
|
|
if (runtimeAuthRefreshCancelled) {
|
|
clearTimeout(timer);
|
|
runtimeAuthState.refreshTimer = undefined;
|
|
}
|
|
};
|
|
|
|
const resolveAuthProfileFailoverReason = (params: {
|
|
allInCooldown: boolean;
|
|
message: string;
|
|
profileIds?: Array<string | undefined>;
|
|
}): FailoverReason => {
|
|
if (params.allInCooldown) {
|
|
const profileIds = (params.profileIds ?? profileCandidates).filter(
|
|
(id): id is string => typeof id === "string" && id.length > 0,
|
|
);
|
|
return (
|
|
resolveProfilesUnavailableReason({
|
|
store: authStore,
|
|
profileIds,
|
|
}) ?? "unknown"
|
|
);
|
|
}
|
|
const classified = classifyFailoverReason(params.message);
|
|
return classified ?? "auth";
|
|
};
|
|
|
|
const throwAuthProfileFailover = (params: {
|
|
allInCooldown: boolean;
|
|
message?: string;
|
|
error?: unknown;
|
|
}): never => {
|
|
const fallbackMessage = `No available auth profile for ${provider} (all in cooldown or unavailable).`;
|
|
const message =
|
|
params.message?.trim() ||
|
|
(params.error ? describeUnknownError(params.error).trim() : "") ||
|
|
fallbackMessage;
|
|
const reason = resolveAuthProfileFailoverReason({
|
|
allInCooldown: params.allInCooldown,
|
|
message,
|
|
profileIds: profileCandidates,
|
|
});
|
|
if (fallbackConfigured) {
|
|
throw new FailoverError(message, {
|
|
reason,
|
|
provider,
|
|
model: modelId,
|
|
status: resolveFailoverStatus(reason),
|
|
cause: params.error,
|
|
});
|
|
}
|
|
if (params.error instanceof Error) {
|
|
throw params.error;
|
|
}
|
|
throw new Error(message);
|
|
};
|
|
|
|
const resolveApiKeyForCandidate = async (candidate?: string) => {
|
|
return getApiKeyForModel({
|
|
model: runtimeModel,
|
|
cfg: params.config,
|
|
profileId: candidate,
|
|
store: authStore,
|
|
agentDir,
|
|
});
|
|
};
|
|
|
|
const applyApiKeyInfo = async (candidate?: string): Promise<void> => {
|
|
apiKeyInfo = await resolveApiKeyForCandidate(candidate);
|
|
const resolvedProfileId = apiKeyInfo.profileId ?? candidate;
|
|
if (!apiKeyInfo.apiKey) {
|
|
if (apiKeyInfo.mode !== "aws-sdk") {
|
|
throw new Error(
|
|
`No API key resolved for provider "${runtimeModel.provider}" (auth mode: ${apiKeyInfo.mode}).`,
|
|
);
|
|
}
|
|
lastProfileId = resolvedProfileId;
|
|
return;
|
|
}
|
|
let runtimeAuthHandled = false;
|
|
const preparedAuth = await prepareProviderRuntimeAuth({
|
|
provider: runtimeModel.provider,
|
|
config: params.config,
|
|
workspaceDir: resolvedWorkspace,
|
|
env: process.env,
|
|
context: {
|
|
config: params.config,
|
|
agentDir,
|
|
workspaceDir: resolvedWorkspace,
|
|
env: process.env,
|
|
provider: runtimeModel.provider,
|
|
modelId,
|
|
model: runtimeModel,
|
|
apiKey: apiKeyInfo.apiKey,
|
|
authMode: apiKeyInfo.mode,
|
|
profileId: apiKeyInfo.profileId,
|
|
},
|
|
});
|
|
if (preparedAuth?.baseUrl) {
|
|
runtimeModel = { ...runtimeModel, baseUrl: preparedAuth.baseUrl };
|
|
effectiveModel = { ...effectiveModel, baseUrl: preparedAuth.baseUrl };
|
|
}
|
|
if (preparedAuth?.apiKey) {
|
|
authStorage.setRuntimeApiKey(runtimeModel.provider, preparedAuth.apiKey);
|
|
runtimeAuthState = {
|
|
sourceApiKey: apiKeyInfo.apiKey,
|
|
authMode: apiKeyInfo.mode,
|
|
profileId: apiKeyInfo.profileId,
|
|
expiresAt: preparedAuth.expiresAt,
|
|
};
|
|
if (preparedAuth.expiresAt) {
|
|
scheduleRuntimeAuthRefresh();
|
|
}
|
|
runtimeAuthHandled = true;
|
|
}
|
|
if (runtimeAuthHandled) {
|
|
// Plugin-owned runtime auth already stored the exchanged credential.
|
|
} else {
|
|
authStorage.setRuntimeApiKey(runtimeModel.provider, apiKeyInfo.apiKey);
|
|
runtimeAuthState = null;
|
|
}
|
|
lastProfileId = apiKeyInfo.profileId;
|
|
};
|
|
|
|
const advanceAuthProfile = async (): Promise<boolean> => {
|
|
if (lockedProfileId) {
|
|
return false;
|
|
}
|
|
let nextIndex = profileIndex + 1;
|
|
while (nextIndex < profileCandidates.length) {
|
|
const candidate = profileCandidates[nextIndex];
|
|
if (candidate && isProfileInCooldown(authStore, candidate, undefined, modelId)) {
|
|
nextIndex += 1;
|
|
continue;
|
|
}
|
|
try {
|
|
await applyApiKeyInfo(candidate);
|
|
profileIndex = nextIndex;
|
|
thinkLevel = initialThinkLevel;
|
|
attemptedThinking.clear();
|
|
return true;
|
|
} catch (err) {
|
|
if (candidate && candidate === lockedProfileId) {
|
|
throw err;
|
|
}
|
|
nextIndex += 1;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
|
|
try {
|
|
const autoProfileCandidates = profileCandidates.filter(
|
|
(candidate): candidate is string =>
|
|
typeof candidate === "string" && candidate.length > 0 && candidate !== lockedProfileId,
|
|
);
|
|
const allAutoProfilesInCooldown =
|
|
autoProfileCandidates.length > 0 &&
|
|
autoProfileCandidates.every((candidate) =>
|
|
isProfileInCooldown(authStore, candidate, undefined, modelId),
|
|
);
|
|
const unavailableReason = allAutoProfilesInCooldown
|
|
? (resolveProfilesUnavailableReason({
|
|
store: authStore,
|
|
profileIds: autoProfileCandidates,
|
|
}) ?? "unknown")
|
|
: null;
|
|
const allowTransientCooldownProbe =
|
|
params.allowTransientCooldownProbe === true &&
|
|
allAutoProfilesInCooldown &&
|
|
shouldAllowCooldownProbeForReason(unavailableReason);
|
|
let didTransientCooldownProbe = false;
|
|
|
|
while (profileIndex < profileCandidates.length) {
|
|
const candidate = profileCandidates[profileIndex];
|
|
const inCooldown =
|
|
candidate &&
|
|
candidate !== lockedProfileId &&
|
|
isProfileInCooldown(authStore, candidate, undefined, modelId);
|
|
if (inCooldown) {
|
|
if (allowTransientCooldownProbe && !didTransientCooldownProbe) {
|
|
didTransientCooldownProbe = true;
|
|
log.warn(
|
|
`probing cooldowned auth profile for ${provider}/${modelId} due to ${unavailableReason ?? "transient"} unavailability`,
|
|
);
|
|
} else {
|
|
profileIndex += 1;
|
|
continue;
|
|
}
|
|
}
|
|
await applyApiKeyInfo(profileCandidates[profileIndex]);
|
|
break;
|
|
}
|
|
if (profileIndex >= profileCandidates.length) {
|
|
throwAuthProfileFailover({ allInCooldown: true });
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof FailoverError) {
|
|
throw err;
|
|
}
|
|
if (profileCandidates[profileIndex] === lockedProfileId) {
|
|
throwAuthProfileFailover({ allInCooldown: false, error: err });
|
|
}
|
|
const advanced = await advanceAuthProfile();
|
|
if (!advanced) {
|
|
throwAuthProfileFailover({ allInCooldown: false, error: err });
|
|
}
|
|
}
|
|
|
|
const maybeRefreshRuntimeAuthForAuthError = async (
|
|
errorText: string,
|
|
retried: boolean,
|
|
): Promise<boolean> => {
|
|
if (!runtimeAuthState || retried) {
|
|
return false;
|
|
}
|
|
if (!isFailoverErrorMessage(errorText)) {
|
|
return false;
|
|
}
|
|
if (classifyFailoverReason(errorText) !== "auth") {
|
|
return false;
|
|
}
|
|
try {
|
|
await refreshRuntimeAuth("auth-error");
|
|
scheduleRuntimeAuthRefresh();
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
const MAX_TIMEOUT_COMPACTION_ATTEMPTS = 2;
|
|
const MAX_OVERFLOW_COMPACTION_ATTEMPTS = 3;
|
|
const MAX_RUN_LOOP_ITERATIONS = resolveMaxRunRetryIterations(profileCandidates.length);
|
|
let overflowCompactionAttempts = 0;
|
|
let toolResultTruncationAttempted = false;
|
|
let bootstrapPromptWarningSignaturesSeen =
|
|
params.bootstrapPromptWarningSignaturesSeen ??
|
|
(params.bootstrapPromptWarningSignature ? [params.bootstrapPromptWarningSignature] : []);
|
|
const usageAccumulator = createUsageAccumulator();
|
|
let lastRunPromptUsage: ReturnType<typeof normalizeUsage> | undefined;
|
|
let autoCompactionCount = 0;
|
|
let runLoopIterations = 0;
|
|
let overloadFailoverAttempts = 0;
|
|
let timeoutCompactionAttempts = 0;
|
|
const maybeMarkAuthProfileFailure = async (failure: {
|
|
profileId?: string;
|
|
reason?: AuthProfileFailureReason | null;
|
|
config?: RunEmbeddedPiAgentParams["config"];
|
|
agentDir?: RunEmbeddedPiAgentParams["agentDir"];
|
|
modelId?: string;
|
|
}) => {
|
|
const { profileId, reason } = failure;
|
|
if (!profileId || !reason || reason === "timeout") {
|
|
return;
|
|
}
|
|
await markAuthProfileFailure({
|
|
store: authStore,
|
|
profileId,
|
|
reason,
|
|
cfg: params.config,
|
|
agentDir,
|
|
runId: params.runId,
|
|
modelId: failure.modelId,
|
|
});
|
|
};
|
|
const resolveAuthProfileFailureReason = (
|
|
failoverReason: FailoverReason | null,
|
|
): AuthProfileFailureReason | null => {
|
|
// Timeouts are transport/model-path failures, not auth health signals,
|
|
// so they should not persist auth-profile failure state.
|
|
if (!failoverReason || failoverReason === "timeout") {
|
|
return null;
|
|
}
|
|
return failoverReason;
|
|
};
|
|
const maybeBackoffBeforeOverloadFailover = async (reason: FailoverReason | null) => {
|
|
if (reason !== "overloaded") {
|
|
return;
|
|
}
|
|
overloadFailoverAttempts += 1;
|
|
const delayMs = computeBackoff(OVERLOAD_FAILOVER_BACKOFF_POLICY, overloadFailoverAttempts);
|
|
log.warn(
|
|
`overload backoff before failover for ${provider}/${modelId}: attempt=${overloadFailoverAttempts} delayMs=${delayMs}`,
|
|
);
|
|
try {
|
|
await sleepWithAbort(delayMs, params.abortSignal);
|
|
} catch (err) {
|
|
if (params.abortSignal?.aborted) {
|
|
const abortErr = new Error("Operation aborted", { cause: err });
|
|
abortErr.name = "AbortError";
|
|
throw abortErr;
|
|
}
|
|
throw err;
|
|
}
|
|
};
|
|
// Resolve the context engine once and reuse across retries to avoid
|
|
// repeated initialization/connection overhead per attempt.
|
|
ensureContextEnginesInitialized();
|
|
const contextEngine = await resolveContextEngine(params.config);
|
|
try {
|
|
// When the engine owns compaction, compactEmbeddedPiSessionDirect is
|
|
// bypassed. Fire lifecycle hooks here so recovery paths still notify
|
|
// subscribers like memory extensions and usage trackers.
|
|
const runOwnsCompactionBeforeHook = async (reason: string) => {
|
|
if (
|
|
contextEngine.info.ownsCompaction !== true ||
|
|
!hookRunner?.hasHooks("before_compaction")
|
|
) {
|
|
return;
|
|
}
|
|
try {
|
|
await hookRunner.runBeforeCompaction(
|
|
{ messageCount: -1, sessionFile: params.sessionFile },
|
|
hookCtx,
|
|
);
|
|
} catch (hookErr) {
|
|
log.warn(`before_compaction hook failed during ${reason}: ${String(hookErr)}`);
|
|
}
|
|
};
|
|
const runOwnsCompactionAfterHook = async (
|
|
reason: string,
|
|
compactResult: Awaited<ReturnType<typeof contextEngine.compact>>,
|
|
) => {
|
|
if (
|
|
contextEngine.info.ownsCompaction !== true ||
|
|
!compactResult.ok ||
|
|
!compactResult.compacted ||
|
|
!hookRunner?.hasHooks("after_compaction")
|
|
) {
|
|
return;
|
|
}
|
|
try {
|
|
await hookRunner.runAfterCompaction(
|
|
{
|
|
messageCount: -1,
|
|
compactedCount: -1,
|
|
tokenCount: compactResult.result?.tokensAfter,
|
|
sessionFile: params.sessionFile,
|
|
},
|
|
hookCtx,
|
|
);
|
|
} catch (hookErr) {
|
|
log.warn(`after_compaction hook failed during ${reason}: ${String(hookErr)}`);
|
|
}
|
|
};
|
|
let authRetryPending = false;
|
|
// Hoisted so the retry-limit error path can use the most recent API total.
|
|
let lastTurnTotal: number | undefined;
|
|
while (true) {
|
|
if (runLoopIterations >= MAX_RUN_LOOP_ITERATIONS) {
|
|
const message =
|
|
`Exceeded retry limit after ${runLoopIterations} attempts ` +
|
|
`(max=${MAX_RUN_LOOP_ITERATIONS}).`;
|
|
log.error(
|
|
`[run-retry-limit] sessionKey=${params.sessionKey ?? params.sessionId} ` +
|
|
`provider=${provider}/${modelId} attempts=${runLoopIterations} ` +
|
|
`maxAttempts=${MAX_RUN_LOOP_ITERATIONS}`,
|
|
);
|
|
return {
|
|
payloads: [
|
|
{
|
|
text:
|
|
"Request failed after repeated internal retries. " +
|
|
"Please try again, or use /new to start a fresh session.",
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta: buildErrorAgentMeta({
|
|
sessionId: params.sessionId,
|
|
provider,
|
|
model: model.id,
|
|
usageAccumulator,
|
|
lastRunPromptUsage,
|
|
lastTurnTotal,
|
|
}),
|
|
error: { kind: "retry_limit", message },
|
|
},
|
|
};
|
|
}
|
|
runLoopIterations += 1;
|
|
const runtimeAuthRetry = authRetryPending;
|
|
authRetryPending = false;
|
|
attemptedThinking.add(thinkLevel);
|
|
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
|
|
|
const prompt =
|
|
provider === "anthropic" ? scrubAnthropicRefusalMagic(params.prompt) : params.prompt;
|
|
|
|
const attempt = await runEmbeddedAttempt({
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
trigger: params.trigger,
|
|
memoryFlushWritePath: params.memoryFlushWritePath,
|
|
messageChannel: params.messageChannel,
|
|
messageProvider: params.messageProvider,
|
|
agentAccountId: params.agentAccountId,
|
|
messageTo: params.messageTo,
|
|
messageThreadId: params.messageThreadId,
|
|
groupId: params.groupId,
|
|
groupChannel: params.groupChannel,
|
|
groupSpace: params.groupSpace,
|
|
spawnedBy: params.spawnedBy,
|
|
senderId: params.senderId,
|
|
senderName: params.senderName,
|
|
senderUsername: params.senderUsername,
|
|
senderE164: params.senderE164,
|
|
senderIsOwner: params.senderIsOwner,
|
|
currentChannelId: params.currentChannelId,
|
|
currentThreadTs: params.currentThreadTs,
|
|
currentMessageId: params.currentMessageId,
|
|
replyToMode: params.replyToMode,
|
|
hasRepliedRef: params.hasRepliedRef,
|
|
sessionFile: params.sessionFile,
|
|
workspaceDir: resolvedWorkspace,
|
|
agentDir,
|
|
config: params.config,
|
|
allowGatewaySubagentBinding: params.allowGatewaySubagentBinding,
|
|
contextEngine,
|
|
contextTokenBudget: ctxInfo.tokens,
|
|
skillsSnapshot: params.skillsSnapshot,
|
|
prompt,
|
|
images: params.images,
|
|
clientTools: params.clientTools,
|
|
disableTools: params.disableTools,
|
|
provider,
|
|
modelId,
|
|
model: applyLocalNoAuthHeaderOverride(effectiveModel, apiKeyInfo),
|
|
authProfileId: lastProfileId,
|
|
authProfileIdSource: lockedProfileId ? "user" : "auto",
|
|
authStorage,
|
|
modelRegistry,
|
|
agentId: workspaceResolution.agentId,
|
|
legacyBeforeAgentStartResult,
|
|
thinkLevel,
|
|
fastMode: params.fastMode,
|
|
verboseLevel: params.verboseLevel,
|
|
reasoningLevel: params.reasoningLevel,
|
|
toolResultFormat: resolvedToolResultFormat,
|
|
execOverrides: params.execOverrides,
|
|
bashElevated: params.bashElevated,
|
|
timeoutMs: params.timeoutMs,
|
|
runId: params.runId,
|
|
abortSignal: params.abortSignal,
|
|
shouldEmitToolResult: params.shouldEmitToolResult,
|
|
shouldEmitToolOutput: params.shouldEmitToolOutput,
|
|
onPartialReply: params.onPartialReply,
|
|
onAssistantMessageStart: params.onAssistantMessageStart,
|
|
onBlockReply: params.onBlockReply,
|
|
onBlockReplyFlush: params.onBlockReplyFlush,
|
|
blockReplyBreak: params.blockReplyBreak,
|
|
blockReplyChunking: params.blockReplyChunking,
|
|
onReasoningStream: params.onReasoningStream,
|
|
onReasoningEnd: params.onReasoningEnd,
|
|
onToolResult: params.onToolResult,
|
|
onAgentEvent: params.onAgentEvent,
|
|
extraSystemPrompt: params.extraSystemPrompt,
|
|
inputProvenance: params.inputProvenance,
|
|
streamParams: params.streamParams,
|
|
ownerNumbers: params.ownerNumbers,
|
|
enforceFinalTag: params.enforceFinalTag,
|
|
bootstrapPromptWarningSignaturesSeen,
|
|
bootstrapPromptWarningSignature:
|
|
bootstrapPromptWarningSignaturesSeen[bootstrapPromptWarningSignaturesSeen.length - 1],
|
|
});
|
|
|
|
const {
|
|
aborted,
|
|
promptError,
|
|
timedOut,
|
|
timedOutDuringCompaction,
|
|
sessionIdUsed,
|
|
lastAssistant,
|
|
} = attempt;
|
|
bootstrapPromptWarningSignaturesSeen =
|
|
attempt.bootstrapPromptWarningSignaturesSeen ??
|
|
(attempt.bootstrapPromptWarningSignature
|
|
? Array.from(
|
|
new Set([
|
|
...bootstrapPromptWarningSignaturesSeen,
|
|
attempt.bootstrapPromptWarningSignature,
|
|
]),
|
|
)
|
|
: bootstrapPromptWarningSignaturesSeen);
|
|
const lastAssistantUsage = normalizeUsage(lastAssistant?.usage as UsageLike);
|
|
const attemptUsage = attempt.attemptUsage ?? lastAssistantUsage;
|
|
mergeUsageIntoAccumulator(usageAccumulator, attemptUsage);
|
|
// Keep prompt size from the latest model call so session totalTokens
|
|
// reflects current context usage, not accumulated tool-loop usage.
|
|
lastRunPromptUsage = lastAssistantUsage ?? attemptUsage;
|
|
lastTurnTotal = lastAssistantUsage?.total ?? attemptUsage?.total;
|
|
const attemptCompactionCount = Math.max(0, attempt.compactionCount ?? 0);
|
|
autoCompactionCount += attemptCompactionCount;
|
|
const activeErrorContext = resolveActiveErrorContext({
|
|
lastAssistant,
|
|
provider,
|
|
model: modelId,
|
|
});
|
|
const formattedAssistantErrorText = lastAssistant
|
|
? formatAssistantErrorText(lastAssistant, {
|
|
cfg: params.config,
|
|
sessionKey: params.sessionKey ?? params.sessionId,
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
})
|
|
: undefined;
|
|
const assistantErrorText =
|
|
lastAssistant?.stopReason === "error"
|
|
? lastAssistant.errorMessage?.trim() || formattedAssistantErrorText
|
|
: undefined;
|
|
|
|
// ── Timeout-triggered compaction ──────────────────────────────────
|
|
// When the LLM times out with high context usage, compact before
|
|
// retrying to break the death spiral of repeated timeouts.
|
|
if (timedOut && !timedOutDuringCompaction) {
|
|
// Only consider prompt-side tokens here. API totals include output
|
|
// tokens, which can make a long generation look like high context
|
|
// pressure even when the prompt itself was small.
|
|
const lastTurnPromptTokens = derivePromptTokens(lastRunPromptUsage);
|
|
const tokenUsedRatio =
|
|
lastTurnPromptTokens != null && ctxInfo.tokens > 0
|
|
? lastTurnPromptTokens / ctxInfo.tokens
|
|
: 0;
|
|
if (timeoutCompactionAttempts >= MAX_TIMEOUT_COMPACTION_ATTEMPTS) {
|
|
log.warn(
|
|
`[timeout-compaction] already attempted timeout compaction ${timeoutCompactionAttempts} time(s); falling through to failover rotation`,
|
|
);
|
|
} else if (tokenUsedRatio > 0.65) {
|
|
const timeoutDiagId = createCompactionDiagId();
|
|
timeoutCompactionAttempts++;
|
|
log.warn(
|
|
`[timeout-compaction] LLM timed out with high prompt token usage (${Math.round(tokenUsedRatio * 100)}%); ` +
|
|
`attempting compaction before retry (attempt ${timeoutCompactionAttempts}/${MAX_TIMEOUT_COMPACTION_ATTEMPTS}) diagId=${timeoutDiagId}`,
|
|
);
|
|
let timeoutCompactResult: Awaited<ReturnType<typeof contextEngine.compact>>;
|
|
await runOwnsCompactionBeforeHook("timeout recovery");
|
|
try {
|
|
const timeoutCompactionRuntimeContext = {
|
|
...buildEmbeddedCompactionRuntimeContext({
|
|
sessionKey: params.sessionKey,
|
|
messageChannel: params.messageChannel,
|
|
messageProvider: params.messageProvider,
|
|
agentAccountId: params.agentAccountId,
|
|
currentChannelId: params.currentChannelId,
|
|
currentThreadTs: params.currentThreadTs,
|
|
currentMessageId: params.currentMessageId,
|
|
authProfileId: lastProfileId,
|
|
workspaceDir: resolvedWorkspace,
|
|
agentDir,
|
|
config: params.config,
|
|
skillsSnapshot: params.skillsSnapshot,
|
|
senderIsOwner: params.senderIsOwner,
|
|
senderId: params.senderId,
|
|
provider,
|
|
modelId,
|
|
thinkLevel,
|
|
reasoningLevel: params.reasoningLevel,
|
|
bashElevated: params.bashElevated,
|
|
extraSystemPrompt: params.extraSystemPrompt,
|
|
ownerNumbers: params.ownerNumbers,
|
|
}),
|
|
runId: params.runId,
|
|
trigger: "timeout_recovery",
|
|
diagId: timeoutDiagId,
|
|
attempt: timeoutCompactionAttempts,
|
|
maxAttempts: MAX_TIMEOUT_COMPACTION_ATTEMPTS,
|
|
};
|
|
timeoutCompactResult = await contextEngine.compact({
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
sessionFile: params.sessionFile,
|
|
tokenBudget: ctxInfo.tokens,
|
|
force: true,
|
|
compactionTarget: "budget",
|
|
runtimeContext: timeoutCompactionRuntimeContext,
|
|
});
|
|
} catch (compactErr) {
|
|
log.warn(
|
|
`[timeout-compaction] contextEngine.compact() threw during timeout recovery for ${provider}/${modelId}: ${String(compactErr)}`,
|
|
);
|
|
timeoutCompactResult = {
|
|
ok: false,
|
|
compacted: false,
|
|
reason: String(compactErr),
|
|
};
|
|
}
|
|
await runOwnsCompactionAfterHook("timeout recovery", timeoutCompactResult);
|
|
if (timeoutCompactResult.compacted) {
|
|
autoCompactionCount += 1;
|
|
if (contextEngine.info.ownsCompaction === true) {
|
|
await runPostCompactionSideEffects({
|
|
config: params.config,
|
|
sessionKey: params.sessionKey,
|
|
sessionFile: params.sessionFile,
|
|
});
|
|
}
|
|
log.info(
|
|
`[timeout-compaction] compaction succeeded for ${provider}/${modelId}; retrying prompt`,
|
|
);
|
|
continue;
|
|
} else {
|
|
log.warn(
|
|
`[timeout-compaction] compaction did not reduce context for ${provider}/${modelId}; falling through to normal handling`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
const contextOverflowError = !aborted
|
|
? (() => {
|
|
if (promptError) {
|
|
const errorText = describeUnknownError(promptError);
|
|
if (isLikelyContextOverflowError(errorText)) {
|
|
return { text: errorText, source: "promptError" as const };
|
|
}
|
|
// Prompt submission failed with a non-overflow error. Do not
|
|
// inspect prior assistant errors from history for this attempt.
|
|
return null;
|
|
}
|
|
if (assistantErrorText && isLikelyContextOverflowError(assistantErrorText)) {
|
|
return {
|
|
text: assistantErrorText,
|
|
source: "assistantError" as const,
|
|
};
|
|
}
|
|
return null;
|
|
})()
|
|
: null;
|
|
|
|
if (contextOverflowError) {
|
|
const overflowDiagId = createCompactionDiagId();
|
|
const errorText = contextOverflowError.text;
|
|
const msgCount = attempt.messagesSnapshot?.length ?? 0;
|
|
const observedOverflowTokens = extractObservedOverflowTokenCount(errorText);
|
|
log.warn(
|
|
`[context-overflow-diag] sessionKey=${params.sessionKey ?? params.sessionId} ` +
|
|
`provider=${provider}/${modelId} source=${contextOverflowError.source} ` +
|
|
`messages=${msgCount} sessionFile=${params.sessionFile} ` +
|
|
`diagId=${overflowDiagId} compactionAttempts=${overflowCompactionAttempts} ` +
|
|
`observedTokens=${observedOverflowTokens ?? "unknown"} ` +
|
|
`error=${errorText.slice(0, 200)}`,
|
|
);
|
|
const isCompactionFailure = isCompactionFailureError(errorText);
|
|
const hadAttemptLevelCompaction = attemptCompactionCount > 0;
|
|
// If this attempt already compacted (SDK auto-compaction), avoid immediately
|
|
// running another explicit compaction for the same overflow trigger.
|
|
if (
|
|
!isCompactionFailure &&
|
|
hadAttemptLevelCompaction &&
|
|
overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS
|
|
) {
|
|
overflowCompactionAttempts++;
|
|
log.warn(
|
|
`context overflow persisted after in-attempt compaction (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); retrying prompt without additional compaction for ${provider}/${modelId}`,
|
|
);
|
|
continue;
|
|
}
|
|
// Attempt explicit overflow compaction only when this attempt did not
|
|
// already auto-compact.
|
|
if (
|
|
!isCompactionFailure &&
|
|
!hadAttemptLevelCompaction &&
|
|
overflowCompactionAttempts < MAX_OVERFLOW_COMPACTION_ATTEMPTS
|
|
) {
|
|
if (log.isEnabled("debug")) {
|
|
log.debug(
|
|
`[compaction-diag] decision diagId=${overflowDiagId} branch=compact ` +
|
|
`isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` +
|
|
`attempt=${overflowCompactionAttempts + 1} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`,
|
|
);
|
|
}
|
|
overflowCompactionAttempts++;
|
|
log.warn(
|
|
`context overflow detected (attempt ${overflowCompactionAttempts}/${MAX_OVERFLOW_COMPACTION_ATTEMPTS}); attempting auto-compaction for ${provider}/${modelId}`,
|
|
);
|
|
let compactResult: Awaited<ReturnType<typeof contextEngine.compact>>;
|
|
await runOwnsCompactionBeforeHook("overflow recovery");
|
|
try {
|
|
const overflowCompactionRuntimeContext = {
|
|
...buildEmbeddedCompactionRuntimeContext({
|
|
sessionKey: params.sessionKey,
|
|
messageChannel: params.messageChannel,
|
|
messageProvider: params.messageProvider,
|
|
agentAccountId: params.agentAccountId,
|
|
currentChannelId: params.currentChannelId,
|
|
currentThreadTs: params.currentThreadTs,
|
|
currentMessageId: params.currentMessageId,
|
|
authProfileId: lastProfileId,
|
|
workspaceDir: resolvedWorkspace,
|
|
agentDir,
|
|
config: params.config,
|
|
skillsSnapshot: params.skillsSnapshot,
|
|
senderIsOwner: params.senderIsOwner,
|
|
senderId: params.senderId,
|
|
provider,
|
|
modelId,
|
|
thinkLevel,
|
|
reasoningLevel: params.reasoningLevel,
|
|
bashElevated: params.bashElevated,
|
|
extraSystemPrompt: params.extraSystemPrompt,
|
|
ownerNumbers: params.ownerNumbers,
|
|
}),
|
|
runId: params.runId,
|
|
trigger: "overflow",
|
|
...(observedOverflowTokens !== undefined
|
|
? { currentTokenCount: observedOverflowTokens }
|
|
: {}),
|
|
diagId: overflowDiagId,
|
|
attempt: overflowCompactionAttempts,
|
|
maxAttempts: MAX_OVERFLOW_COMPACTION_ATTEMPTS,
|
|
};
|
|
compactResult = await contextEngine.compact({
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
sessionFile: params.sessionFile,
|
|
tokenBudget: ctxInfo.tokens,
|
|
...(observedOverflowTokens !== undefined
|
|
? { currentTokenCount: observedOverflowTokens }
|
|
: {}),
|
|
force: true,
|
|
compactionTarget: "budget",
|
|
runtimeContext: overflowCompactionRuntimeContext,
|
|
});
|
|
if (compactResult.ok && compactResult.compacted) {
|
|
await runContextEngineMaintenance({
|
|
contextEngine,
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
sessionFile: params.sessionFile,
|
|
reason: "compaction",
|
|
runtimeContext: overflowCompactionRuntimeContext,
|
|
});
|
|
}
|
|
} catch (compactErr) {
|
|
log.warn(
|
|
`contextEngine.compact() threw during overflow recovery for ${provider}/${modelId}: ${String(compactErr)}`,
|
|
);
|
|
compactResult = {
|
|
ok: false,
|
|
compacted: false,
|
|
reason: String(compactErr),
|
|
};
|
|
}
|
|
await runOwnsCompactionAfterHook("overflow recovery", compactResult);
|
|
if (compactResult.compacted) {
|
|
autoCompactionCount += 1;
|
|
log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`);
|
|
continue;
|
|
}
|
|
log.warn(
|
|
`auto-compaction failed for ${provider}/${modelId}: ${compactResult.reason ?? "nothing to compact"}`,
|
|
);
|
|
}
|
|
// Fallback: try truncating oversized tool results in the session.
|
|
// This handles the case where a single tool result exceeds the
|
|
// context window and compaction cannot reduce it further.
|
|
if (!toolResultTruncationAttempted) {
|
|
const contextWindowTokens = ctxInfo.tokens;
|
|
const hasOversized = attempt.messagesSnapshot
|
|
? sessionLikelyHasOversizedToolResults({
|
|
messages: attempt.messagesSnapshot,
|
|
contextWindowTokens,
|
|
})
|
|
: false;
|
|
|
|
if (hasOversized) {
|
|
if (log.isEnabled("debug")) {
|
|
log.debug(
|
|
`[compaction-diag] decision diagId=${overflowDiagId} branch=truncate_tool_results ` +
|
|
`isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=${hasOversized} ` +
|
|
`attempt=${overflowCompactionAttempts} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`,
|
|
);
|
|
}
|
|
toolResultTruncationAttempted = true;
|
|
log.warn(
|
|
`[context-overflow-recovery] Attempting tool result truncation for ${provider}/${modelId} ` +
|
|
`(contextWindow=${contextWindowTokens} tokens)`,
|
|
);
|
|
const truncResult = await truncateOversizedToolResultsInSession({
|
|
sessionFile: params.sessionFile,
|
|
contextWindowTokens,
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
});
|
|
if (truncResult.truncated) {
|
|
log.info(
|
|
`[context-overflow-recovery] Truncated ${truncResult.truncatedCount} tool result(s); retrying prompt`,
|
|
);
|
|
// Do NOT reset overflowCompactionAttempts here — the global cap must remain
|
|
// enforced across all iterations to prevent unbounded compaction cycles (OC-65).
|
|
continue;
|
|
}
|
|
log.warn(
|
|
`[context-overflow-recovery] Tool result truncation did not help: ${truncResult.reason ?? "unknown"}`,
|
|
);
|
|
} else if (log.isEnabled("debug")) {
|
|
log.debug(
|
|
`[compaction-diag] decision diagId=${overflowDiagId} branch=give_up ` +
|
|
`isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=${hasOversized} ` +
|
|
`attempt=${overflowCompactionAttempts} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`,
|
|
);
|
|
}
|
|
}
|
|
if (
|
|
(isCompactionFailure ||
|
|
overflowCompactionAttempts >= MAX_OVERFLOW_COMPACTION_ATTEMPTS ||
|
|
toolResultTruncationAttempted) &&
|
|
log.isEnabled("debug")
|
|
) {
|
|
log.debug(
|
|
`[compaction-diag] decision diagId=${overflowDiagId} branch=give_up ` +
|
|
`isCompactionFailure=${isCompactionFailure} hasOversizedToolResults=unknown ` +
|
|
`attempt=${overflowCompactionAttempts} maxAttempts=${MAX_OVERFLOW_COMPACTION_ATTEMPTS}`,
|
|
);
|
|
}
|
|
const kind = isCompactionFailure ? "compaction_failure" : "context_overflow";
|
|
return {
|
|
payloads: [
|
|
{
|
|
text:
|
|
"Context overflow: prompt too large for the model. " +
|
|
"Try /reset (or /new) to start a fresh session, or use a larger-context model.",
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta: buildErrorAgentMeta({
|
|
sessionId: sessionIdUsed,
|
|
provider,
|
|
model: model.id,
|
|
usageAccumulator,
|
|
lastRunPromptUsage,
|
|
lastAssistant,
|
|
lastTurnTotal,
|
|
}),
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
error: { kind, message: errorText },
|
|
},
|
|
};
|
|
}
|
|
|
|
if (promptError && !aborted) {
|
|
// Normalize wrapped errors (e.g. abort-wrapped RESOURCE_EXHAUSTED) into
|
|
// FailoverError so rate-limit classification works even for nested shapes.
|
|
const normalizedPromptFailover = coerceToFailoverError(promptError, {
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
profileId: lastProfileId,
|
|
});
|
|
const promptErrorDetails = normalizedPromptFailover
|
|
? describeFailoverError(normalizedPromptFailover)
|
|
: describeFailoverError(promptError);
|
|
const errorText = promptErrorDetails.message || describeUnknownError(promptError);
|
|
if (await maybeRefreshRuntimeAuthForAuthError(errorText, runtimeAuthRetry)) {
|
|
authRetryPending = true;
|
|
continue;
|
|
}
|
|
// Handle role ordering errors with a user-friendly message
|
|
if (/incorrect role information|roles must alternate/i.test(errorText)) {
|
|
return {
|
|
payloads: [
|
|
{
|
|
text:
|
|
"Message ordering conflict - please try again. " +
|
|
"If this persists, use /new to start a fresh session.",
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta: buildErrorAgentMeta({
|
|
sessionId: sessionIdUsed,
|
|
provider,
|
|
model: model.id,
|
|
usageAccumulator,
|
|
lastRunPromptUsage,
|
|
lastAssistant,
|
|
lastTurnTotal,
|
|
}),
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
error: { kind: "role_ordering", message: errorText },
|
|
},
|
|
};
|
|
}
|
|
// Handle image size errors with a user-friendly message (no retry needed)
|
|
const imageSizeError = parseImageSizeError(errorText);
|
|
if (imageSizeError) {
|
|
const maxMb = imageSizeError.maxMb;
|
|
const maxMbLabel =
|
|
typeof maxMb === "number" && Number.isFinite(maxMb) ? `${maxMb}` : null;
|
|
const maxBytesHint = maxMbLabel ? ` (max ${maxMbLabel}MB)` : "";
|
|
return {
|
|
payloads: [
|
|
{
|
|
text:
|
|
`Image too large for the model${maxBytesHint}. ` +
|
|
"Please compress or resize the image and try again.",
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta: buildErrorAgentMeta({
|
|
sessionId: sessionIdUsed,
|
|
provider,
|
|
model: model.id,
|
|
usageAccumulator,
|
|
lastRunPromptUsage,
|
|
lastAssistant,
|
|
lastTurnTotal,
|
|
}),
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
error: { kind: "image_size", message: errorText },
|
|
},
|
|
};
|
|
}
|
|
const promptFailoverReason =
|
|
promptErrorDetails.reason ?? classifyFailoverReason(errorText);
|
|
const promptProfileFailureReason =
|
|
resolveAuthProfileFailureReason(promptFailoverReason);
|
|
await maybeMarkAuthProfileFailure({
|
|
profileId: lastProfileId,
|
|
reason: promptProfileFailureReason,
|
|
modelId,
|
|
});
|
|
const promptFailoverFailure =
|
|
promptFailoverReason !== null || isFailoverErrorMessage(errorText);
|
|
// Capture the failing profile before auth-profile rotation mutates `lastProfileId`.
|
|
const failedPromptProfileId = lastProfileId;
|
|
const logPromptFailoverDecision = createFailoverDecisionLogger({
|
|
stage: "prompt",
|
|
runId: params.runId,
|
|
rawError: errorText,
|
|
failoverReason: promptFailoverReason,
|
|
profileFailureReason: promptProfileFailureReason,
|
|
provider,
|
|
model: modelId,
|
|
profileId: failedPromptProfileId,
|
|
fallbackConfigured,
|
|
aborted,
|
|
});
|
|
if (
|
|
promptFailoverFailure &&
|
|
promptFailoverReason !== "timeout" &&
|
|
(await advanceAuthProfile())
|
|
) {
|
|
logPromptFailoverDecision("rotate_profile");
|
|
await maybeBackoffBeforeOverloadFailover(promptFailoverReason);
|
|
continue;
|
|
}
|
|
const fallbackThinking = pickFallbackThinkingLevel({
|
|
message: errorText,
|
|
attempted: attemptedThinking,
|
|
});
|
|
if (fallbackThinking) {
|
|
log.warn(
|
|
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
|
);
|
|
thinkLevel = fallbackThinking;
|
|
continue;
|
|
}
|
|
// Throw FailoverError for prompt-side failover reasons when fallbacks
|
|
// are configured so outer model fallback can continue on overload,
|
|
// rate-limit, auth, or billing failures.
|
|
if (fallbackConfigured && promptFailoverFailure) {
|
|
const status = resolveFailoverStatus(promptFailoverReason ?? "unknown");
|
|
logPromptFailoverDecision("fallback_model", { status });
|
|
await maybeBackoffBeforeOverloadFailover(promptFailoverReason);
|
|
throw (
|
|
normalizedPromptFailover ??
|
|
new FailoverError(errorText, {
|
|
reason: promptFailoverReason ?? "unknown",
|
|
provider,
|
|
model: modelId,
|
|
profileId: lastProfileId,
|
|
status: resolveFailoverStatus(promptFailoverReason ?? "unknown"),
|
|
})
|
|
);
|
|
}
|
|
if (promptFailoverFailure || promptFailoverReason) {
|
|
logPromptFailoverDecision("surface_error");
|
|
}
|
|
throw promptError;
|
|
}
|
|
|
|
const fallbackThinking = pickFallbackThinkingLevel({
|
|
message: lastAssistant?.errorMessage,
|
|
attempted: attemptedThinking,
|
|
});
|
|
if (fallbackThinking && !aborted) {
|
|
log.warn(
|
|
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
|
);
|
|
thinkLevel = fallbackThinking;
|
|
continue;
|
|
}
|
|
|
|
const authFailure = isAuthAssistantError(lastAssistant);
|
|
const rateLimitFailure = isRateLimitAssistantError(lastAssistant);
|
|
const billingFailure = isBillingAssistantError(lastAssistant);
|
|
const failoverFailure = isFailoverAssistantError(lastAssistant);
|
|
const assistantFailoverReason = classifyFailoverReason(lastAssistant?.errorMessage ?? "");
|
|
const assistantProfileFailureReason =
|
|
resolveAuthProfileFailureReason(assistantFailoverReason);
|
|
const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError;
|
|
const imageDimensionError = parseImageDimensionError(lastAssistant?.errorMessage ?? "");
|
|
// Capture the failing profile before auth-profile rotation mutates `lastProfileId`.
|
|
const failedAssistantProfileId = lastProfileId;
|
|
const logAssistantFailoverDecision = createFailoverDecisionLogger({
|
|
stage: "assistant",
|
|
runId: params.runId,
|
|
rawError: lastAssistant?.errorMessage?.trim(),
|
|
failoverReason: assistantFailoverReason,
|
|
profileFailureReason: assistantProfileFailureReason,
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
profileId: failedAssistantProfileId,
|
|
fallbackConfigured,
|
|
timedOut,
|
|
aborted,
|
|
});
|
|
|
|
if (
|
|
authFailure &&
|
|
(await maybeRefreshRuntimeAuthForAuthError(
|
|
lastAssistant?.errorMessage ?? "",
|
|
runtimeAuthRetry,
|
|
))
|
|
) {
|
|
authRetryPending = true;
|
|
continue;
|
|
}
|
|
if (imageDimensionError && lastProfileId) {
|
|
const details = [
|
|
imageDimensionError.messageIndex !== undefined
|
|
? `message=${imageDimensionError.messageIndex}`
|
|
: null,
|
|
imageDimensionError.contentIndex !== undefined
|
|
? `content=${imageDimensionError.contentIndex}`
|
|
: null,
|
|
imageDimensionError.maxDimensionPx !== undefined
|
|
? `limit=${imageDimensionError.maxDimensionPx}px`
|
|
: null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
log.warn(
|
|
`Profile ${lastProfileId} rejected image payload${details ? ` (${details})` : ""}.`,
|
|
);
|
|
}
|
|
|
|
// Rotate on timeout to try another account/model path in this turn,
|
|
// but exclude post-prompt compaction timeouts (model succeeded; no profile issue).
|
|
const shouldRotate =
|
|
(!aborted && failoverFailure) || (timedOut && !timedOutDuringCompaction);
|
|
|
|
if (shouldRotate) {
|
|
if (lastProfileId) {
|
|
const reason = timedOut ? "timeout" : assistantProfileFailureReason;
|
|
// Skip cooldown for timeouts: a timeout is model/network-specific,
|
|
// not an auth issue. Marking the profile would poison fallback models
|
|
// on the same provider (e.g. gpt-5.3 timeout blocks gpt-5.2).
|
|
await maybeMarkAuthProfileFailure({
|
|
profileId: lastProfileId,
|
|
reason,
|
|
modelId,
|
|
});
|
|
if (timedOut && !isProbeSession) {
|
|
log.warn(`Profile ${lastProfileId} timed out. Trying next account...`);
|
|
}
|
|
if (cloudCodeAssistFormatError) {
|
|
log.warn(
|
|
`Profile ${lastProfileId} hit Cloud Code Assist format error. Tool calls will be sanitized on retry.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const rotated = await advanceAuthProfile();
|
|
if (rotated) {
|
|
logAssistantFailoverDecision("rotate_profile");
|
|
await maybeBackoffBeforeOverloadFailover(assistantFailoverReason);
|
|
continue;
|
|
}
|
|
|
|
if (fallbackConfigured) {
|
|
await maybeBackoffBeforeOverloadFailover(assistantFailoverReason);
|
|
// Prefer formatted error message (user-friendly) over raw errorMessage
|
|
const message =
|
|
(lastAssistant
|
|
? formatAssistantErrorText(lastAssistant, {
|
|
cfg: params.config,
|
|
sessionKey: params.sessionKey ?? params.sessionId,
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
})
|
|
: undefined) ||
|
|
lastAssistant?.errorMessage?.trim() ||
|
|
(timedOut
|
|
? "LLM request timed out."
|
|
: rateLimitFailure
|
|
? "LLM request rate limited."
|
|
: billingFailure
|
|
? formatBillingErrorMessage(
|
|
activeErrorContext.provider,
|
|
activeErrorContext.model,
|
|
)
|
|
: authFailure
|
|
? "LLM request unauthorized."
|
|
: "LLM request failed.");
|
|
const status =
|
|
resolveFailoverStatus(assistantFailoverReason ?? "unknown") ??
|
|
(isTimeoutErrorMessage(message) ? 408 : undefined);
|
|
logAssistantFailoverDecision("fallback_model", { status });
|
|
throw new FailoverError(message, {
|
|
reason: assistantFailoverReason ?? "unknown",
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
profileId: lastProfileId,
|
|
status,
|
|
});
|
|
}
|
|
logAssistantFailoverDecision("surface_error");
|
|
}
|
|
|
|
const usageMeta = buildUsageAgentMetaFields({
|
|
usageAccumulator,
|
|
lastAssistantUsage: lastAssistant?.usage as UsageLike | undefined,
|
|
lastRunPromptUsage,
|
|
lastTurnTotal,
|
|
});
|
|
const agentMeta: EmbeddedPiAgentMeta = {
|
|
sessionId: sessionIdUsed,
|
|
provider: lastAssistant?.provider ?? provider,
|
|
model: lastAssistant?.model ?? model.id,
|
|
usage: usageMeta.usage,
|
|
lastCallUsage: usageMeta.lastCallUsage,
|
|
promptTokens: usageMeta.promptTokens,
|
|
compactionCount: autoCompactionCount > 0 ? autoCompactionCount : undefined,
|
|
};
|
|
|
|
const payloads = buildEmbeddedRunPayloads({
|
|
assistantTexts: attempt.assistantTexts,
|
|
toolMetas: attempt.toolMetas,
|
|
lastAssistant: attempt.lastAssistant,
|
|
lastToolError: attempt.lastToolError,
|
|
config: params.config,
|
|
sessionKey: params.sessionKey ?? params.sessionId,
|
|
provider: activeErrorContext.provider,
|
|
model: activeErrorContext.model,
|
|
verboseLevel: params.verboseLevel,
|
|
reasoningLevel: params.reasoningLevel,
|
|
toolResultFormat: resolvedToolResultFormat,
|
|
suppressToolErrorWarnings: params.suppressToolErrorWarnings,
|
|
inlineToolResultsAllowed: false,
|
|
didSendViaMessagingTool: attempt.didSendViaMessagingTool,
|
|
didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt,
|
|
});
|
|
|
|
// Timeout aborts can leave the run without any assistant payloads.
|
|
// Emit an explicit timeout error instead of silently completing, so
|
|
// callers do not lose the turn as an orphaned user message.
|
|
if (timedOut && !timedOutDuringCompaction && payloads.length === 0) {
|
|
return {
|
|
payloads: [
|
|
{
|
|
text:
|
|
"Request timed out before a response was generated. " +
|
|
"Please try again, or increase `agents.defaults.timeoutSeconds` in your config.",
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta,
|
|
aborted,
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
},
|
|
didSendViaMessagingTool: attempt.didSendViaMessagingTool,
|
|
didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt,
|
|
messagingToolSentTexts: attempt.messagingToolSentTexts,
|
|
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls,
|
|
messagingToolSentTargets: attempt.messagingToolSentTargets,
|
|
successfulCronAdds: attempt.successfulCronAdds,
|
|
};
|
|
}
|
|
|
|
// Detect incomplete turns where prompt() resolved prematurely due to
|
|
// pi-agent-core's auto-retry timing issue: when a mid-turn 429/overload
|
|
// triggers an internal retry, waitForRetry() resolves on the next
|
|
// assistant message *before* tool execution completes in the retried
|
|
// loop (see #8643). The captured lastAssistant has a non-terminal
|
|
// stopReason (e.g. "toolUse") with no text content, producing empty
|
|
// payloads. Surface an error instead of silently dropping the reply.
|
|
//
|
|
// Exclusions:
|
|
// - didSendDeterministicApprovalPrompt: approval-prompt turns
|
|
// intentionally produce empty payloads with stopReason=toolUse
|
|
// - lastToolError: suppressed/recoverable tool failures also produce
|
|
// empty payloads with stopReason=toolUse; those are handled by
|
|
// buildEmbeddedRunPayloads' own warning policy
|
|
if (
|
|
payloads.length === 0 &&
|
|
!aborted &&
|
|
!timedOut &&
|
|
!attempt.clientToolCall &&
|
|
!attempt.yieldDetected &&
|
|
!attempt.didSendDeterministicApprovalPrompt &&
|
|
!attempt.lastToolError
|
|
) {
|
|
const incompleteStopReason = lastAssistant?.stopReason;
|
|
// Only trigger for non-terminal stop reasons (toolUse, etc.) to
|
|
// avoid false positives when the model legitimately produces no text.
|
|
// StopReason union: "aborted" | "error" | "length" | "toolUse"
|
|
// "toolUse" is the key signal that prompt() resolved mid-turn.
|
|
if (incompleteStopReason === "toolUse" || incompleteStopReason === "error") {
|
|
log.warn(
|
|
`incomplete turn detected: runId=${params.runId} sessionId=${params.sessionId} ` +
|
|
`stopReason=${incompleteStopReason} payloads=0 — surfacing error to user`,
|
|
);
|
|
|
|
// Mark the failing profile for cooldown so multi-profile setups
|
|
// rotate away from the exhausted credential on the next turn.
|
|
if (lastProfileId) {
|
|
const failoverReason = classifyFailoverReason(lastAssistant?.errorMessage ?? "");
|
|
await maybeMarkAuthProfileFailure({
|
|
profileId: lastProfileId,
|
|
reason: resolveAuthProfileFailureReason(failoverReason),
|
|
});
|
|
}
|
|
|
|
// Warn about potential side-effects when mutating tools executed
|
|
// before the turn was interrupted, so users don't blindly retry.
|
|
const hadMutatingTools = attempt.toolMetas.some((t) =>
|
|
isLikelyMutatingToolName(t.toolName),
|
|
);
|
|
const errorText = hadMutatingTools
|
|
? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying."
|
|
: "⚠️ Agent couldn't generate a response. Please try again.";
|
|
|
|
return {
|
|
payloads: [
|
|
{
|
|
text: errorText,
|
|
isError: true,
|
|
},
|
|
],
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta,
|
|
aborted,
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
},
|
|
didSendViaMessagingTool: attempt.didSendViaMessagingTool,
|
|
didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt,
|
|
messagingToolSentTexts: attempt.messagingToolSentTexts,
|
|
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls,
|
|
messagingToolSentTargets: attempt.messagingToolSentTargets,
|
|
successfulCronAdds: attempt.successfulCronAdds,
|
|
};
|
|
}
|
|
}
|
|
|
|
log.debug(
|
|
`embedded run done: runId=${params.runId} sessionId=${params.sessionId} durationMs=${Date.now() - started} aborted=${aborted}`,
|
|
);
|
|
if (lastProfileId) {
|
|
await markAuthProfileGood({
|
|
store: authStore,
|
|
provider,
|
|
profileId: lastProfileId,
|
|
agentDir: params.agentDir,
|
|
});
|
|
await markAuthProfileUsed({
|
|
store: authStore,
|
|
profileId: lastProfileId,
|
|
agentDir: params.agentDir,
|
|
});
|
|
}
|
|
return {
|
|
payloads: payloads.length ? payloads : undefined,
|
|
meta: {
|
|
durationMs: Date.now() - started,
|
|
agentMeta,
|
|
aborted,
|
|
systemPromptReport: attempt.systemPromptReport,
|
|
// Handle client tool calls (OpenResponses hosted tools)
|
|
// Propagate the LLM stop reason so callers (lifecycle events,
|
|
// ACP bridge) can distinguish end_turn from max_tokens.
|
|
stopReason: attempt.clientToolCall
|
|
? "tool_calls"
|
|
: attempt.yieldDetected
|
|
? "end_turn"
|
|
: (lastAssistant?.stopReason as string | undefined),
|
|
pendingToolCalls: attempt.clientToolCall
|
|
? [
|
|
{
|
|
id: randomBytes(5).toString("hex").slice(0, 9),
|
|
name: attempt.clientToolCall.name,
|
|
arguments: JSON.stringify(attempt.clientToolCall.params),
|
|
},
|
|
]
|
|
: undefined,
|
|
},
|
|
didSendViaMessagingTool: attempt.didSendViaMessagingTool,
|
|
didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt,
|
|
messagingToolSentTexts: attempt.messagingToolSentTexts,
|
|
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls,
|
|
messagingToolSentTargets: attempt.messagingToolSentTargets,
|
|
successfulCronAdds: attempt.successfulCronAdds,
|
|
};
|
|
}
|
|
} finally {
|
|
await contextEngine.dispose?.();
|
|
stopRuntimeAuthRefreshTimer();
|
|
}
|
|
}),
|
|
);
|
|
}
|