mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-29 05:11:12 +00:00
4ed28c3559
* refactor(agents): trim shell and session exports * refactor(agents): trim loop and model exports * chore(deadcode): refresh unused export baseline
822 lines
25 KiB
TypeScript
822 lines
25 KiB
TypeScript
/**
|
|
* Tool-call loop detection.
|
|
*
|
|
* Watches recent tool history for repeated no-progress patterns and circuit-breaker thresholds.
|
|
*/
|
|
import { createHash } from "node:crypto";
|
|
import {
|
|
normalizeNullableString as nonEmptyStringField,
|
|
normalizeOptionalString as normalizeRunId,
|
|
} from "@openclaw/normalization-core/string-coerce";
|
|
import type { ToolLoopDetectionConfig } from "../config/types.tools.js";
|
|
import type { SessionState, ToolCallRecord } from "../logging/diagnostic-session-state.js";
|
|
import { createSubsystemLogger } from "../logging/subsystem.js";
|
|
import { isPlainObject } from "../utils.js";
|
|
import { isMessagingToolSendAction } from "./embedded-agent-messaging.js";
|
|
import { stableStringify } from "./stable-stringify.js";
|
|
|
|
const log = createSubsystemLogger("agents/loop-detection");
|
|
|
|
type LoopDetectorKind =
|
|
| "generic_repeat"
|
|
| "unknown_tool_repeat"
|
|
| "known_poll_no_progress"
|
|
| "global_circuit_breaker"
|
|
| "ping_pong";
|
|
|
|
type LoopDetectionResult =
|
|
| { stuck: false }
|
|
| {
|
|
stuck: true;
|
|
level: "warning" | "critical";
|
|
detector: LoopDetectorKind;
|
|
count: number;
|
|
message: string;
|
|
pairedToolName?: string;
|
|
warningKey?: string;
|
|
};
|
|
|
|
const TOOL_CALL_HISTORY_SIZE = 30;
|
|
const WARNING_THRESHOLD = 10;
|
|
export const UNKNOWN_TOOL_THRESHOLD = 10;
|
|
const CRITICAL_THRESHOLD = 20;
|
|
const GLOBAL_CIRCUIT_BREAKER_THRESHOLD = 30;
|
|
const DEFAULT_LOOP_DETECTION_CONFIG = {
|
|
enabled: false,
|
|
historySize: TOOL_CALL_HISTORY_SIZE,
|
|
warningThreshold: WARNING_THRESHOLD,
|
|
unknownToolThreshold: UNKNOWN_TOOL_THRESHOLD,
|
|
criticalThreshold: CRITICAL_THRESHOLD,
|
|
globalCircuitBreakerThreshold: GLOBAL_CIRCUIT_BREAKER_THRESHOLD,
|
|
detectors: {
|
|
genericRepeat: true,
|
|
knownPollNoProgress: true,
|
|
pingPong: true,
|
|
},
|
|
};
|
|
|
|
type ResolvedLoopDetectionConfig = {
|
|
enabled: boolean;
|
|
historySize: number;
|
|
warningThreshold: number;
|
|
unknownToolThreshold: number;
|
|
criticalThreshold: number;
|
|
globalCircuitBreakerThreshold: number;
|
|
detectors: {
|
|
genericRepeat: boolean;
|
|
knownPollNoProgress: boolean;
|
|
pingPong: boolean;
|
|
};
|
|
};
|
|
|
|
type ToolLoopDetectionScope = {
|
|
runId?: string;
|
|
};
|
|
|
|
function selectHistoryForScope(
|
|
history: readonly ToolCallRecord[],
|
|
scope?: ToolLoopDetectionScope,
|
|
): ToolCallRecord[] {
|
|
const runId = normalizeRunId(scope?.runId);
|
|
return history.filter((record) => normalizeRunId(record.runId) === runId);
|
|
}
|
|
|
|
function asPositiveInt(value: number | undefined, fallback: number): number {
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
|
|
return fallback;
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function resolveLoopDetectionConfig(config?: ToolLoopDetectionConfig): ResolvedLoopDetectionConfig {
|
|
const warningThreshold = asPositiveInt(
|
|
config?.warningThreshold,
|
|
DEFAULT_LOOP_DETECTION_CONFIG.warningThreshold,
|
|
);
|
|
let criticalThreshold = asPositiveInt(
|
|
config?.criticalThreshold,
|
|
DEFAULT_LOOP_DETECTION_CONFIG.criticalThreshold,
|
|
);
|
|
let globalCircuitBreakerThreshold = asPositiveInt(
|
|
config?.globalCircuitBreakerThreshold,
|
|
DEFAULT_LOOP_DETECTION_CONFIG.globalCircuitBreakerThreshold,
|
|
);
|
|
|
|
if (criticalThreshold <= warningThreshold) {
|
|
criticalThreshold = warningThreshold + 1;
|
|
}
|
|
if (globalCircuitBreakerThreshold <= criticalThreshold) {
|
|
globalCircuitBreakerThreshold = criticalThreshold + 1;
|
|
}
|
|
|
|
return {
|
|
enabled: config?.enabled ?? DEFAULT_LOOP_DETECTION_CONFIG.enabled,
|
|
historySize: asPositiveInt(config?.historySize, DEFAULT_LOOP_DETECTION_CONFIG.historySize),
|
|
warningThreshold,
|
|
unknownToolThreshold: asPositiveInt(
|
|
config?.unknownToolThreshold,
|
|
DEFAULT_LOOP_DETECTION_CONFIG.unknownToolThreshold,
|
|
),
|
|
criticalThreshold,
|
|
globalCircuitBreakerThreshold,
|
|
detectors: {
|
|
genericRepeat:
|
|
config?.detectors?.genericRepeat ?? DEFAULT_LOOP_DETECTION_CONFIG.detectors.genericRepeat,
|
|
knownPollNoProgress:
|
|
config?.detectors?.knownPollNoProgress ??
|
|
DEFAULT_LOOP_DETECTION_CONFIG.detectors.knownPollNoProgress,
|
|
pingPong: config?.detectors?.pingPong ?? DEFAULT_LOOP_DETECTION_CONFIG.detectors.pingPong,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Hash a tool call for pattern matching.
|
|
* Uses tool name + deterministic JSON serialization digest of params.
|
|
*/
|
|
function hashToolCall(toolName: string, params: unknown): string {
|
|
return `${toolName}:${digestStable(params)}`;
|
|
}
|
|
|
|
function digestStable(value: unknown): string {
|
|
const serialized = stableStringify(value);
|
|
return createHash("sha256").update(serialized).digest("hex");
|
|
}
|
|
|
|
function isKnownPollToolCall(toolName: string, params: unknown): boolean {
|
|
if (toolName === "command_status") {
|
|
return true;
|
|
}
|
|
if (toolName !== "process" || !isPlainObject(params)) {
|
|
return false;
|
|
}
|
|
const action = params.action;
|
|
return action === "poll" || action === "log";
|
|
}
|
|
|
|
function extractTextContent(result: unknown): string {
|
|
if (!isPlainObject(result) || !Array.isArray(result.content)) {
|
|
return "";
|
|
}
|
|
return result.content
|
|
.filter(
|
|
(entry): entry is { type: string; text: string } =>
|
|
isPlainObject(entry) && typeof entry.type === "string" && typeof entry.text === "string",
|
|
)
|
|
.map((entry) => entry.text)
|
|
.join("\n")
|
|
.trim();
|
|
}
|
|
|
|
function formatErrorForHash(error: unknown): string {
|
|
if (error instanceof Error) {
|
|
return error.message || error.name;
|
|
}
|
|
if (typeof error === "string") {
|
|
return error;
|
|
}
|
|
if (typeof error === "number" || typeof error === "boolean" || typeof error === "bigint") {
|
|
return `${error}`;
|
|
}
|
|
return stableStringify(error);
|
|
}
|
|
|
|
function extractUnknownToolName(error: unknown): string | undefined {
|
|
const raw = formatErrorForHash(error).trim();
|
|
if (!raw) {
|
|
return undefined;
|
|
}
|
|
const match =
|
|
raw.match(/unknown tool[:\s]+["']?([a-z0-9_.-]+)["']?/i) ??
|
|
raw.match(/tool\s+["']?([a-z0-9_.-]+)["']?\s+(?:not found|is not available)/i);
|
|
const toolName = match?.[1]?.trim();
|
|
return toolName ? toolName.toLowerCase() : undefined;
|
|
}
|
|
|
|
function stringField(value: unknown): string | null {
|
|
return typeof value === "string" ? value : null;
|
|
}
|
|
|
|
function hashExecToolOutcome(details: Record<string, unknown>, text: string): string | undefined {
|
|
const status = stringField(details.status);
|
|
if (!status) {
|
|
return undefined;
|
|
}
|
|
|
|
if (status === "running") {
|
|
return digestStable({
|
|
status,
|
|
tail: stringField(details.tail) ?? "",
|
|
});
|
|
}
|
|
|
|
if (status === "completed" || status === "failed") {
|
|
return digestStable({
|
|
status,
|
|
exitCode: typeof details.exitCode === "number" ? details.exitCode : null,
|
|
timedOut: details.timedOut === true,
|
|
output: nonEmptyStringField(details.aggregated) ?? text,
|
|
});
|
|
}
|
|
|
|
if (status === "approval-pending" || status === "approval-unavailable") {
|
|
return digestStable({
|
|
status,
|
|
reason: stringField(details.reason),
|
|
host: stringField(details.host),
|
|
command: stringField(details.command) ?? "",
|
|
warningText: stringField(details.warningText) ?? "",
|
|
});
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
// Delivery results carry fresh per-call ids (messageId/runId) in details and text, so
|
|
// hashing them defeats no-progress loop blocking (#89090). Hash only id-stripped facts
|
|
// for outbound-message actions; other `message` actions keep full hashing (real progress).
|
|
const SEND_LIKE_MESSAGE_ACTIONS = new Set([
|
|
"send",
|
|
"broadcast",
|
|
"reply",
|
|
"thread-reply",
|
|
"sendWithEffect",
|
|
"sendAttachment",
|
|
"upload-file",
|
|
"sticker",
|
|
"poll",
|
|
]);
|
|
// Denylist of per-call volatile delivery ids/timestamps stripped before hashing. Must
|
|
// cover the id/timestamp fields a channel's delivery result can carry; a new channel
|
|
// emitting a volatile field name outside this set silently regresses its loop blocking.
|
|
const VOLATILE_SEND_RESULT_KEYS = new Set([
|
|
"messageId",
|
|
"message_id",
|
|
"messageIds",
|
|
"platformMessageId",
|
|
"platformMessageIds",
|
|
"fileId",
|
|
"file_id",
|
|
"fileKey",
|
|
"pollId",
|
|
"poll_id",
|
|
"receipt",
|
|
"runId",
|
|
"idempotencyKey",
|
|
"ts",
|
|
"timestamp",
|
|
"sentAt",
|
|
"deliveredAt",
|
|
"createdAt",
|
|
]);
|
|
|
|
// A message object's own `id` is its volatile per-send id; a bare `id` elsewhere
|
|
// (route/conversation) is a stable fact, so only strip `id` on the message object.
|
|
function isMessageDeliveryObject(value: Record<string, unknown>): boolean {
|
|
return (
|
|
typeof value.id === "string" &&
|
|
typeof value.text === "string" &&
|
|
(typeof value.direction === "string" ||
|
|
typeof value.senderId === "string" ||
|
|
typeof value.accountId === "string" ||
|
|
isPlainObject(value.conversation))
|
|
);
|
|
}
|
|
|
|
function stripVolatileSendIds(value: unknown): unknown {
|
|
if (Array.isArray(value)) {
|
|
return value.map(stripVolatileSendIds);
|
|
}
|
|
if (!isPlainObject(value)) {
|
|
return value;
|
|
}
|
|
const dropMessageObjectId = isMessageDeliveryObject(value);
|
|
const stripped: Record<string, unknown> = {};
|
|
for (const [key, nested] of Object.entries(value)) {
|
|
if (VOLATILE_SEND_RESULT_KEYS.has(key) || (key === "id" && dropMessageObjectId)) {
|
|
continue;
|
|
}
|
|
stripped[key] = stripVolatileSendIds(nested);
|
|
}
|
|
return stripped;
|
|
}
|
|
|
|
function isVolatileSendResult(toolName: string, params: unknown): boolean {
|
|
if (toolName === "sessions_send") {
|
|
return true;
|
|
}
|
|
const args = isPlainObject(params) ? params : {};
|
|
if (toolName === "message") {
|
|
return typeof args.action === "string" && SEND_LIKE_MESSAGE_ACTIONS.has(args.action);
|
|
}
|
|
// Provider-docked send tools (telegram/discord/...) return the same volatile-id shape.
|
|
// SEND_LIKE_MESSAGE_ACTIONS stays broader than the terminal-send set on purpose:
|
|
// broadcast/reply/sticker/poll carry volatile ids but are not terminal sends.
|
|
return isMessagingToolSendAction(toolName, args);
|
|
}
|
|
|
|
// Only the loop detector's own veto must not reset the streak; other blocked results
|
|
// (plugin/approval vetoes) keep a hash so repeated identical denials still escalate.
|
|
function isLoopVetoResult(details: Record<string, unknown>): boolean {
|
|
return details.status === "blocked" && details.deniedReason === "tool-loop";
|
|
}
|
|
|
|
function hashToolOutcome(
|
|
toolName: string,
|
|
params: unknown,
|
|
result: unknown,
|
|
error: unknown,
|
|
): { resultHash?: string; unknownToolName?: string } {
|
|
if (error !== undefined) {
|
|
const unknownToolName = extractUnknownToolName(error);
|
|
return {
|
|
resultHash: `error:${digestStable(formatErrorForHash(error))}`,
|
|
unknownToolName,
|
|
};
|
|
}
|
|
if (!isPlainObject(result)) {
|
|
return { resultHash: result === undefined ? undefined : digestStable(result) };
|
|
}
|
|
|
|
const details = isPlainObject(result.details) ? result.details : {};
|
|
const text = extractTextContent(result);
|
|
// The loop detector's own veto is not real progress; giving it no result hash keeps a
|
|
// critical loop block sticky instead of letting the block reset the streak (#89090).
|
|
if (isLoopVetoResult(details)) {
|
|
return { resultHash: undefined };
|
|
}
|
|
if (toolName === "exec") {
|
|
const execHash = hashExecToolOutcome(details, text);
|
|
if (execHash) {
|
|
return { resultHash: execHash };
|
|
}
|
|
}
|
|
if (isKnownPollToolCall(toolName, params) && toolName === "process" && isPlainObject(params)) {
|
|
const action = params.action;
|
|
if (action === "poll") {
|
|
return {
|
|
resultHash: digestStable({
|
|
action,
|
|
status: details.status,
|
|
exitCode: details.exitCode ?? null,
|
|
exitSignal: details.exitSignal ?? null,
|
|
aggregated: details.aggregated ?? null,
|
|
text,
|
|
}),
|
|
};
|
|
}
|
|
if (action === "log") {
|
|
return {
|
|
resultHash: digestStable({
|
|
action,
|
|
status: details.status,
|
|
totalLines: details.totalLines ?? null,
|
|
totalChars: details.totalChars ?? null,
|
|
truncated: details.truncated ?? null,
|
|
exitCode: details.exitCode ?? null,
|
|
exitSignal: details.exitSignal ?? null,
|
|
text,
|
|
}),
|
|
};
|
|
}
|
|
}
|
|
|
|
if (isVolatileSendResult(toolName, params)) {
|
|
return { resultHash: digestStable(stripVolatileSendIds(details)) };
|
|
}
|
|
|
|
return {
|
|
resultHash: digestStable({
|
|
details,
|
|
text,
|
|
}),
|
|
};
|
|
}
|
|
|
|
function getUnknownToolRepeatStreak(
|
|
history: Array<{ toolName: string; unknownToolName?: string }>,
|
|
toolName: string,
|
|
): { count: number; unknownToolName?: string } {
|
|
let streak = 0;
|
|
let repeatedUnknownToolName: string | undefined;
|
|
|
|
for (let i = history.length - 1; i >= 0; i -= 1) {
|
|
const record = history[i];
|
|
if (!record || record.toolName !== toolName || !record.unknownToolName) {
|
|
break;
|
|
}
|
|
if (!repeatedUnknownToolName) {
|
|
repeatedUnknownToolName = record.unknownToolName;
|
|
streak = 1;
|
|
continue;
|
|
}
|
|
if (record.unknownToolName !== repeatedUnknownToolName) {
|
|
break;
|
|
}
|
|
streak += 1;
|
|
}
|
|
|
|
return { count: streak, unknownToolName: repeatedUnknownToolName };
|
|
}
|
|
|
|
function getNoProgressStreak(
|
|
history: Array<{ toolName: string; argsHash: string; resultHash?: string }>,
|
|
toolName: string,
|
|
argsHash: string,
|
|
): { count: number; latestResultHash?: string } {
|
|
let streak = 0;
|
|
let latestResultHash: string | undefined;
|
|
|
|
for (let i = history.length - 1; i >= 0; i -= 1) {
|
|
const record = history[i];
|
|
if (!record || record.toolName !== toolName || record.argsHash !== argsHash) {
|
|
continue;
|
|
}
|
|
if (typeof record.resultHash !== "string" || !record.resultHash) {
|
|
continue;
|
|
}
|
|
if (!latestResultHash) {
|
|
latestResultHash = record.resultHash;
|
|
streak = 1;
|
|
continue;
|
|
}
|
|
if (record.resultHash !== latestResultHash) {
|
|
break;
|
|
}
|
|
streak += 1;
|
|
}
|
|
|
|
return { count: streak, latestResultHash };
|
|
}
|
|
|
|
function getPingPongStreak(
|
|
history: Array<{ toolName: string; argsHash: string; resultHash?: string }>,
|
|
currentSignature: string,
|
|
): {
|
|
count: number;
|
|
pairedToolName?: string;
|
|
pairedSignature?: string;
|
|
noProgressEvidence: boolean;
|
|
} {
|
|
const last = history.at(-1);
|
|
if (!last) {
|
|
return { count: 0, noProgressEvidence: false };
|
|
}
|
|
|
|
let otherSignature: string | undefined;
|
|
let otherToolName: string | undefined;
|
|
for (let i = history.length - 2; i >= 0; i -= 1) {
|
|
const call = history[i];
|
|
if (!call) {
|
|
continue;
|
|
}
|
|
if (call.argsHash !== last.argsHash) {
|
|
otherSignature = call.argsHash;
|
|
otherToolName = call.toolName;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!otherSignature || !otherToolName) {
|
|
return { count: 0, noProgressEvidence: false };
|
|
}
|
|
|
|
let alternatingTailCount = 0;
|
|
for (let i = history.length - 1; i >= 0; i -= 1) {
|
|
const call = history[i];
|
|
if (!call) {
|
|
continue;
|
|
}
|
|
const expected = alternatingTailCount % 2 === 0 ? last.argsHash : otherSignature;
|
|
if (call.argsHash !== expected) {
|
|
break;
|
|
}
|
|
alternatingTailCount += 1;
|
|
}
|
|
|
|
if (alternatingTailCount < 2) {
|
|
return { count: 0, noProgressEvidence: false };
|
|
}
|
|
|
|
const expectedCurrentSignature = otherSignature;
|
|
if (currentSignature !== expectedCurrentSignature) {
|
|
return { count: 0, noProgressEvidence: false };
|
|
}
|
|
|
|
const tailStart = Math.max(0, history.length - alternatingTailCount);
|
|
let firstHashA: string | undefined;
|
|
let firstHashB: string | undefined;
|
|
let noProgressEvidence = true;
|
|
for (let i = tailStart; i < history.length; i += 1) {
|
|
const call = history[i];
|
|
if (!call) {
|
|
continue;
|
|
}
|
|
if (!call.resultHash) {
|
|
noProgressEvidence = false;
|
|
break;
|
|
}
|
|
if (call.argsHash === last.argsHash) {
|
|
if (!firstHashA) {
|
|
firstHashA = call.resultHash;
|
|
} else if (firstHashA !== call.resultHash) {
|
|
noProgressEvidence = false;
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
if (call.argsHash === otherSignature) {
|
|
if (!firstHashB) {
|
|
firstHashB = call.resultHash;
|
|
} else if (firstHashB !== call.resultHash) {
|
|
noProgressEvidence = false;
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
noProgressEvidence = false;
|
|
break;
|
|
}
|
|
|
|
// Need repeated stable outcomes on both sides before treating ping-pong as no-progress.
|
|
if (!firstHashA || !firstHashB) {
|
|
noProgressEvidence = false;
|
|
}
|
|
|
|
return {
|
|
count: alternatingTailCount + 1,
|
|
pairedToolName: last.toolName,
|
|
pairedSignature: last.argsHash,
|
|
noProgressEvidence,
|
|
};
|
|
}
|
|
|
|
function canonicalPairKey(signatureA: string, signatureB: string): string {
|
|
return [signatureA, signatureB].toSorted().join("|");
|
|
}
|
|
|
|
/**
|
|
* Detect if an agent is stuck in a repetitive tool call loop.
|
|
* Checks if the same tool+params combination has been called excessively.
|
|
*/
|
|
export function detectToolCallLoop(
|
|
state: SessionState,
|
|
toolName: string,
|
|
params: unknown,
|
|
config?: ToolLoopDetectionConfig,
|
|
scope?: ToolLoopDetectionScope,
|
|
): LoopDetectionResult {
|
|
const resolvedConfig = resolveLoopDetectionConfig(config);
|
|
if (!resolvedConfig.enabled) {
|
|
return { stuck: false };
|
|
}
|
|
const history = selectHistoryForScope(state.toolCallHistory ?? [], scope);
|
|
const currentHash = hashToolCall(toolName, params);
|
|
const unknownToolStreak = getUnknownToolRepeatStreak(history, toolName);
|
|
const noProgress = getNoProgressStreak(history, toolName, currentHash);
|
|
const noProgressStreak = noProgress.count;
|
|
const knownPollTool = isKnownPollToolCall(toolName, params);
|
|
const pingPong = getPingPongStreak(history, currentHash);
|
|
|
|
if (unknownToolStreak.count >= resolvedConfig.unknownToolThreshold) {
|
|
return {
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "unknown_tool_repeat",
|
|
count: unknownToolStreak.count,
|
|
message: `CRITICAL: attempted unavailable tool ${unknownToolStreak.unknownToolName ?? toolName} ${unknownToolStreak.count} times. Stop retrying that missing tool and answer without it.`,
|
|
warningKey: `unknown-tool:${toolName}:${unknownToolStreak.unknownToolName ?? "unknown"}`,
|
|
};
|
|
}
|
|
|
|
if (noProgressStreak >= resolvedConfig.globalCircuitBreakerThreshold) {
|
|
log.error(
|
|
`Global circuit breaker triggered: ${toolName} repeated ${noProgressStreak} times with no progress`,
|
|
);
|
|
return {
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "global_circuit_breaker",
|
|
count: noProgressStreak,
|
|
message: `CRITICAL: ${toolName} has repeated identical no-progress outcomes ${noProgressStreak} times. Session execution blocked by global circuit breaker to prevent runaway loops.`,
|
|
warningKey: `global:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
|
};
|
|
}
|
|
|
|
if (
|
|
knownPollTool &&
|
|
resolvedConfig.detectors.knownPollNoProgress &&
|
|
noProgressStreak >= resolvedConfig.criticalThreshold
|
|
) {
|
|
log.error(`Critical polling loop detected: ${toolName} repeated ${noProgressStreak} times`);
|
|
return {
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "known_poll_no_progress",
|
|
count: noProgressStreak,
|
|
message: `CRITICAL: Called ${toolName} with identical arguments and no progress ${noProgressStreak} times. This appears to be a stuck polling loop. Session execution blocked to prevent resource waste.`,
|
|
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
|
};
|
|
}
|
|
|
|
if (
|
|
knownPollTool &&
|
|
resolvedConfig.detectors.knownPollNoProgress &&
|
|
noProgressStreak >= resolvedConfig.warningThreshold
|
|
) {
|
|
log.warn(`Polling loop warning: ${toolName} repeated ${noProgressStreak} times`);
|
|
return {
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "known_poll_no_progress",
|
|
count: noProgressStreak,
|
|
message: `WARNING: You have called ${toolName} ${noProgressStreak} times with identical arguments and no progress. Stop polling and either (1) increase wait time between checks, or (2) report the task as failed if the process is stuck.`,
|
|
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
|
};
|
|
}
|
|
|
|
const pingPongWarningKey = pingPong.pairedSignature
|
|
? `pingpong:${canonicalPairKey(currentHash, pingPong.pairedSignature)}`
|
|
: `pingpong:${toolName}:${currentHash}`;
|
|
|
|
if (
|
|
resolvedConfig.detectors.pingPong &&
|
|
pingPong.count >= resolvedConfig.criticalThreshold &&
|
|
pingPong.noProgressEvidence
|
|
) {
|
|
log.error(
|
|
`Critical ping-pong loop detected: alternating calls count=${pingPong.count} currentTool=${toolName}`,
|
|
);
|
|
return {
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "ping_pong",
|
|
count: pingPong.count,
|
|
message: `CRITICAL: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls) with no progress. This appears to be a stuck ping-pong loop. Session execution blocked to prevent resource waste.`,
|
|
pairedToolName: pingPong.pairedToolName,
|
|
warningKey: pingPongWarningKey,
|
|
};
|
|
}
|
|
|
|
if (resolvedConfig.detectors.pingPong && pingPong.count >= resolvedConfig.warningThreshold) {
|
|
log.warn(
|
|
`Ping-pong loop warning: alternating calls count=${pingPong.count} currentTool=${toolName}`,
|
|
);
|
|
return {
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "ping_pong",
|
|
count: pingPong.count,
|
|
message: `WARNING: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls). This looks like a ping-pong loop; stop retrying and report the task as failed.`,
|
|
pairedToolName: pingPong.pairedToolName,
|
|
warningKey: pingPongWarningKey,
|
|
};
|
|
}
|
|
|
|
// Generic detector: warn on repeated identical calls, then block only after
|
|
// outcomes prove the calls are not making progress.
|
|
const recentCount = history.filter(
|
|
(h) => h.toolName === toolName && h.argsHash === currentHash,
|
|
).length;
|
|
|
|
if (
|
|
!knownPollTool &&
|
|
resolvedConfig.detectors.genericRepeat &&
|
|
noProgressStreak >= resolvedConfig.criticalThreshold
|
|
) {
|
|
log.error(`Critical generic loop detected: ${toolName} repeated ${noProgressStreak} times`);
|
|
return {
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "generic_repeat",
|
|
count: noProgressStreak,
|
|
message: `CRITICAL: Called ${toolName} with identical arguments and identical outcomes ${noProgressStreak} times. Session execution blocked to prevent runaway loops.`,
|
|
warningKey: `generic:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
|
};
|
|
}
|
|
|
|
if (
|
|
!knownPollTool &&
|
|
resolvedConfig.detectors.genericRepeat &&
|
|
recentCount >= resolvedConfig.warningThreshold
|
|
) {
|
|
log.warn(`Loop warning: ${toolName} called ${recentCount} times with identical arguments`);
|
|
return {
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "generic_repeat",
|
|
count: recentCount,
|
|
message: `WARNING: You have called ${toolName} ${recentCount} times with identical arguments. If this is not making progress, stop retrying and report the task as failed.`,
|
|
warningKey: `generic:${toolName}:${currentHash}`,
|
|
};
|
|
}
|
|
|
|
return { stuck: false };
|
|
}
|
|
|
|
/**
|
|
* Record a tool call in the session's history for loop detection.
|
|
* Maintains sliding window of last N calls.
|
|
*/
|
|
export function recordToolCall(
|
|
state: SessionState,
|
|
toolName: string,
|
|
params: unknown,
|
|
toolCallId?: string,
|
|
config?: ToolLoopDetectionConfig,
|
|
scope?: ToolLoopDetectionScope,
|
|
): void {
|
|
const resolvedConfig = resolveLoopDetectionConfig(config);
|
|
const runId = normalizeRunId(scope?.runId);
|
|
if (!state.toolCallHistory) {
|
|
state.toolCallHistory = [];
|
|
}
|
|
|
|
state.toolCallHistory.push({
|
|
toolName,
|
|
argsHash: hashToolCall(toolName, params),
|
|
toolCallId,
|
|
...(runId && { runId }),
|
|
timestamp: Date.now(),
|
|
});
|
|
|
|
if (state.toolCallHistory.length > resolvedConfig.historySize) {
|
|
state.toolCallHistory.splice(0, state.toolCallHistory.length - resolvedConfig.historySize);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Record a completed tool call outcome so loop detection can identify no-progress repeats.
|
|
*/
|
|
export function recordToolCallOutcome(
|
|
state: SessionState,
|
|
params: {
|
|
toolName: string;
|
|
toolParams: unknown;
|
|
toolCallId?: string;
|
|
result?: unknown;
|
|
error?: unknown;
|
|
config?: ToolLoopDetectionConfig;
|
|
runId?: string;
|
|
},
|
|
): ToolCallRecord | undefined {
|
|
const resolvedConfig = resolveLoopDetectionConfig(params.config);
|
|
const runId = normalizeRunId(params.runId);
|
|
const outcome = hashToolOutcome(params.toolName, params.toolParams, params.result, params.error);
|
|
const resultHash = outcome.resultHash;
|
|
if (!resultHash) {
|
|
return undefined;
|
|
}
|
|
|
|
if (!state.toolCallHistory) {
|
|
state.toolCallHistory = [];
|
|
}
|
|
|
|
const argsHash = hashToolCall(params.toolName, params.toolParams);
|
|
let matched = false;
|
|
let recordedOutcome: ToolCallRecord | undefined;
|
|
for (let i = state.toolCallHistory.length - 1; i >= 0; i -= 1) {
|
|
const call = state.toolCallHistory[i];
|
|
if (!call) {
|
|
continue;
|
|
}
|
|
if (normalizeRunId(call.runId) !== runId) {
|
|
continue;
|
|
}
|
|
if (params.toolCallId && call.toolCallId !== params.toolCallId) {
|
|
continue;
|
|
}
|
|
if (call.toolName !== params.toolName || call.argsHash !== argsHash) {
|
|
continue;
|
|
}
|
|
if (call.resultHash !== undefined) {
|
|
continue;
|
|
}
|
|
call.resultHash = resultHash;
|
|
call.unknownToolName = outcome.unknownToolName;
|
|
matched = true;
|
|
recordedOutcome = call;
|
|
break;
|
|
}
|
|
|
|
if (!matched) {
|
|
const record: ToolCallRecord = {
|
|
toolName: params.toolName,
|
|
argsHash,
|
|
toolCallId: params.toolCallId,
|
|
...(runId && { runId }),
|
|
resultHash,
|
|
unknownToolName: outcome.unknownToolName,
|
|
timestamp: Date.now(),
|
|
};
|
|
state.toolCallHistory.push(record);
|
|
recordedOutcome = record;
|
|
}
|
|
|
|
if (state.toolCallHistory.length > resolvedConfig.historySize) {
|
|
state.toolCallHistory.splice(0, state.toolCallHistory.length - resolvedConfig.historySize);
|
|
}
|
|
return recordedOutcome;
|
|
}
|
|
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|