fix: coalesce loop warning events

This commit is contained in:
Gustavo Madeira Santana
2026-02-16 14:36:05 -05:00
parent de3f446b92
commit ae0249c27a
5 changed files with 80 additions and 16 deletions

View File

@@ -20,7 +20,7 @@ Docs: https://docs.openclaw.ai
- Gateway/Config: prevent `config.patch` object-array merges from falling back to full-array replacement when some patch entries lack `id`, so partial `agents.list` updates no longer drop unrelated agents. (#17989) Thanks @stakeswky.
- Config/Discord: require string IDs in Discord allowlists, keep onboarding inputs string-only, and add doctor repair for numeric entries. (#18220) Thanks @thewilloftheshadow.
- Agents/Models: probe the primary model when its auth-profile cooldown is near expiry (with per-provider throttling), so runs recover from temporary rate limits without staying on fallback models until restart. (#17478) Thanks @PlayerGhost.
- Agents/Tools: make loop detection progress-aware and phased by hard-blocking known `process(action=poll|log)` no-progress loops, warning on generic identical-call repeats, warning + blocking ping-pong alternation loops (10/20), adding a global circuit breaker at 30 no-progress repeats, and emitting structured diagnostic `tool.loop` warning/error events for loop actions. (#16808) Thanks @akramcodez and @beca-oc.
- Agents/Tools: make loop detection progress-aware and phased by hard-blocking known `process(action=poll|log)` no-progress loops, warning on generic identical-call repeats, warning + blocking ping-pong alternation loops (10/20), coalescing repeated warning spam into threshold buckets, adding a global circuit breaker at 30 no-progress repeats, and emitting structured diagnostic `tool.loop` warning/error events for loop actions. (#16808) Thanks @akramcodez and @beca-oc.
- Agents/Tools: scope the `message` tool schema to the active channel so Telegram uses `buttons` and Discord uses `components`. (#18215) Thanks @obviyus.
- Discord: optimize reaction notification handling to skip unnecessary message fetches in `off`/`all`/`allowlist` modes, streamline reaction routing, and improve reaction emoji formatting. (#18248) Thanks @thewilloftheshadow and @victorGPT.
- Telegram: keep draft-stream preview replies attached to the user message for `replyToMode: "all"` in groups and DMs, preserving threaded reply context from preview through finalization. (#17880) Thanks @yinghaosang.

View File

@@ -112,6 +112,36 @@ describe("before_tool_call loop detection behavior", () => {
).rejects.toThrow("global circuit breaker");
});
it("coalesces repeated generic warning events into threshold buckets", async () => {
const emitted: DiagnosticToolLoopEvent[] = [];
const stop = onDiagnosticEvent((evt) => {
if (evt.type === "tool.loop" && evt.level === "warning") {
emitted.push(evt);
}
});
try {
const execute = vi.fn().mockResolvedValue({
content: [{ type: "text", text: "same output" }],
details: { ok: true },
});
// oxlint-disable-next-line typescript/no-explicit-any
const tool = wrapToolWithBeforeToolCallHook({ name: "read", execute } as any, {
agentId: "main",
sessionKey: "main",
});
const params = { path: "/tmp/file" };
for (let i = 0; i < 21; i += 1) {
await tool.execute(`read-bucket-${i}`, params, undefined, undefined);
}
const genericWarns = emitted.filter((evt) => evt.detector === "generic_repeat");
expect(genericWarns.map((evt) => evt.count)).toEqual([10, 20]);
} finally {
stop();
}
});
it("emits structured warning diagnostic events for ping-pong loops", async () => {
const emitted: DiagnosticToolLoopEvent[] = [];
const stop = onDiagnosticEvent((evt) => {

View File

@@ -1,3 +1,4 @@
import type { SessionState } from "../logging/diagnostic-session-state.js";
import type { AnyAgentTool } from "./tools/common.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
@@ -15,6 +16,27 @@ const log = createSubsystemLogger("agents/tools");
const BEFORE_TOOL_CALL_WRAPPED = Symbol("beforeToolCallWrapped");
const adjustedParamsByToolCallId = new Map<string, unknown>();
const MAX_TRACKED_ADJUSTED_PARAMS = 1024;
const LOOP_WARNING_BUCKET_SIZE = 10;
const MAX_LOOP_WARNING_KEYS = 256;
function shouldEmitLoopWarning(state: SessionState, warningKey: string, count: number): boolean {
if (!state.toolLoopWarningBuckets) {
state.toolLoopWarningBuckets = new Map();
}
const bucket = Math.floor(count / LOOP_WARNING_BUCKET_SIZE);
const lastBucket = state.toolLoopWarningBuckets.get(warningKey) ?? 0;
if (bucket <= lastBucket) {
return false;
}
state.toolLoopWarningBuckets.set(warningKey, bucket);
if (state.toolLoopWarningBuckets.size > MAX_LOOP_WARNING_KEYS) {
const oldest = state.toolLoopWarningBuckets.keys().next().value;
if (oldest) {
state.toolLoopWarningBuckets.delete(oldest);
}
}
return true;
}
async function recordLoopOutcome(args: {
ctx?: HookContext;
@@ -86,18 +108,21 @@ export async function runBeforeToolCallHook(args: {
reason: loopResult.message,
};
} else {
log.warn(`Loop warning for ${toolName}: ${loopResult.message}`);
logToolLoopAction({
sessionKey: args.ctx.sessionKey,
sessionId: args.ctx?.agentId,
toolName,
level: "warning",
action: "warn",
detector: loopResult.detector,
count: loopResult.count,
message: loopResult.message,
pairedToolName: loopResult.pairedToolName,
});
const warningKey = loopResult.warningKey ?? `${loopResult.detector}:${toolName}`;
if (shouldEmitLoopWarning(sessionState, warningKey, loopResult.count)) {
log.warn(`Loop warning for ${toolName}: ${loopResult.message}`);
logToolLoopAction({
sessionKey: args.ctx.sessionKey,
sessionId: args.ctx?.agentId,
toolName,
level: "warning",
action: "warn",
detector: loopResult.detector,
count: loopResult.count,
message: loopResult.message,
pairedToolName: loopResult.pairedToolName,
});
}
}
}

View File

@@ -20,6 +20,7 @@ export type LoopDetectionResult =
count: number;
message: string;
pairedToolName?: string;
warningKey?: string;
};
export const TOOL_CALL_HISTORY_SIZE = 30;
@@ -161,7 +162,7 @@ function getNoProgressStreak(
history: Array<{ toolName: string; argsHash: string; resultHash?: string }>,
toolName: string,
argsHash: string,
): number {
): { count: number; latestResultHash?: string } {
let streak = 0;
let latestResultHash: string | undefined;
@@ -184,7 +185,7 @@ function getNoProgressStreak(
streak += 1;
}
return streak;
return { count: streak, latestResultHash };
}
function getPingPongStreak(
@@ -253,7 +254,8 @@ export function detectToolCallLoop(
): LoopDetectionResult {
const history = state.toolCallHistory ?? [];
const currentHash = hashToolCall(toolName, params);
const noProgressStreak = getNoProgressStreak(history, toolName, currentHash);
const noProgress = getNoProgressStreak(history, toolName, currentHash);
const noProgressStreak = noProgress.count;
const knownPollTool = isKnownPollToolCall(toolName, params);
const pingPong = getPingPongStreak(history, currentHash);
@@ -267,6 +269,7 @@ export function detectToolCallLoop(
detector: "global_circuit_breaker",
count: noProgressStreak,
message: `CRITICAL: ${toolName} has repeated identical no-progress outcomes ${noProgressStreak} times. Session execution blocked by global circuit breaker to prevent runaway loops.`,
warningKey: `global:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
};
}
@@ -278,6 +281,7 @@ export function detectToolCallLoop(
detector: "known_poll_no_progress",
count: noProgressStreak,
message: `CRITICAL: Called ${toolName} with identical arguments and no progress ${noProgressStreak} times. This appears to be a stuck polling loop. Session execution blocked to prevent resource waste.`,
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
};
}
@@ -289,6 +293,7 @@ export function detectToolCallLoop(
detector: "known_poll_no_progress",
count: noProgressStreak,
message: `WARNING: You have called ${toolName} ${noProgressStreak} times with identical arguments and no progress. Stop polling and either (1) increase wait time between checks, or (2) report the task as failed if the process is stuck.`,
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
};
}
@@ -303,6 +308,7 @@ export function detectToolCallLoop(
count: pingPong.count,
message: `CRITICAL: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls). This appears to be a stuck ping-pong loop. Session execution blocked to prevent resource waste.`,
pairedToolName: pingPong.pairedToolName,
warningKey: `pingpong:${toolName}:${currentHash}:${pingPong.pairedToolName ?? "unknown"}`,
};
}
@@ -317,6 +323,7 @@ export function detectToolCallLoop(
count: pingPong.count,
message: `WARNING: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls). This looks like a ping-pong loop; stop retrying and report the task as failed.`,
pairedToolName: pingPong.pairedToolName,
warningKey: `pingpong:${toolName}:${currentHash}:${pingPong.pairedToolName ?? "unknown"}`,
};
}
@@ -333,6 +340,7 @@ export function detectToolCallLoop(
detector: "generic_repeat",
count: recentCount,
message: `WARNING: You have called ${toolName} ${recentCount} times with identical arguments. If this is not making progress, stop retrying and report the task as failed.`,
warningKey: `generic:${toolName}:${currentHash}`,
};
}

View File

@@ -7,6 +7,7 @@ export type SessionState = {
state: SessionStateValue;
queueDepth: number;
toolCallHistory?: ToolCallRecord[];
toolLoopWarningBuckets?: Map<string, number>;
commandPollCounts?: Map<string, { count: number; lastPollAt: number }>;
};