mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-29 01:52:04 +00:00
fix: coalesce loop warning events
This commit is contained in:
@@ -20,7 +20,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Gateway/Config: prevent `config.patch` object-array merges from falling back to full-array replacement when some patch entries lack `id`, so partial `agents.list` updates no longer drop unrelated agents. (#17989) Thanks @stakeswky.
|
||||
- Config/Discord: require string IDs in Discord allowlists, keep onboarding inputs string-only, and add doctor repair for numeric entries. (#18220) Thanks @thewilloftheshadow.
|
||||
- Agents/Models: probe the primary model when its auth-profile cooldown is near expiry (with per-provider throttling), so runs recover from temporary rate limits without staying on fallback models until restart. (#17478) Thanks @PlayerGhost.
|
||||
- Agents/Tools: make loop detection progress-aware and phased by hard-blocking known `process(action=poll|log)` no-progress loops, warning on generic identical-call repeats, warning + blocking ping-pong alternation loops (10/20), adding a global circuit breaker at 30 no-progress repeats, and emitting structured diagnostic `tool.loop` warning/error events for loop actions. (#16808) Thanks @akramcodez and @beca-oc.
|
||||
- Agents/Tools: make loop detection progress-aware and phased by hard-blocking known `process(action=poll|log)` no-progress loops, warning on generic identical-call repeats, warning + blocking ping-pong alternation loops (10/20), coalescing repeated warning spam into threshold buckets, adding a global circuit breaker at 30 no-progress repeats, and emitting structured diagnostic `tool.loop` warning/error events for loop actions. (#16808) Thanks @akramcodez and @beca-oc.
|
||||
- Agents/Tools: scope the `message` tool schema to the active channel so Telegram uses `buttons` and Discord uses `components`. (#18215) Thanks @obviyus.
|
||||
- Discord: optimize reaction notification handling to skip unnecessary message fetches in `off`/`all`/`allowlist` modes, streamline reaction routing, and improve reaction emoji formatting. (#18248) Thanks @thewilloftheshadow and @victorGPT.
|
||||
- Telegram: keep draft-stream preview replies attached to the user message for `replyToMode: "all"` in groups and DMs, preserving threaded reply context from preview through finalization. (#17880) Thanks @yinghaosang.
|
||||
|
||||
@@ -112,6 +112,36 @@ describe("before_tool_call loop detection behavior", () => {
|
||||
).rejects.toThrow("global circuit breaker");
|
||||
});
|
||||
|
||||
it("coalesces repeated generic warning events into threshold buckets", async () => {
|
||||
const emitted: DiagnosticToolLoopEvent[] = [];
|
||||
const stop = onDiagnosticEvent((evt) => {
|
||||
if (evt.type === "tool.loop" && evt.level === "warning") {
|
||||
emitted.push(evt);
|
||||
}
|
||||
});
|
||||
try {
|
||||
const execute = vi.fn().mockResolvedValue({
|
||||
content: [{ type: "text", text: "same output" }],
|
||||
details: { ok: true },
|
||||
});
|
||||
// oxlint-disable-next-line typescript/no-explicit-any
|
||||
const tool = wrapToolWithBeforeToolCallHook({ name: "read", execute } as any, {
|
||||
agentId: "main",
|
||||
sessionKey: "main",
|
||||
});
|
||||
const params = { path: "/tmp/file" };
|
||||
|
||||
for (let i = 0; i < 21; i += 1) {
|
||||
await tool.execute(`read-bucket-${i}`, params, undefined, undefined);
|
||||
}
|
||||
|
||||
const genericWarns = emitted.filter((evt) => evt.detector === "generic_repeat");
|
||||
expect(genericWarns.map((evt) => evt.count)).toEqual([10, 20]);
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("emits structured warning diagnostic events for ping-pong loops", async () => {
|
||||
const emitted: DiagnosticToolLoopEvent[] = [];
|
||||
const stop = onDiagnosticEvent((evt) => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SessionState } from "../logging/diagnostic-session-state.js";
|
||||
import type { AnyAgentTool } from "./tools/common.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
@@ -15,6 +16,27 @@ const log = createSubsystemLogger("agents/tools");
|
||||
const BEFORE_TOOL_CALL_WRAPPED = Symbol("beforeToolCallWrapped");
|
||||
const adjustedParamsByToolCallId = new Map<string, unknown>();
|
||||
const MAX_TRACKED_ADJUSTED_PARAMS = 1024;
|
||||
const LOOP_WARNING_BUCKET_SIZE = 10;
|
||||
const MAX_LOOP_WARNING_KEYS = 256;
|
||||
|
||||
function shouldEmitLoopWarning(state: SessionState, warningKey: string, count: number): boolean {
|
||||
if (!state.toolLoopWarningBuckets) {
|
||||
state.toolLoopWarningBuckets = new Map();
|
||||
}
|
||||
const bucket = Math.floor(count / LOOP_WARNING_BUCKET_SIZE);
|
||||
const lastBucket = state.toolLoopWarningBuckets.get(warningKey) ?? 0;
|
||||
if (bucket <= lastBucket) {
|
||||
return false;
|
||||
}
|
||||
state.toolLoopWarningBuckets.set(warningKey, bucket);
|
||||
if (state.toolLoopWarningBuckets.size > MAX_LOOP_WARNING_KEYS) {
|
||||
const oldest = state.toolLoopWarningBuckets.keys().next().value;
|
||||
if (oldest) {
|
||||
state.toolLoopWarningBuckets.delete(oldest);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function recordLoopOutcome(args: {
|
||||
ctx?: HookContext;
|
||||
@@ -86,18 +108,21 @@ export async function runBeforeToolCallHook(args: {
|
||||
reason: loopResult.message,
|
||||
};
|
||||
} else {
|
||||
log.warn(`Loop warning for ${toolName}: ${loopResult.message}`);
|
||||
logToolLoopAction({
|
||||
sessionKey: args.ctx.sessionKey,
|
||||
sessionId: args.ctx?.agentId,
|
||||
toolName,
|
||||
level: "warning",
|
||||
action: "warn",
|
||||
detector: loopResult.detector,
|
||||
count: loopResult.count,
|
||||
message: loopResult.message,
|
||||
pairedToolName: loopResult.pairedToolName,
|
||||
});
|
||||
const warningKey = loopResult.warningKey ?? `${loopResult.detector}:${toolName}`;
|
||||
if (shouldEmitLoopWarning(sessionState, warningKey, loopResult.count)) {
|
||||
log.warn(`Loop warning for ${toolName}: ${loopResult.message}`);
|
||||
logToolLoopAction({
|
||||
sessionKey: args.ctx.sessionKey,
|
||||
sessionId: args.ctx?.agentId,
|
||||
toolName,
|
||||
level: "warning",
|
||||
action: "warn",
|
||||
detector: loopResult.detector,
|
||||
count: loopResult.count,
|
||||
message: loopResult.message,
|
||||
pairedToolName: loopResult.pairedToolName,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ export type LoopDetectionResult =
|
||||
count: number;
|
||||
message: string;
|
||||
pairedToolName?: string;
|
||||
warningKey?: string;
|
||||
};
|
||||
|
||||
export const TOOL_CALL_HISTORY_SIZE = 30;
|
||||
@@ -161,7 +162,7 @@ function getNoProgressStreak(
|
||||
history: Array<{ toolName: string; argsHash: string; resultHash?: string }>,
|
||||
toolName: string,
|
||||
argsHash: string,
|
||||
): number {
|
||||
): { count: number; latestResultHash?: string } {
|
||||
let streak = 0;
|
||||
let latestResultHash: string | undefined;
|
||||
|
||||
@@ -184,7 +185,7 @@ function getNoProgressStreak(
|
||||
streak += 1;
|
||||
}
|
||||
|
||||
return streak;
|
||||
return { count: streak, latestResultHash };
|
||||
}
|
||||
|
||||
function getPingPongStreak(
|
||||
@@ -253,7 +254,8 @@ export function detectToolCallLoop(
|
||||
): LoopDetectionResult {
|
||||
const history = state.toolCallHistory ?? [];
|
||||
const currentHash = hashToolCall(toolName, params);
|
||||
const noProgressStreak = getNoProgressStreak(history, toolName, currentHash);
|
||||
const noProgress = getNoProgressStreak(history, toolName, currentHash);
|
||||
const noProgressStreak = noProgress.count;
|
||||
const knownPollTool = isKnownPollToolCall(toolName, params);
|
||||
const pingPong = getPingPongStreak(history, currentHash);
|
||||
|
||||
@@ -267,6 +269,7 @@ export function detectToolCallLoop(
|
||||
detector: "global_circuit_breaker",
|
||||
count: noProgressStreak,
|
||||
message: `CRITICAL: ${toolName} has repeated identical no-progress outcomes ${noProgressStreak} times. Session execution blocked by global circuit breaker to prevent runaway loops.`,
|
||||
warningKey: `global:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -278,6 +281,7 @@ export function detectToolCallLoop(
|
||||
detector: "known_poll_no_progress",
|
||||
count: noProgressStreak,
|
||||
message: `CRITICAL: Called ${toolName} with identical arguments and no progress ${noProgressStreak} times. This appears to be a stuck polling loop. Session execution blocked to prevent resource waste.`,
|
||||
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -289,6 +293,7 @@ export function detectToolCallLoop(
|
||||
detector: "known_poll_no_progress",
|
||||
count: noProgressStreak,
|
||||
message: `WARNING: You have called ${toolName} ${noProgressStreak} times with identical arguments and no progress. Stop polling and either (1) increase wait time between checks, or (2) report the task as failed if the process is stuck.`,
|
||||
warningKey: `poll:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -303,6 +308,7 @@ export function detectToolCallLoop(
|
||||
count: pingPong.count,
|
||||
message: `CRITICAL: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls). This appears to be a stuck ping-pong loop. Session execution blocked to prevent resource waste.`,
|
||||
pairedToolName: pingPong.pairedToolName,
|
||||
warningKey: `pingpong:${toolName}:${currentHash}:${pingPong.pairedToolName ?? "unknown"}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -317,6 +323,7 @@ export function detectToolCallLoop(
|
||||
count: pingPong.count,
|
||||
message: `WARNING: You are alternating between repeated tool-call patterns (${pingPong.count} consecutive calls). This looks like a ping-pong loop; stop retrying and report the task as failed.`,
|
||||
pairedToolName: pingPong.pairedToolName,
|
||||
warningKey: `pingpong:${toolName}:${currentHash}:${pingPong.pairedToolName ?? "unknown"}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -333,6 +340,7 @@ export function detectToolCallLoop(
|
||||
detector: "generic_repeat",
|
||||
count: recentCount,
|
||||
message: `WARNING: You have called ${toolName} ${recentCount} times with identical arguments. If this is not making progress, stop retrying and report the task as failed.`,
|
||||
warningKey: `generic:${toolName}:${currentHash}`,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ export type SessionState = {
|
||||
state: SessionStateValue;
|
||||
queueDepth: number;
|
||||
toolCallHistory?: ToolCallRecord[];
|
||||
toolLoopWarningBuckets?: Map<string, number>;
|
||||
commandPollCounts?: Map<string, { count: number; lastPollAt: number }>;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user