mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-03 06:51:38 +00:00
f0371e8d88
* fix(agents): make global loop breaker reachable Record loop-detector vetoes as typed no-progress outcomes so repeated blocked calls continue the existing streak without colliding with plugin or approval denials. Extract streak accounting into its own owner module and keep completed veto records out of argument reconciliation.\n\nFixes #109435. * test: harden isolated project routing Route registry-sensitive UI tests through the isolated project for both focused and broad runs, centralize the isolated file list, and register the Codex prewarm test in the full extension shard. * test(qa): prove global loop breaker runtime Drive 31 identical read attempts through the real QA Gateway agent loop and verify the typed veto streak reaches the global circuit breaker before the turn returns a final marker. * test: keep isolated UI files out of shared runs Always exclude registry-sensitive files from the shared UI project and reject broad watch targets that would span shared and isolated projects.
1717 lines
56 KiB
TypeScript
1717 lines
56 KiB
TypeScript
// Tool loop detection tests cover repeated-call hashing, ping-pong detection,
|
|
// unknown-tool thresholds, and circuit-breaker escalation.
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import type { ToolLoopDetectionConfig } from "../config/types.tools.js";
|
|
import type { SessionState } from "../logging/diagnostic-session-state.js";
|
|
|
|
// Recognize a provider-docked send tool by name (only "telegram" here) so the
|
|
// volatility strip applies to it without pulling in the channel-plugin registry; the
|
|
// real detector is covered by embedded-agent-messaging's own tests.
|
|
const isMessagingToolSendActionMock = vi.hoisted(() =>
|
|
vi.fn((toolName: string): boolean => toolName === "telegram"),
|
|
);
|
|
vi.mock("./embedded-agent-messaging.js", () => ({
|
|
isMessagingToolSendAction: isMessagingToolSendActionMock,
|
|
}));
|
|
import { reconcileToolCallExecutionParams } from "./tool-loop-call-reconciliation.js";
|
|
import {
|
|
UNKNOWN_TOOL_THRESHOLD,
|
|
detectToolCallLoop,
|
|
recordToolCall,
|
|
recordToolCallOutcome,
|
|
} from "./tool-loop-detection.js";
|
|
|
|
const TOOL_CALL_HISTORY_SIZE = 30;
|
|
const WARNING_THRESHOLD = 10;
|
|
const CRITICAL_THRESHOLD = 20;
|
|
const GLOBAL_CIRCUIT_BREAKER_THRESHOLD = 30;
|
|
|
|
function createState(): SessionState {
|
|
return {
|
|
lastActivity: Date.now(),
|
|
state: "processing",
|
|
queueDepth: 0,
|
|
};
|
|
}
|
|
|
|
function recordArgsHash(toolName: string, params: unknown): string {
|
|
const state = createState();
|
|
recordToolCall(state, toolName, params, "hash-test");
|
|
const hash = state.toolCallHistory?.[0]?.argsHash;
|
|
if (!hash) {
|
|
throw new Error("recordToolCall did not record an argument hash");
|
|
}
|
|
return hash;
|
|
}
|
|
|
|
const enabledLoopDetectionConfig: ToolLoopDetectionConfig = { enabled: true };
|
|
|
|
function recordSuccessfulCall(
|
|
state: SessionState,
|
|
toolName: string,
|
|
params: unknown,
|
|
result: unknown,
|
|
index: number,
|
|
): void {
|
|
const toolCallId = `${toolName}-${index}`;
|
|
recordToolCall(state, toolName, params, toolCallId);
|
|
recordToolCallOutcome(state, {
|
|
toolName,
|
|
toolParams: params,
|
|
toolCallId,
|
|
result,
|
|
});
|
|
}
|
|
|
|
function recordFailedCall(
|
|
state: SessionState,
|
|
toolName: string,
|
|
params: unknown,
|
|
error: unknown,
|
|
index: number,
|
|
): void {
|
|
const toolCallId = `${toolName}-error-${index}`;
|
|
recordToolCall(state, toolName, params, toolCallId);
|
|
recordToolCallOutcome(state, {
|
|
toolName,
|
|
toolParams: params,
|
|
toolCallId,
|
|
error,
|
|
});
|
|
}
|
|
|
|
function recordRepeatedSuccessfulCalls(params: {
|
|
state: SessionState;
|
|
toolName: string;
|
|
toolParams: unknown;
|
|
result: unknown;
|
|
count: number;
|
|
startIndex?: number;
|
|
}) {
|
|
const startIndex = params.startIndex ?? 0;
|
|
for (let i = 0; i < params.count; i += 1) {
|
|
recordSuccessfulCall(
|
|
params.state,
|
|
params.toolName,
|
|
params.toolParams,
|
|
params.result,
|
|
startIndex + i,
|
|
);
|
|
}
|
|
}
|
|
|
|
function createNoProgressPollFixture(sessionId: string) {
|
|
return {
|
|
params: { action: "poll", sessionId },
|
|
result: {
|
|
content: [{ type: "text", text: "(no new output)\n\nProcess still running." }],
|
|
details: { status: "running", aggregated: "steady" },
|
|
},
|
|
};
|
|
}
|
|
|
|
function createReadNoProgressFixture() {
|
|
return {
|
|
toolName: "read",
|
|
params: { path: "/same.txt" },
|
|
result: {
|
|
content: [{ type: "text", text: "same output" }],
|
|
details: { ok: true },
|
|
},
|
|
} as const;
|
|
}
|
|
|
|
function createPingPongFixture() {
|
|
return {
|
|
state: createState(),
|
|
readParams: { path: "/a.txt" },
|
|
listParams: { dir: "/workspace" },
|
|
};
|
|
}
|
|
|
|
function detectLoopAfterRepeatedCalls(params: {
|
|
toolName: string;
|
|
toolParams: unknown;
|
|
result: unknown;
|
|
count: number;
|
|
config?: ToolLoopDetectionConfig;
|
|
}) {
|
|
const state = createState();
|
|
recordRepeatedSuccessfulCalls({
|
|
state,
|
|
toolName: params.toolName,
|
|
toolParams: params.toolParams,
|
|
result: params.result,
|
|
count: params.count,
|
|
});
|
|
return detectToolCallLoop(
|
|
state,
|
|
params.toolName,
|
|
params.toolParams,
|
|
params.config ?? enabledLoopDetectionConfig,
|
|
);
|
|
}
|
|
|
|
function recordSuccessfulPingPongCalls(params: {
|
|
state: SessionState;
|
|
readParams: { path: string };
|
|
listParams: { dir: string };
|
|
count: number;
|
|
textAtIndex: (toolName: "read" | "list", index: number) => string;
|
|
}) {
|
|
// Alternating successful calls with unchanged output exercise the ping-pong
|
|
// detector independently from same-tool repetition.
|
|
for (let i = 0; i < params.count; i += 1) {
|
|
if (i % 2 === 0) {
|
|
recordSuccessfulCall(
|
|
params.state,
|
|
"read",
|
|
params.readParams,
|
|
{ content: [{ type: "text", text: params.textAtIndex("read", i) }], details: { ok: true } },
|
|
i,
|
|
);
|
|
} else {
|
|
recordSuccessfulCall(
|
|
params.state,
|
|
"list",
|
|
params.listParams,
|
|
{ content: [{ type: "text", text: params.textAtIndex("list", i) }], details: { ok: true } },
|
|
i,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
function expectPingPongLoop(
|
|
loopResult: ReturnType<typeof detectToolCallLoop>,
|
|
expected: { level: "warning" | "critical"; count: number; expectCriticalText?: boolean },
|
|
) {
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (!loopResult.stuck) {
|
|
return;
|
|
}
|
|
expect(loopResult.level).toBe(expected.level);
|
|
expect(loopResult.detector).toBe("ping_pong");
|
|
expect(loopResult.count).toBe(expected.count);
|
|
if (expected.expectCriticalText) {
|
|
expect(loopResult.message).toContain("CRITICAL");
|
|
}
|
|
}
|
|
|
|
describe("tool-loop-detection", () => {
|
|
describe("recordToolCall argument hashing", () => {
|
|
it("creates consistent hash for same tool and params", () => {
|
|
const hash1 = recordArgsHash("read", { path: "/file.txt" });
|
|
const hash2 = recordArgsHash("read", { path: "/file.txt" });
|
|
expect(hash1).toBe(hash2);
|
|
});
|
|
|
|
it("creates different hashes for different params", () => {
|
|
const hash1 = recordArgsHash("read", { path: "/file1.txt" });
|
|
const hash2 = recordArgsHash("read", { path: "/file2.txt" });
|
|
expect(hash1).not.toBe(hash2);
|
|
});
|
|
|
|
it("creates different hashes for different tools", () => {
|
|
const hash1 = recordArgsHash("read", { path: "/file.txt" });
|
|
const hash2 = recordArgsHash("write", { path: "/file.txt" });
|
|
expect(hash1).not.toBe(hash2);
|
|
});
|
|
|
|
it("hashes non-object params with the same digest shape", () => {
|
|
const hashes = [
|
|
recordArgsHash("tool", "string-param"),
|
|
recordArgsHash("tool", 123),
|
|
recordArgsHash("tool", null),
|
|
];
|
|
expect(hashes).toHaveLength(3);
|
|
for (const hash of hashes) {
|
|
expect(hash.startsWith("tool:")).toBe(true);
|
|
expect(hash.length).toBe("tool:".length + 64);
|
|
expect(/^[a-f0-9]+$/.test(hash.slice("tool:".length))).toBe(true);
|
|
}
|
|
});
|
|
|
|
it("produces deterministic hashes regardless of key order", () => {
|
|
const hash1 = recordArgsHash("tool", { a: 1, b: 2 });
|
|
const hash2 = recordArgsHash("tool", { b: 2, a: 1 });
|
|
expect(hash1).toBe(hash2);
|
|
});
|
|
|
|
it("keeps hashes fixed-size even for large params", () => {
|
|
const payload = { data: "x".repeat(20_000) };
|
|
const hash = recordArgsHash("read", payload);
|
|
expect(hash.startsWith("read:")).toBe(true);
|
|
expect(hash.length).toBe("read:".length + 64);
|
|
});
|
|
|
|
it("hashes circular params without collapsing repeated references", () => {
|
|
const shared = { id: "shared" };
|
|
const payload: Record<string, unknown> = { first: shared, second: shared };
|
|
payload.self = payload;
|
|
|
|
const equivalentShared = { id: "shared" };
|
|
const equivalentPayload: Record<string, unknown> = {
|
|
second: equivalentShared,
|
|
first: equivalentShared,
|
|
};
|
|
equivalentPayload.self = equivalentPayload;
|
|
|
|
expect(recordArgsHash("tool", payload)).toBe(recordArgsHash("tool", equivalentPayload));
|
|
expect(recordArgsHash("tool", payload)).toEqual(expect.stringMatching(/^tool:[a-f0-9]{64}$/));
|
|
});
|
|
});
|
|
|
|
describe("recordToolCall", () => {
|
|
it("adds tool call to empty history", () => {
|
|
const state = createState();
|
|
|
|
recordToolCall(state, "read", { path: "/file.txt" }, "call-1");
|
|
|
|
expect(state.toolCallHistory).toHaveLength(1);
|
|
expect(state.toolCallHistory?.[0]?.toolName).toBe("read");
|
|
expect(state.toolCallHistory?.[0]?.toolCallId).toBe("call-1");
|
|
});
|
|
|
|
it("maintains sliding window of last N calls", () => {
|
|
const state = createState();
|
|
|
|
for (let i = 0; i < TOOL_CALL_HISTORY_SIZE + 10; i += 1) {
|
|
recordToolCall(state, "tool", { iteration: i }, `call-${i}`);
|
|
}
|
|
|
|
expect(state.toolCallHistory).toHaveLength(TOOL_CALL_HISTORY_SIZE);
|
|
|
|
const oldestCall = state.toolCallHistory?.[0];
|
|
expect(oldestCall?.argsHash).toBe(recordArgsHash("tool", { iteration: 10 }));
|
|
});
|
|
|
|
it("records timestamp for each call", () => {
|
|
const state = createState();
|
|
const before = Date.now();
|
|
recordToolCall(state, "tool", { arg: 1 }, "call-ts");
|
|
const after = Date.now();
|
|
|
|
const timestamp = state.toolCallHistory?.[0]?.timestamp ?? 0;
|
|
expect(timestamp).toBeGreaterThanOrEqual(before);
|
|
expect(timestamp).toBeLessThanOrEqual(after);
|
|
});
|
|
|
|
it("records run id when provided", () => {
|
|
const state = createState();
|
|
|
|
recordToolCall(state, "tool", { arg: 1 }, "call-run", enabledLoopDetectionConfig, {
|
|
runId: "run-1",
|
|
});
|
|
|
|
expect(state.toolCallHistory?.[0]?.runId).toBe("run-1");
|
|
});
|
|
});
|
|
|
|
describe("detectToolCallLoop", () => {
|
|
it("is disabled by default", () => {
|
|
const state = createState();
|
|
|
|
for (let i = 0; i < 20; i += 1) {
|
|
recordToolCall(state, "read", { path: "/same.txt" }, `default-${i}`);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "read", { path: "/same.txt" });
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("does not flag unique tool calls", () => {
|
|
const state = createState();
|
|
|
|
for (let i = 0; i < 15; i += 1) {
|
|
recordToolCall(state, "read", { path: `/file${i}.txt` }, `call-${i}`);
|
|
}
|
|
|
|
const result = detectToolCallLoop(
|
|
state,
|
|
"read",
|
|
{ path: "/new-file.txt" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(result.stuck).toBe(false);
|
|
});
|
|
|
|
it("ignores repeated history from other runs", () => {
|
|
const state = createState();
|
|
const params = { path: "/same.txt" };
|
|
|
|
for (let i = 0; i < WARNING_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "read", params, `old-run-${i}`, enabledLoopDetectionConfig, {
|
|
runId: "heartbeat-1",
|
|
});
|
|
}
|
|
|
|
const result = detectToolCallLoop(state, "read", params, enabledLoopDetectionConfig, {
|
|
runId: "heartbeat-2",
|
|
});
|
|
|
|
expect(result.stuck).toBe(false);
|
|
});
|
|
|
|
it("detects repeated history within the same run", () => {
|
|
const state = createState();
|
|
const params = { path: "/same.txt" };
|
|
|
|
for (let i = 0; i < WARNING_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "read", params, `same-run-${i}`, enabledLoopDetectionConfig, {
|
|
runId: "run-1",
|
|
});
|
|
}
|
|
|
|
const result = detectToolCallLoop(state, "read", params, enabledLoopDetectionConfig, {
|
|
runId: "run-1",
|
|
});
|
|
|
|
expect(result.stuck).toBe(true);
|
|
if (result.stuck) {
|
|
expect(result.detector).toBe("generic_repeat");
|
|
expect(result.count).toBe(WARNING_THRESHOLD);
|
|
}
|
|
});
|
|
|
|
it("keeps scoped and unscoped history isolated", () => {
|
|
const state = createState();
|
|
const params = { path: "/same.txt" };
|
|
|
|
for (let i = 0; i < WARNING_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "read", params, `scoped-${i}`, enabledLoopDetectionConfig, {
|
|
runId: "run-1",
|
|
});
|
|
}
|
|
|
|
const result = detectToolCallLoop(state, "read", params, enabledLoopDetectionConfig);
|
|
|
|
expect(result.stuck).toBe(false);
|
|
});
|
|
|
|
it("warns on generic repeated tool+args calls", () => {
|
|
const state = createState();
|
|
for (let i = 0; i < WARNING_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "read", { path: "/same.txt" }, `warn-${i}`);
|
|
}
|
|
|
|
const result = detectToolCallLoop(
|
|
state,
|
|
"read",
|
|
{ path: "/same.txt" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(result.stuck).toBe(true);
|
|
if (result.stuck) {
|
|
expect(result.level).toBe("warning");
|
|
expect(result.detector).toBe("generic_repeat");
|
|
expect(result.count).toBe(WARNING_THRESHOLD);
|
|
expect(result.message).toContain("WARNING");
|
|
expect(result.message).toContain(`${WARNING_THRESHOLD} times`);
|
|
}
|
|
});
|
|
|
|
it("blocks generic no-progress loops at critical threshold", () => {
|
|
const fixture = createReadNoProgressFixture();
|
|
const loopResult = detectLoopAfterRepeatedCalls({
|
|
toolName: fixture.toolName,
|
|
toolParams: fixture.params,
|
|
result: fixture.result,
|
|
count: CRITICAL_THRESHOLD,
|
|
});
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
expect(loopResult.detector).toBe("generic_repeat");
|
|
expect(loopResult.message).toContain("identical outcomes");
|
|
}
|
|
});
|
|
|
|
it("warns for known polling no-progress loops", () => {
|
|
const { params, result } = createNoProgressPollFixture("sess-1");
|
|
const loopResult = detectLoopAfterRepeatedCalls({
|
|
toolName: "process",
|
|
toolParams: params,
|
|
result,
|
|
count: WARNING_THRESHOLD,
|
|
});
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("warning");
|
|
expect(loopResult.detector).toBe("known_poll_no_progress");
|
|
expect(loopResult.message).toContain("no progress");
|
|
}
|
|
});
|
|
|
|
it("blocks known polling no-progress loops at critical threshold", () => {
|
|
const { params, result } = createNoProgressPollFixture("sess-1");
|
|
const loopResult = detectLoopAfterRepeatedCalls({
|
|
toolName: "process",
|
|
toolParams: params,
|
|
result,
|
|
count: CRITICAL_THRESHOLD,
|
|
});
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
expect(loopResult.detector).toBe("known_poll_no_progress");
|
|
expect(loopResult.message).toContain("CRITICAL");
|
|
}
|
|
});
|
|
|
|
it("does not block known polling when output progresses", () => {
|
|
const state = createState();
|
|
const params = { action: "poll", sessionId: "sess-1" };
|
|
|
|
for (let i = 0; i < CRITICAL_THRESHOLD + 5; i += 1) {
|
|
const result = {
|
|
content: [{ type: "text", text: `line ${i}` }],
|
|
details: { status: "running", aggregated: `line ${i}` },
|
|
};
|
|
recordSuccessfulCall(state, "process", params, result, i);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "process", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("blocks any tool with global no-progress breaker at 30", () => {
|
|
const fixture = createReadNoProgressFixture();
|
|
const loopResult = detectLoopAfterRepeatedCalls({
|
|
toolName: fixture.toolName,
|
|
toolParams: fixture.params,
|
|
result: fixture.result,
|
|
count: GLOBAL_CIRCUIT_BREAKER_THRESHOLD,
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
expect(loopResult.detector).toBe("global_circuit_breaker");
|
|
expect(loopResult.message).toContain("global circuit breaker");
|
|
}
|
|
});
|
|
|
|
it("warns on repeated stable argument churn without vetoing the next call", () => {
|
|
const state = createState();
|
|
const paths = ["/tmp/a.md", "/tmp/b.md", "/tmp/a.md", "/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "write made no changes" }],
|
|
details: { ok: true, changed: false },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("warning");
|
|
expect(loopResult.detector).toBe("argument_churn");
|
|
expect(loopResult.livenessSignal).toBe("argument_churn");
|
|
expect(loopResult.count).toBe(GLOBAL_CIRCUIT_BREAKER_THRESHOLD);
|
|
expect(loopResult.message).toContain("tool call remains allowed");
|
|
}
|
|
|
|
const escapeResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/c.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(escapeResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("normalizes built-in write no-ops that only differ by echoed path", () => {
|
|
const state = createState();
|
|
const content = "same content";
|
|
const paths = ["/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (const [index, targetPath] of paths.entries()) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content },
|
|
{
|
|
content: [
|
|
{
|
|
type: "text",
|
|
text: `No changes made to ${targetPath}. The file already has identical content.`,
|
|
},
|
|
],
|
|
details: { changed: false },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const hashes = state.toolCallHistory?.map((record) => record.resultHash);
|
|
expect(hashes?.[0]).toBeTypeOf("string");
|
|
expect(hashes?.[0]).toBe(hashes?.[1]);
|
|
expect(state.toolCallHistory?.every((record) => record.noProgress === true)).toBe(true);
|
|
});
|
|
|
|
it("preserves target identity for successful write outcomes", () => {
|
|
const state = createState();
|
|
const content = "same content";
|
|
const paths = ["/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (const [index, targetPath] of paths.entries()) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content },
|
|
{
|
|
content: [
|
|
{
|
|
type: "text",
|
|
text: `Successfully wrote ${Buffer.byteLength(content, "utf8")} bytes to ${targetPath}`,
|
|
},
|
|
],
|
|
details: {
|
|
changed: true,
|
|
created: true,
|
|
diff: "+same content",
|
|
patch: `--- ${targetPath}\n+++ ${targetPath}\n+same content`,
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const history = state.toolCallHistory ?? [];
|
|
expect(history[0]?.resultHash).not.toBe(history[1]?.resultHash);
|
|
expect(history.every((record) => record.noProgress === undefined)).toBe(true);
|
|
});
|
|
|
|
it("uses the supplied warning threshold when reconciling rewritten calls", () => {
|
|
const state = createState();
|
|
const paths = ["/tmp/a.md", "/tmp/b.md"];
|
|
for (let index = 0; index < 6; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "write made no changes" }],
|
|
details: { changed: false },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
recordToolCall(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/original.md", content: "same content" },
|
|
"rewritten-call",
|
|
);
|
|
|
|
const reconciled = reconcileToolCallExecutionParams(state, {
|
|
toolName: "write",
|
|
toolParams: { path: "/tmp/a.md", content: "same content" },
|
|
toolCallId: "rewritten-call",
|
|
warningThreshold: 6,
|
|
});
|
|
|
|
expect(reconciled).toEqual({ active: true, count: 6, variantCount: 2 });
|
|
});
|
|
|
|
it("does not reconcile a completed loop veto as a pending call", () => {
|
|
const state = createState();
|
|
state.toolCallHistory = [
|
|
{
|
|
toolName: "write",
|
|
argsHash: "pending-args",
|
|
timestamp: 1,
|
|
},
|
|
{
|
|
toolName: "write",
|
|
argsHash: "vetoed-args",
|
|
outcomeKind: "tool-loop-veto",
|
|
timestamp: 2,
|
|
},
|
|
];
|
|
|
|
expect(
|
|
reconcileToolCallExecutionParams(state, {
|
|
toolName: "write",
|
|
toolParams: { path: "/tmp/rewritten.md", content: "same content" },
|
|
warningThreshold: 6,
|
|
}),
|
|
).toEqual({ active: false, count: 0, variantCount: 0 });
|
|
expect(state.toolCallHistory[0]?.argsHash).not.toBe("pending-args");
|
|
expect(state.toolCallHistory[1]?.argsHash).toBe("vetoed-args");
|
|
});
|
|
|
|
it("keeps completed churn evidence across a pending same-tool sibling", () => {
|
|
const state = createState();
|
|
const paths = ["/tmp/a.md", "/tmp/b.md", "/tmp/a.md", "/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "write made no changes" }],
|
|
details: { ok: true, changed: false },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
recordToolCall(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
"pending-sibling",
|
|
);
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/b.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult).toMatchObject({
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "argument_churn",
|
|
});
|
|
});
|
|
|
|
it("does not treat generic stable successes as semantic no-progress", () => {
|
|
const state = createState();
|
|
const paths = ["/tmp/a.md", "/tmp/b.md", "/tmp/a.md", "/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"side_effect",
|
|
{ path: targetPath },
|
|
{
|
|
content: [{ type: "text", text: "done" }],
|
|
details: { ok: true },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"side_effect",
|
|
{ path: "/tmp/a.md" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult).toMatchObject({
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "generic_repeat",
|
|
});
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.livenessSignal).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it("keeps repeated stable errors eligible for argument-churn liveness", () => {
|
|
const state = createState();
|
|
const paths = ["/tmp/a.md", "/tmp/b.md", "/tmp/a.md", "/tmp/a.md", "/tmp/b.md"];
|
|
|
|
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
const toolCallId = `failed-${index}`;
|
|
const params = { path: targetPath };
|
|
recordToolCall(state, "side_effect", params, toolCallId, enabledLoopDetectionConfig);
|
|
recordToolCallOutcome(state, {
|
|
toolName: "side_effect",
|
|
toolParams: params,
|
|
toolCallId,
|
|
error: new Error("permission denied"),
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"side_effect",
|
|
{ path: "/tmp/a.md" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult).toMatchObject({
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "argument_churn",
|
|
livenessSignal: "argument_churn",
|
|
});
|
|
});
|
|
|
|
it("keeps generic critical repeats ahead of warning-only argument churn", () => {
|
|
const state = createState();
|
|
|
|
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "wrote /tmp/a.md" }],
|
|
details: { ok: true, path: "/tmp/a.md" },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
for (let index = 0; index < WARNING_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/b.md", content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "wrote /tmp/b.md" }],
|
|
details: { ok: true, path: "/tmp/b.md" },
|
|
},
|
|
CRITICAL_THRESHOLD + index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(loopResult).toMatchObject({
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "generic_repeat",
|
|
});
|
|
});
|
|
|
|
it("preserves churn liveness when strict alternation owns the primary warning", () => {
|
|
const state = createState();
|
|
|
|
for (let index = 0; index < WARNING_THRESHOLD; index += 1) {
|
|
const targetPath = index % 2 === 0 ? "/tmp/a.md" : "/tmp/b.md";
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "write made no changes" }],
|
|
details: { ok: true, changed: false },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("warning");
|
|
expect(loopResult.detector).toBe("ping_pong");
|
|
expect(loopResult.livenessSignal).toBe("argument_churn");
|
|
}
|
|
});
|
|
|
|
it("does not carry argument-churn liveness across singleton probes", () => {
|
|
const state = createState();
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
const targetPath =
|
|
index === 8 || index === 19 ? `/tmp/probe-${index}.md` : `/tmp/${index % 2}.md`;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: `wrote ${targetPath}` }],
|
|
details: { ok: true, path: targetPath },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/a.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("does not block a one-shot batch of distinct arguments", () => {
|
|
const state = createState();
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: `/tmp/file-${index}.md`, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: "write complete" }],
|
|
details: { ok: true },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/next.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("does not block a legitimate two-pass batch", () => {
|
|
const state = createState();
|
|
const paths = Array.from({ length: 15 }, (_, index) => `/tmp/batch-${index}.md`);
|
|
const content = "same content";
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
const targetPath = paths[index % paths.length]!;
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content },
|
|
{
|
|
content: [
|
|
{
|
|
type: "text",
|
|
text: `Successfully wrote ${Buffer.byteLength(content, "utf8")} bytes to ${targetPath}`,
|
|
},
|
|
],
|
|
details: {
|
|
changed: true,
|
|
created: true,
|
|
diff: "+same content",
|
|
patch: `--- ${targetPath}\n+++ ${targetPath}\n+same content`,
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/next.md", content },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("does not block argument churn when a repeated variant makes progress", () => {
|
|
const state = createState();
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
const targetPath = index % 2 === 0 ? "/tmp/a.md" : "/tmp/b.md";
|
|
recordSuccessfulCall(
|
|
state,
|
|
"write",
|
|
{ path: targetPath, content: "same content" },
|
|
{
|
|
content: [{ type: "text", text: `write ${targetPath} revision ${index}` }],
|
|
details: { ok: true, revision: index },
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"write",
|
|
{ path: "/tmp/c.md", content: "same content" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("blocks repeated completed exec calls despite volatile runtime details", () => {
|
|
const state = createState();
|
|
const params = { command: "grafana-api.sh datasources" };
|
|
|
|
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"exec",
|
|
params,
|
|
{
|
|
content: [{ type: "text", text: "Loki\nPrometheus" }],
|
|
details: {
|
|
status: "completed",
|
|
exitCode: 0,
|
|
durationMs: 100 + index,
|
|
cwd: `/tmp/run-${index}`,
|
|
aggregated: "Loki\nPrometheus",
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "exec", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
expect(loopResult.detector).toBe("generic_repeat");
|
|
}
|
|
});
|
|
|
|
it("blocks repeated running exec calls despite volatile session details and text", () => {
|
|
const state = createState();
|
|
const params = { command: "tail -f /var/log/app.log", yieldMs: 1000 };
|
|
|
|
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"exec",
|
|
params,
|
|
{
|
|
content: [
|
|
{
|
|
type: "text",
|
|
text: `Command still running (session sess-${index}, pid ${1000 + index})`,
|
|
},
|
|
],
|
|
details: {
|
|
status: "running",
|
|
sessionId: `sess-${index}`,
|
|
pid: 1000 + index,
|
|
startedAt: Date.now() + index,
|
|
cwd: `/tmp/run-${index}`,
|
|
tail: "(no new output)",
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "exec", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
expect(loopResult.detector).toBe("generic_repeat");
|
|
}
|
|
});
|
|
|
|
it("keeps changing exec output below the global no-progress breaker", () => {
|
|
const state = createState();
|
|
const params = { command: "date" };
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"exec",
|
|
params,
|
|
{
|
|
content: [{ type: "text", text: `tick ${index}` }],
|
|
details: {
|
|
status: "completed",
|
|
exitCode: 0,
|
|
durationMs: 100 + index,
|
|
aggregated: `tick ${index}`,
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "exec", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("warning");
|
|
expect(loopResult.detector).toBe("generic_repeat");
|
|
}
|
|
});
|
|
|
|
it("keeps changing empty-output exec failures below the global no-progress breaker", () => {
|
|
const state = createState();
|
|
const params = { command: "openclaw flaky-helper" };
|
|
|
|
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
|
|
recordSuccessfulCall(
|
|
state,
|
|
"exec",
|
|
params,
|
|
{
|
|
content: [{ type: "text", text: `Runtime failed before spawn: attempt ${index}` }],
|
|
details: {
|
|
status: "failed",
|
|
exitCode: null,
|
|
durationMs: 100 + index,
|
|
aggregated: "",
|
|
},
|
|
},
|
|
index,
|
|
);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "exec", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("warning");
|
|
expect(loopResult.detector).toBe("generic_repeat");
|
|
}
|
|
});
|
|
|
|
it("does not block repeated unknown-tool failures before the unknown-tool threshold", () => {
|
|
const state = createState();
|
|
const toolName = "exec";
|
|
const unknownToolError = new Error("Tool exec not found");
|
|
|
|
for (let index = 0; index < UNKNOWN_TOOL_THRESHOLD - 1; index += 1) {
|
|
recordFailedCall(state, toolName, { command: `echo ${index}` }, unknownToolError, index);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
toolName,
|
|
{ command: "echo still allowed" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("blocks repeated unknown-tool failures even when the args keep changing", () => {
|
|
const state = createState();
|
|
const toolName = "exec";
|
|
const unknownToolError = new Error("Tool exec not found");
|
|
|
|
const attempts = [
|
|
{ command: "ls" },
|
|
{ command: "pwd" },
|
|
{ input: "whoami" },
|
|
{ cmd: "env" },
|
|
{ shell: "bash -lc ls" },
|
|
{ command: "printf ok" },
|
|
{ cwd: "/tmp", command: "ls" },
|
|
{ args: ["ls", "/tmp"] },
|
|
{ command: "find . -maxdepth 1" },
|
|
{ text: "run ls" },
|
|
{ command: "uname -a" },
|
|
{ command: "id" },
|
|
{ command: "date" },
|
|
{ command: "ps" },
|
|
{ command: "df -h" },
|
|
{ command: "free -m" },
|
|
{ command: "ls /tmp" },
|
|
{ command: "ls -la" },
|
|
{ command: "cat /etc/hostname" },
|
|
{ command: "echo done" },
|
|
];
|
|
|
|
for (const [index, params] of attempts.entries()) {
|
|
recordFailedCall(state, toolName, params, unknownToolError, index);
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
toolName,
|
|
{ command: "echo still looping" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.detector).toBe("unknown_tool_repeat");
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("warns on ping-pong alternating patterns", () => {
|
|
const state = createState();
|
|
const readParams = { path: "/a.txt" };
|
|
const listParams = { dir: "/workspace" };
|
|
|
|
for (let i = 0; i < WARNING_THRESHOLD - 1; i += 1) {
|
|
if (i % 2 === 0) {
|
|
recordToolCall(state, "read", readParams, `read-${i}`);
|
|
} else {
|
|
recordToolCall(state, "list", listParams, `list-${i}`);
|
|
}
|
|
}
|
|
|
|
const loopResult = detectToolCallLoop(state, "list", listParams, enabledLoopDetectionConfig);
|
|
expectPingPongLoop(loopResult, { level: "warning", count: WARNING_THRESHOLD });
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.message).toContain("ping-pong loop");
|
|
}
|
|
});
|
|
|
|
it("blocks ping-pong alternating patterns at critical threshold", () => {
|
|
const { state, readParams, listParams } = createPingPongFixture();
|
|
|
|
recordSuccessfulPingPongCalls({
|
|
state,
|
|
readParams,
|
|
listParams,
|
|
count: CRITICAL_THRESHOLD - 1,
|
|
textAtIndex: (toolName) => (toolName === "read" ? "read stable" : "list stable"),
|
|
});
|
|
|
|
const loopResult = detectToolCallLoop(state, "list", listParams, enabledLoopDetectionConfig);
|
|
expectPingPongLoop(loopResult, {
|
|
level: "critical",
|
|
count: CRITICAL_THRESHOLD,
|
|
expectCriticalText: true,
|
|
});
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.message).toContain("ping-pong loop");
|
|
}
|
|
});
|
|
|
|
it("does not block ping-pong at critical threshold when outcomes are progressing", () => {
|
|
const { state, readParams, listParams } = createPingPongFixture();
|
|
|
|
recordSuccessfulPingPongCalls({
|
|
state,
|
|
readParams,
|
|
listParams,
|
|
count: CRITICAL_THRESHOLD - 1,
|
|
textAtIndex: (toolName, index) => `${toolName} ${index}`,
|
|
});
|
|
|
|
const loopResult = detectToolCallLoop(state, "list", listParams, enabledLoopDetectionConfig);
|
|
expectPingPongLoop(loopResult, { level: "warning", count: CRITICAL_THRESHOLD });
|
|
});
|
|
|
|
it("does not flag ping-pong when alternation is broken", () => {
|
|
const state = createState();
|
|
recordToolCall(state, "read", { path: "/a.txt" }, "a1");
|
|
recordToolCall(state, "list", { dir: "/workspace" }, "b1");
|
|
recordToolCall(state, "read", { path: "/a.txt" }, "a2");
|
|
recordToolCall(state, "write", { path: "/tmp/out.txt" }, "c1"); // breaks alternation
|
|
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"list",
|
|
{ dir: "/workspace" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("records fixed-size result hashes for large tool outputs", () => {
|
|
const state = createState();
|
|
const params = { action: "log", sessionId: "sess-big" };
|
|
const toolCallId = "log-big";
|
|
recordToolCall(state, "process", params, toolCallId);
|
|
recordToolCallOutcome(state, {
|
|
toolName: "process",
|
|
toolParams: params,
|
|
toolCallId,
|
|
result: {
|
|
content: [{ type: "text", text: "y".repeat(40_000) }],
|
|
details: { status: "running", totalLines: 1, totalChars: 40_000 },
|
|
},
|
|
});
|
|
|
|
const entry = state.toolCallHistory?.find((call) => call.toolCallId === toolCallId);
|
|
expect(typeof entry?.resultHash).toBe("string");
|
|
expect(entry?.resultHash?.length).toBe(64);
|
|
});
|
|
|
|
it("returns the recorded call when a pre-recorded tool call receives its result", () => {
|
|
const state = createState();
|
|
const params = { action: "lookup", path: "cron.enabled" };
|
|
|
|
recordToolCall(state, "gateway", params, "call-1");
|
|
|
|
const recorded = recordToolCallOutcome(state, {
|
|
toolName: "gateway",
|
|
toolParams: params,
|
|
toolCallId: "call-1",
|
|
result: { content: [{ type: "text", text: "same schema" }] },
|
|
});
|
|
|
|
expect(recorded?.toolCallId).toBe("call-1");
|
|
expect(state.toolCallHistory).toHaveLength(1);
|
|
expect(state.toolCallHistory?.[0]?.resultHash).toBeTypeOf("string");
|
|
});
|
|
|
|
it("returns the recorded call while trimming production call/outcome records", () => {
|
|
const state = createState();
|
|
let lastRecordedToolCallId: string | undefined;
|
|
|
|
for (let i = 0; i < TOOL_CALL_HISTORY_SIZE + 3; i += 1) {
|
|
const params = { action: "lookup", path: `config.${i}` };
|
|
const toolCallId = `call-${i}`;
|
|
recordToolCall(state, "gateway", params, toolCallId);
|
|
const recorded = recordToolCallOutcome(state, {
|
|
toolName: "gateway",
|
|
toolParams: params,
|
|
toolCallId,
|
|
result: { content: [{ type: "text", text: `schema-${i}` }] },
|
|
});
|
|
lastRecordedToolCallId = recorded?.toolCallId;
|
|
}
|
|
|
|
expect(lastRecordedToolCallId).toBe(`call-${TOOL_CALL_HISTORY_SIZE + 2}`);
|
|
expect(state.toolCallHistory).toHaveLength(TOOL_CALL_HISTORY_SIZE);
|
|
expect(state.toolCallHistory?.[0]?.toolCallId).toBe("call-3");
|
|
});
|
|
|
|
it("does not attach outcomes to matching calls from other runs", () => {
|
|
const state = createState();
|
|
const params = { path: "/same.txt" };
|
|
recordToolCall(state, "read", params, "call-1", enabledLoopDetectionConfig, {
|
|
runId: "run-1",
|
|
});
|
|
|
|
recordToolCallOutcome(state, {
|
|
toolName: "read",
|
|
toolParams: params,
|
|
toolCallId: "call-1",
|
|
result: { content: [{ type: "text", text: "same output" }] },
|
|
config: enabledLoopDetectionConfig,
|
|
runId: "run-2",
|
|
});
|
|
|
|
expect(state.toolCallHistory).toHaveLength(2);
|
|
expect(state.toolCallHistory?.[0]?.resultHash).toBeUndefined();
|
|
expect(state.toolCallHistory?.[1]?.runId).toBe("run-2");
|
|
expect(state.toolCallHistory?.[1]?.resultHash).toBeTypeOf("string");
|
|
});
|
|
|
|
it("handles empty history", () => {
|
|
const state = createState();
|
|
|
|
const result = detectToolCallLoop(state, "tool", { arg: 1 }, enabledLoopDetectionConfig);
|
|
expect(result.stuck).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("message send loop detection (#89090)", () => {
|
|
// Mirror jsonResult(payload): text is the stringified payload (so it carries the
|
|
// volatile id too) and details is the payload — the shape a real send returns.
|
|
function sendResult(payload: Record<string, unknown>) {
|
|
return {
|
|
content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
|
|
details: payload,
|
|
};
|
|
}
|
|
|
|
function recordSend(
|
|
state: SessionState,
|
|
toolName: string,
|
|
params: unknown,
|
|
payload: Record<string, unknown>,
|
|
index: number,
|
|
): void {
|
|
const toolCallId = `${toolName}-${index}`;
|
|
recordToolCall(state, toolName, params, toolCallId, enabledLoopDetectionConfig);
|
|
recordToolCallOutcome(state, {
|
|
toolName,
|
|
toolParams: params,
|
|
toolCallId,
|
|
result: sendResult(payload),
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
}
|
|
|
|
function sendPayload(index: number): Record<string, unknown> {
|
|
return {
|
|
ok: true,
|
|
channel: "feishu",
|
|
chatId: "oc_chat",
|
|
runId: `run_${index}`,
|
|
messageId: `om_${index}`,
|
|
receipt: { platformMessageId: `p_${index}` },
|
|
};
|
|
}
|
|
|
|
it("gives duplicate sends a stable result hash despite per-call ids in details and text", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
recordSend(state, "message", params, sendPayload(0), 0);
|
|
recordSend(state, "message", params, sendPayload(1), 1);
|
|
const hashes = state.toolCallHistory
|
|
?.filter((call) => call.toolName === "message")
|
|
.map((call) => call.resultHash);
|
|
expect(hashes?.[0]).toBeTypeOf("string");
|
|
expect(hashes?.[0]).toBe(hashes?.[1]);
|
|
});
|
|
|
|
it("strips nested ids so broadcast results with per-call ids share a result hash", () => {
|
|
const state = createState();
|
|
const params = { action: "broadcast", text: "ping" };
|
|
const broadcast = (index: number) => ({
|
|
results: [
|
|
{ channel: "feishu", ok: true, result: { messageId: `om_${index}`, receipt: index } },
|
|
],
|
|
});
|
|
recordSend(state, "message", params, broadcast(0), 0);
|
|
recordSend(state, "message", params, broadcast(1), 1);
|
|
const hashes = state.toolCallHistory
|
|
?.filter((call) => call.toolName === "message")
|
|
.map((call) => call.resultHash);
|
|
expect(hashes?.[0]).toBe(hashes?.[1]);
|
|
});
|
|
|
|
it("escalates identical-arg send loops to critical even though every id differs", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "message", params, sendPayload(i), i);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("also blocks sessions_send loops whose result carries a fresh runId", () => {
|
|
const state = createState();
|
|
const params = { sessionKey: "agent:main:peer", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "sessions_send", params, { ok: true, runId: `run_${i}` }, i);
|
|
}
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"sessions_send",
|
|
params,
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("escalates provider-docked send-tool loops (e.g. telegram) whose result carries fresh ids", () => {
|
|
const state = createState();
|
|
const params = { to: "telegram:123", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "telegram", params, sendPayload(i), i);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "telegram", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("escalates sibling delivery actions (reply) the same as send", () => {
|
|
const state = createState();
|
|
const params = { action: "reply", target: "feishu:oc_chat", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "message", params, sendPayload(i), i);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("blocks upload-file loops whose result carries a fresh file id", () => {
|
|
const state = createState();
|
|
const params = { action: "upload-file", target: "feishu:oc_chat", media: "img.png" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{ ok: true, channel: "feishu", fileId: `f_${i}`, messageId: `om_${i}` },
|
|
i,
|
|
);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("blocks poll loops whose result carries a fresh poll id", () => {
|
|
const state = createState();
|
|
const params = { action: "poll", target: "tg:chat-1", question: "?" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{ ok: true, channel: "telegram", messageId: `om_${i}`, chatId: "c1", pollId: `pl_${i}` },
|
|
i,
|
|
);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("does not flag distinct messages (different args) as a no-progress loop", () => {
|
|
const state = createState();
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
{ action: "send", target: "feishu:oc_chat", text: `line ${i}` },
|
|
sendPayload(i),
|
|
i,
|
|
);
|
|
}
|
|
const loopResult = detectToolCallLoop(
|
|
state,
|
|
"message",
|
|
{ action: "send", target: "feishu:oc_chat", text: "line final" },
|
|
enabledLoopDetectionConfig,
|
|
);
|
|
expect(loopResult.stuck).toBe(false);
|
|
});
|
|
|
|
it("keeps full result hashing for non-send message actions so id/timestamp progress is not masked", () => {
|
|
const state = createState();
|
|
const params = { action: "read", target: "feishu:oc_chat" };
|
|
// Results differ only in id/timestamp: a non-send action must still treat that as
|
|
// progress (distinct hashes), unlike a send where those are stripped as volatile.
|
|
recordSend(state, "message", params, { ok: true, messageId: "m_0", ts: 1000 }, 0);
|
|
recordSend(state, "message", params, { ok: true, messageId: "m_1", ts: 2000 }, 1);
|
|
const hashes = state.toolCallHistory
|
|
?.filter((call) => call.toolName === "message")
|
|
.map((call) => call.resultHash);
|
|
expect(hashes?.[0]).toBeTypeOf("string");
|
|
expect(hashes?.[0]).not.toBe(hashes?.[1]);
|
|
});
|
|
|
|
it("keeps full result hashing for administrative message mutations", () => {
|
|
const state = createState();
|
|
const params = { action: "channel-create", name: "support" };
|
|
recordSend(state, "message", params, { ok: true, messageId: "m_0", timestamp: 1000 }, 0);
|
|
recordSend(state, "message", params, { ok: true, messageId: "m_1", timestamp: 2000 }, 1);
|
|
const hashes = state.toolCallHistory
|
|
?.filter((call) => call.toolName === "message")
|
|
.map((call) => call.resultHash);
|
|
expect(hashes?.[0]).toBeTypeOf("string");
|
|
expect(hashes?.[0]).not.toBe(hashes?.[1]);
|
|
});
|
|
|
|
it("preserves stable nested route ids so distinct routes stay distinguishable", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
// messageId is volatile (stripped); a nested route id is a stable fact that must survive,
|
|
// so two sends resolving to different routes keep distinct hashes.
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{ ok: true, messageId: "om_0", route: { id: "conv-A" } },
|
|
0,
|
|
);
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{ ok: true, messageId: "om_1", route: { id: "conv-B" } },
|
|
1,
|
|
);
|
|
const hashes = state.toolCallHistory
|
|
?.filter((call) => call.toolName === "message")
|
|
.map((call) => call.resultHash);
|
|
expect(hashes?.[0]).not.toBe(hashes?.[1]);
|
|
});
|
|
|
|
it("counts loop vetoes until the global circuit breaker becomes reachable", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "message", params, sendPayload(i), i);
|
|
}
|
|
for (let i = CRITICAL_THRESHOLD; i < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; i += 1) {
|
|
const before = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(before).toMatchObject({
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "generic_repeat",
|
|
count: i,
|
|
});
|
|
const recorded = recordToolCallOutcome(state, {
|
|
toolName: "message",
|
|
toolParams: params,
|
|
toolCallId: `message-veto-${i}`,
|
|
result: {
|
|
content: [{ type: "text", text: "blocked" }],
|
|
details: { status: "blocked", deniedReason: "tool-loop" },
|
|
},
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
expect(recorded).toMatchObject({
|
|
toolCallId: `message-veto-${i}`,
|
|
outcomeKind: "tool-loop-veto",
|
|
resultHash: undefined,
|
|
});
|
|
}
|
|
const after = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(after).toMatchObject({
|
|
stuck: true,
|
|
level: "critical",
|
|
detector: "global_circuit_breaker",
|
|
count: GLOBAL_CIRCUIT_BREAKER_THRESHOLD,
|
|
});
|
|
});
|
|
|
|
it("does not count unrelated hashless calls as no-progress outcomes", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "message", params, sendPayload(i), i);
|
|
}
|
|
for (let i = CRITICAL_THRESHOLD; i < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "message", params, `pending-${i}`, enabledLoopDetectionConfig);
|
|
}
|
|
|
|
expect(
|
|
detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig),
|
|
).toMatchObject({
|
|
stuck: true,
|
|
detector: "generic_repeat",
|
|
count: CRITICAL_THRESHOLD,
|
|
});
|
|
});
|
|
|
|
it("does not carry older loop vetoes across a later progressing outcome", () => {
|
|
const state = createState();
|
|
const params = { action: "send", target: "feishu:oc_chat", text: "ping" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(state, "message", params, sendPayload(i), i);
|
|
}
|
|
for (let i = 0; i < 5; i += 1) {
|
|
recordToolCallOutcome(state, {
|
|
toolName: "message",
|
|
toolParams: params,
|
|
toolCallId: `old-veto-${i}`,
|
|
result: {
|
|
content: [{ type: "text", text: "blocked" }],
|
|
details: { status: "blocked", deniedReason: "tool-loop" },
|
|
},
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
}
|
|
recordSend(state, "message", params, { ...sendPayload(25), route: { id: "new-route" } }, 25);
|
|
|
|
expect(
|
|
detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig),
|
|
).toMatchObject({
|
|
stuck: true,
|
|
level: "warning",
|
|
detector: "generic_repeat",
|
|
count: 26,
|
|
});
|
|
});
|
|
|
|
it("still escalates repeated plugin/approval vetoes to a critical loop", () => {
|
|
const state = createState();
|
|
const params = { action: "read", target: "feishu:oc_chat" };
|
|
// A non-loop veto (plugin/approval) keeps a stable result hash, so repeated identical
|
|
// denials still accumulate a no-progress streak and reach a critical block.
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordToolCall(state, "message", params, `veto-${i}`, enabledLoopDetectionConfig);
|
|
recordToolCallOutcome(state, {
|
|
toolName: "message",
|
|
toolParams: params,
|
|
toolCallId: `veto-${i}`,
|
|
result: {
|
|
content: [{ type: "text", text: "blocked" }],
|
|
details: { status: "blocked", deniedReason: "plugin-before-tool-call" },
|
|
},
|
|
config: enabledLoopDetectionConfig,
|
|
});
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("blocks plugin-shaped send results whose message object carries a bare per-send id", () => {
|
|
const state = createState();
|
|
const params = { action: "send", to: "feishu:chat-1", content: "hello" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
// The volatile id is the message object's own `id`; conversation.id is stable.
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{
|
|
message: {
|
|
id: `qa_${i}`,
|
|
accountId: "default",
|
|
direction: "outbound",
|
|
conversation: { id: "loop-room", chatType: "channel" },
|
|
senderId: "openclaw",
|
|
text: "hello",
|
|
timestamp: 1_800_000_000_000 + i,
|
|
},
|
|
},
|
|
i,
|
|
);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck).toBe(true);
|
|
if (loopResult.stuck) {
|
|
expect(loopResult.level).toBe("critical");
|
|
}
|
|
});
|
|
|
|
it("does not escalate when a stable conversation id changes between sends", () => {
|
|
const state = createState();
|
|
const params = { action: "send", to: "feishu:chat-1", content: "hello" };
|
|
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
|
|
recordSend(
|
|
state,
|
|
"message",
|
|
params,
|
|
{
|
|
message: {
|
|
id: `qa_${i}`,
|
|
direction: "outbound",
|
|
conversation: { id: `loop-room-${i}`, chatType: "channel" },
|
|
text: "hello",
|
|
},
|
|
},
|
|
i,
|
|
);
|
|
}
|
|
const loopResult = detectToolCallLoop(state, "message", params, enabledLoopDetectionConfig);
|
|
expect(loopResult.stuck && loopResult.level).not.toBe("critical");
|
|
});
|
|
});
|
|
});
|
|
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|