test(agents): tolerate Anthropic cache tool drift

This commit is contained in:
Vincent Koc
2026-05-17 02:15:04 +08:00
parent d5035bad62
commit ec38e96884
2 changed files with 31 additions and 2 deletions

View File

@@ -143,6 +143,16 @@ describe("live cache regression runner", () => {
).toBe(1024);
});
it("classifies Anthropic tool-only probe misses as provider drift", () => {
expect(__testing.isAnthropicToolProbeDrift(new Error("expected tool call for noop"))).toBe(true);
expect(
__testing.isAnthropicToolProbeDrift(
new Error('expected tool-only response for noop, got "ok"'),
),
).toBe(true);
expect(__testing.isAnthropicToolProbeDrift(new Error("other failure"))).toBe(false);
});
it("accepts empty cache probe text only when usage is observable", () => {
expect(
__testing.shouldAcceptEmptyCacheProbe({

View File

@@ -605,6 +605,16 @@ function isAnthropicEmptyCacheProbe(error: unknown): boolean {
return error instanceof CacheProbeTextMismatchError && error.text.trim().length === 0;
}
function isAnthropicToolProbeDrift(error: unknown): boolean {
if (!(error instanceof Error)) {
return false;
}
return (
error.message.startsWith("expected tool call for ") ||
error.message.startsWith("expected tool-only response for ")
);
}
function shouldSkipAnthropicCacheProviderDrift(error: unknown): boolean {
return Boolean(
shouldSkipLiveProviderDrift({
@@ -646,8 +656,16 @@ async function runAnthropicCacheLane(params: {
}
}
if (shouldSkipAnthropicCacheProviderDrift(lastError) || isAnthropicEmptyCacheProbe(lastError)) {
const reason = isAnthropicEmptyCacheProbe(lastError) ? "empty response" : "account drift";
if (
shouldSkipAnthropicCacheProviderDrift(lastError) ||
isAnthropicEmptyCacheProbe(lastError) ||
isAnthropicToolProbeDrift(lastError)
) {
const reason = isAnthropicEmptyCacheProbe(lastError)
? "empty response"
: isAnthropicToolProbeDrift(lastError)
? "tool probe drift"
: "account drift";
const warning = `anthropic ${params.lane} skipped: ${reason}`;
params.warnings.push(warning);
logLiveCache(warning);
@@ -682,6 +700,7 @@ export const __testing = {
assertAgainstBaseline,
evaluateAgainstBaseline,
resolveCacheProbeMaxTokens,
isAnthropicToolProbeDrift,
shouldAcceptEmptyCacheProbe,
shouldRetryCacheProbeText,
shouldRetryBaselineFindings,