fix(cron): classify denied isolated runs

This commit is contained in:
Peter Steinberger
2026-04-27 03:01:47 +01:00
parent 22c9e82e83
commit fc3abc139b
5 changed files with 149 additions and 5 deletions

View File

@@ -1,5 +1,31 @@
import { describe, expect, it } from "vitest";
import { resolveCronPayloadOutcome } from "./isolated-agent/helpers.js";
import { detectCronDenialToken, resolveCronPayloadOutcome } from "./isolated-agent/helpers.js";
describe("detectCronDenialToken", () => {
it("matches host denial markers case-sensitively", () => {
expect(detectCronDenialToken("SYSTEM_RUN_DENIED: approval blocked")).toBe("SYSTEM_RUN_DENIED");
expect(detectCronDenialToken("INVALID_REQUEST: denied")).toBe("INVALID_REQUEST");
expect(detectCronDenialToken("system_run_denied: approval blocked")).toBeUndefined();
expect(detectCronDenialToken("invalid_request: denied")).toBeUndefined();
});
it("matches model-narrated denial phrases case-insensitively", () => {
expect(detectCronDenialToken("Approval Cannot Safely Bind this runtime command")).toBe(
"approval cannot safely bind",
);
expect(detectCronDenialToken("The runtime denied the operation.")).toBe("runtime denied");
expect(detectCronDenialToken("I could not run the script.")).toBe("could not run");
expect(detectCronDenialToken("The command did not run to completion.")).toBe("did not run");
expect(detectCronDenialToken("The request was denied by policy.")).toBe("was denied");
});
it("ignores empty and non-token text", () => {
expect(detectCronDenialToken(undefined)).toBeUndefined();
expect(
detectCronDenialToken("The denied claim was reviewed, then the job succeeded."),
).toBeUndefined();
});
});
describe("resolveCronPayloadOutcome", () => {
it("uses the last non-empty non-error payload as summary and output", () => {
@@ -134,4 +160,47 @@ describe("resolveCronPayloadOutcome", () => {
{ text: "Final weather summary" },
]);
});
it("promotes narrated denial markers in summary text to fatal errors", () => {
const result = resolveCronPayloadOutcome({
payloads: [
{
text: "SYSTEM_RUN_DENIED: approval cannot safely bind this interpreter/runtime command",
},
],
});
expect(result.hasFatalErrorPayload).toBe(true);
expect(result.embeddedRunError).toBe(
'cron classifier: denial token "SYSTEM_RUN_DENIED" detected in summary',
);
});
it("promotes narrated denial markers from final assistant visible text", () => {
const result = resolveCronPayloadOutcome({
payloads: [{ text: "Working on it..." }],
finalAssistantVisibleText: "I could not run the requested script.",
preferFinalAssistantVisibleText: true,
});
expect(result.hasFatalErrorPayload).toBe(true);
expect(result.outputText).toBe("I could not run the requested script.");
expect(result.embeddedRunError).toBe(
'cron classifier: denial token "could not run" detected in summary',
);
});
it("keeps structured error payload reasons ahead of denial-token reasons", () => {
const result = resolveCronPayloadOutcome({
payloads: [
{
text: "Exec failed before SYSTEM_RUN_DENIED could be retried",
isError: true,
},
],
});
expect(result.hasFatalErrorPayload).toBe(true);
expect(result.embeddedRunError).toBe("Exec failed before SYSTEM_RUN_DENIED could be retried");
});
});

View File

@@ -21,6 +21,60 @@ export type CronPayloadOutcome = {
embeddedRunError?: string;
};
type CronDenialSignal = {
token: string;
field: string;
};
const CRON_DENIAL_EXACT_TOKENS = ["SYSTEM_RUN_DENIED", "INVALID_REQUEST"] as const;
const CRON_DENIAL_CASE_INSENSITIVE_TOKENS = [
"approval cannot safely bind",
"runtime denied",
"could not run",
"did not run",
"was denied",
] as const;
export function detectCronDenialToken(text: string | undefined): string | undefined {
const normalized = normalizeOptionalString(text);
if (!normalized) {
return undefined;
}
for (const token of CRON_DENIAL_EXACT_TOKENS) {
if (normalized.includes(token)) {
return token;
}
}
const lowerText = normalized.toLowerCase();
for (const token of CRON_DENIAL_CASE_INSENSITIVE_TOKENS) {
if (lowerText.includes(token)) {
return token;
}
}
return undefined;
}
function resolveCronDenialSignal(
fields: Array<{ field: string; text?: string | undefined }>,
): CronDenialSignal | undefined {
const seen = new Set<string>();
for (const { field, text } of fields) {
if (seen.has(field)) {
continue;
}
seen.add(field);
const token = detectCronDenialToken(text);
if (token) {
return { token, field };
}
}
return undefined;
}
function formatCronDenialSignal(signal: CronDenialSignal): string {
return `cron classifier: denial token "${signal.token}" detected in ${signal.field}`;
}
export function pickSummaryFromOutput(text: string | undefined) {
const clean = (text ?? "").trim();
if (!clean) {
@@ -157,7 +211,7 @@ export function resolveCronPayloadOutcome(params: {
params.payloads
.slice(lastErrorPayloadIndex + 1)
.some((payload) => payload?.isError !== true && Boolean(payload?.text?.trim()));
const hasFatalErrorPayload = hasErrorPayload && !hasSuccessfulPayloadAfterLastError;
const hasFatalStructuredErrorPayload = hasErrorPayload && !hasSuccessfulPayloadAfterLastError;
const normalizedFinalAssistantVisibleText = normalizeOptionalString(
params.finalAssistantVisibleText,
);
@@ -169,7 +223,7 @@ export function resolveCronPayloadOutcome(params: {
const shouldUseFinalAssistantVisibleText =
params.preferFinalAssistantVisibleText === true &&
normalizedFinalAssistantVisibleText !== undefined &&
!hasFatalErrorPayload &&
!hasFatalStructuredErrorPayload &&
!hasStructuredDeliveryPayloads;
const summary = shouldUseFinalAssistantVisibleText
? (pickSummaryFromOutput(normalizedFinalAssistantVisibleText) ?? fallbackSummary)
@@ -189,6 +243,18 @@ export function resolveCronPayloadOutcome(params: {
.toReversed()
.find((payload) => payload?.isError === true && Boolean(payload?.text?.trim()))
?.text?.trim();
const denialSignal = resolveCronDenialSignal([
{ field: "summary", text: summary },
{ field: "outputText", text: outputText },
{ field: "synthesizedText", text: synthesizedText },
{ field: "fallbackSummary", text: fallbackSummary },
{ field: "fallbackOutputText", text: fallbackOutputText },
...params.payloads.map((payload, index) => ({
field: `payloads[${index}].text`,
text: payload?.text,
})),
]);
const hasFatalErrorPayload = hasFatalStructuredErrorPayload || denialSignal !== undefined;
return {
summary,
outputText,
@@ -197,8 +263,10 @@ export function resolveCronPayloadOutcome(params: {
deliveryPayloads: resolvedDeliveryPayloads,
deliveryPayloadHasStructuredContent,
hasFatalErrorPayload,
embeddedRunError: hasFatalErrorPayload
embeddedRunError: hasFatalStructuredErrorPayload
? (lastErrorPayloadText ?? "cron isolated run returned an error payload")
: undefined,
: denialSignal
? formatCronDenialSignal(denialSignal)
: undefined,
};
}