From e1eac7a79dac04099328a72fef48faee02fdccc2 Mon Sep 17 00:00:00 2001 From: Chunyue Wang <80630709+openperf@users.noreply.github.com> Date: Sun, 5 Jul 2026 16:54:01 +0800 Subject: [PATCH] fix(agents): strip system-event prefix from modelPrompt when before_prompt_build hooks add context (#95349) * fix(agents): strip system-event prefix from modelPrompt when before_prompt_build hooks add context * fix(agents): preserve prompt boundaries when stripping runtime context * test(agents): assert nested prompt text in boundary regression * fix(agents): anchor runtime context stripping to prompt boundaries * fix(agents): prefer the active prompt boundary * fix(agents): avoid rewriting prompts without hidden context * fix(agents): make prompt stripping helper total * test(agents): drop synthetic prompt boundary proof * fix(agents): anchor prompt stripping to hook boundaries * fix(agents): preserve normalized prompt boundaries * fix(agents): carry prompt build boundaries --------- Co-authored-by: Vincent Koc Co-authored-by: Peter Steinberger --- .../embedded-agent-runner/run/attempt.ts | 11 + .../run/runtime-context-prompt.test.ts | 291 ++++++++++++++++++ .../run/runtime-context-prompt.ts | 135 +++++++- 3 files changed, 420 insertions(+), 17 deletions(-) diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index 782c344a0ef8..b94d038f9286 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -4185,6 +4185,7 @@ export async function runEmbeddedAttempt( } } const promptForModelBeforeRuntimeContextSplit = effectivePrompt; + const promptForRuntimeContextBeforeAnnotation = promptForRuntimeContextSplit; if (!isRawModelRun) { promptForRuntimeContextSplit = annotateInterSessionPromptText( promptForRuntimeContextSplit, @@ -4265,6 +4266,16 @@ export async function runEmbeddedAttempt( modelPrompt: hasPromptBuildContext ? promptForModelBeforeRuntimeContextSplit : undefined, + modelPromptBuildContext: + hasPromptBuildContext && effectiveTranscriptPrompt !== undefined + ? { + promptBeforeHooks: promptBeforePromptBuildHooks, + transcriptPromptBeforeTransforms: effectiveTranscriptPrompt, + promptBeforeAnnotation: promptForRuntimeContextBeforeAnnotation, + prependContext: promptBuildPrependContext ?? "", + appendContext: promptBuildAppendContext ?? "", + } + : undefined, emptyTranscriptMode: params.suppressNextUserMessagePersistence ? "model-prompt" : "runtime-event", diff --git a/src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts b/src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts index 1f8d1595e7d9..45f0934fb3d3 100644 --- a/src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts +++ b/src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts @@ -7,6 +7,24 @@ import { resolveRuntimeContextPromptParts, } from "./runtime-context-prompt.js"; +function withModelPromptBuildContext(params: { + promptBeforeHooks: string; + transcriptPrompt: string; + promptBeforeAnnotation?: string; + prependContext?: string; + appendContext?: string; +}) { + return { + modelPromptBuildContext: { + promptBeforeHooks: params.promptBeforeHooks, + transcriptPromptBeforeTransforms: params.transcriptPrompt, + promptBeforeAnnotation: params.promptBeforeAnnotation ?? params.promptBeforeHooks, + prependContext: params.prependContext ?? "", + appendContext: params.appendContext ?? "", + }, + }; +} + describe("runtime context prompt submission", () => { it("keeps unchanged prompts as a normal user prompt", () => { expect( @@ -99,6 +117,260 @@ describe("runtime context prompt submission", () => { }); }); + it("strips system-event prefix from modelPrompt when hooks add prepend context", () => { + // Regression: before_prompt_build hooks that add prependContext set hasPromptBuildContext=true, + // causing modelPrompt=effectivePrompt (with system-event prefix). Without this fix the event + // appeared in both runtimeContext (Message A) and modelPrompt (Message B). #95323 + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello, what can you do?"; + const prependContext = "Hook injected context"; + const queuedBody = [systemEvent, "", userText].join("\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: queuedBody, + transcriptPrompt: userText, + modelPrompt: [prependContext, "", queuedBody].join("\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: queuedBody, + transcriptPrompt: userText, + prependContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [prependContext, "", userText].join("\n"), + runtimeContext: systemEvent, + }); + }); + + it("strips system-event prefix from modelPrompt when hooks add append context", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const appendContext = "Hook tail context"; + const queuedBody = [systemEvent, "", userText].join("\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: queuedBody, + transcriptPrompt: userText, + modelPrompt: [queuedBody, "", appendContext].join("\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: queuedBody, + transcriptPrompt: userText, + appendContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [userText, "", appendContext].join("\n"), + runtimeContext: systemEvent, + }); + }); + + it("strips hidden prompt context on both sides without removing repeated hook text", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const untrustedContext = "Untrusted channel metadata"; + const hookContext = systemEvent; + const effectivePrompt = [systemEvent, userText, untrustedContext].join("\n\n"); + const modelPrompt = [hookContext, effectivePrompt, hookContext].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: userText, + modelPrompt, + ...withModelPromptBuildContext({ + promptBeforeHooks: effectivePrompt, + transcriptPrompt: userText, + prependContext: hookContext, + appendContext: hookContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [hookContext, userText, hookContext].join("\n\n"), + runtimeContext: [systemEvent, untrustedContext].join("\n\n"), + }); + }); + + it("anchors hidden-context removal before append hooks that repeat the prompt", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const appendContext = "Hook summary: Hello"; + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: [systemEvent, userText].join("\n\n"), + transcriptPrompt: userText, + modelPrompt: [systemEvent, userText, appendContext].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: [systemEvent, userText].join("\n\n"), + transcriptPrompt: userText, + appendContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [userText, appendContext].join("\n\n"), + runtimeContext: systemEvent, + }); + }); + + it("strips the last matching prompt occurrence when prepend hooks quote the body", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const untrustedContext = "Untrusted channel metadata"; + const effectivePrompt = [systemEvent, userText, untrustedContext].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: userText, + modelPrompt: [effectivePrompt, effectivePrompt].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: effectivePrompt, + transcriptPrompt: userText, + prependContext: effectivePrompt, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [effectivePrompt, userText].join("\n\n"), + runtimeContext: [systemEvent, untrustedContext].join("\n\n"), + }); + }); + + it("strips the active prompt before append hooks that quote the body", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const effectivePrompt = [systemEvent, userText].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: userText, + modelPrompt: [effectivePrompt, effectivePrompt].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: effectivePrompt, + transcriptPrompt: userText, + appendContext: effectivePrompt, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [userText, effectivePrompt].join("\n\n"), + runtimeContext: systemEvent, + }); + }); + + it("normalizes quoted hook prompts before locating the active prompt", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const internalContext = [ + "<<>>", + "private runtime note", + "<<>>", + ].join("\n"); + const effectivePrompt = [systemEvent, userText, internalContext].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: userText, + modelPrompt: [effectivePrompt, effectivePrompt].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: effectivePrompt, + transcriptPrompt: userText, + appendContext: effectivePrompt, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [userText, systemEvent, userText].join("\n\n"), + runtimeContext: [systemEvent, internalContext].join("\n\n"), + }); + }); + + it("preserves user prompt edge whitespace while removing hidden context", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = " leading text "; + const appendContext = "Hook tail"; + const effectivePrompt = [systemEvent, userText].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt, + transcriptPrompt: userText, + modelPrompt: [effectivePrompt, appendContext].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks: effectivePrompt, + transcriptPrompt: userText, + appendContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: `${userText}\n\n${appendContext}`, + runtimeContext: systemEvent, + }); + }); + + it("strips hidden context after prompt transforms decorate the active hook body", () => { + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const prependContext = "Hook injected context"; + const queuedContext = "[Queued messages while agent was busy]"; + const promptBeforeHooks = [systemEvent, userText].join("\n\n"); + const promptBeforeAnnotation = [queuedContext, promptBeforeHooks].join("\n\n"); + const transcriptPrompt = [queuedContext, userText].join("\n\n"); + const modelPrompt = [queuedContext, prependContext, promptBeforeHooks].join("\n\n"); + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: promptBeforeAnnotation, + transcriptPrompt, + modelPrompt, + ...withModelPromptBuildContext({ + promptBeforeHooks, + transcriptPrompt: userText, + promptBeforeAnnotation, + prependContext, + }), + }), + ).toEqual({ + prompt: transcriptPrompt, + modelPrompt: [queuedContext, prependContext, userText].join("\n\n"), + runtimeContext: systemEvent, + }); + }); + + it("keeps outer provenance context ahead of source runtime context", () => { + const provenance = "[Inter-session message] sourceTool=sessions_send isUser=false"; + const systemEvent = "System: [2026-06-20 13:59:51] Slack DM from Alice"; + const userText = "Hello"; + const promptBeforeHooks = [systemEvent, userText].join("\n\n"); + const prependContext = "Hook injected context"; + + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: [provenance, promptBeforeHooks].join("\n"), + transcriptPrompt: userText, + modelPrompt: [prependContext, promptBeforeHooks].join("\n\n"), + ...withModelPromptBuildContext({ + promptBeforeHooks, + transcriptPrompt: userText, + prependContext, + }), + }), + ).toEqual({ + prompt: userText, + modelPrompt: [prependContext, userText].join("\n\n"), + runtimeContext: [provenance, systemEvent].join("\n\n"), + }); + }); + it("does not extract no-transcript delimiter text", () => { const effectivePrompt = [ "visible ask", @@ -180,6 +452,25 @@ describe("runtime context prompt submission", () => { ); }); + it("preserves repeated hook text when there is no hidden runtime context", () => { + const modelPrompt = "Hello\n\nHook summary: Hello"; + expect( + resolveRuntimeContextPromptParts({ + effectivePrompt: "Hello", + transcriptPrompt: "Hello", + modelPrompt, + ...withModelPromptBuildContext({ + promptBeforeHooks: "Hello", + transcriptPrompt: "Hello", + appendContext: "Hook summary: Hello", + }), + }), + ).toEqual({ + prompt: "Hello", + modelPrompt, + }); + }); + it("fails closed for unterminated hidden runtime context blocks", () => { // Unterminated internal context is ambiguous; keep only the known transcript // prompt rather than leaking partial hidden content. diff --git a/src/agents/embedded-agent-runner/run/runtime-context-prompt.ts b/src/agents/embedded-agent-runner/run/runtime-context-prompt.ts index 5406fa52ca3b..c5e69dfe74c2 100644 --- a/src/agents/embedded-agent-runner/run/runtime-context-prompt.ts +++ b/src/agents/embedded-agent-runner/run/runtime-context-prompt.ts @@ -34,6 +34,14 @@ export type RuntimeContextCustomMessage = { type EmptyTranscriptMode = "model-prompt" | "runtime-event"; +type ModelPromptBuildContext = { + promptBeforeHooks: string; + transcriptPromptBeforeTransforms: string; + promptBeforeAnnotation: string; + prependContext: string; + appendContext: string; +}; + /** Combines inbound context and the current prompt using the channel-provided joiner. */ export function buildCurrentInboundPrompt(params: { context: CurrentInboundPromptContext | undefined; @@ -54,17 +62,53 @@ export function buildCurrentInboundPrompt(params: { return [prefix, params.prompt].join(params.context?.promptJoiner ?? "\n\n"); } -function removeLastPromptOccurrence(text: string, prompt: string): string | null { +function splitLastPromptOccurrence( + text: string, + prompt: string, +): { before: string; after: string } | null { const index = text.lastIndexOf(prompt); if (index === -1) { return null; } - const before = text.slice(0, index).trimEnd(); - const after = text.slice(index + prompt.length).trimStart(); - return [before, after] - .filter((part) => part.length > 0) - .join("\n\n") - .trim(); + return { + before: text.slice(0, index), + after: text.slice(index + prompt.length), + }; +} + +function replacePromptOccurrenceWithinHookBounds(params: { + text: string; + promptBeforeHooks: string; + transcriptPrompt: string; + prependContext: string; + appendContext: string; +}): string | null { + if (!params.promptBeforeHooks) { + return null; + } + const prependIndex = params.prependContext ? params.text.indexOf(params.prependContext) : -1; + if (params.prependContext && prependIndex === -1) { + return null; + } + const searchStart = prependIndex === -1 ? 0 : prependIndex + params.prependContext.length; + const appendIndex = params.appendContext ? params.text.lastIndexOf(params.appendContext) : -1; + if (params.appendContext && appendIndex < searchStart) { + return null; + } + const searchEnd = appendIndex === -1 ? params.text.length : appendIndex; + const occurrenceIndex = params.text.lastIndexOf( + params.promptBeforeHooks, + searchEnd - params.promptBeforeHooks.length, + ); + if ( + occurrenceIndex < searchStart || + occurrenceIndex + params.promptBeforeHooks.length > searchEnd + ) { + return null; + } + return `${params.text.slice(0, occurrenceIndex)}${params.transcriptPrompt}${params.text.slice( + occurrenceIndex + params.promptBeforeHooks.length, + )}`; } /** @@ -76,6 +120,7 @@ export function resolveRuntimeContextPromptParts(params: { effectivePrompt: string; transcriptPrompt?: string; modelPrompt?: string; + modelPromptBuildContext?: ModelPromptBuildContext; emptyTranscriptMode?: EmptyTranscriptMode; }): RuntimeContextPromptParts { const transcriptPrompt = params.transcriptPrompt; @@ -89,6 +134,23 @@ export function resolveRuntimeContextPromptParts(params: { : shouldExtractInternalRuntimeContext ? extractInternalRuntimeContext(params.modelPrompt) : { text: params.modelPrompt }; + const modelPromptBuildContext = params.modelPromptBuildContext + ? { + promptBeforeHooks: extractInternalRuntimeContext( + params.modelPromptBuildContext.promptBeforeHooks, + ).text, + transcriptPromptBeforeTransforms: extractInternalRuntimeContext( + params.modelPromptBuildContext.transcriptPromptBeforeTransforms, + ).text, + promptBeforeAnnotation: extractInternalRuntimeContext( + params.modelPromptBuildContext.promptBeforeAnnotation, + ).text, + prependContext: extractInternalRuntimeContext(params.modelPromptBuildContext.prependContext) + .text, + appendContext: extractInternalRuntimeContext(params.modelPromptBuildContext.appendContext) + .text, + } + : undefined; const modelPromptText = modelPrompt?.text ?? transcriptPrompt ?? extracted.text; const prompt = transcriptPrompt ?? extracted.text; if (!prompt.trim() && params.emptyTranscriptMode === "model-prompt") { @@ -100,14 +162,36 @@ export function resolveRuntimeContextPromptParts(params: { ...(extracted.runtimeContext ? { runtimeContext: extracted.runtimeContext } : {}), }; } - const hiddenRuntimeContext = modelPrompt - ? (removeLastPromptOccurrence(extracted.text, modelPrompt.text)?.trim() ?? - (transcriptPrompt - ? removeLastPromptOccurrence(extracted.text, transcriptPrompt)?.trim() - : undefined)) - : transcriptPrompt - ? removeLastPromptOccurrence(extracted.text, transcriptPrompt)?.trim() - : undefined; + const sourcePromptParts = modelPromptBuildContext + ? splitLastPromptOccurrence( + modelPromptBuildContext.promptBeforeHooks, + modelPromptBuildContext.transcriptPromptBeforeTransforms, + ) + : undefined; + const outerPromptParts = modelPromptBuildContext + ? splitLastPromptOccurrence(extracted.text, modelPromptBuildContext.promptBeforeAnnotation) + : undefined; + const fallbackPromptParts = !modelPromptBuildContext + ? modelPrompt + ? (splitLastPromptOccurrence(extracted.text, modelPrompt.text) ?? + (transcriptPrompt + ? splitLastPromptOccurrence(extracted.text, transcriptPrompt) + : undefined)) + : transcriptPrompt + ? splitLastPromptOccurrence(extracted.text, transcriptPrompt) + : undefined + : undefined; + // Source context sits inside the active prompt; provenance sits outside all + // prompt transforms. Preserve that nesting order when hiding both. + const hiddenRuntimeContext = [ + outerPromptParts?.before, + sourcePromptParts?.before ?? fallbackPromptParts?.before, + sourcePromptParts?.after ?? fallbackPromptParts?.after, + outerPromptParts?.after, + ] + .map((part) => part?.trim()) + .filter((part): part is string => Boolean(part)) + .join("\n\n"); // The hidden context is whatever remains after removing the last visible // prompt occurrence, plus any explicit internal runtime-context block. const runtimeContext = @@ -134,10 +218,27 @@ export function resolveRuntimeContextPromptParts(params: { }; } + // When hooks added pre-prompt context, modelPromptText still contains the + // system-event prefix that was separated into runtimeContext. Strip it so + // events aren't delivered to the model twice (Message A and Message B). + const hasHiddenSourceContext = Boolean( + sourcePromptParts?.before.trim() || sourcePromptParts?.after.trim(), + ); + const returnModelPromptText = + hasHiddenSourceContext && modelPromptBuildContext && modelPrompt + ? (replacePromptOccurrenceWithinHookBounds({ + text: modelPromptText, + promptBeforeHooks: modelPromptBuildContext.promptBeforeHooks, + transcriptPrompt: modelPromptBuildContext.transcriptPromptBeforeTransforms, + prependContext: modelPromptBuildContext.prependContext, + appendContext: modelPromptBuildContext.appendContext, + }) ?? modelPromptText) + : modelPromptText; + return { prompt, - ...(modelPromptText.trim() && modelPromptText !== prompt - ? { modelPrompt: modelPromptText } + ...(returnModelPromptText.trim() && returnModelPromptText !== prompt + ? { modelPrompt: returnModelPromptText } : {}), ...(runtimeContext ? { runtimeContext } : {}), };