feat(plugins): add sanitized model call hooks

This commit is contained in:
Vincent Koc
2026-04-25 10:56:18 -07:00
parent 9ffe764416
commit 275c128e99
8 changed files with 321 additions and 44 deletions

View File

@@ -59,6 +59,8 @@ export type PluginHookName =
| "before_prompt_build"
| "before_agent_start"
| "before_agent_reply"
| "model_call_started"
| "model_call_ended"
| "llm_input"
| "llm_output"
| "agent_end"
@@ -90,6 +92,8 @@ export const PLUGIN_HOOK_NAMES = [
"before_prompt_build",
"before_agent_start",
"before_agent_reply",
"model_call_started",
"model_call_ended",
"llm_input",
"llm_output",
"agent_end",
@@ -187,6 +191,26 @@ export type PluginHookLlmInputEvent = {
imagesCount: number;
};
export type PluginHookModelCallBaseEvent = {
runId: string;
callId: string;
sessionKey?: string;
sessionId?: string;
provider: string;
model: string;
api?: string;
transport?: string;
};
export type PluginHookModelCallStartedEvent = PluginHookModelCallBaseEvent;
export type PluginHookModelCallEndedEvent = PluginHookModelCallBaseEvent & {
durationMs: number;
outcome: "completed" | "error";
errorCategory?: string;
upstreamRequestIdHash?: string;
};
export type PluginHookLlmOutputEvent = {
runId: string;
sessionId: string;
@@ -676,6 +700,14 @@ export type PluginHookHandlerMap = {
event: PluginHookBeforeAgentReplyEvent,
ctx: PluginHookAgentContext,
) => Promise<PluginHookBeforeAgentReplyResult | void> | PluginHookBeforeAgentReplyResult | void;
model_call_started: (
event: PluginHookModelCallStartedEvent,
ctx: PluginHookAgentContext,
) => Promise<void> | void;
model_call_ended: (
event: PluginHookModelCallEndedEvent,
ctx: PluginHookAgentContext,
) => Promise<void> | void;
llm_input: (event: PluginHookLlmInputEvent, ctx: PluginHookAgentContext) => Promise<void> | void;
llm_output: (
event: PluginHookLlmOutputEvent,

View File

@@ -29,6 +29,8 @@ import type {
PluginHookBeforePromptBuildEvent,
PluginHookBeforePromptBuildResult,
PluginHookBeforeCompactionEvent,
PluginHookModelCallEndedEvent,
PluginHookModelCallStartedEvent,
PluginHookInboundClaimContext,
PluginHookInboundClaimEvent,
PluginHookInboundClaimResult,
@@ -85,6 +87,8 @@ export type {
PluginHookBeforeModelResolveResult,
PluginHookBeforePromptBuildEvent,
PluginHookBeforePromptBuildResult,
PluginHookModelCallEndedEvent,
PluginHookModelCallStartedEvent,
PluginHookLlmInputEvent,
PluginHookLlmOutputEvent,
PluginHookAgentEndEvent,
@@ -588,6 +592,30 @@ export function createHookRunner(
);
}
/**
* Run model_call_started hook.
* Allows plugins to observe sanitized model-call metadata.
* Runs in parallel (fire-and-forget).
*/
async function runModelCallStarted(
event: PluginHookModelCallStartedEvent,
ctx: PluginHookAgentContext,
): Promise<void> {
return runVoidHook("model_call_started", event, ctx);
}
/**
* Run model_call_ended hook.
* Allows plugins to observe sanitized terminal model-call metadata.
* Runs in parallel (fire-and-forget).
*/
async function runModelCallEnded(
event: PluginHookModelCallEndedEvent,
ctx: PluginHookAgentContext,
): Promise<void> {
return runVoidHook("model_call_ended", event, ctx);
}
/**
* Run agent_end hook.
* Allows plugins to analyze completed conversations.
@@ -1124,6 +1152,8 @@ export function createHookRunner(
runBeforePromptBuild,
runBeforeAgentStart,
runBeforeAgentReply,
runModelCallStarted,
runModelCallEnded,
runLlmInput,
runLlmOutput,
runAgentEnd,

View File

@@ -7,14 +7,24 @@ const hookCtx = {
};
async function expectLlmHookCall(params: {
hookName: "llm_input" | "llm_output";
hookName: "model_call_started" | "model_call_ended" | "llm_input" | "llm_output";
event: Record<string, unknown>;
expectedEvent: Record<string, unknown>;
}) {
const handler = vi.fn();
const { runner } = createHookRunnerWithRegistry([{ hookName: params.hookName, handler }]);
if (params.hookName === "llm_input") {
if (params.hookName === "model_call_started") {
await runner.runModelCallStarted(
params.event as Parameters<typeof runner.runModelCallStarted>[0],
hookCtx,
);
} else if (params.hookName === "model_call_ended") {
await runner.runModelCallEnded(
params.event as Parameters<typeof runner.runModelCallEnded>[0],
hookCtx,
);
} else if (params.hookName === "llm_input") {
await runner.runLlmInput(
{
...params.event,
@@ -40,6 +50,38 @@ async function expectLlmHookCall(params: {
describe("llm hook runner methods", () => {
it.each([
{
name: "runModelCallStarted invokes registered model_call_started hooks",
hookName: "model_call_started" as const,
methodName: "runModelCallStarted" as const,
event: {
runId: "run-1",
callId: "call-1",
sessionId: "session-1",
provider: "openai",
model: "gpt-5",
api: "openai-responses",
transport: "http",
},
expectedEvent: { runId: "run-1", callId: "call-1", provider: "openai" },
},
{
name: "runModelCallEnded invokes registered model_call_ended hooks",
hookName: "model_call_ended" as const,
methodName: "runModelCallEnded" as const,
event: {
runId: "run-1",
callId: "call-1",
sessionId: "session-1",
provider: "openai",
model: "gpt-5",
durationMs: 42,
outcome: "error",
errorCategory: "TimeoutError",
upstreamRequestIdHash: "sha256:abcdef123456",
},
expectedEvent: { runId: "run-1", callId: "call-1", outcome: "error" },
},
{
name: "runLlmInput invokes registered llm_input hooks",
hookName: "llm_input" as const,
@@ -80,8 +122,13 @@ describe("llm hook runner methods", () => {
});
it("hasHooks returns true for registered llm hooks", () => {
const { runner } = createHookRunnerWithRegistry([{ hookName: "llm_input", handler: vi.fn() }]);
const { runner } = createHookRunnerWithRegistry([
{ hookName: "model_call_started", handler: vi.fn() },
{ hookName: "llm_input", handler: vi.fn() },
]);
expect(runner.hasHooks("model_call_started")).toBe(true);
expect(runner.hasHooks("model_call_ended")).toBe(false);
expect(runner.hasHooks("llm_input")).toBe(true);
expect(runner.hasHooks("llm_output")).toBe(false);
});