From 8af0ba970301727c905151c7e5ed991e1afdffc4 Mon Sep 17 00:00:00 2001 From: sallyom Date: Sun, 19 Apr 2026 23:42:50 -0400 Subject: [PATCH] Plugins: pass attachments to model resolve hooks Signed-off-by: sallyom --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- src/agents/pi-embedded-runner/run.ts | 13 ++-- .../pi-embedded-runner/run/setup.test.ts | 75 +++++++++++++++++++ src/agents/pi-embedded-runner/run/setup.ts | 30 ++++++-- src/plugins/hook-types.ts | 1 + 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 src/agents/pi-embedded-runner/run/setup.test.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 7765bfd0d2b..615f4670385 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -20db3f5afb93db334ad7456d26303c81b2a3eeaa5c1f8846a459eec72be20b96 plugin-sdk-api-baseline.json -d02926e9facb3321a1018804d4c0370d9627963bee5e478942dda469e529c20b plugin-sdk-api-baseline.jsonl +40d6f3ba88037ba0ef7d51743f28cc996b9951137fbe65553473e71b054c6510 plugin-sdk-api-baseline.json +869e0b705e48001a98d85c574ad1e6ec8aef11393cc5f13b936f6004f58213dd plugin-sdk-api-baseline.jsonl diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 6cab79ea815..927ce7f3998 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -113,7 +113,11 @@ import { import type { RunEmbeddedPiAgentParams } from "./run/params.js"; import { buildEmbeddedRunPayloads } from "./run/payloads.js"; import { handleRetryLimitExhaustion } from "./run/retry-limit.js"; -import { resolveEffectiveRuntimeModel, resolveHookModelSelection } from "./run/setup.js"; +import { + buildBeforeModelResolveAttachments, + resolveEffectiveRuntimeModel, + resolveHookModelSelection, +} from "./run/setup.js"; import { mergeAttemptToolMediaPayloads } from "./run/tool-media-payloads.js"; import { resolveLiveToolResultMaxChars, @@ -300,14 +304,9 @@ export async function runEmbeddedPiAgent( channelId: params.messageChannel ?? params.messageProvider ?? undefined, }; - const attachments = (params.images ?? []).map((img) => ({ - kind: "image" as const, - mimeType: img.mimeType, - })); - const hookSelection = await resolveHookModelSelection({ prompt: params.prompt, - attachments: attachments.length > 0 ? attachments : undefined, + attachments: buildBeforeModelResolveAttachments(params.images), provider, modelId, hookRunner, diff --git a/src/agents/pi-embedded-runner/run/setup.test.ts b/src/agents/pi-embedded-runner/run/setup.test.ts new file mode 100644 index 00000000000..17ca0c80673 --- /dev/null +++ b/src/agents/pi-embedded-runner/run/setup.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildBeforeModelResolveAttachments, resolveHookModelSelection } from "./setup.js"; + +const hookContext = { + sessionId: "session-1", + workspaceDir: "/tmp/workspace", +}; + +describe("buildBeforeModelResolveAttachments", () => { + it("maps prompt image metadata to before_model_resolve attachments", () => { + expect( + buildBeforeModelResolveAttachments([{ mimeType: "image/png" }, { mimeType: "image/jpeg" }]), + ).toEqual([ + { kind: "image", mimeType: "image/png" }, + { kind: "image", mimeType: "image/jpeg" }, + ]); + }); + + it("omits attachments when there are no images", () => { + expect(buildBeforeModelResolveAttachments(undefined)).toBeUndefined(); + expect(buildBeforeModelResolveAttachments([])).toBeUndefined(); + }); +}); + +describe("resolveHookModelSelection", () => { + it("passes attachment metadata to before_model_resolve hooks", async () => { + const attachments = [{ kind: "image" as const, mimeType: "image/png" }]; + const hookRunner = { + hasHooks: vi.fn((hookName: string) => hookName === "before_model_resolve"), + runBeforeModelResolve: vi.fn(async () => ({ + providerOverride: "vision-provider", + modelOverride: "vision-model", + })), + runBeforeAgentStart: vi.fn(), + }; + + const result = await resolveHookModelSelection({ + prompt: "describe this image", + attachments, + provider: "default-provider", + modelId: "default-model", + hookRunner, + hookContext, + }); + + expect(hookRunner.runBeforeModelResolve).toHaveBeenCalledWith( + { prompt: "describe this image", attachments }, + hookContext, + ); + expect(hookRunner.runBeforeAgentStart).not.toHaveBeenCalled(); + expect(result.provider).toBe("vision-provider"); + expect(result.modelId).toBe("vision-model"); + }); + + it("omits the attachments key for text-only before_model_resolve hooks", async () => { + const hookRunner = { + hasHooks: vi.fn((hookName: string) => hookName === "before_model_resolve"), + runBeforeModelResolve: vi.fn(async () => undefined), + runBeforeAgentStart: vi.fn(), + }; + + await resolveHookModelSelection({ + prompt: "text only", + provider: "default-provider", + modelId: "default-model", + hookRunner, + hookContext, + }); + + expect(hookRunner.runBeforeModelResolve).toHaveBeenCalledWith( + { prompt: "text only" }, + hookContext, + ); + }); +}); diff --git a/src/agents/pi-embedded-runner/run/setup.ts b/src/agents/pi-embedded-runner/run/setup.ts index 2134dc6f631..aa9e099c3be 100644 --- a/src/agents/pi-embedded-runner/run/setup.ts +++ b/src/agents/pi-embedded-runner/run/setup.ts @@ -1,6 +1,10 @@ import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import type { ProviderRuntimeModel } from "../../../plugins/provider-runtime-model.types.js"; -import type { PluginHookBeforeAgentStartResult } from "../../../plugins/types.js"; +import type { + PluginHookBeforeAgentStartResult, + PluginHookBeforeModelResolveAttachment, + PluginHookBeforeModelResolveEvent, +} from "../../../plugins/types.js"; import { CONTEXT_WINDOW_HARD_MIN_TOKENS, CONTEXT_WINDOW_WARN_BELOW_TOKENS, @@ -28,7 +32,7 @@ type HookContext = { type HookRunnerLike = { hasHooks(hookName: string): boolean; runBeforeModelResolve( - input: { prompt: string; attachments?: { kind: string; mimeType?: string }[] }, + input: PluginHookBeforeModelResolveEvent, context: HookContext, ): Promise<{ providerOverride?: string; modelOverride?: string } | undefined>; runBeforeAgentStart( @@ -39,7 +43,7 @@ type HookRunnerLike = { export async function resolveHookModelSelection(params: { prompt: string; - attachments?: { kind: string; mimeType?: string }[]; + attachments?: PluginHookBeforeModelResolveAttachment[]; provider: string; modelId: string; hookRunner?: HookRunnerLike | null; @@ -58,10 +62,10 @@ export async function resolveHookModelSelection(params: { // fields if present. New hook takes precedence when both are set. if (hookRunner?.hasHooks("before_model_resolve")) { try { - modelResolveOverride = await hookRunner.runBeforeModelResolve( - { prompt: params.prompt, attachments: params.attachments }, - params.hookContext, - ); + const event: PluginHookBeforeModelResolveEvent = params.attachments + ? { prompt: params.prompt, attachments: params.attachments } + : { prompt: params.prompt }; + modelResolveOverride = await hookRunner.runBeforeModelResolve(event, params.hookContext); } catch (hookErr) { log.warn(`before_model_resolve hook failed: ${String(hookErr)}`); } @@ -100,6 +104,18 @@ export async function resolveHookModelSelection(params: { }; } +export function buildBeforeModelResolveAttachments( + images: readonly { mimeType?: string }[] | undefined, +): PluginHookBeforeModelResolveAttachment[] | undefined { + if (!images?.length) { + return undefined; + } + return images.map((img) => ({ + kind: "image", + mimeType: img.mimeType, + })); +} + export function resolveEffectiveRuntimeModel(params: { cfg: OpenClawConfig | undefined; provider: string; diff --git a/src/plugins/hook-types.ts b/src/plugins/hook-types.ts index 765c1f3a1f2..64667ee68cf 100644 --- a/src/plugins/hook-types.ts +++ b/src/plugins/hook-types.ts @@ -33,6 +33,7 @@ export type { PluginHookBeforeAgentStartEvent, PluginHookBeforeAgentStartOverrideResult, PluginHookBeforeAgentStartResult, + PluginHookBeforeModelResolveAttachment, PluginHookBeforeModelResolveEvent, PluginHookBeforeModelResolveResult, PluginHookBeforePromptBuildEvent,