feat(plugins): pass attachment metadata to before_model_resolve hook (#67322)

Merged via squash.

Prepared head SHA: 8af0ba9703
Co-authored-by: estack-takeda-yorichika <47170408+estack-takeda-yorichika@users.noreply.github.com>
Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com>
Reviewed-by: @sallyom

docs/.generated/plugin-sdk-api-baseline.sha256

@@ -1,2 +1,2 @@
-20db3f5afb93db334ad7456d26303c81b2a3eeaa5c1f8846a459eec72be20b96  plugin-sdk-api-baseline.json
-d02926e9facb3321a1018804d4c0370d9627963bee5e478942dda469e529c20b  plugin-sdk-api-baseline.jsonl
+40d6f3ba88037ba0ef7d51743f28cc996b9951137fbe65553473e71b054c6510  plugin-sdk-api-baseline.json
+869e0b705e48001a98d85c574ad1e6ec8aef11393cc5f13b936f6004f58213dd  plugin-sdk-api-baseline.jsonl

src/agents/pi-embedded-runner/run.ts

@@ -113,7 +113,11 @@ import {
 import type { RunEmbeddedPiAgentParams } from "./run/params.js";
 import { buildEmbeddedRunPayloads } from "./run/payloads.js";
 import { handleRetryLimitExhaustion } from "./run/retry-limit.js";
-import { resolveEffectiveRuntimeModel, resolveHookModelSelection } from "./run/setup.js";
+import {
+  buildBeforeModelResolveAttachments,
+  resolveEffectiveRuntimeModel,
+  resolveHookModelSelection,
+} from "./run/setup.js";
 import { mergeAttemptToolMediaPayloads } from "./run/tool-media-payloads.js";
 import {
   resolveLiveToolResultMaxChars,
@@ -302,6 +306,7 @@ export async function runEmbeddedPiAgent(
 
       const hookSelection = await resolveHookModelSelection({
         prompt: params.prompt,
+        attachments: buildBeforeModelResolveAttachments(params.images),
         provider,
         modelId,
         hookRunner,

src/agents/pi-embedded-runner/run/setup.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it, vi } from "vitest";
+import { buildBeforeModelResolveAttachments, resolveHookModelSelection } from "./setup.js";
+
+const hookContext = {
+  sessionId: "session-1",
+  workspaceDir: "/tmp/workspace",
+};
+
+describe("buildBeforeModelResolveAttachments", () => {
+  it("maps prompt image metadata to before_model_resolve attachments", () => {
+    expect(
+      buildBeforeModelResolveAttachments([{ mimeType: "image/png" }, { mimeType: "image/jpeg" }]),
+    ).toEqual([
+      { kind: "image", mimeType: "image/png" },
+      { kind: "image", mimeType: "image/jpeg" },
+    ]);
+  });
+
+  it("omits attachments when there are no images", () => {
+    expect(buildBeforeModelResolveAttachments(undefined)).toBeUndefined();
+    expect(buildBeforeModelResolveAttachments([])).toBeUndefined();
+  });
+});
+
+describe("resolveHookModelSelection", () => {
+  it("passes attachment metadata to before_model_resolve hooks", async () => {
+    const attachments = [{ kind: "image" as const, mimeType: "image/png" }];
+    const hookRunner = {
+      hasHooks: vi.fn((hookName: string) => hookName === "before_model_resolve"),
+      runBeforeModelResolve: vi.fn(async () => ({
+        providerOverride: "vision-provider",
+        modelOverride: "vision-model",
+      })),
+      runBeforeAgentStart: vi.fn(),
+    };
+
+    const result = await resolveHookModelSelection({
+      prompt: "describe this image",
+      attachments,
+      provider: "default-provider",
+      modelId: "default-model",
+      hookRunner,
+      hookContext,
+    });
+
+    expect(hookRunner.runBeforeModelResolve).toHaveBeenCalledWith(
+      { prompt: "describe this image", attachments },
+      hookContext,
+    );
+    expect(hookRunner.runBeforeAgentStart).not.toHaveBeenCalled();
+    expect(result.provider).toBe("vision-provider");
+    expect(result.modelId).toBe("vision-model");
+  });
+
+  it("omits the attachments key for text-only before_model_resolve hooks", async () => {
+    const hookRunner = {
+      hasHooks: vi.fn((hookName: string) => hookName === "before_model_resolve"),
+      runBeforeModelResolve: vi.fn(async () => undefined),
+      runBeforeAgentStart: vi.fn(),
+    };
+
+    await resolveHookModelSelection({
+      prompt: "text only",
+      provider: "default-provider",
+      modelId: "default-model",
+      hookRunner,
+      hookContext,
+    });
+
+    expect(hookRunner.runBeforeModelResolve).toHaveBeenCalledWith(
+      { prompt: "text only" },
+      hookContext,
+    );
+  });
+});

src/agents/pi-embedded-runner/run/setup.ts

@@ -1,6 +1,10 @@
 import type { OpenClawConfig } from "../../../config/types.openclaw.js";
 import type { ProviderRuntimeModel } from "../../../plugins/provider-runtime-model.types.js";
-import type { PluginHookBeforeAgentStartResult } from "../../../plugins/types.js";
+import type {
+  PluginHookBeforeAgentStartResult,
+  PluginHookBeforeModelResolveAttachment,
+  PluginHookBeforeModelResolveEvent,
+} from "../../../plugins/types.js";
 import {
   CONTEXT_WINDOW_HARD_MIN_TOKENS,
   CONTEXT_WINDOW_WARN_BELOW_TOKENS,
@@ -28,7 +32,7 @@ type HookContext = {
 type HookRunnerLike = {
   hasHooks(hookName: string): boolean;
   runBeforeModelResolve(
-    input: { prompt: string },
+    input: PluginHookBeforeModelResolveEvent,
     context: HookContext,
   ): Promise<{ providerOverride?: string; modelOverride?: string } | undefined>;
   runBeforeAgentStart(
@@ -39,6 +43,7 @@ type HookRunnerLike = {
 
 export async function resolveHookModelSelection(params: {
   prompt: string;
+  attachments?: PluginHookBeforeModelResolveAttachment[];
   provider: string;
   modelId: string;
   hookRunner?: HookRunnerLike | null;
@@ -57,10 +62,10 @@ export async function resolveHookModelSelection(params: {
   // fields if present. New hook takes precedence when both are set.
   if (hookRunner?.hasHooks("before_model_resolve")) {
     try {
-      modelResolveOverride = await hookRunner.runBeforeModelResolve(
-        { prompt: params.prompt },
-        params.hookContext,
-      );
+      const event: PluginHookBeforeModelResolveEvent = params.attachments
+        ? { prompt: params.prompt, attachments: params.attachments }
+        : { prompt: params.prompt };
+      modelResolveOverride = await hookRunner.runBeforeModelResolve(event, params.hookContext);
     } catch (hookErr) {
       log.warn(`before_model_resolve hook failed: ${String(hookErr)}`);
     }
@@ -99,6 +104,18 @@ export async function resolveHookModelSelection(params: {
   };
 }
 
+export function buildBeforeModelResolveAttachments(
+  images: readonly { mimeType?: string }[] | undefined,
+): PluginHookBeforeModelResolveAttachment[] | undefined {
+  if (!images?.length) {
+    return undefined;
+  }
+  return images.map((img) => ({
+    kind: "image",
+    mimeType: img.mimeType,
+  }));
+}
+
 export function resolveEffectiveRuntimeModel(params: {
   cfg: OpenClawConfig | undefined;
   provider: string;

src/plugins/hook-before-agent-start.types.ts

@@ -1,7 +1,14 @@
 // before_model_resolve hook
+export type PluginHookBeforeModelResolveAttachment = {
+  kind: "image" | "video" | "audio" | "document" | "other";
+  mimeType?: string;
+};
+
 export type PluginHookBeforeModelResolveEvent = {
   /** User prompt for this run. No session messages are available yet in this phase. */
   prompt: string;
+  /** Attachment metadata for file-aware model routing. */
+  attachments?: PluginHookBeforeModelResolveAttachment[];
 };
 
 export type PluginHookBeforeModelResolveResult = {

src/plugins/hook-types.ts

@@ -33,6 +33,7 @@ export type {
   PluginHookBeforeAgentStartEvent,
   PluginHookBeforeAgentStartOverrideResult,
   PluginHookBeforeAgentStartResult,
+  PluginHookBeforeModelResolveAttachment,
   PluginHookBeforeModelResolveEvent,
   PluginHookBeforeModelResolveResult,
   PluginHookBeforePromptBuildEvent,