From 2d80bbc43de0fb5f212dcad3a41722d878642ef7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Apr 2026 18:26:31 +0100 Subject: [PATCH] feat(agents): allow disabling PI harness fallback --- docs/.generated/config-baseline.sha256 | 8 +- src/agents/harness/registry.test.ts | 6 + src/agents/harness/selection.test.ts | 72 +++++++++- src/agents/harness/selection.ts | 130 ++++++++++++++++-- .../pi-embedded-runner/run/backend.test.ts | 19 ++- src/agents/pi-embedded-runner/runtime.ts | 43 ++++-- src/config/types.agent-defaults.ts | 8 +- src/config/types.agents-shared.ts | 7 + src/config/types.agents.ts | 8 +- src/config/zod-schema.agent-defaults.ts | 2 + src/config/zod-schema.agent-runtime.ts | 9 ++ src/gateway/server-startup-post-attach.ts | 2 +- 12 files changed, 278 insertions(+), 36 deletions(-) diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index c8e85b02dbb..9324a7056db 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -a962c1d7ddffa15f2333854f77b03da4f6db07fada16f288377ee1daf50afc08 config-baseline.json -3c8455d44a63d495ad295d2c9d76fed7a190b80344dabaa0e78ba433bf2d253b config-baseline.core.json -df55c673a1cdbebc4fe68baaaf9d0d4289313be5034be92f0d510726a086b1d6 config-baseline.channel.json -3f6fccab66a9abe7e1dd412fb01b13b944ed24edbe09df55ada3323acc7f76fe config-baseline.plugin.json +c2705b6fbb297a6f06aefa6036db71aa5dbfea5a21ec3dafd53ed631cdc558f9 config-baseline.json +b8e245d02a00b696af2b4f0447553dd3b5bb98ca805aca650fb2ce5c0487eacb config-baseline.core.json +e1f94346a8507ce3dec763b598e79f3bb89ff2e33189ce977cc87d3b05e71c1d config-baseline.channel.json +9153501720ea74f9356432a011fa9b41c9b700084bfe0d156feb5647624b35ad config-baseline.plugin.json diff --git a/src/agents/harness/registry.test.ts b/src/agents/harness/registry.test.ts index 2d225c6b149..db2d2c2d014 100644 --- a/src/agents/harness/registry.test.ts +++ b/src/agents/harness/registry.test.ts @@ -13,6 +13,7 @@ import { selectAgentHarness } from "./selection.js"; import type { AgentHarness } from "./types.js"; const originalRuntime = process.env.OPENCLAW_AGENT_RUNTIME; +const originalHarnessFallback = process.env.OPENCLAW_AGENT_HARNESS_FALLBACK; afterEach(() => { clearAgentHarnesses(); @@ -21,6 +22,11 @@ afterEach(() => { } else { process.env.OPENCLAW_AGENT_RUNTIME = originalRuntime; } + if (originalHarnessFallback == null) { + delete process.env.OPENCLAW_AGENT_HARNESS_FALLBACK; + } else { + process.env.OPENCLAW_AGENT_HARNESS_FALLBACK = originalHarnessFallback; + } }); function makeHarness( diff --git a/src/agents/harness/selection.test.ts b/src/agents/harness/selection.test.ts index e21ade2e145..92d8d4094d3 100644 --- a/src/agents/harness/selection.test.ts +++ b/src/agents/harness/selection.test.ts @@ -1,11 +1,12 @@ import type { Api, Model } from "@mariozechner/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/config.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult, } from "../pi-embedded-runner/run/types.js"; import { clearAgentHarnesses, registerAgentHarness } from "./registry.js"; -import { runAgentHarnessAttemptWithFallback } from "./selection.js"; +import { runAgentHarnessAttemptWithFallback, selectAgentHarness } from "./selection.js"; import type { AgentHarness } from "./types.js"; const piRunAttempt = vi.fn(async () => createAttemptResult("pi")); @@ -20,6 +21,7 @@ vi.mock("./builtin-pi.js", () => ({ })); const originalRuntime = process.env.OPENCLAW_AGENT_RUNTIME; +const originalHarnessFallback = process.env.OPENCLAW_AGENT_HARNESS_FALLBACK; afterEach(() => { clearAgentHarnesses(); @@ -29,9 +31,14 @@ afterEach(() => { } else { process.env.OPENCLAW_AGENT_RUNTIME = originalRuntime; } + if (originalHarnessFallback == null) { + delete process.env.OPENCLAW_AGENT_HARNESS_FALLBACK; + } else { + process.env.OPENCLAW_AGENT_HARNESS_FALLBACK = originalHarnessFallback; + } }); -function createAttemptParams(): EmbeddedRunAttemptParams { +function createAttemptParams(config?: OpenClawConfig): EmbeddedRunAttemptParams { return { prompt: "hello", sessionId: "session-1", @@ -45,6 +52,7 @@ function createAttemptParams(): EmbeddedRunAttemptParams { authStorage: {} as never, modelRegistry: {} as never, thinkLevel: "low", + config, } as EmbeddedRunAttemptParams; } @@ -115,4 +123,64 @@ describe("runAgentHarnessAttemptWithFallback", () => { ); expect(piRunAttempt).not.toHaveBeenCalled(); }); + + it("disables PI retry fallback when auto-selected harness fails and fallback is none", async () => { + process.env.OPENCLAW_AGENT_RUNTIME = "auto"; + registerFailingCodexHarness(); + + await expect( + runAgentHarnessAttemptWithFallback( + createAttemptParams({ agents: { defaults: { embeddedHarness: { fallback: "none" } } } }), + ), + ).rejects.toThrow("codex startup failed"); + expect(piRunAttempt).not.toHaveBeenCalled(); + }); + + it("honors env fallback override over config fallback", async () => { + process.env.OPENCLAW_AGENT_RUNTIME = "auto"; + process.env.OPENCLAW_AGENT_HARNESS_FALLBACK = "none"; + registerFailingCodexHarness(); + + await expect(runAgentHarnessAttemptWithFallback(createAttemptParams())).rejects.toThrow( + "codex startup failed", + ); + expect(piRunAttempt).not.toHaveBeenCalled(); + }); +}); + +describe("selectAgentHarness", () => { + it("fails instead of choosing PI when no plugin harness matches and fallback is none", () => { + expect(() => + selectAgentHarness({ + provider: "anthropic", + modelId: "sonnet-4.6", + config: { agents: { defaults: { embeddedHarness: { fallback: "none" } } } }, + }), + ).toThrow("PI fallback is disabled"); + expect(piRunAttempt).not.toHaveBeenCalled(); + }); + + it("allows per-agent embedded harness policy overrides", () => { + const config: OpenClawConfig = { + agents: { + defaults: { embeddedHarness: { fallback: "pi" } }, + list: [ + { id: "main", default: true }, + { id: "strict", embeddedHarness: { fallback: "none" } }, + ], + }, + }; + + expect(() => + selectAgentHarness({ + provider: "anthropic", + modelId: "sonnet-4.6", + config, + sessionKey: "agent:strict:session-1", + }), + ).toThrow("PI fallback is disabled"); + expect(selectAgentHarness({ provider: "anthropic", modelId: "sonnet-4.6", config }).id).toBe( + "pi", + ); + }); }); diff --git a/src/agents/harness/selection.ts b/src/agents/harness/selection.ts index 18d7bc9eca7..90fa7b7f3ff 100644 --- a/src/agents/harness/selection.ts +++ b/src/agents/harness/selection.ts @@ -1,10 +1,20 @@ +import type { OpenClawConfig } from "../../config/config.js"; +import type { AgentEmbeddedHarnessConfig } from "../../config/types.agents-shared.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; +import { normalizeAgentId } from "../../routing/session-key.js"; +import { listAgentEntries, resolveSessionAgentIds } from "../agent-scope.js"; import type { CompactEmbeddedPiSessionParams } from "../pi-embedded-runner/compact.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult, } from "../pi-embedded-runner/run/types.js"; -import { resolveEmbeddedAgentRuntime } from "../pi-embedded-runner/runtime.js"; +import { + normalizeEmbeddedAgentRuntime, + resolveEmbeddedAgentHarnessFallback, + resolveEmbeddedAgentRuntime, + type EmbeddedAgentHarnessFallback, + type EmbeddedAgentRuntime, +} from "../pi-embedded-runner/runtime.js"; import type { EmbeddedPiCompactResult } from "../pi-embedded-runner/types.js"; import { createPiAgentHarness } from "./builtin-pi.js"; import { listRegisteredAgentHarnesses } from "./registry.js"; @@ -12,8 +22,13 @@ import type { AgentHarness, AgentHarnessSupport } from "./types.js"; const log = createSubsystemLogger("agents/harness"); -function listAvailableAgentHarnesses(): AgentHarness[] { - return [...listRegisteredAgentHarnesses().map((entry) => entry.harness), createPiAgentHarness()]; +type AgentHarnessPolicy = { + runtime: EmbeddedAgentRuntime; + fallback: EmbeddedAgentHarnessFallback; +}; + +function listPluginAgentHarnesses(): AgentHarness[] { + return listRegisteredAgentHarnesses().map((entry) => entry.harness); } function compareHarnessSupport( @@ -27,21 +42,39 @@ function compareHarnessSupport( return left.harness.id.localeCompare(right.harness.id); } -export function selectAgentHarness(params: { provider: string; modelId?: string }): AgentHarness { - const runtime = resolveEmbeddedAgentRuntime(); - const harnesses = listAvailableAgentHarnesses(); +export function selectAgentHarness(params: { + provider: string; + modelId?: string; + config?: OpenClawConfig; + agentId?: string; + sessionKey?: string; +}): AgentHarness { + const policy = resolveAgentHarnessPolicy(params); + // PI is intentionally not part of the plugin candidate list. It is the legacy + // fallback path, so `fallback: "none"` can prove that only plugin harnesses run. + const pluginHarnesses = listPluginAgentHarnesses(); + const piHarness = createPiAgentHarness(); + const runtime = policy.runtime; + if (runtime === "pi") { + return piHarness; + } if (runtime !== "auto") { - const forced = harnesses.find((entry) => entry.id === runtime); + const forced = pluginHarnesses.find((entry) => entry.id === runtime); if (forced) { return forced; } + if (policy.fallback === "none") { + throw new Error( + `Requested agent harness "${runtime}" is not registered and PI fallback is disabled.`, + ); + } log.warn("requested agent harness is not registered; falling back to embedded PI backend", { requestedRuntime: runtime, }); - return createPiAgentHarness(); + return piHarness; } - const supported = harnesses + const supported = pluginHarnesses .map((harness) => ({ harness, support: harness.supports({ @@ -60,16 +93,34 @@ export function selectAgentHarness(params: { provider: string; modelId?: string ) .toSorted(compareHarnessSupport); - return supported[0]?.harness ?? createPiAgentHarness(); + const selected = supported[0]?.harness; + if (selected) { + return selected; + } + if (policy.fallback === "none") { + throw new Error( + `No registered agent harness supports ${formatProviderModel(params)} and PI fallback is disabled.`, + ); + } + return piHarness; } export async function runAgentHarnessAttemptWithFallback( params: EmbeddedRunAttemptParams, ): Promise { - const runtime = resolveEmbeddedAgentRuntime(); + const policy = resolveAgentHarnessPolicy({ + provider: params.provider, + modelId: params.modelId, + config: params.config, + agentId: params.agentId, + sessionKey: params.sessionKey, + }); const harness = selectAgentHarness({ provider: params.provider, modelId: params.modelId, + config: params.config, + agentId: params.agentId, + sessionKey: params.sessionKey, }); if (harness.id === "pi") { return harness.runAttempt(params); @@ -78,7 +129,7 @@ export async function runAgentHarnessAttemptWithFallback( try { return await harness.runAttempt(params); } catch (error) { - if (runtime !== "auto") { + if (policy.runtime !== "auto" || policy.fallback === "none") { throw error; } log.warn(`${harness.label} failed; falling back to embedded PI backend`, { error }); @@ -92,9 +143,64 @@ export async function maybeCompactAgentHarnessSession( const harness = selectAgentHarness({ provider: params.provider ?? "", modelId: params.model, + config: params.config, + sessionKey: params.sessionKey, }); if (!harness.compact) { return undefined; } return harness.compact(params); } + +export function resolveAgentHarnessPolicy(params: { + provider?: string; + modelId?: string; + config?: OpenClawConfig; + agentId?: string; + sessionKey?: string; + env?: NodeJS.ProcessEnv; +}): AgentHarnessPolicy { + const env = params.env ?? process.env; + // Harness policy can be session-scoped because users may switch between agents + // with different strictness requirements inside the same gateway process. + const agentPolicy = resolveAgentEmbeddedHarnessConfig(params.config, { + agentId: params.agentId, + sessionKey: params.sessionKey, + }); + const defaultsPolicy = params.config?.agents?.defaults?.embeddedHarness; + const runtime = env.OPENCLAW_AGENT_RUNTIME?.trim() + ? resolveEmbeddedAgentRuntime(env) + : normalizeEmbeddedAgentRuntime(agentPolicy?.runtime ?? defaultsPolicy?.runtime); + return { + runtime, + fallback: + resolveEmbeddedAgentHarnessFallback(env) ?? + normalizeAgentHarnessFallback(agentPolicy?.fallback ?? defaultsPolicy?.fallback), + }; +} + +function resolveAgentEmbeddedHarnessConfig( + config: OpenClawConfig | undefined, + params: { agentId?: string; sessionKey?: string }, +): AgentEmbeddedHarnessConfig | undefined { + if (!config) { + return undefined; + } + const { sessionAgentId } = resolveSessionAgentIds({ + config, + agentId: params.agentId, + sessionKey: params.sessionKey, + }); + return listAgentEntries(config).find((entry) => normalizeAgentId(entry.id) === sessionAgentId) + ?.embeddedHarness; +} + +function normalizeAgentHarnessFallback( + value: AgentEmbeddedHarnessConfig["fallback"] | undefined, +): EmbeddedAgentHarnessFallback { + return value === "none" ? "none" : "pi"; +} + +function formatProviderModel(params: { provider: string; modelId?: string }): string { + return params.modelId ? `${params.provider}/${params.modelId}` : params.provider; +} diff --git a/src/agents/pi-embedded-runner/run/backend.test.ts b/src/agents/pi-embedded-runner/run/backend.test.ts index 80fb4f8b4b8..e659e555303 100644 --- a/src/agents/pi-embedded-runner/run/backend.test.ts +++ b/src/agents/pi-embedded-runner/run/backend.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { resolveEmbeddedAgentRuntime } from "../runtime.js"; +import { resolveEmbeddedAgentHarnessFallback, resolveEmbeddedAgentRuntime } from "../runtime.js"; describe("resolveEmbeddedAgentRuntime", () => { it("uses auto mode by default", () => { @@ -28,3 +28,20 @@ describe("resolveEmbeddedAgentRuntime", () => { ); }); }); + +describe("resolveEmbeddedAgentHarnessFallback", () => { + it("accepts the PI fallback kill switch", () => { + expect(resolveEmbeddedAgentHarnessFallback({ OPENCLAW_AGENT_HARNESS_FALLBACK: "none" })).toBe( + "none", + ); + expect(resolveEmbeddedAgentHarnessFallback({ OPENCLAW_AGENT_HARNESS_FALLBACK: "pi" })).toBe( + "pi", + ); + }); + + it("ignores unknown fallback values", () => { + expect( + resolveEmbeddedAgentHarnessFallback({ OPENCLAW_AGENT_HARNESS_FALLBACK: "custom" }), + ).toBeUndefined(); + }); +}); diff --git a/src/agents/pi-embedded-runner/runtime.ts b/src/agents/pi-embedded-runner/runtime.ts index 6d03a963ba9..639d6adc17a 100644 --- a/src/agents/pi-embedded-runner/runtime.ts +++ b/src/agents/pi-embedded-runner/runtime.ts @@ -1,20 +1,35 @@ export type EmbeddedAgentRuntime = "pi" | "auto" | (string & {}); +export type EmbeddedAgentHarnessFallback = "pi" | "none"; + +export function normalizeEmbeddedAgentRuntime(raw: string | undefined): EmbeddedAgentRuntime { + const value = raw?.trim(); + if (!value) { + return "auto"; + } + if (value === "pi") { + return "pi"; + } + if (value === "codex" || value === "codex-app-server" || value === "app-server") { + return "codex"; + } + if (value === "auto") { + return "auto"; + } + return value; +} export function resolveEmbeddedAgentRuntime( env: NodeJS.ProcessEnv = process.env, ): EmbeddedAgentRuntime { - const raw = env.OPENCLAW_AGENT_RUNTIME?.trim(); - if (!raw) { - return "auto"; - } - if (raw === "pi") { - return "pi"; - } - if (raw === "codex" || raw === "codex-app-server" || raw === "app-server") { - return "codex"; - } - if (raw === "auto") { - return "auto"; - } - return raw; + return normalizeEmbeddedAgentRuntime(env.OPENCLAW_AGENT_RUNTIME?.trim()); +} + +export function resolveEmbeddedAgentHarnessFallback( + env: NodeJS.ProcessEnv = process.env, +): EmbeddedAgentHarnessFallback | undefined { + const raw = env.OPENCLAW_AGENT_HARNESS_FALLBACK?.trim().toLowerCase(); + if (raw === "pi" || raw === "none") { + return raw; + } + return undefined; } diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 30943073f96..e3336a893e5 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -1,4 +1,8 @@ -import type { AgentModelConfig, AgentSandboxConfig } from "./types.agents-shared.js"; +import type { + AgentEmbeddedHarnessConfig, + AgentModelConfig, + AgentSandboxConfig, +} from "./types.agents-shared.js"; import type { BlockStreamingChunkConfig, BlockStreamingCoalesceConfig, @@ -127,6 +131,8 @@ export type CliBackendConfig = { export type AgentDefaultsConfig = { /** Global default provider params applied to all models before per-model and per-agent overrides. */ params?: Record; + /** Default embedded agent harness policy. */ + embeddedHarness?: AgentEmbeddedHarnessConfig; /** Primary model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */ model?: AgentModelConfig; /** Optional image-capable model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */ diff --git a/src/config/types.agents-shared.ts b/src/config/types.agents-shared.ts index 351aba6eb10..078dc7233e2 100644 --- a/src/config/types.agents-shared.ts +++ b/src/config/types.agents-shared.ts @@ -14,6 +14,13 @@ export type AgentModelConfig = fallbacks?: string[]; }; +export type AgentEmbeddedHarnessConfig = { + /** Embedded harness id: "auto", "pi", or a registered plugin harness id. */ + runtime?: string; + /** Fallback when no plugin harness matches or an auto-selected plugin harness fails. */ + fallback?: "pi" | "none"; +}; + export type AgentSandboxConfig = { mode?: "off" | "non-main" | "all"; /** Sandbox runtime backend id. Default: "docker". */ diff --git a/src/config/types.agents.ts b/src/config/types.agents.ts index c3d19a3bf74..24c6903c35e 100644 --- a/src/config/types.agents.ts +++ b/src/config/types.agents.ts @@ -1,6 +1,10 @@ import type { ChatType } from "../channels/chat-type.js"; import type { AgentDefaultsConfig } from "./types.agent-defaults.js"; -import type { AgentModelConfig, AgentSandboxConfig } from "./types.agents-shared.js"; +import type { + AgentEmbeddedHarnessConfig, + AgentModelConfig, + AgentSandboxConfig, +} from "./types.agents-shared.js"; import type { HumanDelayConfig, IdentityConfig } from "./types.base.js"; import type { GroupChatConfig } from "./types.messages.js"; import type { AgentToolsConfig, MemorySearchConfig } from "./types.tools.js"; @@ -66,6 +70,8 @@ export type AgentConfig = { agentDir?: string; /** Optional per-agent full system prompt replacement. */ systemPromptOverride?: AgentDefaultsConfig["systemPromptOverride"]; + /** Optional per-agent embedded harness policy override. */ + embeddedHarness?: AgentEmbeddedHarnessConfig; model?: AgentModelConfig; /** Optional per-agent default thinking level (overrides agents.defaults.thinkingDefault). */ thinkingDefault?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "adaptive"; diff --git a/src/config/zod-schema.agent-defaults.ts b/src/config/zod-schema.agent-defaults.ts index 19ee009a1f4..16b5cfcd8b8 100644 --- a/src/config/zod-schema.agent-defaults.ts +++ b/src/config/zod-schema.agent-defaults.ts @@ -3,6 +3,7 @@ import { isValidNonNegativeByteSizeString } from "./byte-size.js"; import { HeartbeatSchema, AgentSandboxSchema, + AgentEmbeddedHarnessSchema, AgentModelSchema, MemorySearchSchema, } from "./zod-schema.agent-runtime.js"; @@ -18,6 +19,7 @@ export const AgentDefaultsSchema = z .object({ /** Global default provider params applied to all models before per-model and per-agent overrides. */ params: z.record(z.string(), z.unknown()).optional(), + embeddedHarness: AgentEmbeddedHarnessSchema, model: AgentModelSchema.optional(), imageModel: AgentModelSchema.optional(), imageGenerationModel: AgentModelSchema.optional(), diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 3ab0b095030..bb56cf971c2 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -781,6 +781,14 @@ const AgentRuntimeSchema = z ]) .optional(); +export const AgentEmbeddedHarnessSchema = z + .object({ + runtime: z.string().optional(), + fallback: z.enum(["pi", "none"]).optional(), + }) + .strict() + .optional(); + export const AgentEntrySchema = z .object({ id: z.string(), @@ -789,6 +797,7 @@ export const AgentEntrySchema = z workspace: z.string().optional(), agentDir: z.string().optional(), systemPromptOverride: z.string().optional(), + embeddedHarness: AgentEmbeddedHarnessSchema, model: AgentModelSchema.optional(), thinkingDefault: z .enum(["off", "minimal", "low", "medium", "high", "xhigh", "adaptive"]) diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 22190a2c755..fdf54e37034 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -67,7 +67,7 @@ async function prewarmConfiguredPrimaryModel(params: { if (runtime !== "auto" && runtime !== "pi") { return; } - if (selectAgentHarness({ provider, modelId: model }).id !== "pi") { + if (selectAgentHarness({ provider, modelId: model, config: params.cfg }).id !== "pi") { return; } const agentDir = resolveOpenClawAgentDir();