From 137dbc71f05e1e357074660f62526e1522855b8f Mon Sep 17 00:00:00 2001 From: EVA Date: Fri, 24 Apr 2026 03:57:09 +0700 Subject: [PATCH] feat: expose harness selection decisions (#70760) Codex harness selection now keeps the decision helper internal, logs debug-only selection reasons and candidates, and documents `/status` as the primary user-facing signal. Thanks @100yenadmin. Co-authored-by: Eva --- CHANGELOG.md | 1 + docs/plugins/codex-harness.md | 6 + docs/plugins/sdk-agent-harness.md | 4 + src/agents/harness/selection.test.ts | 74 +++++++++++++ src/agents/harness/selection.ts | 159 ++++++++++++++++++++++++--- 5 files changed, 227 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730da7b3847..d291bec6669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai ### Changes - Agents/subagents: add optional forked context for native `sessions_spawn` runs so agents can let a child inherit the requester transcript when needed, while keeping clean isolated sessions as the default; includes prompt guidance, context-engine hook metadata, docs, and QA coverage. +- Codex harness: add structured debug logging for embedded harness selection decisions so `/status` stays simple while gateway logs explain auto-selection and Pi fallback reasons. (#70760) Thanks @100yenadmin. - Providers/OpenAI: add forward-compatible `gpt-5.5` and `gpt-5.5-pro` support for OpenAI API keys, OpenAI Codex OAuth, and the Codex CLI default model. - Providers/OpenAI Codex: add image generation and reference-image editing through Codex OAuth, so `openai-codex/gpt-image-2` works without an `OPENAI_API_KEY`. Fixes #70703. diff --git a/docs/plugins/codex-harness.md b/docs/plugins/codex-harness.md index b96c9798ca5..d8086fd95d3 100644 --- a/docs/plugins/codex-harness.md +++ b/docs/plugins/codex-harness.md @@ -47,6 +47,12 @@ OpenClaw now keeps OpenAI GPT model refs canonical as `openai/*`: Legacy `openai-codex/gpt-*` and `codex/gpt-*` refs remain accepted as compatibility aliases, but new docs/config examples should use `openai/gpt-*`. +Use `/status` to confirm the effective harness for the current session. If the +selection is surprising, enable debug logging for the `agents/harness` subsystem +and inspect the gateway's structured `agent harness selected` record. It +includes the selected harness id, selection reason, runtime/fallback policy, and, +in `auto` mode, each plugin candidate's support result. + Harness selection is not a live session control. When an embedded turn runs, OpenClaw records the selected harness id on that session and keeps using it for later turns in the same session id. Change `embeddedHarness` config or diff --git a/docs/plugins/sdk-agent-harness.md b/docs/plugins/sdk-agent-harness.md index 059d0e67090..741c0e5ab88 100644 --- a/docs/plugins/sdk-agent-harness.md +++ b/docs/plugins/sdk-agent-harness.md @@ -106,6 +106,10 @@ Legacy sessions created before harness pins are treated as PI-pinned once they have transcript history. Use a new/reset session when changing between PI and a native plugin harness. `/status` shows non-default harness ids such as `codex` next to `Fast`; PI stays hidden because it is the default compatibility path. +If the selected harness is surprising, enable `agents/harness` debug logging and +inspect the gateway's structured `agent harness selected` record. It includes +the selected harness id, selection reason, runtime/fallback policy, and, in +`auto` mode, each plugin candidate's support result. The bundled Codex plugin registers `codex` as its harness id. Core treats that as an ordinary plugin harness id; Codex-specific aliases belong in the plugin diff --git a/src/agents/harness/selection.test.ts b/src/agents/harness/selection.test.ts index 02780f01d7d..59c246da792 100644 --- a/src/agents/harness/selection.test.ts +++ b/src/agents/harness/selection.test.ts @@ -152,6 +152,80 @@ describe("runAgentHarnessAttemptWithFallback", () => { }); describe("selectAgentHarness", () => { + it("auto-selects the highest-priority plugin harness without duplicate support probes", () => { + process.env.OPENCLAW_AGENT_RUNTIME = "auto"; + const lowPrioritySupports = vi.fn(() => ({ + supported: true as const, + priority: 10, + reason: "generic codex support", + })); + const highPrioritySupports = vi.fn(() => ({ + supported: true as const, + priority: 100, + reason: "native codex app-server", + })); + const unsupportedSupports = vi.fn(() => ({ + supported: false as const, + reason: "provider mismatch", + })); + registerAgentHarness( + { + id: "codex-low", + label: "Low Codex", + supports: lowPrioritySupports, + runAttempt: vi.fn(async () => createAttemptResult("codex-low")), + }, + { ownerPluginId: "codex-low" }, + ); + registerAgentHarness( + { + id: "codex-high", + label: "High Codex", + supports: highPrioritySupports, + runAttempt: vi.fn(async () => createAttemptResult("codex-high")), + }, + { ownerPluginId: "codex-high" }, + ); + registerAgentHarness( + { + id: "other", + label: "Other Harness", + supports: unsupportedSupports, + runAttempt: vi.fn(async () => createAttemptResult("other")), + }, + { ownerPluginId: "other" }, + ); + + const harness = selectAgentHarness({ + provider: "codex", + modelId: "gpt-5.4", + }); + + expect(harness.id).toBe("codex-high"); + expect(lowPrioritySupports).toHaveBeenCalledTimes(1); + expect(highPrioritySupports).toHaveBeenCalledTimes(1); + expect(unsupportedSupports).toHaveBeenCalledTimes(1); + }); + + it("keeps pinned PI selection from probing plugin support", () => { + const supports = vi.fn(() => ({ supported: true as const, priority: 100 })); + registerAgentHarness({ + id: "codex", + label: "Codex", + supports, + runAttempt: vi.fn(async () => createAttemptResult("codex")), + }); + + const harness = selectAgentHarness({ + provider: "codex", + modelId: "gpt-5.4", + agentHarnessId: "pi", + }); + + expect(harness.id).toBe("pi"); + expect(supports).not.toHaveBeenCalled(); + }); + it("fails instead of choosing PI when no plugin harness matches and fallback is none", () => { expect(() => selectAgentHarness({ diff --git a/src/agents/harness/selection.ts b/src/agents/harness/selection.ts index 02bb7d53602..8d9b151c24d 100644 --- a/src/agents/harness/selection.ts +++ b/src/agents/harness/selection.ts @@ -28,6 +28,31 @@ type AgentHarnessPolicy = { fallback: EmbeddedAgentHarnessFallback; }; +type AgentHarnessSelectionCandidate = { + id: string; + label: string; + pluginId?: string; + supported?: boolean; + priority?: number; + reason?: string; +}; + +type AgentHarnessSelectionDecision = { + harness: AgentHarness; + policy: AgentHarnessPolicy; + selectedHarnessId: string; + selectedReason: + | "pinned" + | "forced_pi" + | "forced_plugin" + | "forced_plugin_fallback_to_pi" + // Auto mode chose a registered plugin harness that supports the provider/model. + | "auto_plugin" + // Auto mode found no supporting plugin harness, so PI handled the run. + | "auto_pi_fallback"; + candidates: AgentHarnessSelectionCandidate[]; +}; + function listPluginAgentHarnesses(): AgentHarness[] { return listRegisteredAgentHarnesses().map((entry) => entry.harness); } @@ -51,20 +76,41 @@ export function selectAgentHarness(params: { sessionKey?: string; agentHarnessId?: string; }): AgentHarness { - const policy = - resolvePinnedAgentHarnessPolicy(params.agentHarnessId) ?? resolveAgentHarnessPolicy(params); + return selectAgentHarnessDecision(params).harness; +} + +function selectAgentHarnessDecision(params: { + provider: string; + modelId?: string; + config?: OpenClawConfig; + agentId?: string; + sessionKey?: string; + agentHarnessId?: string; +}): AgentHarnessSelectionDecision { + const pinnedPolicy = resolvePinnedAgentHarnessPolicy(params.agentHarnessId); + const policy = pinnedPolicy ?? resolveAgentHarnessPolicy(params); // PI is intentionally not part of the plugin candidate list. It is the legacy // fallback path, so `fallback: "none"` can prove that only plugin harnesses run. const pluginHarnesses = listPluginAgentHarnesses(); const piHarness = createPiAgentHarness(); const runtime = policy.runtime; if (runtime === "pi") { - return piHarness; + return buildSelectionDecision({ + harness: piHarness, + policy, + selectedReason: pinnedPolicy ? "pinned" : "forced_pi", + candidates: listHarnessCandidates(pluginHarnesses), + }); } if (runtime !== "auto") { const forced = pluginHarnesses.find((entry) => entry.id === runtime); if (forced) { - return forced; + return buildSelectionDecision({ + harness: forced, + policy, + selectedReason: pinnedPolicy ? "pinned" : "forced_plugin", + candidates: listHarnessCandidates(pluginHarnesses), + }); } if (policy.fallback === "none") { throw new Error( @@ -74,18 +120,23 @@ export function selectAgentHarness(params: { log.warn("requested agent harness is not registered; falling back to embedded PI backend", { requestedRuntime: runtime, }); - return piHarness; + return buildSelectionDecision({ + harness: piHarness, + policy, + selectedReason: "forced_plugin_fallback_to_pi", + candidates: listHarnessCandidates(pluginHarnesses), + }); } - const supported = pluginHarnesses - .map((harness) => ({ - harness, - support: harness.supports({ - provider: params.provider, - modelId: params.modelId, - requestedRuntime: runtime, - }), - })) + const candidates = pluginHarnesses.map((harness) => ({ + harness, + support: harness.supports({ + provider: params.provider, + modelId: params.modelId, + requestedRuntime: runtime, + }), + })); + const supported = candidates .filter( ( entry, @@ -98,20 +149,30 @@ export function selectAgentHarness(params: { const selected = supported[0]?.harness; if (selected) { - return selected; + return buildSelectionDecision({ + harness: selected, + policy, + selectedReason: "auto_plugin", + candidates: candidates.map(toSelectionCandidate), + }); } if (policy.fallback === "none") { throw new Error( `No registered agent harness supports ${formatProviderModel(params)} and PI fallback is disabled.`, ); } - return piHarness; + return buildSelectionDecision({ + harness: piHarness, + policy, + selectedReason: "auto_pi_fallback", + candidates: candidates.map(toSelectionCandidate), + }); } export async function runAgentHarnessAttemptWithFallback( params: EmbeddedRunAttemptParams, ): Promise { - const harness = selectAgentHarness({ + const selection = selectAgentHarnessDecision({ provider: params.provider, modelId: params.modelId, config: params.config, @@ -119,6 +180,13 @@ export async function runAgentHarnessAttemptWithFallback( sessionKey: params.sessionKey, agentHarnessId: params.agentHarnessId, }); + const harness = selection.harness; + logAgentHarnessSelection(selection, { + provider: params.provider, + modelId: params.modelId, + sessionKey: params.sessionKey, + agentId: params.agentId, + }); if (harness.id === "pi") { const result = await harness.runAttempt(params); return { ...result, agentHarnessId: harness.id }; @@ -138,6 +206,63 @@ export async function runAgentHarnessAttemptWithFallback( } } +function listHarnessCandidates(harnesses: AgentHarness[]): AgentHarnessSelectionCandidate[] { + return harnesses.map((harness) => ({ + id: harness.id, + label: harness.label, + pluginId: harness.pluginId, + })); +} + +function toSelectionCandidate(entry: { + harness: AgentHarness; + support: AgentHarnessSupport; +}): AgentHarnessSelectionCandidate { + return { + id: entry.harness.id, + label: entry.harness.label, + pluginId: entry.harness.pluginId, + supported: entry.support.supported, + priority: entry.support.supported ? entry.support.priority : undefined, + reason: entry.support.reason, + }; +} + +function buildSelectionDecision(params: { + harness: AgentHarness; + policy: AgentHarnessPolicy; + selectedReason: AgentHarnessSelectionDecision["selectedReason"]; + candidates: AgentHarnessSelectionCandidate[]; +}): AgentHarnessSelectionDecision { + return { + harness: params.harness, + policy: params.policy, + selectedHarnessId: params.harness.id, + selectedReason: params.selectedReason, + candidates: params.candidates, + }; +} + +function logAgentHarnessSelection( + selection: AgentHarnessSelectionDecision, + params: { provider: string; modelId?: string; sessionKey?: string; agentId?: string }, +) { + if (!log.isEnabled("debug")) { + return; + } + log.debug("agent harness selected", { + provider: params.provider, + modelId: params.modelId, + sessionKey: params.sessionKey, + agentId: params.agentId, + selectedHarnessId: selection.selectedHarnessId, + selectedReason: selection.selectedReason, + runtime: selection.policy.runtime, + fallback: selection.policy.fallback, + candidates: selection.candidates, + }); +} + function resolvePinnedAgentHarnessPolicy( agentHarnessId: string | undefined, ): AgentHarnessPolicy | undefined {