[codex] Add contract-first Pi/Codex runtime plan suite (#71096)

* test: add pi codex runtime contract coverage

* test: expand pi codex tool runtime contracts

* test: tighten tool runtime contracts

* test: reset tool contract param cache

* test: document codex tool middleware fixture

* test: type pi tool contract events

* test: satisfy pi tool contract test types

* test: cover tool media telemetry contracts

* test: reset plugin runtime after tool contracts

* test: add auth profile runtime contracts

* test: strengthen auth profile runtime contracts

* test: clarify auth profile contract fixtures

* test: expand auth profile contract matrix

* test: assert unrelated cli auth isolation

* test: expand auth profile contract matrix

* test: tighten auth profile contract expectations

* test: add outcome fallback runtime contracts

* test: strengthen outcome fallback contracts

* test: isolate outcome fallback contracts

* test: cover codex terminal outcome signals

* test: expand terminal fallback contracts

* test: add delivery no reply runtime contracts

* test: document json no-reply delivery gap

* test: align delivery contract fixtures

* test: add transcript repair runtime contracts

* test: tighten transcript repair contracts

* test: add prompt overlay runtime contracts

* test: tighten prompt overlay contract scope

* test: type prompt overlay contracts

* test: add schema normalization runtime contracts

* test: clarify schema normalization contract gaps

* test: simplify schema normalization contracts

* test: tighten schema normalization contract gaps

* test: cover compaction schema contract

* test: satisfy schema contract lint

* test: add transport params runtime contracts

* test: tighten transport params contract scope

* test: isolate transport params contracts

* test: lock exact transport defaults

* feat: add agent runtime plan foundation

* fix: preserve codex harness auth profiles

* fix: route followup delivery through runtime plan

* fix: normalize parameter-free openai tool schemas

* fix: satisfy runtime plan type checks

* fix: narrow followup delivery runtime planning

* fix: apply codex app-server auth profiles

* fix: classify codex terminal outcomes

* fix: prevent harness auth leakage into unrelated cli providers

* feat: expand agent runtime plan policy contract

* fix: route pi runtime policy through runtime plan

* fix: route codex runtime policy through runtime plan

* fix: route fallback outcome classification through runtime plan

* refactor: make runtime plan contracts topology-safe

* fix: restore runtime plan test type coverage

* fix: align runtime plan schema contract assertions

* fix: stabilize incomplete turn runtime tests

* fix: stabilize codex native web search test

* fix: preserve codex auth profile secret refs

* fix: keep runtime resolved refs canonical

* fix: preserve permissive nested openai schemas

* fix: accept Codex auth provider aliases

* test: update media-only groups mock

* fix: resolve runtime plan rebase checks

* fix: resolve runtime plan rebase checks

---------

Co-authored-by: Eva <eva@100yen.org>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

test/helpers/agents/auth-profile-runtime-contract.ts

@@ -0,0 +1,60 @@
+import {
+  resolveProviderIdForAuth,
+  type ProviderAuthAliasLookupParams,
+} from "../../../src/agents/provider-auth-aliases.js";
+import type { PluginManifestRegistry } from "../../../src/plugins/manifest-registry.js";
+
+export const AUTH_PROFILE_RUNTIME_CONTRACT = {
+  sessionId: "session-auth-contract",
+  sessionKey: "agent:main:auth-contract",
+  runId: "run-auth-contract",
+  workspacePrompt: "continue with the bound Codex profile",
+  openAiProvider: "openai",
+  openAiCodexProvider: "openai-codex",
+  codexCliProvider: "codex-cli",
+  codexHarnessProvider: "codex",
+  claudeCliProvider: "claude-cli",
+  openAiProfileId: "openai:work",
+  openAiCodexProfileId: "openai-codex:work",
+  anthropicProfileId: "anthropic:work",
+} as const;
+
+export function createAuthAliasManifestRegistry(): PluginManifestRegistry {
+  return {
+    plugins: [
+      {
+        id: "openai",
+        origin: "bundled",
+        channels: [],
+        providers: [],
+        cliBackends: [],
+        skills: [],
+        hooks: [],
+        rootDir: "/tmp/openclaw-auth-contract-plugin",
+        source: "test",
+        manifestPath: "/tmp/openclaw-auth-contract-plugin/plugin.json",
+        providerAuthChoices: [
+          {
+            provider: AUTH_PROFILE_RUNTIME_CONTRACT.openAiCodexProvider,
+            method: "oauth",
+            choiceId: AUTH_PROFILE_RUNTIME_CONTRACT.openAiCodexProvider,
+            deprecatedChoiceIds: [AUTH_PROFILE_RUNTIME_CONTRACT.codexCliProvider],
+          },
+        ],
+      },
+    ],
+    diagnostics: [],
+  };
+}
+
+export function expectedForwardedAuthProfile(params: {
+  provider: string;
+  authProfileProvider: string;
+  aliasLookupParams: ProviderAuthAliasLookupParams;
+  sessionAuthProfileId: string | undefined;
+}): string | undefined {
+  return resolveProviderIdForAuth(params.provider, params.aliasLookupParams) ===
+    resolveProviderIdForAuth(params.authProfileProvider, params.aliasLookupParams)
+    ? params.sessionAuthProfileId
+    : undefined;
+}

test/helpers/agents/delivery-no-reply-runtime-contract.ts

@@ -0,0 +1,12 @@
+export const DELIVERY_NO_REPLY_RUNTIME_CONTRACT = {
+  sessionId: "session-delivery-contract",
+  sessionKey: "agent:main:delivery-contract",
+  runId: "run-delivery-contract",
+  prompt: "deliver the follow-up contract turn",
+  originChannel: "discord",
+  originTo: "channel:C1",
+  dispatcherText: "visible dispatcher fallback",
+  visibleText: "visible follow-up",
+  silentText: "NO_REPLY",
+  jsonSilentText: '{"action":"NO_REPLY"}',
+} as const;

test/helpers/agents/openclaw-owned-tool-runtime-contract.ts

@@ -0,0 +1,94 @@
+import type { AgentToolResult } from "@mariozechner/pi-agent-core";
+import { vi } from "vitest";
+import { __testing as beforeToolCallTesting } from "../../../src/agents/pi-tools.before-tool-call.js";
+import type {
+  CodexAppServerExtensionFactory,
+  CodexAppServerToolResultEvent,
+} from "../../../src/plugins/codex-app-server-extension-types.js";
+import {
+  initializeGlobalHookRunner,
+  resetGlobalHookRunner,
+} from "../../../src/plugins/hook-runner-global.js";
+import { createMockPluginRegistry } from "../../../src/plugins/hooks.test-helpers.js";
+import { createEmptyPluginRegistry } from "../../../src/plugins/registry-empty.js";
+import {
+  resetPluginRuntimeStateForTest,
+  setActivePluginRegistry,
+} from "../../../src/plugins/runtime.js";
+
+export function textToolResult(
+  text: string,
+  details: Record<string, unknown> = {},
+): AgentToolResult<unknown> {
+  return {
+    content: [{ type: "text", text }],
+    details,
+  };
+}
+
+export function mediaToolResult(
+  text: string,
+  mediaUrl: string,
+  audioAsVoice = false,
+): AgentToolResult<unknown> {
+  return textToolResult(text, {
+    media: {
+      mediaUrl,
+      ...(audioAsVoice ? { audioAsVoice } : {}),
+    },
+  });
+}
+
+export function installOpenClawOwnedToolHooks(params?: {
+  adjustedParams?: Record<string, unknown>;
+  blockReason?: string;
+}) {
+  const beforeToolCall = vi.fn(async () => {
+    if (params?.blockReason) {
+      return {
+        block: true,
+        blockReason: params.blockReason,
+      };
+    }
+    return params?.adjustedParams ? { params: params.adjustedParams } : {};
+  });
+  const afterToolCall = vi.fn(async () => {});
+  initializeGlobalHookRunner(
+    createMockPluginRegistry([
+      { hookName: "before_tool_call", handler: beforeToolCall },
+      { hookName: "after_tool_call", handler: afterToolCall },
+    ]),
+  );
+  return { beforeToolCall, afterToolCall };
+}
+
+/**
+ * Installs only the Codex app-server `tool_result` middleware fixture.
+ * Pair with `installOpenClawOwnedToolHooks()` when a test asserts before/after hook behavior.
+ */
+export function installCodexToolResultMiddleware(
+  handler: (event: CodexAppServerToolResultEvent) => AgentToolResult<unknown>,
+) {
+  const middleware = vi.fn(async (event: CodexAppServerToolResultEvent) => ({
+    result: handler(event),
+  }));
+  const registry = createEmptyPluginRegistry();
+  const factory: CodexAppServerExtensionFactory = async (codex) => {
+    codex.on("tool_result", middleware);
+  };
+  registry.codexAppServerExtensionFactories.push({
+    pluginId: "runtime-contract",
+    pluginName: "Runtime Contract",
+    rawFactory: factory,
+    factory,
+    source: "test",
+  });
+  setActivePluginRegistry(registry);
+  return { middleware };
+}
+
+export function resetOpenClawOwnedToolHooks(): void {
+  resetGlobalHookRunner();
+  resetPluginRuntimeStateForTest();
+  beforeToolCallTesting.adjustedParamsByToolCallId.clear();
+}

test/helpers/agents/outcome-fallback-runtime-contract.ts

@@ -0,0 +1,48 @@
+import type { EmbeddedPiRunResult } from "../../../src/agents/pi-embedded-runner/types.js";
+
+export const OUTCOME_FALLBACK_RUNTIME_CONTRACT = {
+  primaryProvider: "openai-codex",
+  primaryModel: "gpt-5.4",
+  fallbackProvider: "anthropic",
+  fallbackModel: "claude-haiku-3-5",
+  sessionId: "session-outcome-contract",
+  sessionKey: "agent:main:outcome-contract",
+  runId: "run-outcome-contract",
+  prompt: "finish the contract turn",
+  reasoningOnlyText: "I need to reason about this before answering.",
+  planningOnlyText: "Inspect state, then decide the next step.",
+} as const;
+
+export function createContractRunResult(
+  overrides: Partial<EmbeddedPiRunResult> = {},
+): EmbeddedPiRunResult {
+  const { meta, ...rest } = overrides;
+  return {
+    payloads: [],
+    didSendViaMessagingTool: false,
+    messagingToolSentTexts: [],
+    messagingToolSentMediaUrls: [],
+    messagingToolSentTargets: [],
+    successfulCronAdds: 0,
+    ...rest,
+    meta: {
+      durationMs: 1,
+      ...meta,
+    },
+  };
+}
+
+export function createContractFallbackConfig() {
+  return {
+    agents: {
+      defaults: {
+        model: {
+          primary: `${OUTCOME_FALLBACK_RUNTIME_CONTRACT.primaryProvider}/${OUTCOME_FALLBACK_RUNTIME_CONTRACT.primaryModel}`,
+          fallbacks: [
+            `${OUTCOME_FALLBACK_RUNTIME_CONTRACT.fallbackProvider}/${OUTCOME_FALLBACK_RUNTIME_CONTRACT.fallbackModel}`,
+          ],
+        },
+      },
+    },
+  } as const;
+}

test/helpers/agents/prompt-overlay-runtime-contract.ts

@@ -0,0 +1,48 @@
+import type { OpenClawConfig } from "../../../src/config/types.openclaw.js";
+import type { ProviderSystemPromptContributionContext } from "../../../src/plugins/types.js";
+
+export const GPT5_CONTRACT_MODEL_ID = "gpt-5.4";
+export const GPT5_PREFIXED_CONTRACT_MODEL_ID = "openai/gpt-5.4";
+export const NON_GPT5_CONTRACT_MODEL_ID = "gpt-4.1";
+export const OPENAI_CONTRACT_PROVIDER_ID = "openai";
+export const OPENAI_CODEX_CONTRACT_PROVIDER_ID = "openai-codex";
+export const CODEX_CONTRACT_PROVIDER_ID = "codex";
+export const NON_OPENAI_CONTRACT_PROVIDER_ID = "openrouter";
+
+export function openAiPluginPersonalityConfig(personality: "friendly" | "off"): OpenClawConfig {
+  return {
+    plugins: {
+      entries: {
+        openai: {
+          config: { personality },
+        },
+      },
+    },
+  } satisfies OpenClawConfig;
+}
+
+export function sharedGpt5PersonalityConfig(personality: "friendly" | "off"): OpenClawConfig {
+  return {
+    agents: {
+      defaults: {
+        promptOverlays: {
+          gpt5: { personality },
+        },
+      },
+    },
+  } satisfies OpenClawConfig;
+}
+
+export function codexPromptOverlayContext(params?: {
+  modelId?: string;
+  config?: OpenClawConfig;
+}): ProviderSystemPromptContributionContext {
+  return {
+    provider: CODEX_CONTRACT_PROVIDER_ID,
+    modelId: params?.modelId ?? GPT5_CONTRACT_MODEL_ID,
+    promptMode: "full",
+    agentDir: "/tmp/openclaw-codex-prompt-contract-agent",
+    workspaceDir: "/tmp/openclaw-codex-prompt-contract-workspace",
+    ...(params?.config ? { config: params.config } : {}),
+  };
+}

test/helpers/agents/schema-normalization-runtime-contract.ts

@@ -0,0 +1,92 @@
+export function createParameterFreeTool(name = "ping") {
+  return {
+    name,
+    description: "Parameter-free test tool",
+    parameters: {},
+  };
+}
+
+export function createStrictCompatibleTool(name = "lookup") {
+  return {
+    name,
+    description: "Strict-compatible test tool",
+    parameters: {
+      type: "object",
+      properties: {
+        path: { type: "string" },
+      },
+      required: ["path"],
+      additionalProperties: false,
+    },
+  };
+}
+
+export function createPermissiveTool(name = "schedule") {
+  return {
+    name,
+    description: "Permissive test tool",
+    parameters: {
+      type: "object",
+      properties: {
+        action: { type: "string" },
+        cron: { type: "string" },
+      },
+      required: ["action"],
+      additionalProperties: true,
+    },
+  };
+}
+
+export function createNativeOpenAIResponsesModel() {
+  return {
+    id: "gpt-5.4",
+    name: "GPT-5.4",
+    api: "openai-responses",
+    provider: "openai",
+    baseUrl: "https://api.openai.com/v1",
+    reasoning: true,
+    input: ["text"],
+    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+    contextWindow: 200_000,
+    maxTokens: 8_192,
+  };
+}
+
+export function createNativeOpenAICodexResponsesModel() {
+  return {
+    id: "gpt-5.4",
+    name: "GPT-5.4",
+    api: "openai-codex-responses",
+    provider: "openai-codex",
+    baseUrl: "https://chatgpt.com/backend-api",
+    reasoning: true,
+    input: ["text"],
+    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+    contextWindow: 200_000,
+    maxTokens: 8_192,
+  };
+}
+
+export function createProxyOpenAIResponsesModel() {
+  return {
+    id: "custom-gpt",
+    name: "Custom GPT",
+    api: "openai-responses",
+    provider: "openai",
+    baseUrl: "https://proxy.example.com/v1",
+    reasoning: true,
+    input: ["text"],
+    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+    contextWindow: 200_000,
+    maxTokens: 8_192,
+  };
+}
+
+export function normalizedParameterFreeSchema() {
+  return {
+    type: "object",
+    properties: {},
+    required: [],
+    additionalProperties: false,
+  };
+}

test/helpers/agents/transcript-repair-runtime-contract.ts

@@ -0,0 +1,62 @@
+import type { AgentMessage } from "@mariozechner/pi-agent-core";
+
+export const QUEUED_USER_MESSAGE_MARKER =
+  "[Queued user message that arrived while the previous turn was still active]";
+
+export function textOrphanLeaf(text = "older active-turn message"): { content: string } {
+  return { content: text };
+}
+
+export function structuredOrphanLeaf(): { content: unknown[] } {
+  return {
+    content: [
+      { type: "text", text: "please inspect this" },
+      { type: "image_url", image_url: { url: "https://example.test/cat.png" } },
+      { type: "input_audio", audio_url: "https://example.test/cat.wav" },
+    ],
+  };
+}
+
+export function inlineDataUriOrphanLeaf(): { content: unknown[] } {
+  return {
+    content: [
+      { type: "text", text: "please inspect this inline image" },
+      { type: "image_url", image_url: { url: `data:image/png;base64,${"a".repeat(4096)}` } },
+    ],
+  };
+}
+
+export function mediaOnlyHistoryMessage(): AgentMessage {
+  return {
+    role: "user",
+    content: [{ type: "image", data: "b".repeat(2048), mimeType: "image/png" }],
+    timestamp: 1,
+  } as AgentMessage;
+}
+
+export function structuredHistoryMessage(): AgentMessage {
+  return {
+    role: "user",
+    content: [
+      { type: "text", text: "older structured context" },
+      { type: "image", data: "c".repeat(64), mimeType: "image/png" },
+    ],
+    timestamp: 1,
+  } as AgentMessage;
+}
+
+export function currentPromptHistoryMessage(prompt: string): AgentMessage {
+  return {
+    role: "user",
+    content: [{ type: "text", text: prompt }],
+    timestamp: 2,
+  } as AgentMessage;
+}
+
+export function assistantHistoryMessage(text = "ack"): AgentMessage {
+  return {
+    role: "assistant",
+    content: [{ type: "text", text }],
+    timestamp: 2,
+  } as AgentMessage;
+}

test/helpers/agents/transport-params-runtime-contract.ts

@@ -0,0 +1,33 @@
+export const OPENAI_GPT5_TRANSPORT_DEFAULTS = {
+  parallel_tool_calls: true,
+  text_verbosity: "low",
+  openaiWsWarmup: false,
+} as const;
+
+export const OPENAI_GPT5_TRANSPORT_DEFAULT_CASES = [
+  {
+    provider: "openai",
+    modelId: "gpt-5.4",
+  },
+  {
+    provider: "openai-codex",
+    modelId: "gpt-5.4",
+  },
+] as const;
+
+export const NON_OPENAI_GPT5_TRANSPORT_CASE = {
+  provider: "openrouter",
+  modelId: "gpt-5.4",
+} as const;
+
+export const GPT_PARALLEL_TOOL_CALLS_PAYLOAD_APIS = [
+  "openai-completions",
+  "openai-responses",
+  "openai-codex-responses",
+  "azure-openai-responses",
+] as const;
+
+export const UNRELATED_TOOL_CALLS_PAYLOAD_APIS = [
+  "anthropic-messages",
+  "google-generative-ai",
+] as const;