From edb618c6c4a90a144397e4c56298fd62ec0e5401 Mon Sep 17 00:00:00 2001 From: pash-openai Date: Sat, 25 Apr 2026 16:51:14 -0700 Subject: [PATCH] Manage the Codex app-server binary in OpenClaw (#71808) * Manage Codex app-server binary * Use plugin deps for Codex app-server binary * Stabilize media model registry test * Exclude checkpoint transcripts from memory ingestion --- docs/plugins/codex-harness.md | 21 ++- extensions/codex/openclaw.plugin.json | 7 +- extensions/codex/package.json | 1 + extensions/codex/src/app-server/client.ts | 10 +- .../codex/src/app-server/config.test.ts | 31 +++++ extensions/codex/src/app-server/config.ts | 14 +- .../src/app-server/managed-binary.test.ts | 95 ++++++++++++++ .../codex/src/app-server/managed-binary.ts | 121 ++++++++++++++++++ .../codex/src/app-server/models.test.ts | 11 +- .../src/app-server/shared-client.test.ts | 53 ++++++++ .../codex/src/app-server/shared-client.ts | 21 ++- .../src/app-server/transport-stdio.test.ts | 16 +++ .../codex/src/app-server/transport-stdio.ts | 3 + extensions/codex/src/app-server/version.ts | 3 + extensions/codex/src/manifest.test.ts | 4 + .../memory-core/src/dreaming-phases.test.ts | 2 +- .../src/host/session-files.test.ts | 25 ++++ .../memory-host-sdk/src/host/session-files.ts | 25 +++- pnpm-lock.yaml | 71 ++++++++++ pnpm-workspace.yaml | 2 + src/agents/tools/media-tool-shared.test.ts | 34 +++-- 21 files changed, 537 insertions(+), 33 deletions(-) create mode 100644 extensions/codex/src/app-server/managed-binary.test.ts create mode 100644 extensions/codex/src/app-server/managed-binary.ts create mode 100644 extensions/codex/src/app-server/version.ts diff --git a/docs/plugins/codex-harness.md b/docs/plugins/codex-harness.md index 8798d91f772..f2847597c6b 100644 --- a/docs/plugins/codex-harness.md +++ b/docs/plugins/codex-harness.md @@ -103,8 +103,9 @@ Codex after changing config. ## Requirements - OpenClaw with the bundled `codex` plugin available. -- Codex app-server `0.125.0` or newer. Native MCP hook payloads landed in Codex - `0.124.0`; OpenClaw uses `0.125.0` as the tested support floor. +- Codex app-server `0.125.0` or newer. The bundled plugin manages a compatible + Codex app-server binary by default, so local `codex` commands on `PATH` do + not affect normal harness startup. - Codex auth available to the app-server process. The plugin blocks older or unversioned app-server handshakes. That keeps @@ -340,12 +341,18 @@ fallback catalog: ## App-server connection and policy -By default, the plugin starts Codex locally with: +By default, the plugin starts OpenClaw's managed Codex binary locally with: ```bash codex app-server --listen stdio:// ``` +The managed binary is declared as a bundled plugin runtime dependency and staged +with the rest of the `codex` plugin dependencies. This keeps the app-server +version tied to the bundled plugin instead of whichever separate Codex CLI +happens to be installed locally. Set `appServer.command` only when you +intentionally want to run a different executable. + By default, OpenClaw starts local Codex harness sessions in YOLO mode: `approvalPolicy: "never"`, `approvalsReviewer: "user"`, and `sandbox: "danger-full-access"`. This is the trusted local operator posture used @@ -414,7 +421,7 @@ Supported `appServer` fields: | Field | Default | Meaning | | ------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ | | `transport` | `"stdio"` | `"stdio"` spawns Codex; `"websocket"` connects to `url`. | -| `command` | `"codex"` | Executable for stdio transport. | +| `command` | managed Codex binary | Executable for stdio transport. Leave unset to use the managed binary; set it only for an explicit override. | | `args` | `["app-server", "--listen", "stdio://"]` | Arguments for stdio transport. | | `url` | unset | WebSocket app-server URL. | | `authToken` | unset | Bearer token for WebSocket transport. | @@ -426,8 +433,7 @@ Supported `appServer` fields: | `approvalsReviewer` | `"user"` | Use `"auto_review"` to let Codex review native approval prompts. `guardian_subagent` remains a legacy alias. | | `serviceTier` | unset | Optional Codex app-server service tier: `"fast"`, `"flex"`, or `null`. Invalid legacy values are ignored. | -The older environment variables still work as fallbacks for local testing when -the matching config field is unset: +Environment overrides remain available for local testing: - `OPENCLAW_CODEX_APP_SERVER_BIN` - `OPENCLAW_CODEX_APP_SERVER_ARGS` @@ -435,6 +441,9 @@ the matching config field is unset: - `OPENCLAW_CODEX_APP_SERVER_APPROVAL_POLICY` - `OPENCLAW_CODEX_APP_SERVER_SANDBOX` +`OPENCLAW_CODEX_APP_SERVER_BIN` bypasses the managed binary when +`appServer.command` is unset. + `OPENCLAW_CODEX_APP_SERVER_GUARDIAN=1` was removed. Use `plugins.entries.codex.config.appServer.mode: "guardian"` instead, or `OPENCLAW_CODEX_APP_SERVER_MODE=guardian` for one-off local testing. Config is diff --git a/extensions/codex/openclaw.plugin.json b/extensions/codex/openclaw.plugin.json index b589906658d..c69bba5ef3f 100644 --- a/extensions/codex/openclaw.plugin.json +++ b/extensions/codex/openclaw.plugin.json @@ -57,10 +57,7 @@ "enum": ["stdio", "websocket"], "default": "stdio" }, - "command": { - "type": "string", - "default": "codex" - }, + "command": { "type": "string" }, "args": { "oneOf": [ { @@ -132,7 +129,7 @@ }, "appServer.command": { "label": "Command", - "help": "Executable used for stdio transport.", + "help": "Executable used for stdio transport. Leave unset to use OpenClaw's managed Codex binary.", "advanced": true }, "appServer.args": { diff --git a/extensions/codex/package.json b/extensions/codex/package.json index 1e0a8e906fd..cc840eb415b 100644 --- a/extensions/codex/package.json +++ b/extensions/codex/package.json @@ -5,6 +5,7 @@ "type": "module", "dependencies": { "@mariozechner/pi-coding-agent": "0.70.2", + "@openai/codex": "0.125.0", "ajv": "^8.18.0", "ws": "^8.20.0", "zod": "^4.3.6" diff --git a/extensions/codex/src/app-server/client.ts b/extensions/codex/src/app-server/client.ts index de58d3aad62..aed64ab45d4 100644 --- a/extensions/codex/src/app-server/client.ts +++ b/extensions/codex/src/app-server/client.ts @@ -1,6 +1,7 @@ import { createInterface, type Interface as ReadlineInterface } from "node:readline"; import { embeddedAgentLog, OPENCLAW_VERSION } from "openclaw/plugin-sdk/agent-harness-runtime"; import { resolveCodexAppServerRuntimeOptions, type CodexAppServerStartOptions } from "./config.js"; +import { MIN_CODEX_APP_SERVER_VERSION } from "./version.js"; import { type CodexAppServerRequestMethod, type CodexAppServerRequestParams, @@ -22,7 +23,7 @@ import { type CodexAppServerTransport, } from "./transport.js"; -export const MIN_CODEX_APP_SERVER_VERSION = "0.125.0"; +export { MIN_CODEX_APP_SERVER_VERSION } from "./version.js"; const CODEX_APP_SERVER_PARSE_LOG_MAX = 500; type PendingRequest = { @@ -99,6 +100,9 @@ export class CodexAppServerClient { ...options, headers: options?.headers ?? defaults.headers, }; + if (startOptions.transport === "stdio" && startOptions.commandSource === "managed") { + throw new Error("Managed Codex app-server start options must be resolved before spawn."); + } if (startOptions.transport === "websocket") { return new CodexAppServerClient(createWebSocketTransport(startOptions)); } @@ -407,12 +411,12 @@ function assertSupportedCodexAppServerVersion(response: CodexInitializeResponse) const detectedVersion = readCodexVersionFromUserAgent(response.userAgent); if (!detectedVersion) { throw new Error( - `Codex app-server ${MIN_CODEX_APP_SERVER_VERSION} or newer is required, but OpenClaw could not determine the running Codex version. Upgrade Codex CLI and retry.`, + `Codex app-server ${MIN_CODEX_APP_SERVER_VERSION} or newer is required, but OpenClaw could not determine the running Codex version. Update the configured Codex app-server binary, or remove custom command overrides to use the managed binary.`, ); } if (compareVersions(detectedVersion, MIN_CODEX_APP_SERVER_VERSION) < 0) { throw new Error( - `Codex app-server ${MIN_CODEX_APP_SERVER_VERSION} or newer is required, but detected ${detectedVersion}. Upgrade Codex CLI and retry.`, + `Codex app-server ${MIN_CODEX_APP_SERVER_VERSION} or newer is required, but detected ${detectedVersion}. Update the configured Codex app-server binary, or remove custom command overrides to use the managed binary.`, ); } } diff --git a/extensions/codex/src/app-server/config.test.ts b/extensions/codex/src/app-server/config.test.ts index 1b70514c25a..19a06f16fd3 100644 --- a/extensions/codex/src/app-server/config.test.ts +++ b/extensions/codex/src/app-server/config.test.ts @@ -96,6 +96,36 @@ describe("Codex app-server config", () => { approvalPolicy: "never", sandbox: "danger-full-access", approvalsReviewer: "user", + start: expect.objectContaining({ + command: "codex", + commandSource: "managed", + }), + }), + ); + }); + + it("treats configured and environment commands as explicit overrides", () => { + expect( + resolveCodexAppServerRuntimeOptions({ + pluginConfig: { appServer: { command: "/opt/codex/bin/codex" } }, + env: { OPENCLAW_CODEX_APP_SERVER_BIN: "/usr/local/bin/codex" }, + }).start, + ).toEqual( + expect.objectContaining({ + command: "/opt/codex/bin/codex", + commandSource: "config", + }), + ); + + expect( + resolveCodexAppServerRuntimeOptions({ + pluginConfig: {}, + env: { OPENCLAW_CODEX_APP_SERVER_BIN: "/usr/local/bin/codex" }, + }).start, + ).toEqual( + expect.objectContaining({ + command: "/usr/local/bin/codex", + commandSource: "env", }), ); }); @@ -244,6 +274,7 @@ describe("Codex app-server config", () => { }; const appServerProperties = manifest.configSchema.properties.appServer.properties; + expect(appServerProperties.command?.default).toBeUndefined(); expect(appServerProperties.approvalPolicy?.default).toBeUndefined(); expect(appServerProperties.sandbox?.default).toBeUndefined(); expect(appServerProperties.approvalsReviewer?.default).toBeUndefined(); diff --git a/extensions/codex/src/app-server/config.ts b/extensions/codex/src/app-server/config.ts index def3f666ffd..8cff47f7149 100644 --- a/extensions/codex/src/app-server/config.ts +++ b/extensions/codex/src/app-server/config.ts @@ -7,10 +7,12 @@ export type CodexAppServerPolicyMode = "yolo" | "guardian"; export type CodexAppServerApprovalPolicy = "never" | "on-request" | "on-failure" | "untrusted"; export type CodexAppServerSandboxMode = "read-only" | "workspace-write" | "danger-full-access"; export type CodexAppServerApprovalsReviewer = "user" | "auto_review" | "guardian_subagent"; +export type CodexAppServerCommandSource = "managed" | "resolved-managed" | "config" | "env"; export type CodexAppServerStartOptions = { transport: CodexAppServerTransportMode; command: string; + commandSource?: CodexAppServerCommandSource; args: string[]; url?: string; authToken?: string; @@ -125,8 +127,14 @@ export function resolveCodexAppServerRuntimeOptions( const env = params.env ?? process.env; const config = readCodexPluginConfig(params.pluginConfig).appServer ?? {}; const transport = resolveTransport(config.transport); - const command = - readNonEmptyString(config.command) ?? env.OPENCLAW_CODEX_APP_SERVER_BIN ?? "codex"; + const configCommand = readNonEmptyString(config.command); + const envCommand = readNonEmptyString(env.OPENCLAW_CODEX_APP_SERVER_BIN); + const command = configCommand ?? envCommand ?? "codex"; + const commandSource: CodexAppServerCommandSource = configCommand + ? "config" + : envCommand + ? "env" + : "managed"; const args = resolveArgs(config.args, env.OPENCLAW_CODEX_APP_SERVER_ARGS); const headers = normalizeHeaders(config.headers); const authToken = readNonEmptyString(config.authToken); @@ -146,6 +154,7 @@ export function resolveCodexAppServerRuntimeOptions( start: { transport, command, + commandSource, args: args.length > 0 ? args : ["app-server", "--listen", "stdio://"], ...(url ? { url } : {}), ...(authToken ? { authToken } : {}), @@ -174,6 +183,7 @@ export function codexAppServerStartOptionsKey( return JSON.stringify({ transport: options.transport, command: options.command, + commandSource: options.commandSource ?? null, args: options.args, url: options.url ?? null, authToken: hashSecretForKey(options.authToken), diff --git a/extensions/codex/src/app-server/managed-binary.test.ts b/extensions/codex/src/app-server/managed-binary.test.ts new file mode 100644 index 00000000000..304ce68c721 --- /dev/null +++ b/extensions/codex/src/app-server/managed-binary.test.ts @@ -0,0 +1,95 @@ +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import type { CodexAppServerStartOptions } from "./config.js"; +import { + resolveManagedCodexAppServerPaths, + resolveManagedCodexAppServerStartOptions, +} from "./managed-binary.js"; + +function startOptions( + commandSource: CodexAppServerStartOptions["commandSource"], +): CodexAppServerStartOptions { + return { + transport: "stdio", + command: "codex", + commandSource, + args: ["app-server", "--listen", "stdio://"], + headers: {}, + }; +} + +function managedCommandPath(root: string, platform: NodeJS.Platform): string { + return path.join(root, "node_modules", ".bin", platform === "win32" ? "codex.cmd" : "codex"); +} + +describe("managed Codex app-server binary", () => { + it("leaves explicit command overrides unchanged", async () => { + const explicitOptions = startOptions("config"); + const pathExists = vi.fn(async () => false); + + await expect( + resolveManagedCodexAppServerStartOptions(explicitOptions, { + platform: "darwin", + pathExists, + }), + ).resolves.toBe(explicitOptions); + expect(pathExists).not.toHaveBeenCalled(); + }); + + it("resolves the plugin-local bundled Codex binary", async () => { + const pluginRoot = path.join("/tmp", "openclaw", "extensions", "codex"); + const paths = resolveManagedCodexAppServerPaths({ platform: "darwin", pluginRoot }); + const pathExists = vi.fn(async (filePath: string) => filePath === paths.commandPath); + + await expect( + resolveManagedCodexAppServerStartOptions(startOptions("managed"), { + platform: "darwin", + pluginRoot, + pathExists, + }), + ).resolves.toEqual({ + ...startOptions("managed"), + command: paths.commandPath, + commandSource: "resolved-managed", + }); + expect(paths.commandPath).toBe(managedCommandPath(pluginRoot, "darwin")); + }); + + it("resolves Windows Codex command shims", () => { + const pluginRoot = path.win32.join("C:\\", "OpenClaw", "dist", "extensions", "codex"); + const paths = resolveManagedCodexAppServerPaths({ platform: "win32", pluginRoot }); + + expect(paths.commandPath.endsWith(path.win32.join("node_modules", ".bin", "codex.cmd"))).toBe( + true, + ); + }); + + it("finds Codex in the external runtime-deps install root used by packaged plugins", async () => { + const installRoot = path.join("/tmp", "openclaw-runtime-deps", "codex"); + const pluginRoot = path.join(installRoot, "dist", "extensions", "codex"); + const installedCommand = managedCommandPath(installRoot, "linux"); + const pathExists = vi.fn(async (filePath: string) => filePath === installedCommand); + + await expect( + resolveManagedCodexAppServerStartOptions(startOptions("managed"), { + platform: "linux", + pluginRoot, + pathExists, + }), + ).resolves.toEqual({ + ...startOptions("managed"), + command: installedCommand, + commandSource: "resolved-managed", + }); + }); + + it("fails clearly when bundled runtime deps did not stage Codex", async () => { + await expect( + resolveManagedCodexAppServerStartOptions(startOptions("managed"), { + platform: "darwin", + pluginRoot: path.join("/tmp", "openclaw", "extensions", "codex"), + pathExists: vi.fn(async () => false), + }), + ).rejects.toThrow("Managed Codex app-server binary was not found"); + }); +}); diff --git a/extensions/codex/src/app-server/managed-binary.ts b/extensions/codex/src/app-server/managed-binary.ts new file mode 100644 index 00000000000..c08c3cdc76a --- /dev/null +++ b/extensions/codex/src/app-server/managed-binary.ts @@ -0,0 +1,121 @@ +import { constants as fsConstants } from "node:fs"; +import { access } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CodexAppServerStartOptions } from "./config.js"; +import { MANAGED_CODEX_APP_SERVER_PACKAGE } from "./version.js"; + +const CODEX_PLUGIN_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); + +type ManagedCodexAppServerPaths = { + commandPath: string; + candidateCommandPaths: string[]; +}; + +export type ResolveManagedCodexAppServerOptions = { + platform?: NodeJS.Platform; + pluginRoot?: string; + pathExists?: (filePath: string, platform: NodeJS.Platform) => Promise; +}; + +export async function resolveManagedCodexAppServerStartOptions( + startOptions: CodexAppServerStartOptions, + options: ResolveManagedCodexAppServerOptions = {}, +): Promise { + if (startOptions.transport !== "stdio" || startOptions.commandSource !== "managed") { + return startOptions; + } + + const platform = options.platform ?? process.platform; + const paths = resolveManagedCodexAppServerPaths({ + platform, + pluginRoot: options.pluginRoot, + }); + const pathExists = options.pathExists ?? commandPathExists; + const commandPath = await findManagedCodexAppServerCommandPath({ + candidateCommandPaths: paths.candidateCommandPaths, + pathExists, + platform, + }); + + return { + ...startOptions, + command: commandPath, + commandSource: "resolved-managed", + }; +} + +export function resolveManagedCodexAppServerPaths(params: { + platform?: NodeJS.Platform; + pluginRoot?: string; +}): ManagedCodexAppServerPaths { + const platform = params.platform ?? process.platform; + const candidateCommandPaths = resolveManagedCodexAppServerCommandCandidates( + params.pluginRoot ?? CODEX_PLUGIN_ROOT, + platform, + ); + return { + commandPath: candidateCommandPaths[0] ?? "", + candidateCommandPaths, + }; +} + +function resolveManagedCodexAppServerCommandCandidates( + pluginRoot: string, + platform: NodeJS.Platform, +): string[] { + const pathApi = pathForPlatform(platform); + const commandName = platform === "win32" ? "codex.cmd" : "codex"; + const roots = [ + pluginRoot, + pathApi.dirname(pluginRoot), + pathApi.dirname(pathApi.dirname(pluginRoot)), + isDistExtensionRoot(pluginRoot, platform) + ? pathApi.dirname(pathApi.dirname(pathApi.dirname(pluginRoot))) + : null, + ].filter((root): root is string => Boolean(root)); + return [...new Set(roots.map((root) => pathApi.join(root, "node_modules", ".bin", commandName)))]; +} + +function isDistExtensionRoot(pluginRoot: string, platform: NodeJS.Platform): boolean { + const pathApi = pathForPlatform(platform); + const extensionsDir = pathApi.dirname(pluginRoot); + const distDir = pathApi.dirname(extensionsDir); + return ( + pathApi.basename(extensionsDir) === "extensions" && + (pathApi.basename(distDir) === "dist" || pathApi.basename(distDir) === "dist-runtime") + ); +} + +function pathForPlatform(platform: NodeJS.Platform): typeof path { + return platform === "win32" ? path.win32 : path.posix; +} + +async function findManagedCodexAppServerCommandPath(params: { + candidateCommandPaths: readonly string[]; + pathExists: (filePath: string, platform: NodeJS.Platform) => Promise; + platform: NodeJS.Platform; +}): Promise { + for (const commandPath of params.candidateCommandPaths) { + if (await params.pathExists(commandPath, params.platform)) { + return commandPath; + } + } + + throw new Error( + [ + `Managed Codex app-server binary was not found for ${MANAGED_CODEX_APP_SERVER_PACKAGE}.`, + "Run OpenClaw with bundled plugin runtime dependencies enabled, or run pnpm install in a source checkout.", + "Set plugins.entries.codex.config.appServer.command or OPENCLAW_CODEX_APP_SERVER_BIN to use a custom Codex binary.", + ].join(" "), + ); +} + +async function commandPathExists(filePath: string, platform: NodeJS.Platform): Promise { + try { + await access(filePath, platform === "win32" ? fsConstants.F_OK : fsConstants.X_OK); + return true; + } catch { + return false; + } +} diff --git a/extensions/codex/src/app-server/models.test.ts b/extensions/codex/src/app-server/models.test.ts index 2778bc19f5b..a0b805bf515 100644 --- a/extensions/codex/src/app-server/models.test.ts +++ b/extensions/codex/src/app-server/models.test.ts @@ -7,10 +7,13 @@ const mocks = vi.hoisted(() => { applyAuthProfile: vi.fn(async () => undefined), startOptions: vi.fn(async ({ startOptions }) => startOptions), }; + const managedBinary = { + startOptions: vi.fn(async (startOptions) => startOptions), + }; const providerAuth = { agentDir: vi.fn(() => "/tmp/openclaw-agent"), }; - return { authBridge, providerAuth }; + return { authBridge, managedBinary, providerAuth }; }); vi.mock("./auth-bridge.js", () => ({ @@ -18,6 +21,10 @@ vi.mock("./auth-bridge.js", () => ({ bridgeCodexAppServerStartOptions: mocks.authBridge.startOptions, })); +vi.mock("./managed-binary.js", () => ({ + resolveManagedCodexAppServerStartOptions: mocks.managedBinary.startOptions, +})); + vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ resolveOpenClawAgentDir: mocks.providerAuth.agentDir, })); @@ -38,6 +45,8 @@ describe("listCodexAppServerModels", () => { vi.restoreAllMocks(); mocks.authBridge.applyAuthProfile.mockClear(); mocks.authBridge.startOptions.mockClear(); + mocks.managedBinary.startOptions.mockClear(); + mocks.managedBinary.startOptions.mockImplementation(async (startOptions) => startOptions); mocks.providerAuth.agentDir.mockClear(); }); diff --git a/extensions/codex/src/app-server/shared-client.test.ts b/extensions/codex/src/app-server/shared-client.test.ts index 80e3f1dcf38..7dd24f9f61f 100644 --- a/extensions/codex/src/app-server/shared-client.test.ts +++ b/extensions/codex/src/app-server/shared-client.test.ts @@ -6,6 +6,8 @@ import { createClientHarness } from "./test-support.js"; const mocks = vi.hoisted(() => ({ bridgeCodexAppServerStartOptions: vi.fn(async ({ startOptions }) => startOptions), applyCodexAppServerAuthProfile: vi.fn(async () => undefined), + resolveManagedCodexAppServerStartOptions: vi.fn(async (startOptions) => startOptions), + embeddedAgentLog: { debug: vi.fn(), warn: vi.fn() }, resolveOpenClawAgentDir: vi.fn(() => "/tmp/openclaw-agent"), })); @@ -14,6 +16,15 @@ vi.mock("./auth-bridge.js", () => ({ bridgeCodexAppServerStartOptions: mocks.bridgeCodexAppServerStartOptions, })); +vi.mock("./managed-binary.js", () => ({ + resolveManagedCodexAppServerStartOptions: mocks.resolveManagedCodexAppServerStartOptions, +})); + +vi.mock("openclaw/plugin-sdk/agent-harness-runtime", () => ({ + embeddedAgentLog: mocks.embeddedAgentLog, + OPENCLAW_VERSION: "test", +})); + vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ resolveOpenClawAgentDir: mocks.resolveOpenClawAgentDir, })); @@ -54,6 +65,12 @@ describe("shared Codex app-server client", () => { vi.restoreAllMocks(); mocks.bridgeCodexAppServerStartOptions.mockClear(); mocks.applyCodexAppServerAuthProfile.mockClear(); + mocks.resolveManagedCodexAppServerStartOptions.mockClear(); + mocks.resolveManagedCodexAppServerStartOptions.mockImplementation( + async (startOptions) => startOptions, + ); + mocks.embeddedAgentLog.debug.mockClear(); + mocks.embeddedAgentLog.warn.mockClear(); mocks.resolveOpenClawAgentDir.mockClear(); }); @@ -128,6 +145,42 @@ describe("shared Codex app-server client", () => { ); }); + it("resolves the managed binary before bridging and spawning the shared client", async () => { + const harness = createClientHarness(); + const startSpy = vi.spyOn(CodexAppServerClient, "start").mockReturnValue(harness.client); + mocks.resolveManagedCodexAppServerStartOptions.mockImplementationOnce(async (startOptions) => ({ + ...startOptions, + command: "/cache/openclaw/codex", + commandSource: "resolved-managed", + })); + + const listPromise = listCodexAppServerModels({ timeoutMs: 1000 }); + await sendInitializeResult(harness, "openclaw/0.125.0 (macOS; test)"); + await sendEmptyModelList(harness); + + await expect(listPromise).resolves.toEqual({ models: [] }); + expect(mocks.resolveManagedCodexAppServerStartOptions).toHaveBeenCalledWith( + expect.objectContaining({ + command: "codex", + commandSource: "managed", + }), + ); + expect(mocks.bridgeCodexAppServerStartOptions).toHaveBeenCalledWith( + expect.objectContaining({ + startOptions: expect.objectContaining({ + command: "/cache/openclaw/codex", + commandSource: "resolved-managed", + }), + }), + ); + expect(startSpy).toHaveBeenCalledWith( + expect.objectContaining({ + command: "/cache/openclaw/codex", + commandSource: "resolved-managed", + }), + ); + }); + it("restarts the shared client when the bridged auth token changes", async () => { const first = createClientHarness(); const second = createClientHarness(); diff --git a/extensions/codex/src/app-server/shared-client.ts b/extensions/codex/src/app-server/shared-client.ts index 407e0bc4309..b6d82eeb969 100644 --- a/extensions/codex/src/app-server/shared-client.ts +++ b/extensions/codex/src/app-server/shared-client.ts @@ -6,6 +6,7 @@ import { resolveCodexAppServerRuntimeOptions, type CodexAppServerStartOptions, } from "./config.js"; +import { resolveManagedCodexAppServerStartOptions } from "./managed-binary.js"; import { withTimeout } from "./timeout.js"; type SharedCodexAppServerClientState = { @@ -30,9 +31,13 @@ export async function getSharedCodexAppServerClient(options?: { authProfileId?: string; }): Promise { const state = getSharedCodexAppServerClientState(); + const agentDir = resolveOpenClawAgentDir(); + const requestedStartOptions = + options?.startOptions ?? resolveCodexAppServerRuntimeOptions().start; + const managedStartOptions = await resolveManagedCodexAppServerStartOptions(requestedStartOptions); const startOptions = await bridgeCodexAppServerStartOptions({ - startOptions: options?.startOptions ?? resolveCodexAppServerRuntimeOptions().start, - agentDir: resolveOpenClawAgentDir(), + startOptions: managedStartOptions, + agentDir, authProfileId: options?.authProfileId, }); const key = codexAppServerStartOptionsKey(startOptions, { @@ -52,7 +57,7 @@ export async function getSharedCodexAppServerClient(options?: { await client.initialize(); await applyCodexAppServerAuthProfile({ client, - agentDir: resolveOpenClawAgentDir(), + agentDir, authProfileId: options?.authProfileId, }); return client; @@ -82,9 +87,13 @@ export async function createIsolatedCodexAppServerClient(options?: { timeoutMs?: number; authProfileId?: string; }): Promise { + const agentDir = resolveOpenClawAgentDir(); + const requestedStartOptions = + options?.startOptions ?? resolveCodexAppServerRuntimeOptions().start; + const managedStartOptions = await resolveManagedCodexAppServerStartOptions(requestedStartOptions); const startOptions = await bridgeCodexAppServerStartOptions({ - startOptions: options?.startOptions ?? resolveCodexAppServerRuntimeOptions().start, - agentDir: resolveOpenClawAgentDir(), + startOptions: managedStartOptions, + agentDir, authProfileId: options?.authProfileId, }); const client = CodexAppServerClient.start(startOptions); @@ -93,7 +102,7 @@ export async function createIsolatedCodexAppServerClient(options?: { await withTimeout(initialize, options?.timeoutMs ?? 0, "codex app-server initialize timed out"); await applyCodexAppServerAuthProfile({ client, - agentDir: resolveOpenClawAgentDir(), + agentDir, authProfileId: options?.authProfileId, }); return client; diff --git a/extensions/codex/src/app-server/transport-stdio.test.ts b/extensions/codex/src/app-server/transport-stdio.test.ts index b4e61bd2da2..8d2df0313e8 100644 --- a/extensions/codex/src/app-server/transport-stdio.test.ts +++ b/extensions/codex/src/app-server/transport-stdio.test.ts @@ -44,6 +44,22 @@ describe("resolveCodexAppServerSpawnInvocation", () => { }); }); + it("requires managed Codex commands to be resolved before spawn", () => { + expect(() => + resolveCodexAppServerSpawnInvocation( + { + ...startOptions("codex"), + commandSource: "managed", + }, + { + platform: "darwin", + env: {}, + execPath: "/usr/local/bin/node", + }, + ), + ).toThrow("must be resolved before spawn"); + }); + it("resolves Windows npm .cmd Codex shims through Node instead of raw spawn", async () => { const binDir = await createTempDir(); const entryPath = path.join(binDir, "node_modules", "@openai", "codex", "bin", "codex.js"); diff --git a/extensions/codex/src/app-server/transport-stdio.ts b/extensions/codex/src/app-server/transport-stdio.ts index 129b34e870a..c3c60c110aa 100644 --- a/extensions/codex/src/app-server/transport-stdio.ts +++ b/extensions/codex/src/app-server/transport-stdio.ts @@ -22,6 +22,9 @@ export function resolveCodexAppServerSpawnInvocation( options: CodexAppServerStartOptions, runtime: CodexAppServerSpawnRuntime = DEFAULT_SPAWN_RUNTIME, ): { command: string; args: string[]; shell?: boolean; windowsHide?: boolean } { + if (options.commandSource === "managed") { + throw new Error("Managed Codex app-server start options must be resolved before spawn."); + } const program = resolveWindowsSpawnProgram({ command: options.command, platform: runtime.platform, diff --git a/extensions/codex/src/app-server/version.ts b/extensions/codex/src/app-server/version.ts new file mode 100644 index 00000000000..7897158c9fc --- /dev/null +++ b/extensions/codex/src/app-server/version.ts @@ -0,0 +1,3 @@ +export const MIN_CODEX_APP_SERVER_VERSION = "0.125.0"; +export const MANAGED_CODEX_APP_SERVER_PACKAGE = "@openai/codex"; +export const MANAGED_CODEX_APP_SERVER_PACKAGE_VERSION = MIN_CODEX_APP_SERVER_VERSION; diff --git a/extensions/codex/src/manifest.test.ts b/extensions/codex/src/manifest.test.ts index 530093b2db4..723f7719fdb 100644 --- a/extensions/codex/src/manifest.test.ts +++ b/extensions/codex/src/manifest.test.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import { describe, expect, it } from "vitest"; +import { MANAGED_CODEX_APP_SERVER_PACKAGE_VERSION } from "./app-server/version.js"; type CodexPackageManifest = { dependencies?: Record; @@ -17,6 +18,9 @@ describe("codex package manifest", () => { ) as CodexPackageManifest; expect(packageJson.dependencies?.["@mariozechner/pi-coding-agent"]).toBeDefined(); + expect(packageJson.dependencies?.["@openai/codex"]).toBe( + MANAGED_CODEX_APP_SERVER_PACKAGE_VERSION, + ); expect(packageJson.openclaw?.bundle?.stageRuntimeDependencies).toBe(true); }); }); diff --git a/extensions/memory-core/src/dreaming-phases.test.ts b/extensions/memory-core/src/dreaming-phases.test.ts index e0f992ac5d0..eb3d98f0cb1 100644 --- a/extensions/memory-core/src/dreaming-phases.test.ts +++ b/extensions/memory-core/src/dreaming-phases.test.ts @@ -1215,7 +1215,7 @@ describe("memory-core dreaming phases", () => { "utf-8", ); await fs.writeFile( - path.join(sessionsDir, "ordinary.checkpoint.abc123.jsonl"), + path.join(sessionsDir, "ordinary.checkpoint.11111111-1111-4111-8111-111111111111.jsonl"), JSON.stringify({ type: "message", message: { diff --git a/packages/memory-host-sdk/src/host/session-files.test.ts b/packages/memory-host-sdk/src/host/session-files.test.ts index 0d85589263b..b846bc7e6a2 100644 --- a/packages/memory-host-sdk/src/host/session-files.test.ts +++ b/packages/memory-host-sdk/src/host/session-files.test.ts @@ -43,6 +43,7 @@ describe("listSessionFilesForAgent", () => { "active.jsonl.deleted.2026-02-16T22-27-33.000Z", ]; const excluded = ["active.jsonl.bak.2026-02-16T22-28-33.000Z", "sessions.json", "notes.md"]; + excluded.push("active.checkpoint.11111111-1111-4111-8111-111111111111.jsonl"); for (const fileName of [...included, ...excluded]) { fsSync.writeFileSync(path.join(sessionsDir, fileName), ""); @@ -115,6 +116,30 @@ describe("buildSessionEntry", () => { expect(entry!.lineMap).toEqual([]); }); + it("skips deleted and checkpoint transcripts for dreaming ingestion", async () => { + const deletedPath = path.join(tmpDir, "ordinary.jsonl.deleted.2026-02-16T22-27-33.000Z"); + const checkpointPath = path.join( + tmpDir, + "ordinary.checkpoint.11111111-1111-4111-8111-111111111111.jsonl", + ); + const content = JSON.stringify({ + type: "message", + message: { role: "user", content: "This should never reach the dreaming corpus." }, + }); + fsSync.writeFileSync(deletedPath, content); + fsSync.writeFileSync(checkpointPath, content); + + const deletedEntry = await buildSessionEntry(deletedPath); + const checkpointEntry = await buildSessionEntry(checkpointPath); + + expect(deletedEntry).not.toBeNull(); + expect(deletedEntry?.content).toBe(""); + expect(deletedEntry?.lineMap).toEqual([]); + expect(checkpointEntry).not.toBeNull(); + expect(checkpointEntry?.content).toBe(""); + expect(checkpointEntry?.lineMap).toEqual([]); + }); + it("skips blank lines and invalid JSON without breaking lineMap", async () => { const jsonlLines = [ "", diff --git a/packages/memory-host-sdk/src/host/session-files.ts b/packages/memory-host-sdk/src/host/session-files.ts index 3eb3af5a8e1..7e1b516bec2 100644 --- a/packages/memory-host-sdk/src/host/session-files.ts +++ b/packages/memory-host-sdk/src/host/session-files.ts @@ -1,7 +1,11 @@ import fs from "node:fs/promises"; import path from "node:path"; import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js"; -import { isUsageCountedSessionTranscriptFileName } from "../../../../src/config/sessions/artifacts.js"; +import { + isCompactionCheckpointTranscriptFileName, + isSessionArchiveArtifactName, + isUsageCountedSessionTranscriptFileName, +} from "../../../../src/config/sessions/artifacts.js"; import { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js"; import { redactSensitiveText } from "../../../../src/logging/redact.js"; import { hashText } from "./hash.js"; @@ -41,6 +45,13 @@ function isDreamingNarrativeBootstrapRecord(record: unknown): boolean { return typeof runId === "string" && runId.startsWith("dreaming-narrative-"); } +function shouldSkipTranscriptFileForDreaming(absPath: string): boolean { + const fileName = path.basename(absPath); + return ( + isSessionArchiveArtifactName(fileName) || isCompactionCheckpointTranscriptFileName(fileName) + ); +} + export async function listSessionFilesForAgent(agentId: string): Promise { const dir = resolveSessionTranscriptsDirForAgent(agentId); try { @@ -120,6 +131,18 @@ export function extractSessionText( export async function buildSessionEntry(absPath: string): Promise { try { const stat = await fs.stat(absPath); + if (shouldSkipTranscriptFileForDreaming(absPath)) { + return { + path: sessionPathForFile(absPath), + absPath, + mtimeMs: stat.mtimeMs, + size: stat.size, + hash: hashText("\n\n"), + content: "", + lineMap: [], + generatedByDreamingNarrative: false, + }; + } const raw = await fs.readFile(absPath, "utf-8"); const lines = raw.split("\n"); const collected: string[] = []; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec4c8085a74..eb6c62ef9a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -372,6 +372,9 @@ importers: '@mariozechner/pi-coding-agent': specifier: 0.70.2 version: 0.70.2(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(ws@8.20.0)(zod@4.3.6) + '@openai/codex': + specifier: 0.125.0 + version: 0.125.0 ajv: specifier: ^8.18.0 version: 8.18.0 @@ -2899,6 +2902,47 @@ packages: resolution: {integrity: sha512-tlc/FcYIv5i8RYsl2iDil4A0gOihaas1R5jPcIC4Zw3GhjKsVilw90aHcVlhZPTBLGBzd379S+VcnsDjd9ChiA==} engines: {node: '>=12.4.0'} + '@openai/codex@0.125.0': + resolution: {integrity: sha512-GiE9wlgL95u/5BRirY5d3EaRLU1tu7Y1R09R8lCHHVmcQdSmhS809FdPDWH3gIYHS7ZriAPqXwJ3aLA0WKl40Q==} + engines: {node: '>=16'} + hasBin: true + + '@openai/codex@0.125.0-darwin-arm64': + resolution: {integrity: sha512-Gn2fHiSO0XgyHp1OSd5DWUTm66Bv9UEuipW5pVEj1E+hWZCOrdqnYttllKFWtRGj5yiKefNX3JIxONgh/ZwlOQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + + '@openai/codex@0.125.0-darwin-x64': + resolution: {integrity: sha512-TZ5Lek2X/UXTI9LXFxzarvQaJeuTrqVh4POc7soO/8RclVnCxADnCf15sivxLd5eiFW4t0myGoeVoM4lciRiRg==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + + '@openai/codex@0.125.0-linux-arm64': + resolution: {integrity: sha512-pPnJoJD6rZ2Iin0zNt/up36bO2/EOp2B+1/rPHu/lSq3PJbT3Fmnfut2kJy5LylXb7bGA2XQbtqOogZzIbnlkA==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + + '@openai/codex@0.125.0-linux-x64': + resolution: {integrity: sha512-K2NTTEeBpz/G+N2x17UGWfauRt3So+ir4f+U/60l5PPnYEJB/w3YZrlXo2G9og8Dm9BqtoBAjoPV74sRv9tWWQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + + '@openai/codex@0.125.0-win32-arm64': + resolution: {integrity: sha512-zxoUakw9oIHIFrAyk400XkkLBJFA6nOym0NDq6sQ/jhdcYraKqNSRCII2nsBwZHk+/4zgUvuk52iuutgysY/rQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [win32] + + '@openai/codex@0.125.0-win32-x64': + resolution: {integrity: sha512-ofpOK+OWH5QFuUZ9pTM0d/PcXUXiIP5z5DpRcE9MlucJoyOl4Zy4Nu3NcuHF4YzCkZMQb6x3j0tjDEPHKqNQzw==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@opentelemetry/api-logs@0.215.0': resolution: {integrity: sha512-xrFlqhdhUyO8wSRn6DjE0145/HPWSJ5Nm0C7vWua6TdL/FSEAZvEyvdsa9CRXuxo9ebb7j/NEPhEcO62IJ0qUA==} engines: {node: '>=8.0.0'} @@ -9459,6 +9503,33 @@ snapshots: '@nolyfill/domexception@1.0.28': {} + '@openai/codex@0.125.0': + optionalDependencies: + '@openai/codex-darwin-arm64': '@openai/codex@0.125.0-darwin-arm64' + '@openai/codex-darwin-x64': '@openai/codex@0.125.0-darwin-x64' + '@openai/codex-linux-arm64': '@openai/codex@0.125.0-linux-arm64' + '@openai/codex-linux-x64': '@openai/codex@0.125.0-linux-x64' + '@openai/codex-win32-arm64': '@openai/codex@0.125.0-win32-arm64' + '@openai/codex-win32-x64': '@openai/codex@0.125.0-win32-x64' + + '@openai/codex@0.125.0-darwin-arm64': + optional: true + + '@openai/codex@0.125.0-darwin-x64': + optional: true + + '@openai/codex@0.125.0-linux-arm64': + optional: true + + '@openai/codex@0.125.0-linux-x64': + optional: true + + '@openai/codex@0.125.0-win32-arm64': + optional: true + + '@openai/codex@0.125.0-win32-x64': + optional: true + '@opentelemetry/api-logs@0.215.0': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3f8393aca30..4a7cba44cfd 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,6 +19,8 @@ minimumReleaseAgeExclude: - "@cloudflare/workers-types" - "@hono/node-server" - "@mariozechner/*" + - "@openai/codex" + - "@openai/codex-*" - "@typescript/native-preview*" - "@types/node" - "@rolldown/*" diff --git a/src/agents/tools/media-tool-shared.test.ts b/src/agents/tools/media-tool-shared.test.ts index 85704493f3f..772ea09adad 100644 --- a/src/agents/tools/media-tool-shared.test.ts +++ b/src/agents/tools/media-tool-shared.test.ts @@ -7,6 +7,22 @@ function normalizeHostPath(value: string): string { return path.normalize(path.resolve(value)); } +function createModelRegistryStub(resolve: (provider: string, modelId: string) => unknown): { + calls: Array<[string, string]>; + registry: { find: (provider: string, modelId: string) => unknown }; +} { + const calls: Array<[string, string]> = []; + return { + calls, + registry: { + find(provider, modelId) { + calls.push([provider, modelId]); + return resolve(provider, modelId); + }, + }, + }; +} + describe("resolveMediaToolLocalRoots", () => { afterEach(() => { vi.unstubAllEnvs(); @@ -39,24 +55,24 @@ describe("resolveMediaToolLocalRoots", () => { describe("resolveModelFromRegistry", () => { it("normalizes provider and model refs before registry lookup", () => { const foundModel = { provider: "ollama", id: "qwen3.5:397b-cloud" }; - const find = vi.fn(() => foundModel); + const { calls, registry } = createModelRegistryStub(() => foundModel); const result = resolveModelFromRegistry({ - modelRegistry: { find }, + modelRegistry: registry, provider: " OLLAMA ", modelId: " qwen3.5:397b-cloud ", }); - expect(find).toHaveBeenCalledWith("ollama", "qwen3.5:397b-cloud"); + expect(calls).toEqual([["ollama", "qwen3.5:397b-cloud"]]); expect(result).toBe(foundModel); }); it("reports the normalized ref when the registry lookup misses", () => { - const find = vi.fn(() => null); + const { registry } = createModelRegistryStub(() => null); expect(() => resolveModelFromRegistry({ - modelRegistry: { find }, + modelRegistry: registry, provider: " OLLAMA ", modelId: " qwen3.5:397b-cloud ", }), @@ -65,15 +81,17 @@ describe("resolveModelFromRegistry", () => { it("falls back to provider-prefixed custom model IDs", () => { const foundModel = { provider: "kimchi", id: "kimchi/claude-opus-4-6" }; - const find = vi.fn().mockReturnValueOnce(null).mockReturnValueOnce(foundModel); + const { calls, registry } = createModelRegistryStub((_, modelId) => + modelId === "kimchi/claude-opus-4-6" ? foundModel : null, + ); const result = resolveModelFromRegistry({ - modelRegistry: { find }, + modelRegistry: registry, provider: "kimchi", modelId: "claude-opus-4-6", }); - expect(find.mock.calls).toEqual([ + expect(calls).toEqual([ ["kimchi", "claude-opus-4-6"], ["kimchi", "kimchi/claude-opus-4-6"], ]);