diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b2081946e..b3c32c3f80c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Agents/providers: avoid loading owner plugin runtimes for explicitly configured custom provider models during OpenAI-compatible transport setup. +- Tooling: fail Codex app-server protocol generation before invoking Cargo when local disk headroom is too low. - Release/CI/E2E: fail early when Crabbox sparse-sync full checkouts do not have enough local disk, with guidance for moving the sync root. - Release/CI/E2E: reset shared Crabbox pnpm hydrate state before installs so stale `/var/tmp` stores cannot leave `pnpm install` spinning after completion. - Release/CI/E2E: print heartbeat progress during centralized Docker builds while keeping successful build logs quiet. diff --git a/scripts/check-codex-app-server-protocol.ts b/scripts/check-codex-app-server-protocol.ts index f0b8ff2bef3..0f7e861919e 100644 --- a/scripts/check-codex-app-server-protocol.ts +++ b/scripts/check-codex-app-server-protocol.ts @@ -69,45 +69,52 @@ const checks: Array<{ file: string; snippets: string[] }> = [ ]; const failures: string[] = []; -const source = await generateExperimentalCodexAppServerProtocolSource(); +await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); -try { - await compareGeneratedProtocolMirror(source.jsonRoot); +async function main(): Promise { + const source = await generateExperimentalCodexAppServerProtocolSource(); - for (const check of checks) { - const filePath = path.join(source.typescriptRoot, check.file); - let text: string; - try { - text = await fs.readFile(filePath, "utf8"); - } catch (error) { - failures.push(`${check.file}: missing (${String(error)})`); - continue; - } - for (const snippet of check.snippets) { - if (!text.includes(snippet)) { - failures.push(`${check.file}: missing ${snippet}`); + try { + await compareGeneratedProtocolMirror(source.jsonRoot); + + for (const check of checks) { + const filePath = path.join(source.typescriptRoot, check.file); + let text: string; + try { + text = await fs.readFile(filePath, "utf8"); + } catch (error) { + failures.push(`${check.file}: missing (${String(error)})`); + continue; + } + for (const snippet of check.snippets) { + if (!text.includes(snippet)) { + failures.push(`${check.file}: missing ${snippet}`); + } } } + } finally { + await source.cleanup(); } -} finally { - await source.cleanup(); -} -if (failures.length > 0) { - console.error("Codex app-server generated protocol drift:"); - for (const failure of failures) { - console.error(`- ${failure}`); + if (failures.length > 0) { + console.error("Codex app-server generated protocol drift:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + console.error( + `Run \`pnpm codex-app-server:protocol:sync\` after refreshing the Codex checkout at ${source.codexRepo}.`, + ); + process.exit(1); } - console.error( - `Run \`pnpm codex-app-server:protocol:sync\` after refreshing the Codex checkout at ${source.codexRepo}.`, + + console.log( + `Codex app-server generated protocol matches OpenClaw bridge assumptions: ${source.codexRepo}`, ); - process.exit(1); } -console.log( - `Codex app-server generated protocol matches OpenClaw bridge assumptions: ${source.codexRepo}`, -); - async function compareGeneratedProtocolMirror(sourceJsonRoot: string): Promise { for (const schema of selectedCodexAppServerJsonSchemas) { const sourcePath = path.join(sourceJsonRoot, schema); diff --git a/scripts/lib/codex-app-server-protocol-source.ts b/scripts/lib/codex-app-server-protocol-source.ts index 8743db843b2..b7ba3d465c0 100644 --- a/scripts/lib/codex-app-server-protocol-source.ts +++ b/scripts/lib/codex-app-server-protocol-source.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { resolvePnpmRunner } from "../pnpm-runner.mjs"; const PROTOCOL_SCHEMA_RELATIVE_PATH = "codex-rs/app-server-protocol/schema"; +const DEFAULT_PROTOCOL_GENERATION_MIN_FREE_BYTES = 10 * 1024 * 1024 * 1024; export const selectedCodexAppServerJsonSchemas = [ "DynamicToolCallParams.json", @@ -65,6 +66,69 @@ export function resolveCodexProtocolPnpmCommand( return command; } +export function buildCodexProtocolExportArgs(manifestPath: string, outDir: string): string[] { + return [ + "run", + "--manifest-path", + manifestPath, + "-p", + "codex-app-server-protocol", + "--bin", + "export", + "--", + "--out", + outDir, + "--experimental", + ]; +} + +export function resolveCodexProtocolMinFreeBytes( + env: NodeJS.ProcessEnv = process.env, +): number { + const raw = env.OPENCLAW_CODEX_PROTOCOL_MIN_FREE_BYTES; + if (raw === undefined || raw.trim() === "") { + return DEFAULT_PROTOCOL_GENERATION_MIN_FREE_BYTES; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error( + `OPENCLAW_CODEX_PROTOCOL_MIN_FREE_BYTES must be a non-negative byte count, got ${raw}`, + ); + } + return Math.floor(parsed); +} + +export function resolveCodexProtocolCargoTargetDir( + codexRepo: string, + env: NodeJS.ProcessEnv = process.env, +): string { + const targetDir = env.CARGO_TARGET_DIR ?? env.CARGO_BUILD_TARGET_DIR; + if (targetDir !== undefined && targetDir.trim() !== "") { + return path.isAbsolute(targetDir) ? path.resolve(targetDir) : path.resolve(codexRepo, targetDir); + } + return path.join(codexRepo, "codex-rs", "target"); +} + +export function validateCodexProtocolGenerationHeadroom(params: { + freeBytes: number; + minFreeBytes: number; + pathLabel: string; +}): void { + if (params.minFreeBytes <= 0 || params.freeBytes >= params.minFreeBytes) { + return; + } + + throw new Error( + [ + "Codex app-server protocol generation needs Rust build headroom before running cargo.", + `${params.pathLabel} has ${formatBytes(params.freeBytes)} free; requires at least ${formatBytes( + params.minFreeBytes, + )}.`, + "Run this check on Crabbox/Testbox, free local disk, or set OPENCLAW_CODEX_PROTOCOL_MIN_FREE_BYTES=0 to override intentionally.", + ].join("\n"), + ); +} + export async function resolveCodexAppServerProtocolSource(repoRoot: string): Promise<{ codexRepo: string; sourceRoot: string; @@ -98,6 +162,7 @@ export async function generateExperimentalCodexAppServerProtocolSource( ): Promise { const { codexRepo } = await resolveCodexAppServerProtocolSource(repoRoot); const root = await fs.mkdtemp(path.join(repoRoot, ".tmp-codex-app-server-protocol-")); + const generatedRoot = path.join(root, "generated"); const typescriptRoot = path.join(root, "typescript"); const jsonRoot = path.join(root, "json"); const manifestPath = path.join(codexRepo, "codex-rs/Cargo.toml"); @@ -106,32 +171,9 @@ export async function generateExperimentalCodexAppServerProtocolSource( }; try { - runCargoProtocolGenerator(codexRepo, [ - "run", - "--manifest-path", - manifestPath, - "-p", - "codex-cli", - "--", - "app-server", - "generate-ts", - "--out", - typescriptRoot, - "--experimental", - ]); - runCargoProtocolGenerator(codexRepo, [ - "run", - "--manifest-path", - manifestPath, - "-p", - "codex-cli", - "--", - "app-server", - "generate-json-schema", - "--out", - jsonRoot, - "--experimental", - ]); + await assertCodexProtocolGenerationHeadroom({ codexRepo, repoRoot }); + runCargoProtocolGenerator(codexRepo, buildCodexProtocolExportArgs(manifestPath, generatedRoot)); + await splitGeneratedProtocolOutput(generatedRoot, { jsonRoot, typescriptRoot }); await rewriteTypeScriptImports(typescriptRoot); formatGeneratedTypeScript(repoRoot, typescriptRoot); } catch (error) { @@ -190,6 +232,98 @@ async function isDirectory(candidate: string): Promise { } } +async function assertCodexProtocolGenerationHeadroom(params: { + codexRepo: string; + repoRoot: string; +}): Promise { + const minFreeBytes = resolveCodexProtocolMinFreeBytes(); + if (minFreeBytes <= 0) { + return; + } + + const checks = [ + { path: params.repoRoot, label: "protocol output checkout" }, + { + path: resolveCodexProtocolCargoTargetDir(params.codexRepo), + label: "Cargo target directory", + }, + ]; + for (const check of checks) { + const statsPath = await resolveExistingStatfsPath(check.path); + const stats = await fs.statfs(statsPath); + validateCodexProtocolGenerationHeadroom({ + freeBytes: stats.bavail * stats.bsize, + minFreeBytes, + pathLabel: check.label, + }); + } +} + +function formatBytes(bytes: number): string { + const gib = bytes / (1024 * 1024 * 1024); + if (gib >= 1) { + return `${gib.toFixed(1)} GiB`; + } + return `${Math.floor(bytes / (1024 * 1024))} MiB`; +} + +async function resolveExistingStatfsPath(targetPath: string): Promise { + let currentPath = path.resolve(targetPath); + while (true) { + try { + await fs.stat(currentPath); + return currentPath; + } catch { + const parent = path.dirname(currentPath); + if (parent === currentPath) { + throw new Error(`Cannot find an existing parent directory for ${targetPath}`); + } + currentPath = parent; + } + } +} + +async function splitGeneratedProtocolOutput( + sourceRoot: string, + roots: { jsonRoot: string; typescriptRoot: string }, +): Promise { + await copyGeneratedProtocolFiles(sourceRoot, sourceRoot, roots); +} + +async function copyGeneratedProtocolFiles( + sourceRoot: string, + currentRoot: string, + roots: { jsonRoot: string; typescriptRoot: string }, +): Promise { + const entries = await fs.readdir(currentRoot, { withFileTypes: true }); + await Promise.all( + entries.map(async (entry) => { + const sourcePath = path.join(currentRoot, entry.name); + if (entry.isDirectory()) { + await copyGeneratedProtocolFiles(sourceRoot, sourcePath, roots); + return; + } + if (!entry.isFile()) { + return; + } + + const relativePath = path.relative(sourceRoot, sourcePath); + const targetRoot = entry.name.endsWith(".ts") + ? roots.typescriptRoot + : entry.name.endsWith(".json") + ? roots.jsonRoot + : null; + if (targetRoot === null) { + return; + } + + const targetPath = path.join(targetRoot, relativePath); + await fs.mkdir(path.dirname(targetPath), { recursive: true }); + await fs.copyFile(sourcePath, targetPath); + }), + ); +} + function runCargoProtocolGenerator(codexRepo: string, args: string[]): void { const result = spawnSync("cargo", args, { cwd: codexRepo, diff --git a/scripts/sync-codex-app-server-protocol.ts b/scripts/sync-codex-app-server-protocol.ts index cb82b068da3..99067f7d70c 100644 --- a/scripts/sync-codex-app-server-protocol.ts +++ b/scripts/sync-codex-app-server-protocol.ts @@ -10,21 +10,28 @@ const targetRoot = path.resolve( "extensions/codex/src/app-server/protocol-generated", ); -const source = await generateExperimentalCodexAppServerProtocolSource(); -try { - await fs.rm(targetRoot, { recursive: true, force: true }); - await fs.mkdir(targetRoot, { recursive: true }); +await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); - for (const schema of selectedCodexAppServerJsonSchemas) { - await fs.mkdir(path.dirname(path.join(targetRoot, "json", schema)), { recursive: true }); - const schemaSource = await fs.readFile(path.join(source.jsonRoot, schema), "utf8"); - await fs.writeFile( - path.join(targetRoot, "json", schema), - `${JSON.stringify(JSON.parse(schemaSource), null, 2)}\n`, - ); +async function main(): Promise { + const source = await generateExperimentalCodexAppServerProtocolSource(); + try { + await fs.rm(targetRoot, { recursive: true, force: true }); + await fs.mkdir(targetRoot, { recursive: true }); + + for (const schema of selectedCodexAppServerJsonSchemas) { + await fs.mkdir(path.dirname(path.join(targetRoot, "json", schema)), { recursive: true }); + const schemaSource = await fs.readFile(path.join(source.jsonRoot, schema), "utf8"); + await fs.writeFile( + path.join(targetRoot, "json", schema), + `${JSON.stringify(JSON.parse(schemaSource), null, 2)}\n`, + ); + } + } finally { + await source.cleanup(); } -} finally { - await source.cleanup(); -} -console.log(`Synced Codex app-server generated protocol from ${source.codexRepo}`); + console.log(`Synced Codex app-server generated protocol from ${source.codexRepo}`); +} diff --git a/test/scripts/codex-app-server-protocol-source.test.ts b/test/scripts/codex-app-server-protocol-source.test.ts index 0b35fa3ba7c..ef2eb608817 100644 --- a/test/scripts/codex-app-server-protocol-source.test.ts +++ b/test/scripts/codex-app-server-protocol-source.test.ts @@ -2,8 +2,12 @@ import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { + buildCodexProtocolExportArgs, resolveCodexAppServerProtocolSource, + resolveCodexProtocolCargoTargetDir, + resolveCodexProtocolMinFreeBytes, resolveCodexProtocolPnpmCommand, + validateCodexProtocolGenerationHeadroom, } from "../../scripts/lib/codex-app-server-protocol-source.js"; import { createScriptTestHarness } from "./test-helpers.js"; @@ -19,6 +23,86 @@ afterEach(() => { }); describe("codex app-server protocol source resolver", () => { + it("uses the app-server protocol export binary instead of compiling the full codex cli", () => { + expect(buildCodexProtocolExportArgs("/codex/codex-rs/Cargo.toml", "/tmp/protocol")).toEqual([ + "run", + "--manifest-path", + "/codex/codex-rs/Cargo.toml", + "-p", + "codex-app-server-protocol", + "--bin", + "export", + "--", + "--out", + "/tmp/protocol", + "--experimental", + ]); + }); + + it("fails before cargo protocol generation when local disk headroom is too low", () => { + expect(() => + validateCodexProtocolGenerationHeadroom({ + freeBytes: 6 * 1024 * 1024 * 1024, + minFreeBytes: 10 * 1024 * 1024 * 1024, + pathLabel: "/repo", + }), + ).toThrow(/Run this check on Crabbox\/Testbox/); + }); + + it("allows an explicit local disk headroom override", () => { + expect(resolveCodexProtocolMinFreeBytes({ OPENCLAW_CODEX_PROTOCOL_MIN_FREE_BYTES: "0" })).toBe( + 0, + ); + expect(() => + validateCodexProtocolGenerationHeadroom({ + freeBytes: 1, + minFreeBytes: 0, + pathLabel: "/repo", + }), + ).not.toThrow(); + }); + + it("rejects malformed local disk headroom overrides", () => { + expect(() => + resolveCodexProtocolMinFreeBytes({ OPENCLAW_CODEX_PROTOCOL_MIN_FREE_BYTES: "nope" }), + ).toThrow(/non-negative byte count/); + }); + + it("checks the Codex workspace target dir by default", () => { + expect(resolveCodexProtocolCargoTargetDir("/codex", {})).toBe( + path.join("/codex", "codex-rs", "target"), + ); + }); + + it("checks an explicit Cargo target dir override", () => { + expect(resolveCodexProtocolCargoTargetDir("/codex", { CARGO_TARGET_DIR: "/cache/target" })).toBe( + path.resolve("/cache/target"), + ); + }); + + it("resolves relative Cargo target dir overrides from the Codex checkout", () => { + expect(resolveCodexProtocolCargoTargetDir("/codex", { CARGO_TARGET_DIR: "target-cache" })).toBe( + path.join("/codex", "target-cache"), + ); + }); + + it("checks Cargo's build target dir override", () => { + expect( + resolveCodexProtocolCargoTargetDir("/codex", { + CARGO_BUILD_TARGET_DIR: "/cache/build-target", + }), + ).toBe(path.resolve("/cache/build-target")); + }); + + it("prefers Cargo's target dir override over the build config env override", () => { + expect( + resolveCodexProtocolCargoTargetDir("/codex", { + CARGO_BUILD_TARGET_DIR: "/cache/build-target", + CARGO_TARGET_DIR: "/cache/target", + }), + ).toBe(path.resolve("/cache/target")); + }); + it("wraps Windows pnpm formatting through cmd.exe without shell mode", () => { expect( resolveCodexProtocolPnpmCommand(