refactor: deduplicate Codex prompt snapshot fixtures

This commit is contained in:
Peter Steinberger
2026-08-01 08:35:26 -07:00
parent 7cc2e6d830
commit aa15534e71
8 changed files with 336 additions and 3258 deletions

View File

@@ -24,6 +24,14 @@ const oxfmtPath = path.resolve(
const execFileAsync = promisify(execFile);
type PromptSnapshotFile = Awaited<ReturnType<typeof createHappyPathPromptSnapshotFiles>>[number];
type CodexDynamicToolSnapshotSpec = { name: string };
type CodexDynamicToolSnapshotOverrides = {
base: string;
replace: Record<string, CodexDynamicToolSnapshotSpec>;
};
const CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX = "codex-dynamic-tools.";
const CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT = `${CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX}telegram-direct.json`;
function describeError(error: unknown): string {
return error instanceof Error ? error.message : String(error);
@@ -60,8 +68,49 @@ async function readSnapshotFiles(root: string, files: PromptSnapshotFile[]) {
);
}
export async function createFormattedPromptSnapshotFiles(): Promise<PromptSnapshotFile[]> {
const files = await createHappyPathPromptSnapshotFiles();
/** Keep complete Codex tool specs on the wire; deduplicate only their committed review fixtures. */
function factorCodexDynamicToolSnapshotFiles(files: PromptSnapshotFile[]): PromptSnapshotFile[] {
const base = files.find((file) => path.basename(file.path) === CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT);
if (!base) {
return files;
}
const baseTools = JSON.parse(base.content) as CodexDynamicToolSnapshotSpec[];
if (new Set(baseTools.map((tool) => tool.name)).size !== baseTools.length) {
return files;
}
return files.map((file) => {
const fileName = path.basename(file.path);
if (
fileName === CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT ||
!fileName.startsWith(CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX) ||
!fileName.endsWith(".json")
) {
return file;
}
const tools = JSON.parse(file.content) as CodexDynamicToolSnapshotSpec[];
if (
tools.length !== baseTools.length ||
tools.some((tool, index) => tool.name !== baseTools[index]?.name)
) {
return file;
}
const replace = Object.fromEntries(
tools.flatMap((tool, index) =>
JSON.stringify(tool) === JSON.stringify(baseTools[index]) ? [] : [[tool.name, tool]],
),
);
const overrides: CodexDynamicToolSnapshotOverrides = {
base: CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT,
replace,
};
return { ...file, content: `${JSON.stringify(overrides, null, 2)}\n` };
});
}
async function formatPromptSnapshotFiles(
files: PromptSnapshotFile[],
): Promise<PromptSnapshotFile[]> {
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prompt-snapshots-"));
try {
await writeSnapshotFiles(tmpRoot, files);
@@ -72,6 +121,56 @@ export async function createFormattedPromptSnapshotFiles(): Promise<PromptSnapsh
}
}
export async function createFormattedPromptSnapshotFiles(): Promise<PromptSnapshotFile[]> {
const files = await createHappyPathPromptSnapshotFiles();
return await formatPromptSnapshotFiles(factorCodexDynamicToolSnapshotFiles(files));
}
/** Materialize one complete, formatted Codex dynamic-tool catalog for human review. */
export async function materializeCodexDynamicToolSnapshot(scenario: string): Promise<string> {
if (!/^[a-z][a-z0-9-]*$/u.test(scenario)) {
throw new Error(`Invalid Codex dynamic-tool snapshot scenario: ${scenario}`);
}
const snapshotPath = path.join(
CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR,
`${CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX}${scenario}.json`,
);
const snapshot = JSON.parse(await fs.readFile(path.resolve(repoRoot, snapshotPath), "utf8")) as
| CodexDynamicToolSnapshotSpec[]
| CodexDynamicToolSnapshotOverrides;
let tools: CodexDynamicToolSnapshotSpec[];
if (Array.isArray(snapshot)) {
tools = snapshot;
} else {
if (snapshot.base !== CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT) {
throw new Error(`Unsupported Codex dynamic-tool snapshot base: ${snapshot.base}`);
}
const basePath = path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, snapshot.base);
const base = JSON.parse(await fs.readFile(path.resolve(repoRoot, basePath), "utf8")) as
| CodexDynamicToolSnapshotSpec[]
| CodexDynamicToolSnapshotOverrides;
if (!Array.isArray(base)) {
throw new Error("The canonical Codex dynamic-tool snapshot must contain complete tools");
}
const baseNames = new Set(base.map((tool) => tool.name));
for (const [name, replacement] of Object.entries(snapshot.replace)) {
if (!baseNames.has(name) || replacement.name !== name) {
throw new Error(`Invalid Codex dynamic-tool snapshot replacement: ${name}`);
}
}
tools = base.map((tool) =>
Object.hasOwn(snapshot.replace, tool.name) ? snapshot.replace[tool.name]! : tool,
);
}
const [formatted] = await formatPromptSnapshotFiles([
{ path: snapshotPath, content: `${JSON.stringify(tools, null, 2)}\n` },
]);
if (!formatted) {
throw new Error(`Failed to materialize Codex dynamic-tool snapshot: ${scenario}`);
}
return formatted.content;
}
async function writeSnapshots() {
const files = await createFormattedPromptSnapshotFiles();
await fs.mkdir(path.resolve(repoRoot, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR), {
@@ -117,6 +216,18 @@ async function checkSnapshots() {
}
async function runPromptSnapshotGenerator(argv = process.argv.slice(2)) {
if (argv[0] === "--materialize") {
const scenario = argv[1];
if (!scenario || argv.length !== 2) {
console.error(
"Usage: node --import tsx scripts/generate-prompt-snapshots.ts --materialize <scenario>",
);
process.exitCode = 2;
return;
}
process.stdout.write(await materializeCodexDynamicToolSnapshot(scenario));
return;
}
const mode = argv.includes("--write") ? "write" : argv.includes("--check") ? "check" : undefined;
if (!mode) {

View File

@@ -14,6 +14,16 @@ The workspace bootstrap simulation includes dummy workspace contents so prompt r
The tool catalog is pinned to the canonical happy-path OpenClaw tools so optional locally installed plugin tools do not create fixture churn.
The Telegram JSON is the complete shared tool catalog. Discord and heartbeat JSON fixtures contain readable, complete replacements for their changed top-level tools or namespaces; their `base` field points to the Telegram catalog.
Materialize the complete, formatted tool catalog for a scenario with:
```sh
node --import tsx scripts/generate-prompt-snapshots.ts --materialize discord-group
```
Replace `discord-group` with `heartbeat-turn` to inspect the complete heartbeat catalog.
The Codex model prompt fixture is generated from the same Codex model catalog/cache shape that the Codex runtime uses for remote model metadata. Regenerate it from Codex's runtime cache or, when present, a local Codex checkout with:
```sh

View File

@@ -517,7 +517,7 @@ can you audit whether this prompt path has conflicting silence instructions?
### Tools: Dynamic Tool Catalog
Full JSON: `codex-dynamic-tools.discord-group.json`
Full tool overrides: `codex-dynamic-tools.discord-group.json` (base: `codex-dynamic-tools.telegram-direct.json`)
## Dynamic Tool Names

View File

@@ -512,7 +512,7 @@ Follow the heartbeat monitor scratch context when provided. Recurring tasks are
### Tools: Dynamic Tool Catalog
Full JSON: `codex-dynamic-tools.heartbeat-turn.json`
Full tool overrides: `codex-dynamic-tools.heartbeat-turn.json` (base: `codex-dynamic-tools.telegram-direct.json`)
## Dynamic Tool Names

View File

@@ -829,7 +829,9 @@ function renderModelBoundPromptLayers(params: {
"",
"### Tools: Dynamic Tool Catalog",
"",
`Full JSON: \`${params.scenario.toolSnapshotFile}\``,
params.scenario.toolSnapshotFile === "codex-dynamic-tools.telegram-direct.json"
? `Full JSON: \`${params.scenario.toolSnapshotFile}\``
: `Full tool overrides: \`${params.scenario.toolSnapshotFile}\` (base: \`codex-dynamic-tools.telegram-direct.json\`)`,
"",
];
}
@@ -971,6 +973,17 @@ function renderReadme(scenarios: PromptScenario[]): string {
"",
"The tool catalog is pinned to the canonical happy-path OpenClaw tools so optional locally installed plugin tools do not create fixture churn.",
"",
"The Telegram JSON is the complete shared tool catalog. Discord and heartbeat JSON fixtures contain readable, complete replacements for their changed top-level tools or namespaces; their `base` field points to the Telegram catalog.",
"",
"Materialize the complete, formatted tool catalog for a scenario with:",
"",
markdownFence(
"sh",
"node --import tsx scripts/generate-prompt-snapshots.ts --materialize discord-group",
),
"",
"Replace `discord-group` with `heartbeat-turn` to inspect the complete heartbeat catalog.",
"",
"The Codex model prompt fixture is generated from the same Codex model catalog/cache shape that the Codex runtime uses for remote model metadata. Regenerate it from Codex's runtime cache or, when present, a local Codex checkout with:",
"",
markdownFence("sh", "pnpm prompt:snapshots:sync-codex-model"),

View File

@@ -4,7 +4,10 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { createFormattedPromptSnapshotFiles } from "../../scripts/generate-prompt-snapshots.js";
import {
createFormattedPromptSnapshotFiles,
materializeCodexDynamicToolSnapshot,
} from "../../scripts/generate-prompt-snapshots.js";
import { deleteStalePromptSnapshotFiles } from "../../scripts/prompt-snapshot-files.js";
import {
CODEX_MODEL_PROMPT_FIXTURE_DIR as SYNC_CODEX_MODEL_PROMPT_FIXTURE_DIR,
@@ -133,6 +136,47 @@ describe("happy path prompt snapshots", () => {
]);
});
it("reconstructs complete Codex tool catalogs from readable full-tool overrides", async () => {
const generated = await createHappyPathPromptSnapshotFiles();
const scenarios = [
{ name: "telegram-direct", replacements: [] },
{ name: "discord-group", replacements: ["sessions_spawn"] },
{ name: "heartbeat-turn", replacements: ["openclaw_direct"] },
];
for (const { name, replacements } of scenarios) {
const fileName = `codex-dynamic-tools.${name}.json`;
const expected = generated.find((file) => path.basename(file.path) === fileName);
expect(expected, `missing complete generated tool catalog for ${name}`).toBeDefined();
const committed = JSON.parse(readCommittedSnapshot(fileName)) as
| unknown[]
| {
base: string;
replace: Record<string, { name: string; inputSchema?: unknown; tools?: unknown[] }>;
};
if (Array.isArray(committed)) {
expect(replacements).toEqual([]);
} else {
expect(committed.base).toBe("codex-dynamic-tools.telegram-direct.json");
expect(Object.keys(committed.replace)).toEqual(replacements);
for (const [toolName, tool] of Object.entries(committed.replace)) {
expect(tool.name).toBe(toolName);
expect(tool.inputSchema !== undefined || Array.isArray(tool.tools)).toBe(true);
}
}
const materialized = await materializeCodexDynamicToolSnapshot(name);
expect(JSON.parse(materialized)).toEqual(JSON.parse(expected!.content));
}
});
it("rejects invalid Codex dynamic-tool materialization scenarios", async () => {
await expect(materializeCodexDynamicToolSnapshot("../outside")).rejects.toThrow(
"Invalid Codex dynamic-tool snapshot scenario",
);
});
it("generates snapshots without jiti plugin-loader fallbacks", async () => {
// Perf contract for the check-prompt-snapshots CI lane: scenario channel
// plugins are preloaded through the ambient module graph. A jiti