diff --git a/CHANGELOG.md b/CHANGELOG.md index ea08dc5c48e..7f050e2fc89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Docs: https://docs.openclaw.ai - QA-Lab: include the optional 100-turn runtime parity soak in release-soak artifacts so long-run Codex/Pi transcript drift stays visible outside the default gate. (#80395) Thanks @100yenadmin. - QA-Lab: add a personal-agent failure recovery scenario that checks honest partial status, retry boundaries, and local recovery artifacts. (#83872) Thanks @iFiras-Max1. - QA-Lab: include an opt-in `update.run` package self-upgrade sentinel for destructive latest-package recovery checks. +- QA-Lab: add Codex plugin lifecycle and auth-profile fixture coverage for missing installs, pinned-version drift, first-turn install ordering, and doctor migration safety. (#80323, refs #80174) Thanks @100yenadmin. - Tests/perf: isolate doctor core health check unit coverage from real skills/workspace discovery so `doctor-core-checks` no longer dominates unit perf while keeping one real skills-readiness smoke. (#84493) Thanks @frankekn. ### Fixes diff --git a/extensions/qa-lab/src/auth-profile-fixture.ts b/extensions/qa-lab/src/auth-profile-fixture.ts new file mode 100644 index 00000000000..e2533e4d4d5 --- /dev/null +++ b/extensions/qa-lab/src/auth-profile-fixture.ts @@ -0,0 +1,177 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +export const QA_CODEX_OAUTH_PROFILE_ID = "openai-codex:qa-oauth"; +export const QA_OPENAI_API_KEY_PROFILE_ID = "openai:media-api"; +export const QA_AUTH_PROFILE_STORE_VERSION = 1; + +export type QaAuthProfileShape = "oauth-only" | "apikey-only" | "mixed"; + +export type QaApiKeyAuthProfile = { + type: "api_key"; + provider: "openai"; + key: string; + displayName: string; +}; + +export type QaOAuthAuthProfile = { + type: "oauth"; + provider: "openai-codex"; + access: string; + refresh: string; + expires: number; + email: string; + displayName: string; +}; + +export type QaAuthProfile = QaApiKeyAuthProfile | QaOAuthAuthProfile; + +export type QaAuthProfileSnapshot = { + version: number; + profiles: Record; +}; + +export type QaCodexAuthProfileSelection = + | { + status: "ready"; + profileId: string; + provider: "openai-codex"; + mode: "oauth"; + } + | { + status: "blocked"; + remediation: string; + }; + +const QA_FIXED_OAUTH_EXPIRY_MS = Date.UTC(2036, 0, 1); + +function authProfilesPath(agentDir: string) { + return path.join(agentDir, "auth-profiles.json"); +} + +function buildCodexOAuthProfile(): QaOAuthAuthProfile { + return { + type: "oauth", + provider: "openai-codex", + access: "qa-codex-oauth-access-placeholder", + refresh: "qa-codex-oauth-refresh-placeholder", + expires: QA_FIXED_OAUTH_EXPIRY_MS, + email: "qa-codex@example.test", + displayName: "QA Codex OAuth profile", + }; +} + +function buildOpenAiApiKeyProfile(): QaApiKeyAuthProfile { + return { + type: "api_key", + provider: "openai", + key: "qa-openai-not-a-real-key", + displayName: "QA OpenAI API-key profile", + }; +} + +function buildProfileMap(shape: QaAuthProfileShape): Record { + switch (shape) { + case "oauth-only": + return { + [QA_CODEX_OAUTH_PROFILE_ID]: buildCodexOAuthProfile(), + }; + case "apikey-only": + return { + [QA_OPENAI_API_KEY_PROFILE_ID]: buildOpenAiApiKeyProfile(), + }; + case "mixed": + return { + [QA_CODEX_OAUTH_PROFILE_ID]: buildCodexOAuthProfile(), + [QA_OPENAI_API_KEY_PROFILE_ID]: buildOpenAiApiKeyProfile(), + }; + } + const exhaustive: never = shape; + return exhaustive; +} + +function isQaAuthProfile(value: unknown): value is QaAuthProfile { + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + return ( + (record.type === "oauth" && record.provider === "openai-codex") || + (record.type === "api_key" && record.provider === "openai") + ); +} + +function normalizeAuthProfileSnapshot(value: unknown): QaAuthProfileSnapshot { + if (!value || typeof value !== "object") { + return { version: QA_AUTH_PROFILE_STORE_VERSION, profiles: {} }; + } + const record = value as Record; + const profilesRecord = + record.profiles && typeof record.profiles === "object" + ? (record.profiles as Record) + : {}; + const profiles = Object.fromEntries( + Object.entries(profilesRecord) + .filter((entry): entry is [string, QaAuthProfile] => isQaAuthProfile(entry[1])) + .toSorted(([left], [right]) => left.localeCompare(right)), + ); + return { + version: + typeof record.version === "number" && Number.isFinite(record.version) + ? record.version + : QA_AUTH_PROFILE_STORE_VERSION, + profiles, + }; +} + +export async function seedAuthProfiles( + shape: QaAuthProfileShape, + agentDir: string, +): Promise { + const snapshot = { + version: QA_AUTH_PROFILE_STORE_VERSION, + profiles: buildProfileMap(shape), + }; + await fs.mkdir(agentDir, { recursive: true }); + await fs.writeFile(authProfilesPath(agentDir), `${JSON.stringify(snapshot, null, 2)}\n`, "utf8"); + return snapshot; +} + +export async function snapshotAuthProfiles(agentDir: string): Promise { + const raw = await fs.readFile(authProfilesPath(agentDir), "utf8").catch((error: unknown) => { + if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") { + return null; + } + throw error; + }); + if (!raw) { + return { version: QA_AUTH_PROFILE_STORE_VERSION, profiles: {} }; + } + return normalizeAuthProfileSnapshot(JSON.parse(raw) as unknown); +} + +export function resolveCodexAuthProfile( + snapshot: QaAuthProfileSnapshot, +): QaCodexAuthProfileSelection { + const profileId = Object.keys(snapshot.profiles) + .toSorted((left, right) => left.localeCompare(right)) + .find((candidate) => { + const profile = snapshot.profiles[candidate]; + return profile?.type === "oauth" && profile.provider === "openai-codex"; + }); + + if (!profileId) { + return { + status: "blocked", + remediation: + 'Codex app-server auth requires an openai-codex OAuth profile. Run "openclaw doctor --fix" to repair Codex auth routing before retrying.', + }; + } + + return { + status: "ready", + profileId, + provider: "openai-codex", + mode: "oauth", + }; +} diff --git a/extensions/qa-lab/src/codex-plugin-fixture.ts b/extensions/qa-lab/src/codex-plugin-fixture.ts new file mode 100644 index 00000000000..a286f5bcf4c --- /dev/null +++ b/extensions/qa-lab/src/codex-plugin-fixture.ts @@ -0,0 +1,282 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { resolveCodexAuthProfile, type QaAuthProfileSnapshot } from "./auth-profile-fixture.js"; + +export const CODEX_PLUGIN_CURRENT_VERSION = "2026.5.20"; +export const CODEX_PLUGIN_HEAD_VERSION = "head"; +export const CODEX_PLUGIN_ID = "codex"; + +export const CODEX_PLUGIN_LIFECYCLE_MESSAGES = Object.freeze({ + missingPlugin: + 'Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry.', +}); + +export type CodexPluginFixtureVersion = "missing" | "current" | "head" | (string & {}); + +export type CodexPluginState = { + installed: boolean; + version?: string; +}; + +export type CodexPluginLifecycleStatus = "ready" | "repair-required" | "blocked"; + +export type CodexPluginLifecycleResult = { + status: CodexPluginLifecycleStatus; + pluginState: CodexPluginState; + selectedAuthProfileId?: string; + tokenRoute?: "codex-oauth" | "unavailable"; + remediation?: string; + removedRuntimePins: string[]; +}; + +type CodexPluginPackageJson = { + name: "@openclaw/codex"; + version: string; + openclaw: { + install: { + minHostVersion: string; + }; + compat: { + pluginApi: string; + }; + }; +}; + +type ComparableVersion = { + major: number; + minor: number; + patch: number; +}; + +type CodexPluginInstallGateResult = { + text: string; + inputTokens: number; + responseCount: number; +}; + +function codexPluginDir(agentDir: string) { + return path.join(agentDir, "plugins", CODEX_PLUGIN_ID); +} + +function resolveFixtureVersion(version: CodexPluginFixtureVersion): string { + if (version === "current") { + return CODEX_PLUGIN_CURRENT_VERSION; + } + return version; +} + +function buildPackageJson(version: string): CodexPluginPackageJson { + return { + name: "@openclaw/codex", + version, + openclaw: { + install: { + minHostVersion: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`, + }, + compat: { + pluginApi: `>=${version === CODEX_PLUGIN_HEAD_VERSION ? CODEX_PLUGIN_CURRENT_VERSION : version}`, + }, + }, + }; +} + +function parseComparableVersion(value: string | undefined): ComparableVersion | null { + if (!value || value === CODEX_PLUGIN_HEAD_VERSION) { + return parseComparableVersion(CODEX_PLUGIN_CURRENT_VERSION); + } + const match = value.trim().match(/^(\d+)\.(\d+)\.(\d+)/); + if (!match) { + return null; + } + return { + major: Number.parseInt(match[1] ?? "0", 10), + minor: Number.parseInt(match[2] ?? "0", 10), + patch: Number.parseInt(match[3] ?? "0", 10), + }; +} + +function compareVersions(left: string | undefined, right: string): number { + const leftVersion = parseComparableVersion(left); + const rightVersion = parseComparableVersion(right); + if (!leftVersion || !rightVersion) { + return 0; + } + if (leftVersion.major !== rightVersion.major) { + return leftVersion.major - rightVersion.major; + } + if (leftVersion.minor !== rightVersion.minor) { + return leftVersion.minor - rightVersion.minor; + } + return leftVersion.patch - rightVersion.patch; +} + +function formatPinnedOldRemediation(pluginVersion: string, hostVersion: string) { + return `Codex plugin version ${pluginVersion} is older than OpenClaw ${hostVersion}. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".`; +} + +function formatPinnedNewRemediation(pluginVersion: string, hostVersion: string) { + return `Codex plugin version ${pluginVersion} requires a newer OpenClaw host than ${hostVersion}. Upgrade OpenClaw or install a codex plugin version pinned to ${hostVersion}.`; +} + +function collectStalePiRuntimePins(config: unknown): string[] { + if (!config || typeof config !== "object") { + return []; + } + const root = config as { + agents?: { + defaults?: { agentRuntime?: { id?: unknown } }; + list?: Record; + }; + }; + const hasDefaultsPin = root.agents?.defaults?.agentRuntime?.id === "pi"; + const hasAgentPin = Object.values(root.agents?.list ?? {}).some( + (entry) => entry.agentRuntime?.id === "pi", + ); + return hasDefaultsPin || hasAgentPin ? ["agentRuntime.id=pi"] : []; +} + +export async function seedCodexPluginAt( + version: CodexPluginFixtureVersion, + agentDir: string, +): Promise { + const targetDir = codexPluginDir(agentDir); + await fs.rm(targetDir, { recursive: true, force: true }); + if (version === "missing") { + return; + } + + const resolvedVersion = resolveFixtureVersion(version); + await fs.mkdir(targetDir, { recursive: true }); + await fs.writeFile( + path.join(targetDir, "package.json"), + `${JSON.stringify(buildPackageJson(resolvedVersion), null, 2)}\n`, + "utf8", + ); + await fs.writeFile( + path.join(targetDir, "openclaw.plugin.json"), + `${JSON.stringify({ id: CODEX_PLUGIN_ID, name: "Codex" }, null, 2)}\n`, + "utf8", + ); +} + +export async function snapshotCodexPluginState(agentDir: string): Promise { + const packagePath = path.join(codexPluginDir(agentDir), "package.json"); + const raw = await fs.readFile(packagePath, "utf8").catch((error: unknown) => { + if (error && typeof error === "object" && (error as { code?: unknown }).code === "ENOENT") { + return null; + } + throw error; + }); + if (!raw) { + return { installed: false }; + } + + const parsed = JSON.parse(raw) as { version?: unknown }; + return { + installed: true, + ...(typeof parsed.version === "string" ? { version: parsed.version } : {}), + }; +} + +export function evaluateCodexPluginLifecycle(params: { + plugin: CodexPluginState; + auth: QaAuthProfileSnapshot; + hostVersion: string; + config?: unknown; + doctorFix?: boolean; +}): CodexPluginLifecycleResult { + const authSelection = resolveCodexAuthProfile(params.auth); + const selectedAuthProfileId = + authSelection.status === "ready" ? authSelection.profileId : undefined; + const tokenRoute = authSelection.status === "ready" ? "codex-oauth" : "unavailable"; + const removedRuntimePins = params.doctorFix ? collectStalePiRuntimePins(params.config) : []; + + if (!params.plugin.installed) { + return { + status: "repair-required", + pluginState: params.plugin, + ...(selectedAuthProfileId ? { selectedAuthProfileId } : {}), + tokenRoute, + remediation: CODEX_PLUGIN_LIFECYCLE_MESSAGES.missingPlugin, + removedRuntimePins, + }; + } + + if (authSelection.status === "blocked") { + return { + status: "blocked", + pluginState: params.plugin, + tokenRoute, + remediation: authSelection.remediation, + removedRuntimePins, + }; + } + + const versionDelta = compareVersions(params.plugin.version, params.hostVersion); + if (versionDelta < 0 && params.plugin.version) { + return { + status: "blocked", + pluginState: params.plugin, + selectedAuthProfileId, + tokenRoute, + remediation: formatPinnedOldRemediation(params.plugin.version, params.hostVersion), + removedRuntimePins, + }; + } + if (versionDelta > 0 && params.plugin.version) { + return { + status: "blocked", + pluginState: params.plugin, + selectedAuthProfileId, + tokenRoute, + remediation: formatPinnedNewRemediation(params.plugin.version, params.hostVersion), + removedRuntimePins, + }; + } + + return { + status: "ready", + pluginState: params.plugin, + selectedAuthProfileId, + tokenRoute, + removedRuntimePins, + }; +} + +export function createCodexPluginInstallGate() { + const events: string[] = []; + let installed = false; + let resolveInstall: (() => void) | undefined; + const installedPromise = new Promise((resolve) => { + resolveInstall = resolve; + }); + + return { + events, + markInstalled() { + if (installed) { + return; + } + installed = true; + events.push("codex-plugin:installed"); + resolveInstall?.(); + }, + async runFirstTurnAfterInstall(params: { + inputTokens: number; + run: () => string | Promise; + }): Promise { + if (!installed) { + events.push("agent-turn:waiting-for-codex-plugin"); + await installedPromise; + } + events.push("agent-turn:started"); + const text = await params.run(); + events.push("agent-turn:completed"); + return { + text, + inputTokens: params.inputTokens, + responseCount: 1, + }; + }, + }; +} diff --git a/extensions/qa-lab/src/codex-plugin-lifecycle.test.ts b/extensions/qa-lab/src/codex-plugin-lifecycle.test.ts new file mode 100644 index 00000000000..20cd8b9bf27 --- /dev/null +++ b/extensions/qa-lab/src/codex-plugin-lifecycle.test.ts @@ -0,0 +1,190 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + QA_CODEX_OAUTH_PROFILE_ID, + QA_OPENAI_API_KEY_PROFILE_ID, + resolveCodexAuthProfile, + seedAuthProfiles, + snapshotAuthProfiles, +} from "./auth-profile-fixture.js"; +import { + CODEX_PLUGIN_CURRENT_VERSION, + CODEX_PLUGIN_LIFECYCLE_MESSAGES, + createCodexPluginInstallGate, + evaluateCodexPluginLifecycle, + seedCodexPluginAt, + snapshotCodexPluginState, +} from "./codex-plugin-fixture.js"; +import { createTempDirHarness } from "./temp-dir.test-helper.js"; + +const tempDirs = createTempDirHarness(); + +async function createAgentDir(prefix: string) { + const root = await tempDirs.makeTempDir(prefix); + const agentDir = path.join(root, "agents", "qa", "agent"); + await fs.mkdir(agentDir, { recursive: true }); + return agentDir; +} + +afterEach(async () => { + await tempDirs.cleanup(); +}); + +describe("codex plugin lifecycle: cold install", () => { + it("repairs a missing codex plugin before the retry succeeds without leaking to the API-key path", async () => { + const agentDir = await createAgentDir("qa-codex-plugin-cold-"); + await seedCodexPluginAt("missing", agentDir); + await seedAuthProfiles("mixed", agentDir); + + const missing = evaluateCodexPluginLifecycle({ + plugin: await snapshotCodexPluginState(agentDir), + auth: await snapshotAuthProfiles(agentDir), + hostVersion: CODEX_PLUGIN_CURRENT_VERSION, + }); + + expect(missing.status).toBe("repair-required"); + expect(missing.remediation).toBe(CODEX_PLUGIN_LIFECYCLE_MESSAGES.missingPlugin); + expect(missing.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID); + expect(missing.selectedAuthProfileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID); + + await seedCodexPluginAt("current", agentDir); + const repaired = evaluateCodexPluginLifecycle({ + plugin: await snapshotCodexPluginState(agentDir), + auth: await snapshotAuthProfiles(agentDir), + hostVersion: CODEX_PLUGIN_CURRENT_VERSION, + }); + + expect(repaired.status).toBe("ready"); + expect(repaired.remediation).toBeUndefined(); + expect(repaired.tokenRoute).toBe("codex-oauth"); + }); +}); + +describe("codex plugin lifecycle: OAuth-only with mixed profiles", () => { + it("selects openai-codex OAuth when openai API-key profiles are present", async () => { + const agentDir = await createAgentDir("qa-codex-auth-mixed-"); + await seedAuthProfiles("mixed", agentDir); + + const selection = resolveCodexAuthProfile(await snapshotAuthProfiles(agentDir)); + + expect(selection.status).toBe("ready"); + if (selection.status !== "ready") { + throw new Error(selection.remediation); + } + expect(selection.profileId).toBe(QA_CODEX_OAUTH_PROFILE_ID); + expect(selection.profileId).not.toBe(QA_OPENAI_API_KEY_PROFILE_ID); + expect(selection.provider).toBe("openai-codex"); + expect(selection.mode).toBe("oauth"); + }); +}); + +describe("codex plugin lifecycle: pinned-old codex plugin with new OpenClaw", () => { + it("blocks with a precise update remediation when the plugin is older than the host", async () => { + const agentDir = await createAgentDir("qa-codex-plugin-old-"); + await seedCodexPluginAt("2026.5.19", agentDir); + await seedAuthProfiles("oauth-only", agentDir); + + const result = evaluateCodexPluginLifecycle({ + plugin: await snapshotCodexPluginState(agentDir), + auth: await snapshotAuthProfiles(agentDir), + hostVersion: "2026.5.20", + }); + + expect(result.status).toBe("blocked"); + expect(result.remediation).toBe( + 'Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.20. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix".', + ); + }); +}); + +describe("codex plugin lifecycle: pinned-new codex plugin with old OpenClaw", () => { + it("blocks with a precise host-upgrade remediation when the plugin is newer than the host", async () => { + const agentDir = await createAgentDir("qa-codex-plugin-new-"); + await seedCodexPluginAt("2026.5.21", agentDir); + await seedAuthProfiles("oauth-only", agentDir); + + const result = evaluateCodexPluginLifecycle({ + plugin: await snapshotCodexPluginState(agentDir), + auth: await snapshotAuthProfiles(agentDir), + hostVersion: "2026.5.20", + }); + + expect(result.status).toBe("blocked"); + expect(result.remediation).toBe( + "Codex plugin version 2026.5.21 requires a newer OpenClaw host than 2026.5.20. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.20.", + ); + }); +}); + +describe("codex plugin lifecycle: install racing first agent turn", () => { + it("gates the first turn on install completion without sleeps, lost tokens, or duplicate responses", async () => { + const gate = createCodexPluginInstallGate(); + const turn = gate.runFirstTurnAfterInstall({ + inputTokens: 17, + run: () => "QA_CODEX_PLUGIN_TURN_OK", + }); + + expect(gate.events).toEqual(["agent-turn:waiting-for-codex-plugin"]); + + gate.markInstalled(); + await expect(turn).resolves.toEqual({ + text: "QA_CODEX_PLUGIN_TURN_OK", + inputTokens: 17, + responseCount: 1, + }); + expect(gate.events).toEqual([ + "agent-turn:waiting-for-codex-plugin", + "codex-plugin:installed", + "agent-turn:started", + "agent-turn:completed", + ]); + }); +}); + +describe("codex plugin lifecycle: doctor migration safety matrix", () => { + it.each([ + { + name: "oauth-only host", + profileShape: "oauth-only" as const, + config: {}, + }, + { + name: "mixed profile with no pin", + profileShape: "mixed" as const, + config: {}, + }, + { + name: "mixed profile with defaults pi pin", + profileShape: "mixed" as const, + config: { agents: { defaults: { agentRuntime: { id: "pi" } } } }, + }, + { + name: "mixed profile with main-agent pi pin", + profileShape: "mixed" as const, + config: { agents: { list: { main: { agentRuntime: { id: "pi" } } } } }, + }, + ])( + "keeps codex auth and strips stale pi runtime pins for $name", + async ({ profileShape, config }) => { + const agentDir = await createAgentDir("qa-codex-doctor-matrix-"); + await seedCodexPluginAt("current", agentDir); + await seedAuthProfiles(profileShape, agentDir); + + const result = evaluateCodexPluginLifecycle({ + plugin: await snapshotCodexPluginState(agentDir), + auth: await snapshotAuthProfiles(agentDir), + hostVersion: CODEX_PLUGIN_CURRENT_VERSION, + config, + doctorFix: true, + }); + + expect(result.status).toBe("ready"); + expect(result.selectedAuthProfileId).toBe(QA_CODEX_OAUTH_PROFILE_ID); + expect(result.tokenRoute).toBe("codex-oauth"); + expect(result.removedRuntimePins).toEqual( + Object.keys(config).length === 0 ? [] : ["agentRuntime.id=pi"], + ); + }, + ); +}); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 82e94bcf4cd..a7f06df1e2a 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -213,6 +213,37 @@ describe("qa scenario catalog", () => { ]); }); + it("loads the Codex plugin lifecycle fixture scenarios into the standard runtime tier", () => { + const scenarioIds = [ + "codex-plugin-cold-install", + "codex-plugin-install-race", + "codex-plugin-pinned-old", + "codex-plugin-pinned-new", + "auth-profile-codex-mixed-profiles", + "auth-profile-doctor-migration-safety", + ]; + + for (const scenarioId of scenarioIds) { + const scenario = readQaScenarioById(scenarioId); + expect(scenario.runtimeParityTier).toBe("standard"); + expect(scenario.coverage?.primary.length).toBeGreaterThan(0); + expect(scenario.execution.flow?.steps.length).toBe(1); + } + expect(readQaScenarioExecutionConfig("codex-plugin-pinned-old")).toMatchObject({ + pluginVersion: "2026.5.19", + hostVersion: "2026.5.20", + pluginRelation: "older", + }); + expect(readQaScenarioExecutionConfig("auth-profile-doctor-migration-safety")).toMatchObject({ + matrixCells: [ + "oauth-only", + "mixed-no-pin", + "mixed-defaults-pi-pin", + "mixed-main-agent-pi-pin", + ], + }); + }); + it("keeps the character eval scenario natural and task-shaped", () => { const characterConfig = readQaScenarioExecutionConfig("character-vibes-gollum") as | { diff --git a/qa/scenarios/runtime/auth-profile-codex-mixed-profiles.md b/qa/scenarios/runtime/auth-profile-codex-mixed-profiles.md new file mode 100644 index 00000000000..fb160a0be66 --- /dev/null +++ b/qa/scenarios/runtime/auth-profile-codex-mixed-profiles.md @@ -0,0 +1,70 @@ +# Codex auth profile mixed profiles + +```yaml qa-scenario +id: auth-profile-codex-mixed-profiles +title: Codex auth profile mixed profiles +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.codex-plugin.auth + secondary: + - auth-profiles.provider-selection +objective: Verify mixed openai-codex OAuth and openai API-key profile stores select the Codex OAuth profile for Codex app-server turns. +successCriteria: + - The selected auth profile id is openai-codex:qa-oauth. + - The openai:media-api API-key profile is present but not selected. + - The fixture rejects the residual provider mismatch covered by issue #78499. +docsRefs: + - docs/cli/doctor.md +codeRefs: + - extensions/qa-lab/src/auth-profile-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the auth-profile fixture for mixed OpenAI API-key and Codex OAuth stores. + config: + selectedProfileId: openai-codex:qa-oauth + rejectedProfileId: openai:media-api +``` + +```yaml qa-flow +steps: + - name: validates mixed-profile Codex auth selection + actions: + - set: auth + value: + expr: await qaImport("./auth-profile-fixture.js") + - set: tmpRoot + value: + expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-auth-")) + - try: + actions: + - call: auth.seedAuthProfiles + args: + - mixed + - ref: tmpRoot + - set: selection + value: + expr: auth.resolveCodexAuthProfile(await auth.snapshotAuthProfiles(tmpRoot)) + - assert: + expr: "selection.status === 'ready'" + message: + expr: "`expected ready Codex auth selection, got ${JSON.stringify(selection)}`" + - assert: + expr: "selection.profileId === config.selectedProfileId" + message: mixed profiles must select openai-codex OAuth + - assert: + expr: "selection.profileId !== config.rejectedProfileId" + message: codex profile must not equal openai api-key profile + finally: + - call: fs.rm + args: + - ref: tmpRoot + - recursive: true + force: true + - assert: + expr: "config.selectedProfileId !== config.rejectedProfileId" + message: "codex profile must not equal openai api-key profile" + detailsExpr: "`selected=${selection.profileId} rejected=${config.rejectedProfileId}`" +``` diff --git a/qa/scenarios/runtime/auth-profile-doctor-migration-safety.md b/qa/scenarios/runtime/auth-profile-doctor-migration-safety.md new file mode 100644 index 00000000000..cf4f5ae83f0 --- /dev/null +++ b/qa/scenarios/runtime/auth-profile-doctor-migration-safety.md @@ -0,0 +1,91 @@ +# Codex doctor migration safety matrix + +```yaml qa-scenario +id: auth-profile-doctor-migration-safety +title: Codex doctor migration safety matrix +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.doctor-repair + secondary: + - runtime.codex-plugin.auth +objective: Reproduce the four manual doctor-migration cells as an automated fixture matrix for Codex OAuth selection and stale Pi runtime pin removal. +successCriteria: + - OAuth-only hosts select the openai-codex OAuth profile and use the Codex harness. + - Mixed-profile hosts still select openai-codex OAuth when an openai API-key profile exists. + - Mixed-profile defaults-level pi runtime pins are stripped by doctor repair. + - Mixed-profile per-agent pi runtime pins are stripped by doctor repair. +docsRefs: + - docs/cli/doctor.md +codeRefs: + - extensions/qa-lab/src/auth-profile-fixture.ts + - extensions/qa-lab/src/codex-plugin-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the four-cell doctor migration matrix against Codex auth and stale Pi runtime pins. + config: + matrixCells: + - oauth-only + - mixed-no-pin + - mixed-defaults-pi-pin + - mixed-main-agent-pi-pin +``` + +```yaml qa-flow +steps: + - name: validates doctor migration safety matrix + actions: + - set: auth + value: + expr: await qaImport("./auth-profile-fixture.js") + - set: plugin + value: + expr: await qaImport("./codex-plugin-fixture.js") + - forEach: + items: + ref: config.matrixCells + item: cell + actions: + - set: tmpRoot + value: + expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", `qa-codex-doctor-${cell}-`)) + - set: profileShape + value: + expr: "cell === 'oauth-only' ? 'oauth-only' : 'mixed'" + - set: doctorConfig + value: + expr: "cell === 'mixed-defaults-pi-pin' ? { agents: { defaults: { agentRuntime: { id: 'pi' } } } } : cell === 'mixed-main-agent-pi-pin' ? { agents: { list: { main: { agentRuntime: { id: 'pi' } } } } } : {}" + - try: + actions: + - call: plugin.seedCodexPluginAt + args: + - current + - ref: tmpRoot + - call: auth.seedAuthProfiles + args: + - ref: profileShape + - ref: tmpRoot + - set: result + value: + expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: plugin.CODEX_PLUGIN_CURRENT_VERSION, config: doctorConfig, doctorFix: true })" + - assert: + expr: "result.status === 'ready' && result.selectedAuthProfileId === auth.QA_CODEX_OAUTH_PROFILE_ID && result.tokenRoute === 'codex-oauth'" + message: + expr: "`doctor matrix cell ${cell} failed Codex auth routing: ${JSON.stringify(result)}`" + - assert: + expr: "(Object.keys(doctorConfig).length === 0 && result.removedRuntimePins.length === 0) || result.removedRuntimePins.includes('agentRuntime.id=pi')" + message: + expr: "`doctor matrix cell ${cell} did not report stale Pi pin cleanup: ${JSON.stringify(result)}`" + finally: + - call: fs.rm + args: + - ref: tmpRoot + - recursive: true + force: true + - assert: + expr: "config.matrixCells.length === 4" + message: "expected four doctor migration cells" + detailsExpr: "`cells=${config.matrixCells.join(',')}`" +``` diff --git a/qa/scenarios/runtime/codex-plugin-cold-install.md b/qa/scenarios/runtime/codex-plugin-cold-install.md new file mode 100644 index 00000000000..f4820c6b117 --- /dev/null +++ b/qa/scenarios/runtime/codex-plugin-cold-install.md @@ -0,0 +1,90 @@ +# Codex plugin cold install + +```yaml qa-scenario +id: codex-plugin-cold-install +title: Codex plugin cold install +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.codex-plugin.lifecycle + secondary: + - runtime.doctor-repair +objective: Verify a clean home that needs the Codex runtime reports a clear missing-plugin remediation, installs through doctor repair, and retries through Codex OAuth instead of OpenAI API-key auth. +successCriteria: + - Missing Codex plugin emits the exact remediation string asserted by the fixture test. + - Doctor repair seeds the Codex plugin before retrying the agent turn. + - The retry uses the openai-codex OAuth profile and never routes through the openai API-key profile. +docsRefs: + - docs/cli/doctor.md + - docs/cli/plugins.md + - docs/plugins/install-overrides.md +codeRefs: + - extensions/qa-lab/src/codex-plugin-fixture.ts + - extensions/qa-lab/src/auth-profile-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the Codex lifecycle fixture for missing plugin repair and retry auth routing. + config: + remediation: Codex plugin is required for Codex runtime. Run "openclaw doctor --fix" to install @openclaw/codex, then retry. +``` + +```yaml qa-flow +steps: + - name: validates cold-install repair routing + actions: + - set: auth + value: + expr: await qaImport("./auth-profile-fixture.js") + - set: plugin + value: + expr: await qaImport("./codex-plugin-fixture.js") + - set: tmpRoot + value: + expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-cold-")) + - set: agentDir + value: + expr: path.join(tmpRoot, "agents", "qa", "agent") + - try: + actions: + - call: plugin.seedCodexPluginAt + args: + - missing + - ref: agentDir + - call: auth.seedAuthProfiles + args: + - mixed + - ref: agentDir + - set: missing + value: + expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(agentDir), auth: await auth.snapshotAuthProfiles(agentDir), hostVersion: plugin.CODEX_PLUGIN_CURRENT_VERSION })" + - assert: + expr: "missing.status === 'repair-required'" + message: + expr: "`expected repair-required, got ${JSON.stringify(missing)}`" + - assert: + expr: "missing.remediation === config.remediation" + message: missing Codex plugin remediation drifted + - assert: + expr: "missing.selectedAuthProfileId === auth.QA_CODEX_OAUTH_PROFILE_ID" + message: missing-plugin repair must keep Codex OAuth selected + - call: plugin.seedCodexPluginAt + args: + - current + - ref: agentDir + - set: repaired + value: + expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(agentDir), auth: await auth.snapshotAuthProfiles(agentDir), hostVersion: plugin.CODEX_PLUGIN_CURRENT_VERSION })" + - assert: + expr: "repaired.status === 'ready' && repaired.tokenRoute === 'codex-oauth'" + message: + expr: "`expected repaired Codex OAuth route, got ${JSON.stringify(repaired)}`" + finally: + - call: fs.rm + args: + - ref: tmpRoot + - recursive: true + force: true + detailsExpr: "`missing=${missing.status} repaired=${repaired.status} route=${repaired.tokenRoute}`" +``` diff --git a/qa/scenarios/runtime/codex-plugin-install-race.md b/qa/scenarios/runtime/codex-plugin-install-race.md new file mode 100644 index 00000000000..d8eaeda9cf0 --- /dev/null +++ b/qa/scenarios/runtime/codex-plugin-install-race.md @@ -0,0 +1,64 @@ +# Codex plugin install race + +```yaml qa-scenario +id: codex-plugin-install-race +title: Codex plugin install race +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.codex-plugin.lifecycle + secondary: + - runtime.turn-ordering +objective: Verify first agent turns wait on Codex plugin installation through deterministic ordering primitives, without sleep-based race assertions, lost tokens, or duplicate responses. +successCriteria: + - The first turn records a waiting event before the install completion event. + - The turn starts exactly once after the install completion event. + - Input-token accounting survives the gate and responseCount remains 1. +docsRefs: + - docs/cli/plugins.md +codeRefs: + - extensions/qa-lab/src/codex-plugin-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the deterministic install-vs-first-turn gate. + config: + expectedResponseCount: 1 + expectedText: QA_CODEX_PLUGIN_TURN_OK +``` + +```yaml qa-flow +steps: + - name: validates deterministic install-race gate + actions: + - set: plugin + value: + expr: await qaImport("./codex-plugin-fixture.js") + - set: gate + value: + expr: plugin.createCodexPluginInstallGate() + - set: turn + value: + expr: "gate.runFirstTurnAfterInstall({ inputTokens: 17, run: () => config.expectedText })" + - assert: + expr: "JSON.stringify(gate.events) === JSON.stringify(['agent-turn:waiting-for-codex-plugin'])" + message: + expr: "`expected first turn to wait, got ${JSON.stringify(gate.events)}`" + - call: gate.markInstalled + - set: completed + value: + expr: await turn + - assert: + expr: "completed.text === config.expectedText && completed.responseCount === config.expectedResponseCount && completed.inputTokens === 17" + message: + expr: "`unexpected completed turn: ${JSON.stringify(completed)}`" + - assert: + expr: "JSON.stringify(gate.events) === JSON.stringify(['agent-turn:waiting-for-codex-plugin', 'codex-plugin:installed', 'agent-turn:started', 'agent-turn:completed'])" + message: + expr: "`unexpected install ordering: ${JSON.stringify(gate.events)}`" + - assert: + expr: "config.expectedResponseCount === 1" + message: "first turn must produce one response" + detailsExpr: "`expected=${completed.text} count=${completed.responseCount}`" +``` diff --git a/qa/scenarios/runtime/codex-plugin-pinned-new.md b/qa/scenarios/runtime/codex-plugin-pinned-new.md new file mode 100644 index 00000000000..a9c162013c2 --- /dev/null +++ b/qa/scenarios/runtime/codex-plugin-pinned-new.md @@ -0,0 +1,75 @@ +# Codex plugin pinned new + +```yaml qa-scenario +id: codex-plugin-pinned-new +title: Codex plugin pinned new +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.codex-plugin.version +objective: Verify a Codex plugin pinned ahead of the OpenClaw host version fails closed with a precise host-upgrade remediation. +successCriteria: + - The lifecycle fixture detects the plugin version is newer than the host version. + - The failure remediation points to upgrading OpenClaw or installing a Codex plugin pinned to the host version. + - The remediation string is asserted literally by the Phase 3 test. +docsRefs: + - docs/cli/plugins.md + - docs/cli/update.md +codeRefs: + - extensions/qa-lab/src/codex-plugin-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the lifecycle fixture for pinned-new Codex plugin mismatch. + config: + pluginVersion: 2026.5.21 + hostVersion: 2026.5.20 + pluginRelation: newer + remediation: Codex plugin version 2026.5.21 requires a newer OpenClaw host than 2026.5.20. Upgrade OpenClaw or install a codex plugin version pinned to 2026.5.20. +``` + +```yaml qa-flow +steps: + - name: validates pinned-new remediation + actions: + - set: auth + value: + expr: await qaImport("./auth-profile-fixture.js") + - set: plugin + value: + expr: await qaImport("./codex-plugin-fixture.js") + - set: tmpRoot + value: + expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-new-")) + - try: + actions: + - call: plugin.seedCodexPluginAt + args: + - expr: config.pluginVersion + - ref: tmpRoot + - call: auth.seedAuthProfiles + args: + - oauth-only + - ref: tmpRoot + - set: result + value: + expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })" + - assert: + expr: "result.status === 'blocked'" + message: + expr: "`expected blocked pinned-new plugin, got ${JSON.stringify(result)}`" + - assert: + expr: "result.remediation === config.remediation" + message: pinned-new remediation drifted + finally: + - call: fs.rm + args: + - ref: tmpRoot + - recursive: true + force: true + - assert: + expr: "config.pluginRelation === 'newer'" + message: "expected plugin version to be newer than host" + detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`" +``` diff --git a/qa/scenarios/runtime/codex-plugin-pinned-old.md b/qa/scenarios/runtime/codex-plugin-pinned-old.md new file mode 100644 index 00000000000..261a9794efe --- /dev/null +++ b/qa/scenarios/runtime/codex-plugin-pinned-old.md @@ -0,0 +1,75 @@ +# Codex plugin pinned old + +```yaml qa-scenario +id: codex-plugin-pinned-old +title: Codex plugin pinned old +surface: runtime +runtimeParityTier: standard +coverage: + primary: + - runtime.codex-plugin.version +objective: Verify a Codex plugin pinned behind the OpenClaw host version fails closed with a precise update remediation. +successCriteria: + - The lifecycle fixture detects the plugin version is older than the host version. + - The failure remediation points to openclaw plugins update codex or unpinning the plugin, then rerunning doctor. + - The remediation string is asserted literally by the Phase 3 test. +docsRefs: + - docs/cli/plugins.md + - docs/cli/update.md +codeRefs: + - extensions/qa-lab/src/codex-plugin-fixture.ts + - extensions/qa-lab/src/codex-plugin-lifecycle.test.ts +execution: + kind: flow + summary: Exercise the lifecycle fixture for pinned-old Codex plugin mismatch. + config: + pluginVersion: 2026.5.19 + hostVersion: 2026.5.20 + pluginRelation: older + remediation: Codex plugin version 2026.5.19 is older than OpenClaw 2026.5.20. Run "openclaw plugins update codex" or unpin codex, then rerun "openclaw doctor --fix". +``` + +```yaml qa-flow +steps: + - name: validates pinned-old remediation + actions: + - set: auth + value: + expr: await qaImport("./auth-profile-fixture.js") + - set: plugin + value: + expr: await qaImport("./codex-plugin-fixture.js") + - set: tmpRoot + value: + expr: await fs.mkdtemp(path.join(env.gateway?.workspaceDir ?? "/tmp", "qa-codex-old-")) + - try: + actions: + - call: plugin.seedCodexPluginAt + args: + - expr: config.pluginVersion + - ref: tmpRoot + - call: auth.seedAuthProfiles + args: + - oauth-only + - ref: tmpRoot + - set: result + value: + expr: "plugin.evaluateCodexPluginLifecycle({ plugin: await plugin.snapshotCodexPluginState(tmpRoot), auth: await auth.snapshotAuthProfiles(tmpRoot), hostVersion: config.hostVersion })" + - assert: + expr: "result.status === 'blocked'" + message: + expr: "`expected blocked pinned-old plugin, got ${JSON.stringify(result)}`" + - assert: + expr: "result.remediation === config.remediation" + message: pinned-old remediation drifted + finally: + - call: fs.rm + args: + - ref: tmpRoot + - recursive: true + force: true + - assert: + expr: "config.pluginRelation === 'older'" + message: "expected plugin version to be older than host" + detailsExpr: "`plugin=${config.pluginVersion} host=${config.hostVersion} status=${result.status}`" +```