fix(openai): prevent onboarding from offering unavailable account models (#114356)

* fix(openai): keep account models and onboarding release proof authoritative * fix(openai): document configured model catalog authority
2026-08-02 07:01:34 +00:00 · 2026-07-27 02:23:50 -04:00
parent 604f379203
commit fee6db765d
5 changed files with 344 additions and 21 deletions
--- a/.github/workflows/openclaw-release-checks.yml
+++ b/.github/workflows/openclaw-release-checks.yml
@@ -911,7 +911,7 @@ jobs:
      package_source_sha: ${{ needs.prepare_release_package.outputs.source_sha }}
      package_version: ${{ needs.prepare_release_package.outputs.package_version }}
      suite_profile: custom
-      docker_lanes: doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape
+      docker_lanes: release-typed-onboarding doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape
      published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }}
      published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }}
      telegram_mode: mock-openai
--- a/src/agents/model-catalog.test.ts
+++ b/src/agents/model-catalog.test.ts
@@ -5,6 +5,7 @@ import { resolveOAuthApiKeyMarker } from "./model-auth-markers.js";
 import {
  buildPreparedModelCatalogSnapshot,
  findModelCatalogEntry,
+  loadManifestModelCatalog,
  modelSupportsDocument,
  modelSupportsVision,
 } from "./model-catalog.js";
@@ -28,6 +29,31 @@ vi.mock("../plugins/provider-runtime.runtime.js", () => ({

 const metadataSnapshot = { plugins: [] } as unknown as PluginMetadataSnapshot;

+function providerManifestSnapshot(params: {
+  provider: string;
+  discovery: "static" | "refreshable" | "runtime";
+  modelIds: string[];
+}): PluginMetadataSnapshot {
+  const plugin = {
+    id: params.provider,
+    origin: "bundled",
+    providers: [params.provider],
+    modelCatalog: {
+      providers: {
+        [params.provider]: {
+          api: "openai-responses",
+          models: params.modelIds.map((id) => ({ id, name: id })),
+        },
+      },
+      discovery: { [params.provider]: params.discovery },
+    },
+  };
+  return {
+    plugins: [plugin],
+    manifestRegistry: { plugins: [plugin] },
+  } as unknown as PluginMetadataSnapshot;
+}
+
 function registry(entries: ModelCatalogEntry[]): ModelRegistry {
  return { getAll: () => entries } as unknown as ModelRegistry;
 }
@@ -35,6 +61,7 @@ function registry(entries: ModelCatalogEntry[]): ModelRegistry {
 async function build(params: {
  config?: OpenClawConfig;
  entries?: ModelCatalogEntry[];
+  metadataSnapshot?: PluginMetadataSnapshot;
  readOnly?: boolean;
  includeProviderPluginAugmentation?: boolean;
 }) {
@@ -42,7 +69,7 @@ async function build(params: {
    agentDir: "/tmp/model-catalog-test",
    authCredentials: {},
    config: params.config ?? { plugins: { enabled: false } },
-    metadataSnapshot,
+    metadataSnapshot: params.metadataSnapshot ?? metadataSnapshot,
    modelRegistry: registry(params.entries ?? []),
    readOnly: params.readOnly ?? true,
    ...(params.includeProviderPluginAugmentation !== undefined
@@ -78,6 +105,179 @@ describe("prepared model catalog builder", () => {
    expect(snapshot.routeVariants).toEqual(snapshot.entries);
  });

+  it("keeps account-denied runtime models out of the prepared catalog", async () => {
+    const config: OpenClawConfig = { plugins: { enabled: false } };
+    const runtimeManifest = providerManifestSnapshot({
+      provider: "openai",
+      discovery: "runtime",
+      modelIds: ["gpt-5.5", "gpt-5.6"],
+    });
+
+    const declaredManifestModels = loadManifestModelCatalog({
+      config,
+      metadataSnapshot: runtimeManifest,
+    });
+    expect(declaredManifestModels.map((entry) => entry.id)).toEqual(["gpt-5.5", "gpt-5.6"]);
+
+    const snapshot = await build({ config, metadataSnapshot: runtimeManifest });
+
+    expect(snapshot.entries).toEqual([]);
+    expect(snapshot.routeVariants).toEqual([]);
+    expect(loadManifestModelCatalog({ config, metadataSnapshot: runtimeManifest })).toBe(
+      declaredManifestModels,
+    );
+  });
+
+  it("keeps an account's runtime-discovered model list authoritative", async () => {
+    const snapshot = await build({
+      entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }],
+      metadataSnapshot: providerManifestSnapshot({
+        provider: "openai",
+        discovery: "runtime",
+        modelIds: ["gpt-5.5", "gpt-5.6"],
+      }),
+    });
+
+    expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "openai/gpt-5.5",
+    ]);
+    expect(snapshot.routeVariants.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "openai/gpt-5.5",
+    ]);
+  });
+
+  it("does not augment an account with undiscovered runtime-provider models", async () => {
+    mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([
+      { id: "gpt-5.4", name: "GPT-5.4", provider: "openai" },
+      { id: "gpt-5.5", name: "GPT-5.5", provider: "openai", contextWindow: 128_000 },
+    ]);
+
+    const snapshot = await build({
+      entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }],
+      metadataSnapshot: providerManifestSnapshot({
+        provider: "openai",
+        discovery: "runtime",
+        modelIds: ["gpt-5.5", "gpt-5.6"],
+      }),
+      readOnly: false,
+    });
+
+    expect(mocks.augmentModelCatalogWithProviderPlugins).toHaveBeenCalledOnce();
+    expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "openai/gpt-5.5",
+    ]);
+    expect(snapshot.entries[0]?.contextWindow).toBe(128_000);
+    expect(snapshot.routeVariants.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "openai/gpt-5.5",
+    ]);
+  });
+
+  it("preserves explicitly configured runtime-provider models", async () => {
+    mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([
+      {
+        id: "gpt-5.4",
+        name: "GPT-5.4",
+        provider: "openai",
+        api: "openai-responses",
+        baseUrl: "https://catalog.openai.example.test/v1",
+        contextWindow: 256_000,
+        compat: { supportsTools: false },
+      },
+    ]);
+
+    const snapshot = await build({
+      config: {
+        plugins: { enabled: false },
+        models: {
+          providers: {
+            openai: {
+              api: "openai-responses",
+              baseUrl: "https://api.openai.com/v1",
+              models: [
+                {
+                  id: "gpt-5.4",
+                  name: "Configured GPT-5.4",
+                  contextWindow: 128_000,
+                  maxTokens: 4_096,
+                  reasoning: true,
+                  input: ["text", "image"],
+                  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+                },
+              ],
+            },
+          },
+        },
+      },
+      entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }],
+      metadataSnapshot: providerManifestSnapshot({
+        provider: "openai",
+        discovery: "runtime",
+        modelIds: ["gpt-5.5", "gpt-5.6"],
+      }),
+      readOnly: false,
+    });
+
+    expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "openai/gpt-5.4",
+      "openai/gpt-5.5",
+    ]);
+    expect(snapshot.routeVariants).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          provider: "openai",
+          id: "gpt-5.4",
+          api: "openai-responses",
+          baseUrl: "https://catalog.openai.example.test/v1",
+          contextWindow: 256_000,
+          compat: { supportsTools: false },
+        }),
+        expect.objectContaining({
+          provider: "openai",
+          id: "gpt-5.4",
+          api: "openai-responses",
+          baseUrl: "https://api.openai.com/v1",
+        }),
+      ]),
+    );
+  });
+
+  it.each(["static", "refreshable"] as const)(
+    "keeps %s manifest models available without runtime account discovery",
+    async (discovery) => {
+      const snapshot = await build({
+        metadataSnapshot: providerManifestSnapshot({
+          provider: "manifest-provider",
+          discovery,
+          modelIds: ["manifest-model"],
+        }),
+      });
+
+      expect(snapshot.entries).toMatchObject([
+        { provider: "manifest-provider", id: "manifest-model" },
+      ]);
+    },
+  );
+
+  it("preserves augmentation for providers without runtime account discovery", async () => {
+    mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([
+      { id: "synthetic-model", name: "Synthetic model", provider: "manifest-provider" },
+    ]);
+
+    const snapshot = await build({
+      metadataSnapshot: providerManifestSnapshot({
+        provider: "manifest-provider",
+        discovery: "static",
+        modelIds: ["manifest-model"],
+      }),
+      readOnly: false,
+    });
+
+    expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([
+      "manifest-provider/manifest-model",
+      "manifest-provider/synthetic-model",
+    ]);
+  });
+
  it("overlays configured metadata onto discovered rows", async () => {
    const config: OpenClawConfig = {
      plugins: { enabled: false },
--- a/src/agents/model-catalog.ts
+++ b/src/agents/model-catalog.ts
@@ -328,6 +328,21 @@ function createModelCatalogSnapshot(
  };
 }

+function resolveEligibleManifestCatalogPlugins(
+  snapshot: PluginMetadataSnapshot,
+  config: OpenClawConfig,
+): PluginMetadataSnapshot["plugins"] {
+  return snapshot.plugins.filter(
+    (plugin) =>
+      plugin.modelCatalog &&
+      isManifestPluginAvailableForControlPlane({
+        snapshot,
+        plugin,
+        config,
+      }),
+  );
+}
+
 export function loadManifestModelCatalog(params: {
  config: OpenClawConfig;
  workspaceDir?: string;
@@ -357,17 +372,10 @@ export function loadManifestModelCatalog(params: {
  if (cached?.snapshot === resolvedSnapshot) {
    return cached.rows;
  }
-  const eligiblePlugins = resolvedSnapshot.plugins.filter(
-    (plugin) =>
-      plugin.modelCatalog &&
-      isManifestPluginAvailableForControlPlane({
-        snapshot: resolvedSnapshot,
-        plugin,
-        config: params.config,
-      }),
-  );
  const plan = planEffectiveModelCatalogRows({
-    registry: { plugins: eligiblePlugins },
+    registry: {
+      plugins: resolveEligibleManifestCatalogPlugins(resolvedSnapshot, params.config),
+    },
    config: params.config,
  });
  const rows = plan.rows.map((row) => {
@@ -502,11 +510,30 @@ export async function buildPreparedModelCatalogSnapshot(
      models.push(model);
      mergeCatalogRouteVariants(routeVariants, [model]);
    }
+    const supplementalManifestPlan = planEffectiveModelCatalogRows({
+      registry: {
+        plugins: resolveEligibleManifestCatalogPlugins(manifestMetadataSnapshot, cfg),
+      },
+      config: cfg,
+      selection: "supplemental",
+    });
+    const supplementalManifestKeys = new Set(
+      supplementalManifestPlan.rows.map((entry) => catalogEntryDedupeKey(entry.provider, entry.id)),
+    );
+    const runtimeDiscoveryProviders = new Set(
+      supplementalManifestPlan.entries.flatMap((entry) =>
+        entry.discovery === "runtime" ? [normalizeProviderId(entry.provider)] : [],
+      ),
+    );
+    // Runtime declarations describe possible models, not account entitlement.
+    // Only live registry or refreshed rows may publish those provider models.
    const manifestModels = loadManifestModelCatalog({
      config: cfg,
      env,
      metadataSnapshot: manifestMetadataSnapshot,
-    });
+    }).filter((entry) =>
+      supplementalManifestKeys.has(catalogEntryDedupeKey(entry.provider, entry.id)),
+    );
    mergeCatalogRouteVariants(routeVariants, manifestModels);
    mergeCatalogEntries(models, manifestModels);
    logStage("manifest-models-merged", `entries=${models.length}`);
@@ -553,13 +580,35 @@ export async function buildPreparedModelCatalogSnapshot(
        },
      });
      if (supplemental.length > 0) {
+        // Explicitly configured rows are user-authorized even when live
+        // discovery omits them; normalize both sets to preserve their routes.
+        const accountVisibleModelKeys = new Set(
+          [...models, ...configuredModels].map((entry) =>
+            catalogEntryDedupeKey(
+              entry.provider,
+              normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, {
+                manifestPlugins: getManifestPlugins(),
+              }),
+            ),
+          ),
+        );
        const normalizedSupplemental: ModelCatalogEntry[] = [];
        for (const entry of supplemental) {
+          const id = normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, {
+            manifestPlugins: getManifestPlugins(),
+          });
+          // Account-discovered providers own the visible model set. Synthetic
+          // metadata can enrich an available or explicitly configured model,
+          // but must never advertise a model the account did not discover.
+          if (
+            runtimeDiscoveryProviders.has(normalizeProviderId(entry.provider)) &&
+            !accountVisibleModelKeys.has(catalogEntryDedupeKey(entry.provider, id))
+          ) {
+            continue;
+          }
          normalizedSupplemental.push({
            ...entry,
-            id: normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, {
-              manifestPlugins: getManifestPlugins(),
-            }),
+            id,
          });
        }
        mergeCatalogRouteVariants(routeVariants, normalizedSupplemental);
--- a/test/openai-onboarding.live.test.ts
+++ b/test/openai-onboarding.live.test.ts
@@ -10,6 +10,10 @@ import { extractAgentReplyTexts } from "../scripts/e2e/lib/agent-turn-output.mjs
 import { terminateManagedChild } from "../scripts/lib/managed-child-process.mjs";
 import { readPersistedAuthProfileStoreRaw } from "../src/agents/auth-profiles/sqlite.js";
 import { isLiveTestEnabled } from "../src/agents/live-test-helpers.js";
+import {
+  loadTranscriptEvents,
+  resolveTranscriptSessionKeyBySessionId,
+} from "../src/config/sessions/session-accessor.js";
 import { createOpenClawTestState } from "../src/test-utils/openclaw-test-state.js";
 import { getDeterministicFreePortBlock } from "../src/test-utils/ports.js";

@@ -89,6 +93,39 @@ function summarizeAgentOutput(stdout: string): string {
  }
 }

+function hasPersistedTranscriptMessage(
+  events: unknown[],
+  role: "user" | "assistant",
+  expectedText: string,
+): boolean {
+  return events.some((event) => {
+    if (typeof event !== "object" || event === null) {
+      return false;
+    }
+    const record = event as { type?: unknown; message?: unknown };
+    if (record.type !== "message" || typeof record.message !== "object" || !record.message) {
+      return false;
+    }
+    const message = record.message as { role?: unknown; content?: unknown };
+    if (message.role !== role) {
+      return false;
+    }
+    if (typeof message.content === "string") {
+      return message.content.includes(expectedText);
+    }
+    return (
+      Array.isArray(message.content) &&
+      message.content.some(
+        (part: unknown) =>
+          typeof part === "object" &&
+          part !== null &&
+          typeof (part as { text?: unknown }).text === "string" &&
+          (part as { text: string }).text.includes(expectedText),
+      )
+    );
+  });
+}
+
 async function waitForIsolatedGatewayReady(gateway: ChildProcess, port: number): Promise<void> {
  const deadline = Date.now() + 60_000;
  let startupError = false;
@@ -265,15 +302,17 @@ describeLive("fresh OpenAI onboarding live", () => {
      await waitForIsolatedGatewayReady(gateway, gatewayPort);
      await runOpenClaw(["health", "--json"], state.env);

+      const gatewaySessionId = "openai-onboarding-live-gateway";
+      const gatewayPrompt = `Return exactly ${replyMarker} and no other text.`;
      const gatewayStdout = await runOpenClaw(
        [
          "agent",
          "--agent",
          "main",
          "--session-id",
-          "openai-onboarding-live-gateway",
+          gatewaySessionId,
          "--message",
-          `Return exactly ${replyMarker} and no other text.`,
+          gatewayPrompt,
          "--thinking",
          "off",
          "--json",
@@ -284,6 +323,19 @@ describeLive("fresh OpenAI onboarding live", () => {
        extractAgentReplyTexts(gatewayStdout).some((reply) => reply.includes(replyMarker)),
        `gateway-backed OpenAI agent turn returned ${summarizeAgentOutput(gatewayStdout)}`,
      ).toBe(true);
+      const transcriptScope = {
+        agentId: "main",
+        env: state.env,
+        sessionId: gatewaySessionId,
+      };
+      const persistedSessionKey = resolveTranscriptSessionKeyBySessionId(transcriptScope);
+      expect(typeof persistedSessionKey === "string" && persistedSessionKey.length > 0).toBe(true);
+      const persistedEvents = await loadTranscriptEvents({
+        ...transcriptScope,
+        ...(persistedSessionKey ? { sessionKey: persistedSessionKey } : {}),
+      });
+      expect(hasPersistedTranscriptMessage(persistedEvents, "user", gatewayPrompt)).toBe(true);
+      expect(hasPersistedTranscriptMessage(persistedEvents, "assistant", replyMarker)).toBe(true);
      assertOpenAiEnvProfile(state.agentDir());
    } finally {
      await stopIsolatedGateway(gateway);
--- a/test/scripts/package-acceptance-workflow.test.ts
+++ b/test/scripts/package-acceptance-workflow.test.ts
@@ -2432,6 +2432,17 @@ describe("package artifact reuse", () => {
    expect(packageAcceptanceJob.with).toMatchObject({
      allow_frozen_target_scenario_omissions:
        "${{ inputs.allow_frozen_target_scenario_omissions }}",
+      artifact_digest: "${{ needs.prepare_release_package.outputs.artifact_digest }}",
+      artifact_id: "${{ needs.prepare_release_package.outputs.artifact_id }}",
+      artifact_name: "${{ needs.prepare_release_package.outputs.artifact_name }}",
+      artifact_run_attempt: "${{ needs.prepare_release_package.outputs.artifact_run_attempt }}",
+      artifact_run_id: "${{ needs.prepare_release_package.outputs.artifact_run_id }}",
+      package_file_name: "${{ needs.prepare_release_package.outputs.package_file_name }}",
+      package_sha256:
+        "${{ (needs.resolve_target.outputs.package_acceptance_package_spec == '' && needs.resolve_target.outputs.release_package_spec == '') && needs.prepare_release_package.outputs.package_sha256 || '' }}",
+      package_source_sha: "${{ needs.prepare_release_package.outputs.source_sha }}",
+      package_version: "${{ needs.prepare_release_package.outputs.package_version }}",
+      suite_profile: "custom",
    });
    expect(dockerAcceptanceJob.with).toMatchObject({
      allow_frozen_target_scenario_omissions:
@@ -2463,9 +2474,20 @@ describe("package artifact reuse", () => {
      "package_sha256: ${{ (needs.resolve_target.outputs.package_acceptance_package_spec == '' && needs.resolve_target.outputs.release_package_spec == '') && needs.prepare_release_package.outputs.package_sha256 || '' }}",
    );
    expect(workflow).toContain("suite_profile: custom");
-    expect(workflow).toContain(
-      "docker_lanes: doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape",
-    );
+    expect(String(packageAcceptanceJob.with?.docker_lanes ?? "").split(/\s+/u)).toEqual([
+      "release-typed-onboarding",
+      "doctor-switch",
+      "update-channel-switch",
+      "skill-install",
+      "update-corrupt-plugin",
+      "upgrade-survivor",
+      "published-upgrade-survivor",
+      "root-managed-vps-upgrade",
+      "update-restart-auth",
+      "plugins-offline",
+      "plugin-update",
+      "plugin-binding-command-escape",
+    ]);
    expect(workflow).toContain(
      "published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }}",
    );