diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 1c41a686293c..a0bde0359aef 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -911,7 +911,7 @@ jobs: package_source_sha: ${{ needs.prepare_release_package.outputs.source_sha }} package_version: ${{ needs.prepare_release_package.outputs.package_version }} suite_profile: custom - docker_lanes: doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape + docker_lanes: release-typed-onboarding doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }} published_upgrade_survivor_scenarios: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'reported-issues' || '' }} telegram_mode: mock-openai diff --git a/src/agents/model-catalog.test.ts b/src/agents/model-catalog.test.ts index 095c09731dd8..d33f81d8654c 100644 --- a/src/agents/model-catalog.test.ts +++ b/src/agents/model-catalog.test.ts @@ -5,6 +5,7 @@ import { resolveOAuthApiKeyMarker } from "./model-auth-markers.js"; import { buildPreparedModelCatalogSnapshot, findModelCatalogEntry, + loadManifestModelCatalog, modelSupportsDocument, modelSupportsVision, } from "./model-catalog.js"; @@ -28,6 +29,31 @@ vi.mock("../plugins/provider-runtime.runtime.js", () => ({ const metadataSnapshot = { plugins: [] } as unknown as PluginMetadataSnapshot; +function providerManifestSnapshot(params: { + provider: string; + discovery: "static" | "refreshable" | "runtime"; + modelIds: string[]; +}): PluginMetadataSnapshot { + const plugin = { + id: params.provider, + origin: "bundled", + providers: [params.provider], + modelCatalog: { + providers: { + [params.provider]: { + api: "openai-responses", + models: params.modelIds.map((id) => ({ id, name: id })), + }, + }, + discovery: { [params.provider]: params.discovery }, + }, + }; + return { + plugins: [plugin], + manifestRegistry: { plugins: [plugin] }, + } as unknown as PluginMetadataSnapshot; +} + function registry(entries: ModelCatalogEntry[]): ModelRegistry { return { getAll: () => entries } as unknown as ModelRegistry; } @@ -35,6 +61,7 @@ function registry(entries: ModelCatalogEntry[]): ModelRegistry { async function build(params: { config?: OpenClawConfig; entries?: ModelCatalogEntry[]; + metadataSnapshot?: PluginMetadataSnapshot; readOnly?: boolean; includeProviderPluginAugmentation?: boolean; }) { @@ -42,7 +69,7 @@ async function build(params: { agentDir: "/tmp/model-catalog-test", authCredentials: {}, config: params.config ?? { plugins: { enabled: false } }, - metadataSnapshot, + metadataSnapshot: params.metadataSnapshot ?? metadataSnapshot, modelRegistry: registry(params.entries ?? []), readOnly: params.readOnly ?? true, ...(params.includeProviderPluginAugmentation !== undefined @@ -78,6 +105,179 @@ describe("prepared model catalog builder", () => { expect(snapshot.routeVariants).toEqual(snapshot.entries); }); + it("keeps account-denied runtime models out of the prepared catalog", async () => { + const config: OpenClawConfig = { plugins: { enabled: false } }; + const runtimeManifest = providerManifestSnapshot({ + provider: "openai", + discovery: "runtime", + modelIds: ["gpt-5.5", "gpt-5.6"], + }); + + const declaredManifestModels = loadManifestModelCatalog({ + config, + metadataSnapshot: runtimeManifest, + }); + expect(declaredManifestModels.map((entry) => entry.id)).toEqual(["gpt-5.5", "gpt-5.6"]); + + const snapshot = await build({ config, metadataSnapshot: runtimeManifest }); + + expect(snapshot.entries).toEqual([]); + expect(snapshot.routeVariants).toEqual([]); + expect(loadManifestModelCatalog({ config, metadataSnapshot: runtimeManifest })).toBe( + declaredManifestModels, + ); + }); + + it("keeps an account's runtime-discovered model list authoritative", async () => { + const snapshot = await build({ + entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }], + metadataSnapshot: providerManifestSnapshot({ + provider: "openai", + discovery: "runtime", + modelIds: ["gpt-5.5", "gpt-5.6"], + }), + }); + + expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "openai/gpt-5.5", + ]); + expect(snapshot.routeVariants.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "openai/gpt-5.5", + ]); + }); + + it("does not augment an account with undiscovered runtime-provider models", async () => { + mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([ + { id: "gpt-5.4", name: "GPT-5.4", provider: "openai" }, + { id: "gpt-5.5", name: "GPT-5.5", provider: "openai", contextWindow: 128_000 }, + ]); + + const snapshot = await build({ + entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }], + metadataSnapshot: providerManifestSnapshot({ + provider: "openai", + discovery: "runtime", + modelIds: ["gpt-5.5", "gpt-5.6"], + }), + readOnly: false, + }); + + expect(mocks.augmentModelCatalogWithProviderPlugins).toHaveBeenCalledOnce(); + expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "openai/gpt-5.5", + ]); + expect(snapshot.entries[0]?.contextWindow).toBe(128_000); + expect(snapshot.routeVariants.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "openai/gpt-5.5", + ]); + }); + + it("preserves explicitly configured runtime-provider models", async () => { + mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([ + { + id: "gpt-5.4", + name: "GPT-5.4", + provider: "openai", + api: "openai-responses", + baseUrl: "https://catalog.openai.example.test/v1", + contextWindow: 256_000, + compat: { supportsTools: false }, + }, + ]); + + const snapshot = await build({ + config: { + plugins: { enabled: false }, + models: { + providers: { + openai: { + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + models: [ + { + id: "gpt-5.4", + name: "Configured GPT-5.4", + contextWindow: 128_000, + maxTokens: 4_096, + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + }, + }, + }, + }, + entries: [{ id: "gpt-5.5", name: "GPT-5.5", provider: "openai" }], + metadataSnapshot: providerManifestSnapshot({ + provider: "openai", + discovery: "runtime", + modelIds: ["gpt-5.5", "gpt-5.6"], + }), + readOnly: false, + }); + + expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "openai/gpt-5.4", + "openai/gpt-5.5", + ]); + expect(snapshot.routeVariants).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + provider: "openai", + id: "gpt-5.4", + api: "openai-responses", + baseUrl: "https://catalog.openai.example.test/v1", + contextWindow: 256_000, + compat: { supportsTools: false }, + }), + expect.objectContaining({ + provider: "openai", + id: "gpt-5.4", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + }), + ]), + ); + }); + + it.each(["static", "refreshable"] as const)( + "keeps %s manifest models available without runtime account discovery", + async (discovery) => { + const snapshot = await build({ + metadataSnapshot: providerManifestSnapshot({ + provider: "manifest-provider", + discovery, + modelIds: ["manifest-model"], + }), + }); + + expect(snapshot.entries).toMatchObject([ + { provider: "manifest-provider", id: "manifest-model" }, + ]); + }, + ); + + it("preserves augmentation for providers without runtime account discovery", async () => { + mocks.augmentModelCatalogWithProviderPlugins.mockResolvedValueOnce([ + { id: "synthetic-model", name: "Synthetic model", provider: "manifest-provider" }, + ]); + + const snapshot = await build({ + metadataSnapshot: providerManifestSnapshot({ + provider: "manifest-provider", + discovery: "static", + modelIds: ["manifest-model"], + }), + readOnly: false, + }); + + expect(snapshot.entries.map((entry) => `${entry.provider}/${entry.id}`)).toEqual([ + "manifest-provider/manifest-model", + "manifest-provider/synthetic-model", + ]); + }); + it("overlays configured metadata onto discovered rows", async () => { const config: OpenClawConfig = { plugins: { enabled: false }, diff --git a/src/agents/model-catalog.ts b/src/agents/model-catalog.ts index d06f2479b509..e2dac447d103 100644 --- a/src/agents/model-catalog.ts +++ b/src/agents/model-catalog.ts @@ -328,6 +328,21 @@ function createModelCatalogSnapshot( }; } +function resolveEligibleManifestCatalogPlugins( + snapshot: PluginMetadataSnapshot, + config: OpenClawConfig, +): PluginMetadataSnapshot["plugins"] { + return snapshot.plugins.filter( + (plugin) => + plugin.modelCatalog && + isManifestPluginAvailableForControlPlane({ + snapshot, + plugin, + config, + }), + ); +} + export function loadManifestModelCatalog(params: { config: OpenClawConfig; workspaceDir?: string; @@ -357,17 +372,10 @@ export function loadManifestModelCatalog(params: { if (cached?.snapshot === resolvedSnapshot) { return cached.rows; } - const eligiblePlugins = resolvedSnapshot.plugins.filter( - (plugin) => - plugin.modelCatalog && - isManifestPluginAvailableForControlPlane({ - snapshot: resolvedSnapshot, - plugin, - config: params.config, - }), - ); const plan = planEffectiveModelCatalogRows({ - registry: { plugins: eligiblePlugins }, + registry: { + plugins: resolveEligibleManifestCatalogPlugins(resolvedSnapshot, params.config), + }, config: params.config, }); const rows = plan.rows.map((row) => { @@ -502,11 +510,30 @@ export async function buildPreparedModelCatalogSnapshot( models.push(model); mergeCatalogRouteVariants(routeVariants, [model]); } + const supplementalManifestPlan = planEffectiveModelCatalogRows({ + registry: { + plugins: resolveEligibleManifestCatalogPlugins(manifestMetadataSnapshot, cfg), + }, + config: cfg, + selection: "supplemental", + }); + const supplementalManifestKeys = new Set( + supplementalManifestPlan.rows.map((entry) => catalogEntryDedupeKey(entry.provider, entry.id)), + ); + const runtimeDiscoveryProviders = new Set( + supplementalManifestPlan.entries.flatMap((entry) => + entry.discovery === "runtime" ? [normalizeProviderId(entry.provider)] : [], + ), + ); + // Runtime declarations describe possible models, not account entitlement. + // Only live registry or refreshed rows may publish those provider models. const manifestModels = loadManifestModelCatalog({ config: cfg, env, metadataSnapshot: manifestMetadataSnapshot, - }); + }).filter((entry) => + supplementalManifestKeys.has(catalogEntryDedupeKey(entry.provider, entry.id)), + ); mergeCatalogRouteVariants(routeVariants, manifestModels); mergeCatalogEntries(models, manifestModels); logStage("manifest-models-merged", `entries=${models.length}`); @@ -553,13 +580,35 @@ export async function buildPreparedModelCatalogSnapshot( }, }); if (supplemental.length > 0) { + // Explicitly configured rows are user-authorized even when live + // discovery omits them; normalize both sets to preserve their routes. + const accountVisibleModelKeys = new Set( + [...models, ...configuredModels].map((entry) => + catalogEntryDedupeKey( + entry.provider, + normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, { + manifestPlugins: getManifestPlugins(), + }), + ), + ), + ); const normalizedSupplemental: ModelCatalogEntry[] = []; for (const entry of supplemental) { + const id = normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, { + manifestPlugins: getManifestPlugins(), + }); + // Account-discovered providers own the visible model set. Synthetic + // metadata can enrich an available or explicitly configured model, + // but must never advertise a model the account did not discover. + if ( + runtimeDiscoveryProviders.has(normalizeProviderId(entry.provider)) && + !accountVisibleModelKeys.has(catalogEntryDedupeKey(entry.provider, id)) + ) { + continue; + } normalizedSupplemental.push({ ...entry, - id: normalizeConfiguredProviderCatalogModelId(entry.provider, entry.id, { - manifestPlugins: getManifestPlugins(), - }), + id, }); } mergeCatalogRouteVariants(routeVariants, normalizedSupplemental); diff --git a/test/openai-onboarding.live.test.ts b/test/openai-onboarding.live.test.ts index b3eadcbe004f..f5c23a95f2b0 100644 --- a/test/openai-onboarding.live.test.ts +++ b/test/openai-onboarding.live.test.ts @@ -10,6 +10,10 @@ import { extractAgentReplyTexts } from "../scripts/e2e/lib/agent-turn-output.mjs import { terminateManagedChild } from "../scripts/lib/managed-child-process.mjs"; import { readPersistedAuthProfileStoreRaw } from "../src/agents/auth-profiles/sqlite.js"; import { isLiveTestEnabled } from "../src/agents/live-test-helpers.js"; +import { + loadTranscriptEvents, + resolveTranscriptSessionKeyBySessionId, +} from "../src/config/sessions/session-accessor.js"; import { createOpenClawTestState } from "../src/test-utils/openclaw-test-state.js"; import { getDeterministicFreePortBlock } from "../src/test-utils/ports.js"; @@ -89,6 +93,39 @@ function summarizeAgentOutput(stdout: string): string { } } +function hasPersistedTranscriptMessage( + events: unknown[], + role: "user" | "assistant", + expectedText: string, +): boolean { + return events.some((event) => { + if (typeof event !== "object" || event === null) { + return false; + } + const record = event as { type?: unknown; message?: unknown }; + if (record.type !== "message" || typeof record.message !== "object" || !record.message) { + return false; + } + const message = record.message as { role?: unknown; content?: unknown }; + if (message.role !== role) { + return false; + } + if (typeof message.content === "string") { + return message.content.includes(expectedText); + } + return ( + Array.isArray(message.content) && + message.content.some( + (part: unknown) => + typeof part === "object" && + part !== null && + typeof (part as { text?: unknown }).text === "string" && + (part as { text: string }).text.includes(expectedText), + ) + ); + }); +} + async function waitForIsolatedGatewayReady(gateway: ChildProcess, port: number): Promise { const deadline = Date.now() + 60_000; let startupError = false; @@ -265,15 +302,17 @@ describeLive("fresh OpenAI onboarding live", () => { await waitForIsolatedGatewayReady(gateway, gatewayPort); await runOpenClaw(["health", "--json"], state.env); + const gatewaySessionId = "openai-onboarding-live-gateway"; + const gatewayPrompt = `Return exactly ${replyMarker} and no other text.`; const gatewayStdout = await runOpenClaw( [ "agent", "--agent", "main", "--session-id", - "openai-onboarding-live-gateway", + gatewaySessionId, "--message", - `Return exactly ${replyMarker} and no other text.`, + gatewayPrompt, "--thinking", "off", "--json", @@ -284,6 +323,19 @@ describeLive("fresh OpenAI onboarding live", () => { extractAgentReplyTexts(gatewayStdout).some((reply) => reply.includes(replyMarker)), `gateway-backed OpenAI agent turn returned ${summarizeAgentOutput(gatewayStdout)}`, ).toBe(true); + const transcriptScope = { + agentId: "main", + env: state.env, + sessionId: gatewaySessionId, + }; + const persistedSessionKey = resolveTranscriptSessionKeyBySessionId(transcriptScope); + expect(typeof persistedSessionKey === "string" && persistedSessionKey.length > 0).toBe(true); + const persistedEvents = await loadTranscriptEvents({ + ...transcriptScope, + ...(persistedSessionKey ? { sessionKey: persistedSessionKey } : {}), + }); + expect(hasPersistedTranscriptMessage(persistedEvents, "user", gatewayPrompt)).toBe(true); + expect(hasPersistedTranscriptMessage(persistedEvents, "assistant", replyMarker)).toBe(true); assertOpenAiEnvProfile(state.agentDir()); } finally { await stopIsolatedGateway(gateway); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 8bf4e7d0b3f2..ff386f841951 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -2432,6 +2432,17 @@ describe("package artifact reuse", () => { expect(packageAcceptanceJob.with).toMatchObject({ allow_frozen_target_scenario_omissions: "${{ inputs.allow_frozen_target_scenario_omissions }}", + artifact_digest: "${{ needs.prepare_release_package.outputs.artifact_digest }}", + artifact_id: "${{ needs.prepare_release_package.outputs.artifact_id }}", + artifact_name: "${{ needs.prepare_release_package.outputs.artifact_name }}", + artifact_run_attempt: "${{ needs.prepare_release_package.outputs.artifact_run_attempt }}", + artifact_run_id: "${{ needs.prepare_release_package.outputs.artifact_run_id }}", + package_file_name: "${{ needs.prepare_release_package.outputs.package_file_name }}", + package_sha256: + "${{ (needs.resolve_target.outputs.package_acceptance_package_spec == '' && needs.resolve_target.outputs.release_package_spec == '') && needs.prepare_release_package.outputs.package_sha256 || '' }}", + package_source_sha: "${{ needs.prepare_release_package.outputs.source_sha }}", + package_version: "${{ needs.prepare_release_package.outputs.package_version }}", + suite_profile: "custom", }); expect(dockerAcceptanceJob.with).toMatchObject({ allow_frozen_target_scenario_omissions: @@ -2463,9 +2474,20 @@ describe("package artifact reuse", () => { "package_sha256: ${{ (needs.resolve_target.outputs.package_acceptance_package_spec == '' && needs.resolve_target.outputs.release_package_spec == '') && needs.prepare_release_package.outputs.package_sha256 || '' }}", ); expect(workflow).toContain("suite_profile: custom"); - expect(workflow).toContain( - "docker_lanes: doctor-switch update-channel-switch skill-install update-corrupt-plugin upgrade-survivor published-upgrade-survivor root-managed-vps-upgrade update-restart-auth plugins-offline plugin-update plugin-binding-command-escape", - ); + expect(String(packageAcceptanceJob.with?.docker_lanes ?? "").split(/\s+/u)).toEqual([ + "release-typed-onboarding", + "doctor-switch", + "update-channel-switch", + "skill-install", + "update-corrupt-plugin", + "upgrade-survivor", + "published-upgrade-survivor", + "root-managed-vps-upgrade", + "update-restart-auth", + "plugins-offline", + "plugin-update", + "plugin-binding-command-escape", + ]); expect(workflow).toContain( "published_upgrade_survivor_baselines: ${{ needs.resolve_target.outputs.run_release_soak == 'true' && 'last-stable-4 2026.4.23 2026.5.2 2026.4.15' || '' }}", );